diff --git a/examples/NoiseSensor/NoiseSensor.ino b/examples/NoiseSensor/NoiseSensor.ino index 5f5a4cc..1091751 100644 --- a/examples/NoiseSensor/NoiseSensor.ino +++ b/examples/NoiseSensor/NoiseSensor.ino @@ -1079,7 +1079,7 @@ void setup() if (!soundDetector.begin(16000)) { Serial.println("Sound detector init failed (ES7210 mics)"); } else { - Serial.println("Sound detector ready (dual mics, slots auto-detected)"); + Serial.println("Sound detector ready (dual mics, pair learned at runtime)"); } } diff --git a/examples/NoiseSensor/sound_detector.cpp b/examples/NoiseSensor/sound_detector.cpp index bda619c..e5d20d0 100644 --- a/examples/NoiseSensor/sound_detector.cpp +++ b/examples/NoiseSensor/sound_detector.cpp @@ -2,12 +2,16 @@ * @file sound_detector.cpp * @brief ES7210 dual-microphone capture + stereo balance/loudness detector. * - * The T-Deck Plus has two populated MEMS microphones on the ES7210. The codec - * outputs all four ADC channels as a 4-slot TDM stream on SDOUT1: - * slot 0 = MIC1, slot 1 = MIC2, slot 2 = MIC3, slot 3 = MIC4 - * The board only fits two of the four footprints, and the schematic has all - * four, so the detector finds the live pair at runtime (the first loud sound - * identifies the two slots with signal) and treats the lower slot as left. + * The T-Deck Plus carries an ES7210 ADC with two populated MEMS microphones. + * The codec emits a fixed 4-slot TDM frame ([MIC1, MIC2, MIC3, MIC4]); only two + * of those slots are wired to a real microphone. Which two depends on the board + * revision, so the detector starts from the documented MIC1/MIC3 pair (slots 0 + * and 2) and re-learns the pair from the slots that actually respond to sound. + * + * The ESP32 DMA may pack each 16-bit TDM slot as a tight 16-bit word (4 int16 + * per frame) or pad it into one half of a 32-bit word (8 int16 per frame). + * Reading a padded stream with the packed stride mixes the microphones + * together, so the detector probes the stream at start-up and adapts. * * The detector runs in its own FreeRTOS task so the radar loop never blocks on * I2S. It publishes smoothed, noise-floor-relative levels that the sketch turns @@ -22,7 +26,7 @@ #define MIC_I2S_PORT I2S_NUM_1 #define MIC_BLOCK_FRAMES 256 // 16 ms at 16 kHz #define MIC_SLOTS 4 // 4-slot TDM -#define MIC_GAIN GAIN_18DB +#define MIC_GAIN GAIN_6DB // Set to 1 to print detected levels while tuning the microphones on hardware. #ifndef SOUND_DETECTOR_DEBUG @@ -43,19 +47,27 @@ static constexpr float FLOOR_MARGIN = 1.25f; static constexpr float ABS_GATE = 0.0008f; static constexpr float BALANCE_DEADBAND = 0.02f; static constexpr float BALANCE_GAIN = 1.8f; // widen the usable stereo range -static constexpr float LOUD_MIN_DB = -50.0f; +static constexpr float LOUD_MIN_DB = -55.0f; static constexpr float LOUD_MAX_DB = -6.0f; static constexpr uint32_t HOLD_MS = 260; -// Slot discovery: the codec emits four TDM slots but only two microphones are -// fitted. Whenever a clear sound appears, take the two loudest slots as the mic -// pair. The current pair is only replaced after the new pair repeats, or -// immediately if the current pair is silent while the new one is loud. -static constexpr float SLOT_SELECT_LEVEL = 0.0020f; -static constexpr float SLOT_PAIR_RATIO = 0.10f; -static constexpr float SLOT_DEAD_RATIO = 0.25f; -static constexpr int SLOT_SWITCH_BLOCKS = 3; -static constexpr uint32_t WARMUP_MS = 800; +// Populated-slot discovery. A fitted microphone swings far above its own noise +// floor; a floating or railed input stays flat (the high-pass removes DC). +static constexpr float SLOT_MIN_SWING = 0.010f; // 1% of full scale +static constexpr float SLOT_SECOND_RATIO = 0.15f; +static constexpr float SLOT_THIRD_RATIO = 0.30f; +static constexpr float SLOT_DEAD_RATIO = 0.50f; +static constexpr int SLOT_SWITCH_BLOCKS = 4; +static constexpr float SLOT_MAX_DECAY = 0.997f; // peak hold, ~3.7 s half-life +static constexpr float SLOT_MIN_RISE = 0.002f; // slow, so gaps keep the floor + +// A channel parked at the rails (overdriven analogue gain) carries no usable +// audio; ignore it rather than reporting a permanent maximum. +static constexpr float RAIL_LEVEL = 0.50f; + +// The codec always delivers four TDM slots, but only two are fitted. The +// power-up transient is very loud and can be mistaken for signal. +static constexpr uint32_t WARMUP_MS = 800; // ES7210 SDP interface 2 / high-pass registers. The LilyGO driver leaves the // codec in 2-channel I2S mode and the HPF disabled; enable both here. @@ -91,7 +103,8 @@ bool SoundDetector::begin(uint32_t sampleRate) // Put all four ADCs on SDOUT1 as TDM slots. Without 0x02 on this register // the codec stays in 2-channel mode and MIC3/MIC4 are only sent on SDOUT2, - // which is not wired to the ESP32. + // which is not wired to the ESP32. All four slots are emitted even though + // only two are populated, so the frame boundary never shifts. if (!es7210WriteReg(ES7210_SDP_INTERFACE2, 0x02)) { Serial.println("ES7210 TDM config failed"); return false; @@ -188,65 +201,130 @@ void SoundDetector::taskLoop() continue; } - size_t frames = bytesRead / (MIC_SLOTS * sizeof(int16_t)); - if (frames == 0) continue; + size_t sampleCount = bytesRead / sizeof(int16_t); + if (sampleCount == 0) continue; + + if (!layoutReady_) { + probeLayout(block, sampleCount); + } if ((int32_t)(millis() - warmupUntil_) < 0 || (int32_t)(millis() - suppressUntil_) < 0) { // Still drain I2S so the DMA doesn't overflow while muted. Drop the - // envelopes too, otherwise the device's own ping or the ADC power-up - // transient would be the first thing the detector reports. + // envelopes too, otherwise the device's own ping or the ADC + // power-up transient would be the first thing we report. envL_ = 0.0f; envR_ = 0.0f; publish(0.0f, 0.0f, 0.0f, 0.0f, false); continue; } - processBlock(block, frames); + if (!layoutReady_) { + finaliseLayout(); + } + + processBlock(block, sampleCount); } } -void SoundDetector::processBlock(const int16_t *frames, size_t frameCount) +size_t SoundDetector::sampleOffset(int slot) const { - // RMS of every TDM slot. Two of the four slots are silent because the - // board only fits two microphones. - float rms[MIC_SLOTS] = {}; + if (frameWords_ == 8) { + return (size_t)slot * 2 + sampleHalf_; + } + return (size_t)slot; +} + +void SoundDetector::probeLayout(const int16_t *samples, size_t sampleCount) +{ + if (probeStartMs_ == 0) probeStartMs_ = millis(); + probeSamples_ += sampleCount; + + // Energy per position inside an 8-word (two-frame) window. Packed frames + // light up the low four positions; padded frames light up one half only. + for (size_t i = 0; i < sampleCount; i++) { + uint32_t pos = probeIndex_ & 7; + double v = samples[i]; + posSq_[pos] += v * v; + posCount_[pos]++; + probeIndex_++; + } +} + +void SoundDetector::finaliseLayout() +{ + uint32_t elapsed = millis() - probeStartMs_; + double bytesPerSec = elapsed ? (double)probeSamples_ * 2.0 * 1000.0 / elapsed : 0.0; + double packedRate = (double)sampleRate_ * MIC_SLOTS * 2.0; + double paddedRate = packedRate * 2.0; + + // The sample rate is fixed, so the byte rate directly reveals whether each + // 16-bit slot occupies one word or half of a 32-bit word. + frameWords_ = (bytesPerSec > (packedRate + paddedRate) * 0.5) ? 8 : 4; + sampleHalf_ = 0; + if (frameWords_ == 8) { + double evenEnergy = posSq_[0] + posSq_[2] + posSq_[4] + posSq_[6]; + double oddEnergy = posSq_[1] + posSq_[3] + posSq_[5] + posSq_[7]; + sampleHalf_ = (oddEnergy >= evenEnergy) ? 1 : 0; + } + + layoutReady_ = true; + Serial.printf("Sound detector DMA layout: %s (%s half), " + "%.0f bytes/s (packed %.0f, padded %.0f)\n", + frameWords_ == 8 ? "32-bit words" : "packed 16-bit", + sampleHalf_ ? "high" : "low", + bytesPerSec, packedRate, paddedRate); + Serial.print("Sound detector slot energy:"); + for (int i = 0; i < 8; i++) { + double rms = posCount_[i] ? sqrt(posSq_[i] / posCount_[i]) : 0.0; + Serial.printf(" %d:%.0f", i, rms); + } + Serial.println(); +} + +void SoundDetector::updateSlots(const float *rms) +{ + float swing[MIC_SLOTS]; for (int s = 0; s < MIC_SLOTS; s++) { - int64_t sum = 0, sq = 0; - for (size_t i = 0; i < frameCount; i++) { - int32_t v = frames[i * MIC_SLOTS + s]; - sum += v; - sq += (int64_t)v * v; + // Track a decaying maximum and a slowly rising minimum. A real + // microphone has quiet gaps between sounds, so its minimum stays low + // while its maximum follows the peaks. A floating or railed slot has + // both pinned at the same level and therefore no range. + if (rms[s] > slotMax_[s]) { + slotMax_[s] = rms[s]; + } else { + slotMax_[s] *= SLOT_MAX_DECAY; } - float mean = (float)sum / (float)frameCount; - float var = (float)sq / (float)frameCount - mean * mean; - if (var < 0.0f) var = 0.0f; - rms[s] = sqrtf(var) / 32768.0f; + if (rms[s] < slotMin_[s]) { + slotMin_[s] = rms[s]; + } else { + slotMin_[s] += (rms[s] - slotMin_[s]) * SLOT_MIN_RISE; + } + swing[s] = slotMax_[s] - slotMin_[s]; + if (swing[s] < 0.0f) swing[s] = 0.0f; } - // Find the two loudest slots. With only two microphones fitted, a real - // sound shows up in exactly those slots; the other two stay near the noise - // floor. Re-evaluate on every clear sound so a wrong initial guess (for - // example caused by the codec's power-up transient) cannot stick forever. - int best = 0, second = 1; - if (rms[second] > rms[best]) { - int t = best; best = second; second = t; - } - for (int s = 2; s < MIC_SLOTS; s++) { - if (rms[s] > rms[best]) { - second = best; - best = s; - } else if (rms[s] > rms[second]) { - second = s; + // Rank the slots by how much they respond to sound. + int order[MIC_SLOTS] = { 0, 1, 2, 3 }; + for (int i = 0; i < MIC_SLOTS; i++) { + for (int j = i + 1; j < MIC_SLOTS; j++) { + if (swing[order[j]] > swing[order[i]]) { + int t = order[i]; order[i] = order[j]; order[j] = t; + } } } + float s0 = swing[order[0]]; + float s1 = swing[order[1]]; + float s2 = swing[order[2]]; - if (rms[best] > SLOT_SELECT_LEVEL && - rms[second] > rms[best] * SLOT_PAIR_RATIO) { - int lo = best < second ? best : second; - int hi = best < second ? second : best; + bool valid = s0 > SLOT_MIN_SWING && + s1 > s0 * SLOT_SECOND_RATIO && + s2 < s1 * SLOT_THIRD_RATIO; + + if (valid) { + int lo = order[0] < order[1] ? order[0] : order[1]; + int hi = order[0] < order[1] ? order[1] : order[0]; if (lo != leftSlot_ || hi != rightSlot_) { - float currentLevel = fmaxf(rms[leftSlot_], rms[rightSlot_]); if (lo == candL_ && hi == candR_) { candCount_++; } else { @@ -254,9 +332,8 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount) candR_ = hi; candCount_ = 1; } - // Switch immediately when the current pair is effectively dead, - // otherwise require the new pair to repeat for a few blocks. - if (currentLevel < rms[best] * SLOT_DEAD_RATIO || + float currentSwing = fmaxf(swing[leftSlot_], swing[rightSlot_]); + if (currentSwing < s0 * SLOT_DEAD_RATIO || candCount_ >= SLOT_SWITCH_BLOCKS) { leftSlot_ = lo; rightSlot_ = hi; @@ -265,27 +342,76 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount) envR_ = 0.0f; floorL_ = 0.01f; floorR_ = 0.01f; - Serial.printf("Sound detector using mic slots L=%d R=%d " - "(rms %.4f %.4f %.4f %.4f)\n", + Serial.printf("Sound detector mic slots L=%d R=%d " + "(swing %.4f %.4f %.4f %.4f)\n", leftSlot_, rightSlot_, - rms[0], rms[1], rms[2], rms[3]); + swing[0], swing[1], swing[2], swing[3]); } } else { candCount_ = 0; } + } else { + candCount_ = 0; } +} + +void SoundDetector::processBlock(const int16_t *samples, size_t sampleCount) +{ + size_t frames = sampleCount / frameWords_; + if (frames == 0) return; + + // RMS of every TDM slot, using the word packing discovered at start-up. + float rms[MIC_SLOTS] = {}; + for (int s = 0; s < MIC_SLOTS; s++) { + size_t off = sampleOffset(s); + int64_t sum = 0, sq = 0; + for (size_t i = 0; i < frames; i++) { + int32_t v = samples[i * frameWords_ + off]; + sum += v; + sq += (int64_t)v * v; + } + float mean = (float)sum / (float)frames; + float var = (float)sq / (float)frames - mean * mean; + if (var < 0.0f) var = 0.0f; + rms[s] = sqrtf(var) / 32768.0f; + } + + updateSlots(rms); + +#if SOUND_DETECTOR_DEBUG + if (!rawDumped_) { + rawDumped_ = true; + Serial.print("Sound detector raw:"); + for (int i = 0; i < 24 && i < (int)sampleCount; i++) { + Serial.printf(" %04x", (uint16_t)samples[i]); + } + Serial.println(); + } +#endif float rmsL = rms[MIC_SWAP_LR ? rightSlot_ : leftSlot_]; float rmsR = rms[MIC_SWAP_LR ? leftSlot_ : rightSlot_]; + if (rmsL > RAIL_LEVEL) rmsL = 0.0f; + if (rmsR > RAIL_LEVEL) rmsR = 0.0f; // Normalise to full scale, then smooth with a fast attack / slow release. envL_ += (rmsL - envL_) * (rmsL > envL_ ? ENV_ATTACK : ENV_RELEASE); envR_ += (rmsR - envR_) * (rmsR > envR_ ? ENV_ATTACK : ENV_RELEASE); - // Per-channel noise floor: fall quickly when it gets quieter, creep up very - // slowly so sustained noise does not desensitise the detector. - floorL_ += (envL_ - floorL_) * (envL_ < floorL_ ? FLOOR_FALL : FLOOR_RISE); - floorR_ += (envR_ - floorR_) * (envR_ < floorR_ ? FLOOR_FALL : FLOOR_RISE); + // Per-channel noise floor: fall quickly when it gets quieter, and only + // creep up while the input is actually close to the floor. Without that + // guard a sustained loud sound (or a clipped channel) drags the floor up to + // its own level and the detector goes permanently deaf. + if (envL_ < floorL_) { + floorL_ += (envL_ - floorL_) * FLOOR_FALL; + } else if (envL_ < floorL_ * 1.5f) { + floorL_ += (envL_ - floorL_) * FLOOR_RISE; + } + if (envR_ < floorR_) { + floorR_ += (envR_ - floorR_) * FLOOR_FALL; + } else if (envR_ < floorR_ * 1.5f) { + floorR_ += (envR_ - floorR_) * FLOOR_RISE; + } if (floorL_ < 1e-5f) floorL_ = 1e-5f; if (floorR_ < 1e-5f) floorR_ = 1e-5f; @@ -329,11 +455,13 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount) #if SOUND_DETECTOR_DEBUG static uint32_t lastPrintMs = 0; - if ((uint32_t)(millis() - lastPrintMs) > 1000) { + if ((uint32_t)(millis() - lastPrintMs) > 300) { lastPrintMs = millis(); - Serial.printf("mic rms %.4f %.4f %.4f %.4f | pair L%d R%d | " - "sig %.3f %.3f | bal %+.2f loud %.2f%s\n", + Serial.printf("mic rms %.4f %.4f %.4f %.4f | range %.4f %.4f %.4f %.4f\n", rms[0], rms[1], rms[2], rms[3], + slotMax_[0] - slotMin_[0], slotMax_[1] - slotMin_[1], + slotMax_[2] - slotMin_[2], slotMax_[3] - slotMin_[3]); + Serial.printf(" pair L%d R%d | sig %.3f %.3f | bal %+.2f loud %.2f%s\n", leftSlot_, rightSlot_, sigL, sigR, balance, loudness, active ? "" : " (quiet)"); } diff --git a/examples/NoiseSensor/sound_detector.h b/examples/NoiseSensor/sound_detector.h index c05107b..53fb1ec 100644 --- a/examples/NoiseSensor/sound_detector.h +++ b/examples/NoiseSensor/sound_detector.h @@ -2,10 +2,12 @@ * @file sound_detector.h * @brief Dual-microphone sound localizer for the T-Deck Plus (ES7210). * - * Reads the ES7210's four TDM slots and uses the two slots that carry a - * populated MEMS microphone (the board only fits two of the four footprints). - * Balance is taken from the level difference between the pair, so a sound on - * the left or right side moves a radar blip accordingly. + * The ES7210 emits a 4-slot TDM frame. On the T-Deck/T-Deck Plus only two + * MEMS footprints are populated, and different board revisions populate + * different slots, so the detector learns which two slots actually respond to + * sound instead of assuming a fixed pair. Balance is taken from the level + * difference between those two microphones, so a sound on the left or right + * side moves a radar blip accordingly. */ #pragma once @@ -38,7 +40,11 @@ public: private: static void taskEntry(void *arg); void taskLoop(); - void processBlock(const int16_t *frames, size_t frameCount); + void probeLayout(const int16_t *samples, size_t sampleCount); + void finaliseLayout(); + void updateSlots(const float *rms); + size_t sampleOffset(int slot) const; + void processBlock(const int16_t *samples, size_t sampleCount); void publish(float left, float right, float balance, float loudness, bool active); uint32_t sampleRate_ = 16000; @@ -55,12 +61,34 @@ private: float floorL_ = 0.01f; float floorR_ = 0.01f; - // The codec always delivers four TDM slots, but only two microphones are - // fitted. The pair is (re)discovered from the slots that carry signal. + // The two TDM slots carrying the populated microphones. Seeded with the + // documented T-Deck pair (MIC1 = slot 0, MIC3 = slot 2) and corrected at + // runtime by updateSlots() on the first sounds that arrive. int leftSlot_ = 0; - int rightSlot_ = 1; + int rightSlot_ = 2; + uint32_t warmupUntil_ = 0; + + // Start-up layout probe: the driver may pack the 16-bit TDM slots tightly + // (4 int16 per frame) or pad each sample into one half of a 32-bit word + // (8 int16 per frame). Both the data rate and the energy split between even + // and odd int16 words identify the packing. + uint32_t probeStartMs_ = 0; + uint32_t probeSamples_ = 0; + uint32_t probeIndex_ = 0; + double posSq_[8] = {}; + uint32_t posCount_[8] = {}; + bool layoutReady_ = false; + size_t frameWords_ = 4; + uint8_t sampleHalf_ = 0; // 0 = low half of a 32-bit word, 1 = high half + bool rawDumped_ = false; + + // Per-slot level range used to discover the populated pair. A fitted + // microphone swings between its quiet floor and the sound peaks; a floating + // input stays flat (its DC offset is removed by the high-pass filter), and + // a railed channel stays flat too. Range separates those cases cleanly. + float slotMin_[4] = { 1.0f, 1.0f, 1.0f, 1.0f }; + float slotMax_[4] = {}; int candL_ = -1; int candR_ = -1; int candCount_ = 0; - uint32_t warmupUntil_ = 0; };