Learn the populated microphone slots and DMA packing at runtime

This commit is contained in:
Dejvino
2026-11-12 14:54:47 +01:00
parent 129c670ff4
commit 315cdbd177
3 changed files with 234 additions and 78 deletions
+1 -1
View File
@@ -1079,7 +1079,7 @@ void setup()
if (!soundDetector.begin(16000)) {
Serial.println("Sound detector init failed (ES7210 mics)");
} else {
Serial.println("Sound detector ready (dual mics, slots auto-detected)");
Serial.println("Sound detector ready (dual mics, pair learned at runtime)");
}
}
+196 -68
View File
@@ -2,12 +2,16 @@
* @file sound_detector.cpp
* @brief ES7210 dual-microphone capture + stereo balance/loudness detector.
*
* The T-Deck Plus has two populated MEMS microphones on the ES7210. The codec
* outputs all four ADC channels as a 4-slot TDM stream on SDOUT1:
* slot 0 = MIC1, slot 1 = MIC2, slot 2 = MIC3, slot 3 = MIC4
* The board only fits two of the four footprints, and the schematic has all
* four, so the detector finds the live pair at runtime (the first loud sound
* identifies the two slots with signal) and treats the lower slot as left.
* The T-Deck Plus carries an ES7210 ADC with two populated MEMS microphones.
* The codec emits a fixed 4-slot TDM frame ([MIC1, MIC2, MIC3, MIC4]); only two
* of those slots are wired to a real microphone. Which two depends on the board
* revision, so the detector starts from the documented MIC1/MIC3 pair (slots 0
* and 2) and re-learns the pair from the slots that actually respond to sound.
*
* The ESP32 DMA may pack each 16-bit TDM slot as a tight 16-bit word (4 int16
* per frame) or pad it into one half of a 32-bit word (8 int16 per frame).
* Reading a padded stream with the packed stride mixes the microphones
* together, so the detector probes the stream at start-up and adapts.
*
* The detector runs in its own FreeRTOS task so the radar loop never blocks on
* I2S. It publishes smoothed, noise-floor-relative levels that the sketch turns
@@ -22,7 +26,7 @@
#define MIC_I2S_PORT I2S_NUM_1
#define MIC_BLOCK_FRAMES 256 // 16 ms at 16 kHz
#define MIC_SLOTS 4 // 4-slot TDM
#define MIC_GAIN GAIN_18DB
#define MIC_GAIN GAIN_6DB
// Set to 1 to print detected levels while tuning the microphones on hardware.
#ifndef SOUND_DETECTOR_DEBUG
@@ -43,19 +47,27 @@ static constexpr float FLOOR_MARGIN = 1.25f;
static constexpr float ABS_GATE = 0.0008f;
static constexpr float BALANCE_DEADBAND = 0.02f;
static constexpr float BALANCE_GAIN = 1.8f; // widen the usable stereo range
static constexpr float LOUD_MIN_DB = -50.0f;
static constexpr float LOUD_MIN_DB = -55.0f;
static constexpr float LOUD_MAX_DB = -6.0f;
static constexpr uint32_t HOLD_MS = 260;
// Slot discovery: the codec emits four TDM slots but only two microphones are
// fitted. Whenever a clear sound appears, take the two loudest slots as the mic
// pair. The current pair is only replaced after the new pair repeats, or
// immediately if the current pair is silent while the new one is loud.
static constexpr float SLOT_SELECT_LEVEL = 0.0020f;
static constexpr float SLOT_PAIR_RATIO = 0.10f;
static constexpr float SLOT_DEAD_RATIO = 0.25f;
static constexpr int SLOT_SWITCH_BLOCKS = 3;
static constexpr uint32_t WARMUP_MS = 800;
// Populated-slot discovery. A fitted microphone swings far above its own noise
// floor; a floating or railed input stays flat (the high-pass removes DC).
static constexpr float SLOT_MIN_SWING = 0.010f; // 1% of full scale
static constexpr float SLOT_SECOND_RATIO = 0.15f;
static constexpr float SLOT_THIRD_RATIO = 0.30f;
static constexpr float SLOT_DEAD_RATIO = 0.50f;
static constexpr int SLOT_SWITCH_BLOCKS = 4;
static constexpr float SLOT_MAX_DECAY = 0.997f; // peak hold, ~3.7 s half-life
static constexpr float SLOT_MIN_RISE = 0.002f; // slow, so gaps keep the floor
// A channel parked at the rails (overdriven analogue gain) carries no usable
// audio; ignore it rather than reporting a permanent maximum.
static constexpr float RAIL_LEVEL = 0.50f;
// The codec always delivers four TDM slots, but only two are fitted. The
// power-up transient is very loud and can be mistaken for signal.
static constexpr uint32_t WARMUP_MS = 800;
// ES7210 SDP interface 2 / high-pass registers. The LilyGO driver leaves the
// codec in 2-channel I2S mode and the HPF disabled; enable both here.
@@ -91,7 +103,8 @@ bool SoundDetector::begin(uint32_t sampleRate)
// Put all four ADCs on SDOUT1 as TDM slots. Without 0x02 on this register
// the codec stays in 2-channel mode and MIC3/MIC4 are only sent on SDOUT2,
// which is not wired to the ESP32.
// which is not wired to the ESP32. All four slots are emitted even though
// only two are populated, so the frame boundary never shifts.
if (!es7210WriteReg(ES7210_SDP_INTERFACE2, 0x02)) {
Serial.println("ES7210 TDM config failed");
return false;
@@ -188,65 +201,130 @@ void SoundDetector::taskLoop()
continue;
}
size_t frames = bytesRead / (MIC_SLOTS * sizeof(int16_t));
if (frames == 0) continue;
size_t sampleCount = bytesRead / sizeof(int16_t);
if (sampleCount == 0) continue;
if (!layoutReady_) {
probeLayout(block, sampleCount);
}
if ((int32_t)(millis() - warmupUntil_) < 0 ||
(int32_t)(millis() - suppressUntil_) < 0) {
// Still drain I2S so the DMA doesn't overflow while muted. Drop the
// envelopes too, otherwise the device's own ping or the ADC power-up
// transient would be the first thing the detector reports.
// envelopes too, otherwise the device's own ping or the ADC
// power-up transient would be the first thing we report.
envL_ = 0.0f;
envR_ = 0.0f;
publish(0.0f, 0.0f, 0.0f, 0.0f, false);
continue;
}
processBlock(block, frames);
if (!layoutReady_) {
finaliseLayout();
}
processBlock(block, sampleCount);
}
}
void SoundDetector::processBlock(const int16_t *frames, size_t frameCount)
size_t SoundDetector::sampleOffset(int slot) const
{
// RMS of every TDM slot. Two of the four slots are silent because the
// board only fits two microphones.
float rms[MIC_SLOTS] = {};
if (frameWords_ == 8) {
return (size_t)slot * 2 + sampleHalf_;
}
return (size_t)slot;
}
void SoundDetector::probeLayout(const int16_t *samples, size_t sampleCount)
{
if (probeStartMs_ == 0) probeStartMs_ = millis();
probeSamples_ += sampleCount;
// Energy per position inside an 8-word (two-frame) window. Packed frames
// light up the low four positions; padded frames light up one half only.
for (size_t i = 0; i < sampleCount; i++) {
uint32_t pos = probeIndex_ & 7;
double v = samples[i];
posSq_[pos] += v * v;
posCount_[pos]++;
probeIndex_++;
}
}
void SoundDetector::finaliseLayout()
{
uint32_t elapsed = millis() - probeStartMs_;
double bytesPerSec = elapsed ? (double)probeSamples_ * 2.0 * 1000.0 / elapsed : 0.0;
double packedRate = (double)sampleRate_ * MIC_SLOTS * 2.0;
double paddedRate = packedRate * 2.0;
// The sample rate is fixed, so the byte rate directly reveals whether each
// 16-bit slot occupies one word or half of a 32-bit word.
frameWords_ = (bytesPerSec > (packedRate + paddedRate) * 0.5) ? 8 : 4;
sampleHalf_ = 0;
if (frameWords_ == 8) {
double evenEnergy = posSq_[0] + posSq_[2] + posSq_[4] + posSq_[6];
double oddEnergy = posSq_[1] + posSq_[3] + posSq_[5] + posSq_[7];
sampleHalf_ = (oddEnergy >= evenEnergy) ? 1 : 0;
}
layoutReady_ = true;
Serial.printf("Sound detector DMA layout: %s (%s half), "
"%.0f bytes/s (packed %.0f, padded %.0f)\n",
frameWords_ == 8 ? "32-bit words" : "packed 16-bit",
sampleHalf_ ? "high" : "low",
bytesPerSec, packedRate, paddedRate);
Serial.print("Sound detector slot energy:");
for (int i = 0; i < 8; i++) {
double rms = posCount_[i] ? sqrt(posSq_[i] / posCount_[i]) : 0.0;
Serial.printf(" %d:%.0f", i, rms);
}
Serial.println();
}
void SoundDetector::updateSlots(const float *rms)
{
float swing[MIC_SLOTS];
for (int s = 0; s < MIC_SLOTS; s++) {
int64_t sum = 0, sq = 0;
for (size_t i = 0; i < frameCount; i++) {
int32_t v = frames[i * MIC_SLOTS + s];
sum += v;
sq += (int64_t)v * v;
// Track a decaying maximum and a slowly rising minimum. A real
// microphone has quiet gaps between sounds, so its minimum stays low
// while its maximum follows the peaks. A floating or railed slot has
// both pinned at the same level and therefore no range.
if (rms[s] > slotMax_[s]) {
slotMax_[s] = rms[s];
} else {
slotMax_[s] *= SLOT_MAX_DECAY;
}
float mean = (float)sum / (float)frameCount;
float var = (float)sq / (float)frameCount - mean * mean;
if (var < 0.0f) var = 0.0f;
rms[s] = sqrtf(var) / 32768.0f;
if (rms[s] < slotMin_[s]) {
slotMin_[s] = rms[s];
} else {
slotMin_[s] += (rms[s] - slotMin_[s]) * SLOT_MIN_RISE;
}
swing[s] = slotMax_[s] - slotMin_[s];
if (swing[s] < 0.0f) swing[s] = 0.0f;
}
// Find the two loudest slots. With only two microphones fitted, a real
// sound shows up in exactly those slots; the other two stay near the noise
// floor. Re-evaluate on every clear sound so a wrong initial guess (for
// example caused by the codec's power-up transient) cannot stick forever.
int best = 0, second = 1;
if (rms[second] > rms[best]) {
int t = best; best = second; second = t;
}
for (int s = 2; s < MIC_SLOTS; s++) {
if (rms[s] > rms[best]) {
second = best;
best = s;
} else if (rms[s] > rms[second]) {
second = s;
// Rank the slots by how much they respond to sound.
int order[MIC_SLOTS] = { 0, 1, 2, 3 };
for (int i = 0; i < MIC_SLOTS; i++) {
for (int j = i + 1; j < MIC_SLOTS; j++) {
if (swing[order[j]] > swing[order[i]]) {
int t = order[i]; order[i] = order[j]; order[j] = t;
}
}
}
float s0 = swing[order[0]];
float s1 = swing[order[1]];
float s2 = swing[order[2]];
if (rms[best] > SLOT_SELECT_LEVEL &&
rms[second] > rms[best] * SLOT_PAIR_RATIO) {
int lo = best < second ? best : second;
int hi = best < second ? second : best;
bool valid = s0 > SLOT_MIN_SWING &&
s1 > s0 * SLOT_SECOND_RATIO &&
s2 < s1 * SLOT_THIRD_RATIO;
if (valid) {
int lo = order[0] < order[1] ? order[0] : order[1];
int hi = order[0] < order[1] ? order[1] : order[0];
if (lo != leftSlot_ || hi != rightSlot_) {
float currentLevel = fmaxf(rms[leftSlot_], rms[rightSlot_]);
if (lo == candL_ && hi == candR_) {
candCount_++;
} else {
@@ -254,9 +332,8 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount)
candR_ = hi;
candCount_ = 1;
}
// Switch immediately when the current pair is effectively dead,
// otherwise require the new pair to repeat for a few blocks.
if (currentLevel < rms[best] * SLOT_DEAD_RATIO ||
float currentSwing = fmaxf(swing[leftSlot_], swing[rightSlot_]);
if (currentSwing < s0 * SLOT_DEAD_RATIO ||
candCount_ >= SLOT_SWITCH_BLOCKS) {
leftSlot_ = lo;
rightSlot_ = hi;
@@ -265,27 +342,76 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount)
envR_ = 0.0f;
floorL_ = 0.01f;
floorR_ = 0.01f;
Serial.printf("Sound detector using mic slots L=%d R=%d "
"(rms %.4f %.4f %.4f %.4f)\n",
Serial.printf("Sound detector mic slots L=%d R=%d "
"(swing %.4f %.4f %.4f %.4f)\n",
leftSlot_, rightSlot_,
rms[0], rms[1], rms[2], rms[3]);
swing[0], swing[1], swing[2], swing[3]);
}
} else {
candCount_ = 0;
}
} else {
candCount_ = 0;
}
}
void SoundDetector::processBlock(const int16_t *samples, size_t sampleCount)
{
size_t frames = sampleCount / frameWords_;
if (frames == 0) return;
// RMS of every TDM slot, using the word packing discovered at start-up.
float rms[MIC_SLOTS] = {};
for (int s = 0; s < MIC_SLOTS; s++) {
size_t off = sampleOffset(s);
int64_t sum = 0, sq = 0;
for (size_t i = 0; i < frames; i++) {
int32_t v = samples[i * frameWords_ + off];
sum += v;
sq += (int64_t)v * v;
}
float mean = (float)sum / (float)frames;
float var = (float)sq / (float)frames - mean * mean;
if (var < 0.0f) var = 0.0f;
rms[s] = sqrtf(var) / 32768.0f;
}
updateSlots(rms);
#if SOUND_DETECTOR_DEBUG
if (!rawDumped_) {
rawDumped_ = true;
Serial.print("Sound detector raw:");
for (int i = 0; i < 24 && i < (int)sampleCount; i++) {
Serial.printf(" %04x", (uint16_t)samples[i]);
}
Serial.println();
}
#endif
float rmsL = rms[MIC_SWAP_LR ? rightSlot_ : leftSlot_];
float rmsR = rms[MIC_SWAP_LR ? leftSlot_ : rightSlot_];
if (rmsL > RAIL_LEVEL) rmsL = 0.0f;
if (rmsR > RAIL_LEVEL) rmsR = 0.0f;
// Normalise to full scale, then smooth with a fast attack / slow release.
envL_ += (rmsL - envL_) * (rmsL > envL_ ? ENV_ATTACK : ENV_RELEASE);
envR_ += (rmsR - envR_) * (rmsR > envR_ ? ENV_ATTACK : ENV_RELEASE);
// Per-channel noise floor: fall quickly when it gets quieter, creep up very
// slowly so sustained noise does not desensitise the detector.
floorL_ += (envL_ - floorL_) * (envL_ < floorL_ ? FLOOR_FALL : FLOOR_RISE);
floorR_ += (envR_ - floorR_) * (envR_ < floorR_ ? FLOOR_FALL : FLOOR_RISE);
// Per-channel noise floor: fall quickly when it gets quieter, and only
// creep up while the input is actually close to the floor. Without that
// guard a sustained loud sound (or a clipped channel) drags the floor up to
// its own level and the detector goes permanently deaf.
if (envL_ < floorL_) {
floorL_ += (envL_ - floorL_) * FLOOR_FALL;
} else if (envL_ < floorL_ * 1.5f) {
floorL_ += (envL_ - floorL_) * FLOOR_RISE;
}
if (envR_ < floorR_) {
floorR_ += (envR_ - floorR_) * FLOOR_FALL;
} else if (envR_ < floorR_ * 1.5f) {
floorR_ += (envR_ - floorR_) * FLOOR_RISE;
}
if (floorL_ < 1e-5f) floorL_ = 1e-5f;
if (floorR_ < 1e-5f) floorR_ = 1e-5f;
@@ -329,11 +455,13 @@ void SoundDetector::processBlock(const int16_t *frames, size_t frameCount)
#if SOUND_DETECTOR_DEBUG
static uint32_t lastPrintMs = 0;
if ((uint32_t)(millis() - lastPrintMs) > 1000) {
if ((uint32_t)(millis() - lastPrintMs) > 300) {
lastPrintMs = millis();
Serial.printf("mic rms %.4f %.4f %.4f %.4f | pair L%d R%d | "
"sig %.3f %.3f | bal %+.2f loud %.2f%s\n",
Serial.printf("mic rms %.4f %.4f %.4f %.4f | range %.4f %.4f %.4f %.4f\n",
rms[0], rms[1], rms[2], rms[3],
slotMax_[0] - slotMin_[0], slotMax_[1] - slotMin_[1],
slotMax_[2] - slotMin_[2], slotMax_[3] - slotMin_[3]);
Serial.printf(" pair L%d R%d | sig %.3f %.3f | bal %+.2f loud %.2f%s\n",
leftSlot_, rightSlot_, sigL, sigR, balance, loudness,
active ? "" : " (quiet)");
}
+37 -9
View File
@@ -2,10 +2,12 @@
* @file sound_detector.h
* @brief Dual-microphone sound localizer for the T-Deck Plus (ES7210).
*
* Reads the ES7210's four TDM slots and uses the two slots that carry a
* populated MEMS microphone (the board only fits two of the four footprints).
* Balance is taken from the level difference between the pair, so a sound on
* the left or right side moves a radar blip accordingly.
* The ES7210 emits a 4-slot TDM frame. On the T-Deck/T-Deck Plus only two
* MEMS footprints are populated, and different board revisions populate
* different slots, so the detector learns which two slots actually respond to
* sound instead of assuming a fixed pair. Balance is taken from the level
* difference between those two microphones, so a sound on the left or right
* side moves a radar blip accordingly.
*/
#pragma once
@@ -38,7 +40,11 @@ public:
private:
static void taskEntry(void *arg);
void taskLoop();
void processBlock(const int16_t *frames, size_t frameCount);
void probeLayout(const int16_t *samples, size_t sampleCount);
void finaliseLayout();
void updateSlots(const float *rms);
size_t sampleOffset(int slot) const;
void processBlock(const int16_t *samples, size_t sampleCount);
void publish(float left, float right, float balance, float loudness, bool active);
uint32_t sampleRate_ = 16000;
@@ -55,12 +61,34 @@ private:
float floorL_ = 0.01f;
float floorR_ = 0.01f;
// The codec always delivers four TDM slots, but only two microphones are
// fitted. The pair is (re)discovered from the slots that carry signal.
// The two TDM slots carrying the populated microphones. Seeded with the
// documented T-Deck pair (MIC1 = slot 0, MIC3 = slot 2) and corrected at
// runtime by updateSlots() on the first sounds that arrive.
int leftSlot_ = 0;
int rightSlot_ = 1;
int rightSlot_ = 2;
uint32_t warmupUntil_ = 0;
// Start-up layout probe: the driver may pack the 16-bit TDM slots tightly
// (4 int16 per frame) or pad each sample into one half of a 32-bit word
// (8 int16 per frame). Both the data rate and the energy split between even
// and odd int16 words identify the packing.
uint32_t probeStartMs_ = 0;
uint32_t probeSamples_ = 0;
uint32_t probeIndex_ = 0;
double posSq_[8] = {};
uint32_t posCount_[8] = {};
bool layoutReady_ = false;
size_t frameWords_ = 4;
uint8_t sampleHalf_ = 0; // 0 = low half of a 32-bit word, 1 = high half
bool rawDumped_ = false;
// Per-slot level range used to discover the populated pair. A fitted
// microphone swings between its quiet floor and the sound peaks; a floating
// input stays flat (its DC offset is removed by the high-pass filter), and
// a railed channel stays flat too. Range separates those cases cleanly.
float slotMin_[4] = { 1.0f, 1.0f, 1.0f, 1.0f };
float slotMax_[4] = {};
int candL_ = -1;
int candR_ = -1;
int candCount_ = 0;
uint32_t warmupUntil_ = 0;
};