From 2ba92a65d1c5f0d6f2f19e4ef8fe0c4a4838b031 Mon Sep 17 00:00:00 2001 From: Dejvino Date: Thu, 12 Nov 2026 08:20:38 +0100 Subject: [PATCH] Drive radar blips from the T-Deck Plus dual microphones Add a SoundDetector that reads the ES7210 as a 4-slot TDM stream (slot 0 = MIC1/left, slot 2 = MIC3/right), tracks a per-channel RMS envelope with an adaptive noise floor, and publishes a stereo balance plus a dB-mapped loudness. The radar sketch now slides a single reused blip left/right from the balance and moves it nearer as the loudness rises, instead of firing a hardcoded demo blip. The device's own ping/click mutes the detector for 200 ms so it cannot hear itself. The trackball stays detached: GPIO0 is unavailable while the ES7210 microphones are running. --- examples/NoiseSensor/NoiseSensor.ino | 71 ++++-- examples/NoiseSensor/sound_detector.cpp | 280 ++++++++++++++++++++++++ examples/NoiseSensor/sound_detector.h | 57 +++++ 3 files changed, 391 insertions(+), 17 deletions(-) create mode 100644 examples/NoiseSensor/sound_detector.cpp create mode 100644 examples/NoiseSensor/sound_detector.h diff --git a/examples/NoiseSensor/NoiseSensor.ino b/examples/NoiseSensor/NoiseSensor.ino index 049e1a3..a5affe8 100644 --- a/examples/NoiseSensor/NoiseSensor.ino +++ b/examples/NoiseSensor/NoiseSensor.ino @@ -31,6 +31,7 @@ #include "utilities.h" #include "TouchDrvGT911.hpp" #include "sound_engine.h" +#include "sound_detector.h" // --------------------------------------------------------------------------- // Hardware @@ -38,6 +39,7 @@ TFT_eSPI tft; TouchDrvGT911 touch; SoundEngine sound; +SoundDetector soundDetector; Preferences prefs; static bool touchAvail = false; @@ -391,6 +393,8 @@ static void printPrompt() // --------------------------------------------------------------------------- static void sndBlip() { + // Keep the device's own radar ping out of the microphone detector. + soundDetector.suppress(200); SndEvent e = { SND_TONE, 540, 0, 10, 8000 }; sound.play(e); delay(15); @@ -398,6 +402,7 @@ static void sndBlip() static void sndClick() { + soundDetector.suppress(200); SndEvent e = { SND_TONE, 660, 0, 152, 5000 }; sound.play(e); } @@ -624,24 +629,43 @@ void initRadar() { } } -// Call this when an audio trigger or target is detected -// lrBalance: -1.0 (far left) to +1.0 (far right) -// loudness: mapped to distance (higher volume = closer to origin) -void triggerBlip(float lrBalance, float loudness) { +// Index of the blip currently driven by the microphone. Reused every frame so +// a continuous sound slides around instead of filling all MAX_BLIPS slots. +static int liveBlip = -1; + +static int allocBlip() { for (int i = 0; i < MAX_BLIPS; i++) { - if (!blips[i].active) { - blips[i].active = true; - // Map L/R balance to forward-facing wedge (-45 deg to +45 deg) - blips[i].angle = constrain(lrBalance, -1.0f, 1.0f) * (M_PI / 4.0f); - // Loudness maps to distance: loudest is near origin, quiet is near edge - blips[i].distance = constrain(1.0f - (loudness / 100.0f), 0.15f, 0.95f); - blips[i].alpha = 255; - break; - } + if (!blips[i].active) return i; } + return -1; +} + +// lrBalance: -1.0 (far left) to +1.0 (far right) +// loudness: 0.0 (quiet/far) to 1.0 (loud/near) +static void placeBlip(int i, float lrBalance, float loudness) { + if (i < 0) return; + blips[i].active = true; + // Map L/R balance to forward-facing wedge (-45 deg to +45 deg) + blips[i].angle = constrain(lrBalance, -1.0f, 1.0f) * (M_PI / 4.0f); + // Loudness maps to distance: loudest is near origin, quiet is near edge + blips[i].distance = constrain(1.0f - loudness, 0.15f, 0.95f); + blips[i].alpha = 255; +} + +// One-shot blip (manual trigger / future events) +void triggerBlip(float lrBalance, float loudness) { + placeBlip(allocBlip(), lrBalance, loudness); sndClick(); } +// Continuous update from the dual-microphone detector. +static void updateSoundBlip(float lrBalance, float loudness) { + if (liveBlip < 0 || !blips[liveBlip].active) { + liveBlip = allocBlip(); + } + placeBlip(liveBlip, lrBalance, loudness); +} + static void drawArc(float r, uint32_t color) { for (float a = -M_PI / 4.0; a <= M_PI / 4.0; a += 0.05) { int ax = RADAR_ORIGIN_X + (int)(r * sin(a)); @@ -687,10 +711,13 @@ void radarUpdate() { sndBlip(); } else if (sweepAngle < MAX_RADIUS) { drawArc(sweepAngle, C_GLOW); + } - if (sweepAngle > 109 && sweepAngle < 111) { - triggerBlip(0.2, 50); - } + // Feed the live microphone measurement into the blip layer. Balance moves + // it left/right; loudness pulls it toward the centre (closer target). + SoundReading snd = soundDetector.read(); + if (snd.active) { + updateSoundBlip(snd.balance, snd.loudness); } // 4. Render Active Target Blips with Phosphor Decay @@ -1006,7 +1033,11 @@ static void hardwareInit() Serial.println("No GT911 touch found"); } - attachTrackball(); + // NOTE: GPIO0 (trackball centre / BOOT) is unavailable while the ES7210 + // microphones are running, and leaving its interrupt enabled causes + // spurious captures. Keep it detached; the microphone radar does not need + // the trackball. + // attachTrackball(); } static void bootSequence() @@ -1044,6 +1075,12 @@ void setup() bootSequence(); initRadar(); + + if (!soundDetector.begin(16000)) { + Serial.println("Sound detector init failed (ES7210 mics)"); + } else { + Serial.println("Sound detector ready (MIC1 left / MIC3 right)"); + } } void loop() diff --git a/examples/NoiseSensor/sound_detector.cpp b/examples/NoiseSensor/sound_detector.cpp new file mode 100644 index 0000000..c1a6c66 --- /dev/null +++ b/examples/NoiseSensor/sound_detector.cpp @@ -0,0 +1,280 @@ +/** + * @file sound_detector.cpp + * @brief ES7210 dual-microphone capture + stereo balance/loudness detector. + * + * The T-Deck Plus has two populated MEMS microphones on the ES7210. The codec + * outputs all four ADC channels as a 4-slot TDM stream on SDOUT1: + * slot 0 = MIC1 (left), slot 2 = MIC3 (right) + * We only need slots 0 and 2; slot 1/3 are ignored. + * + * The detector runs in its own FreeRTOS task so the radar loop never blocks on + * I2S. It publishes smoothed, noise-floor-relative levels that the sketch turns + * into a blip angle (balance) and distance (loudness). + */ +#include "sound_detector.h" +#include "utilities.h" +#include "es7210.h" +#include +#include + +#define MIC_I2S_PORT I2S_NUM_1 +#define MIC_BLOCK_FRAMES 256 // 16 ms at 16 kHz +#define MIC_CHANNELS 4 // 4-slot TDM +#define MIC_SLOT_LEFT 0 // MIC1 +#define MIC_SLOT_RIGHT 2 // MIC3 +#define MIC_GAIN GAIN_30DB + +// Set to 1 to print detected levels while tuning the microphones on hardware. +#ifndef SOUND_DETECTOR_DEBUG +#define SOUND_DETECTOR_DEBUG 0 +#endif + +// Envelope / detector tuning (per 16 ms block). +static constexpr float ENV_ATTACK = 0.55f; +static constexpr float ENV_RELEASE = 0.18f; +static constexpr float FLOOR_FALL = 0.12f; // fast follow when quieter +static constexpr float FLOOR_RISE = 0.0008f; // very slow drift upward +static constexpr float FLOOR_MARGIN = 1.25f; +static constexpr float ABS_GATE = 0.0008f; +static constexpr float BALANCE_DEADBAND = 0.06f; +static constexpr float LOUD_MIN_DB = -48.0f; +static constexpr float LOUD_MAX_DB = -6.0f; +static constexpr uint32_t HOLD_MS = 260; + +// ES7210 SDP interface 2 / high-pass registers. The LilyGO driver leaves the +// codec in 2-channel I2S mode and the HPF disabled; enable both here. +static const uint8_t ES7210_SDP_INTERFACE2 = 0x12; +static const uint8_t ES7210_ADC34_HPF2 = 0x20; +static const uint8_t ES7210_ADC34_HPF1 = 0x21; +static const uint8_t ES7210_ADC12_HPF1 = 0x22; +static const uint8_t ES7210_ADC12_HPF2 = 0x23; + +static bool es7210WriteReg(uint8_t reg, uint8_t value) +{ + Wire.beginTransmission(ES7210_ADDR); + Wire.write(reg); + Wire.write(value); + return Wire.endTransmission() == 0; +} + +bool SoundDetector::begin(uint32_t sampleRate) +{ + if (running_) return true; + sampleRate_ = sampleRate; + + audio_hal_codec_config_t cfg = {}; + cfg.adc_input = AUDIO_HAL_ADC_INPUT_ALL; + cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE; + cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE; + cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL; + cfg.i2s_iface.samples = AUDIO_HAL_16K_SAMPLES; + cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS; + + esp_err_t err = es7210_adc_init(&Wire, &cfg); + err |= es7210_adc_config_i2s(cfg.codec_mode, &cfg.i2s_iface); + + // Put all four ADCs on SDOUT1 as TDM slots. Without 0x02 on this register + // only MIC1/MIC2 reach SDOUT1 and the right microphone is missing. + if (!es7210WriteReg(ES7210_SDP_INTERFACE2, 0x02)) { + Serial.println("ES7210 TDM config failed"); + return false; + } + + // High-pass the ADC outputs so the detector sees audio, not DC offset. + es7210WriteReg(ES7210_ADC12_HPF2, 0x2a); + es7210WriteReg(ES7210_ADC12_HPF1, 0x0a); + es7210WriteReg(ES7210_ADC34_HPF2, 0x0a); + es7210WriteReg(ES7210_ADC34_HPF1, 0x2a); + + // Use the same analog gain on both populated mics so the amplitude + // difference between them is meaningful. + err |= es7210_adc_set_gain(ES7210_INPUT_MIC1, MIC_GAIN); + err |= es7210_adc_set_gain(ES7210_INPUT_MIC3, MIC_GAIN); + err |= es7210_adc_ctrl_state(cfg.codec_mode, AUDIO_HAL_CTRL_START); + if (err != ESP_OK) { + Serial.println("ES7210 init failed"); + return false; + } + + i2s_config_t i2s = {}; + i2s.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX); + i2s.sample_rate = sampleRate_; + i2s.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT; + i2s.channel_format = I2S_CHANNEL_FMT_ALL_LEFT; + i2s.communication_format = I2S_COMM_FORMAT_STAND_I2S; + i2s.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1; + i2s.dma_buf_count = 8; + i2s.dma_buf_len = 64; + i2s.use_apll = false; + i2s.tx_desc_auto_clear = true; + i2s.fixed_mclk = 0; + i2s.mclk_multiple = I2S_MCLK_MULTIPLE_256; + i2s.bits_per_chan = I2S_BITS_PER_CHAN_16BIT; + i2s.chan_mask = (i2s_channel_t)(I2S_TDM_ACTIVE_CH0 | I2S_TDM_ACTIVE_CH1 | + I2S_TDM_ACTIVE_CH2 | I2S_TDM_ACTIVE_CH3); + i2s.total_chan = 4; + + if (i2s_driver_install(MIC_I2S_PORT, &i2s, 0, nullptr) != ESP_OK) { + Serial.println("Mic I2S install failed"); + return false; + } + + i2s_pin_config_t pins = {}; + pins.mck_io_num = BOARD_ES7210_MCLK; + pins.bck_io_num = BOARD_ES7210_SCK; + pins.ws_io_num = BOARD_ES7210_LRCK; + pins.data_in_num = BOARD_ES7210_DIN; + pins.data_out_num = I2S_PIN_NO_CHANGE; + i2s_set_pin(MIC_I2S_PORT, &pins); + i2s_zero_dma_buffer(MIC_I2S_PORT); + + running_ = true; + if (xTaskCreatePinnedToCore(taskEntry, "sound-detector", 8192, this, 4, + &task_, 0) != pdPASS) { + running_ = false; + i2s_driver_uninstall(MIC_I2S_PORT); + Serial.println("Mic task create failed"); + return false; + } + return true; +} + +void SoundDetector::taskEntry(void *arg) +{ + static_cast(arg)->taskLoop(); +} + +void SoundDetector::suppress(uint32_t ms) +{ + suppressUntil_ = millis() + ms; +} + +void SoundDetector::taskLoop() +{ + static int16_t block[MIC_BLOCK_FRAMES * MIC_CHANNELS]; + + for (;;) { + size_t bytesRead = 0; + esp_err_t err = i2s_read(MIC_I2S_PORT, block, sizeof(block), &bytesRead, + portMAX_DELAY); + if (err != ESP_OK || bytesRead == 0) { + vTaskDelay(pdMS_TO_TICKS(10)); + continue; + } + + size_t frames = bytesRead / (MIC_CHANNELS * sizeof(int16_t)); + if (frames == 0) continue; + + if ((int32_t)(millis() - suppressUntil_) < 0) { + // Still drain I2S so the DMA doesn't overflow while muted. Drop the + // envelopes too, otherwise the device's own ping would be the first + // thing the detector reports once the mute expires. + envL_ = 0.0f; + envR_ = 0.0f; + publish(0.0f, 0.0f, 0.0f, 0.0f, false); + continue; + } + + processBlock(block, frames); + } +} + +void SoundDetector::processBlock(const int16_t *frames, size_t frameCount) +{ + int64_t sumL = 0, sumR = 0; + int64_t sqL = 0, sqR = 0; + + for (size_t i = 0; i < frameCount; i++) { + int32_t l = frames[i * MIC_CHANNELS + MIC_SLOT_LEFT]; + int32_t r = frames[i * MIC_CHANNELS + MIC_SLOT_RIGHT]; + sumL += l; + sumR += r; + sqL += (int64_t)l * l; + sqR += (int64_t)r * r; + } + + float meanL = (float)sumL / (float)frameCount; + float meanR = (float)sumR / (float)frameCount; + float varL = (float)sqL / (float)frameCount - meanL * meanL; + float varR = (float)sqR / (float)frameCount - meanR * meanR; + if (varL < 0.0f) varL = 0.0f; + if (varR < 0.0f) varR = 0.0f; + + // Normalise to full scale, then smooth with a fast attack / slow release. + float rmsL = sqrtf(varL) / 32768.0f; + float rmsR = sqrtf(varR) / 32768.0f; + envL_ += (rmsL - envL_) * (rmsL > envL_ ? ENV_ATTACK : ENV_RELEASE); + envR_ += (rmsR - envR_) * (rmsR > envR_ ? ENV_ATTACK : ENV_RELEASE); + + // Per-channel noise floor: fall quickly when it gets quieter, creep up very + // slowly so sustained noise does not desensitise the detector. + floorL_ += (envL_ - floorL_) * (envL_ < floorL_ ? FLOOR_FALL : FLOOR_RISE); + floorR_ += (envR_ - floorR_) * (envR_ < floorR_ ? FLOOR_FALL : FLOOR_RISE); + if (floorL_ < 1e-5f) floorL_ = 1e-5f; + if (floorR_ < 1e-5f) floorR_ = 1e-5f; + + float sigL = envL_ - floorL_ * FLOOR_MARGIN; + float sigR = envR_ - floorR_ * FLOOR_MARGIN; + if (sigL < 0.0f) sigL = 0.0f; + if (sigR < 0.0f) sigR = 0.0f; + + // Adaptive gate: a sound must stand clearly above its channel's floor. + float gateL = fmaxf(floorL_ * 2.0f, ABS_GATE); + float gateR = fmaxf(floorR_ * 2.0f, ABS_GATE); + bool present = (sigL > gateL) || (sigR > gateR); + + uint32_t now = millis(); + if (present) lastActiveMs_ = now; + bool active = present || (uint32_t)(now - lastActiveMs_) < HOLD_MS; + + float total = sigL + sigR; + float balance = 0.0f; + if (total > 1e-6f) { + balance = (sigR - sigL) / total; + if (fabsf(balance) < BALANCE_DEADBAND) { + balance = 0.0f; + } else { + // Remove the deadband and rescale the remaining range to +/-1. + float sign = balance < 0.0f ? -1.0f : 1.0f; + balance = (balance - sign * BALANCE_DEADBAND) / (1.0f - BALANCE_DEADBAND); + } + } + + float peak = fmaxf(sigL, sigR); + float db = 20.0f * log10f(peak + 1e-9f); + float loudness = (db - LOUD_MIN_DB) / (LOUD_MAX_DB - LOUD_MIN_DB); + if (loudness < 0.0f) loudness = 0.0f; + if (loudness > 1.0f) loudness = 1.0f; + +#if SOUND_DETECTOR_DEBUG + static uint32_t lastPrintMs = 0; + if (active && (uint32_t)(millis() - lastPrintMs) > 250) { + lastPrintMs = millis(); + Serial.printf("mic L=%.4f R=%.4f bal=%+.2f loud=%.2f\n", + sigL, sigR, balance, loudness); + } +#endif + + publish(sigL, sigR, balance, loudness, active); +} + +void SoundDetector::publish(float left, float right, float balance, float loudness, + bool active) +{ + portENTER_CRITICAL(&mux_); + reading_.left = left; + reading_.right = right; + reading_.balance = balance; + reading_.loudness = loudness; + reading_.active = active; + portEXIT_CRITICAL(&mux_); +} + +SoundReading SoundDetector::read() +{ + SoundReading r; + portENTER_CRITICAL(&mux_); + r = reading_; + portEXIT_CRITICAL(&mux_); + return r; +} diff --git a/examples/NoiseSensor/sound_detector.h b/examples/NoiseSensor/sound_detector.h new file mode 100644 index 0000000..9c99239 --- /dev/null +++ b/examples/NoiseSensor/sound_detector.h @@ -0,0 +1,57 @@ +/** + * @file sound_detector.h + * @brief Dual-microphone sound localizer for the T-Deck Plus (ES7210). + * + * Reads the two populated MEMS microphones as a 4-slot TDM stream: + * slot 0 = MIC1 (left), slot 2 = MIC3 (right) + * and exposes a smoothed left/right level, a stereo balance and a loudness + * value suitable for placing a radar blip. + */ +#pragma once + +#include +#include + +struct SoundReading { + float left; // smoothed level above the noise floor (0..1) + float right; // smoothed level above the noise floor (0..1) + float balance; // -1 = far left, 0 = centre, +1 = far right + float loudness; // 0 = quiet, 1 = loud (dB-mapped) + bool active; // true while a sound is present (with a short hold) +}; + +class SoundDetector { +public: + // Starts the ES7210 codec and the I2S receive task. Returns false if the + // codec or I2S driver could not be initialised. + bool begin(uint32_t sampleRate = 16000); + + // Latest measurement. Safe to call from the main loop. + SoundReading read(); + + bool available() const { return running_; } + + // Ignore microphone input for a while (used so the device's own radar + // ping does not register as a target). + void suppress(uint32_t ms); + +private: + static void taskEntry(void *arg); + void taskLoop(); + void processBlock(const int16_t *frames, size_t frameCount); + void publish(float left, float right, float balance, float loudness, bool active); + + uint32_t sampleRate_ = 16000; + TaskHandle_t task_ = nullptr; + volatile bool running_ = false; + volatile uint32_t suppressUntil_ = 0; + + portMUX_TYPE mux_ = portMUX_INITIALIZER_UNLOCKED; + SoundReading reading_ = {}; + uint32_t lastActiveMs_ = 0; + + float envL_ = 0.0f; + float envR_ = 0.0f; + float floorL_ = 0.01f; + float floorR_ = 0.01f; +};