Drive radar blips from the T-Deck Plus dual microphones
Add a SoundDetector that reads the ES7210 as a 4-slot TDM stream (slot 0 = MIC1/left, slot 2 = MIC3/right), tracks a per-channel RMS envelope with an adaptive noise floor, and publishes a stereo balance plus a dB-mapped loudness. The radar sketch now slides a single reused blip left/right from the balance and moves it nearer as the loudness rises, instead of firing a hardcoded demo blip. The device's own ping/click mutes the detector for 200 ms so it cannot hear itself. The trackball stays detached: GPIO0 is unavailable while the ES7210 microphones are running.
This commit is contained in:
@@ -31,6 +31,7 @@
|
||||
#include "utilities.h"
|
||||
#include "TouchDrvGT911.hpp"
|
||||
#include "sound_engine.h"
|
||||
#include "sound_detector.h"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hardware
|
||||
@@ -38,6 +39,7 @@
|
||||
TFT_eSPI tft;
|
||||
TouchDrvGT911 touch;
|
||||
SoundEngine sound;
|
||||
SoundDetector soundDetector;
|
||||
Preferences prefs;
|
||||
|
||||
static bool touchAvail = false;
|
||||
@@ -391,6 +393,8 @@ static void printPrompt()
|
||||
// ---------------------------------------------------------------------------
|
||||
static void sndBlip()
|
||||
{
|
||||
// Keep the device's own radar ping out of the microphone detector.
|
||||
soundDetector.suppress(200);
|
||||
SndEvent e = { SND_TONE, 540, 0, 10, 8000 };
|
||||
sound.play(e);
|
||||
delay(15);
|
||||
@@ -398,6 +402,7 @@ static void sndBlip()
|
||||
|
||||
static void sndClick()
|
||||
{
|
||||
soundDetector.suppress(200);
|
||||
SndEvent e = { SND_TONE, 660, 0, 152, 5000 };
|
||||
sound.play(e);
|
||||
}
|
||||
@@ -624,24 +629,43 @@ void initRadar() {
|
||||
}
|
||||
}
|
||||
|
||||
// Call this when an audio trigger or target is detected
|
||||
// lrBalance: -1.0 (far left) to +1.0 (far right)
|
||||
// loudness: mapped to distance (higher volume = closer to origin)
|
||||
void triggerBlip(float lrBalance, float loudness) {
|
||||
// Index of the blip currently driven by the microphone. Reused every frame so
|
||||
// a continuous sound slides around instead of filling all MAX_BLIPS slots.
|
||||
static int liveBlip = -1;
|
||||
|
||||
static int allocBlip() {
|
||||
for (int i = 0; i < MAX_BLIPS; i++) {
|
||||
if (!blips[i].active) {
|
||||
if (!blips[i].active) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// lrBalance: -1.0 (far left) to +1.0 (far right)
|
||||
// loudness: 0.0 (quiet/far) to 1.0 (loud/near)
|
||||
static void placeBlip(int i, float lrBalance, float loudness) {
|
||||
if (i < 0) return;
|
||||
blips[i].active = true;
|
||||
// Map L/R balance to forward-facing wedge (-45 deg to +45 deg)
|
||||
blips[i].angle = constrain(lrBalance, -1.0f, 1.0f) * (M_PI / 4.0f);
|
||||
// Loudness maps to distance: loudest is near origin, quiet is near edge
|
||||
blips[i].distance = constrain(1.0f - (loudness / 100.0f), 0.15f, 0.95f);
|
||||
blips[i].distance = constrain(1.0f - loudness, 0.15f, 0.95f);
|
||||
blips[i].alpha = 255;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// One-shot blip (manual trigger / future events)
|
||||
void triggerBlip(float lrBalance, float loudness) {
|
||||
placeBlip(allocBlip(), lrBalance, loudness);
|
||||
sndClick();
|
||||
}
|
||||
|
||||
// Continuous update from the dual-microphone detector.
|
||||
static void updateSoundBlip(float lrBalance, float loudness) {
|
||||
if (liveBlip < 0 || !blips[liveBlip].active) {
|
||||
liveBlip = allocBlip();
|
||||
}
|
||||
placeBlip(liveBlip, lrBalance, loudness);
|
||||
}
|
||||
|
||||
static void drawArc(float r, uint32_t color) {
|
||||
for (float a = -M_PI / 4.0; a <= M_PI / 4.0; a += 0.05) {
|
||||
int ax = RADAR_ORIGIN_X + (int)(r * sin(a));
|
||||
@@ -687,10 +711,13 @@ void radarUpdate() {
|
||||
sndBlip();
|
||||
} else if (sweepAngle < MAX_RADIUS) {
|
||||
drawArc(sweepAngle, C_GLOW);
|
||||
|
||||
if (sweepAngle > 109 && sweepAngle < 111) {
|
||||
triggerBlip(0.2, 50);
|
||||
}
|
||||
|
||||
// Feed the live microphone measurement into the blip layer. Balance moves
|
||||
// it left/right; loudness pulls it toward the centre (closer target).
|
||||
SoundReading snd = soundDetector.read();
|
||||
if (snd.active) {
|
||||
updateSoundBlip(snd.balance, snd.loudness);
|
||||
}
|
||||
|
||||
// 4. Render Active Target Blips with Phosphor Decay
|
||||
@@ -1006,7 +1033,11 @@ static void hardwareInit()
|
||||
Serial.println("No GT911 touch found");
|
||||
}
|
||||
|
||||
attachTrackball();
|
||||
// NOTE: GPIO0 (trackball centre / BOOT) is unavailable while the ES7210
|
||||
// microphones are running, and leaving its interrupt enabled causes
|
||||
// spurious captures. Keep it detached; the microphone radar does not need
|
||||
// the trackball.
|
||||
// attachTrackball();
|
||||
}
|
||||
|
||||
static void bootSequence()
|
||||
@@ -1044,6 +1075,12 @@ void setup()
|
||||
bootSequence();
|
||||
|
||||
initRadar();
|
||||
|
||||
if (!soundDetector.begin(16000)) {
|
||||
Serial.println("Sound detector init failed (ES7210 mics)");
|
||||
} else {
|
||||
Serial.println("Sound detector ready (MIC1 left / MIC3 right)");
|
||||
}
|
||||
}
|
||||
|
||||
void loop()
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* @file sound_detector.cpp
|
||||
* @brief ES7210 dual-microphone capture + stereo balance/loudness detector.
|
||||
*
|
||||
* The T-Deck Plus has two populated MEMS microphones on the ES7210. The codec
|
||||
* outputs all four ADC channels as a 4-slot TDM stream on SDOUT1:
|
||||
* slot 0 = MIC1 (left), slot 2 = MIC3 (right)
|
||||
* We only need slots 0 and 2; slot 1/3 are ignored.
|
||||
*
|
||||
* The detector runs in its own FreeRTOS task so the radar loop never blocks on
|
||||
* I2S. It publishes smoothed, noise-floor-relative levels that the sketch turns
|
||||
* into a blip angle (balance) and distance (loudness).
|
||||
*/
|
||||
#include "sound_detector.h"
|
||||
#include "utilities.h"
|
||||
#include "es7210.h"
|
||||
#include <Wire.h>
|
||||
#include <math.h>
|
||||
|
||||
#define MIC_I2S_PORT I2S_NUM_1
|
||||
#define MIC_BLOCK_FRAMES 256 // 16 ms at 16 kHz
|
||||
#define MIC_CHANNELS 4 // 4-slot TDM
|
||||
#define MIC_SLOT_LEFT 0 // MIC1
|
||||
#define MIC_SLOT_RIGHT 2 // MIC3
|
||||
#define MIC_GAIN GAIN_30DB
|
||||
|
||||
// Set to 1 to print detected levels while tuning the microphones on hardware.
|
||||
#ifndef SOUND_DETECTOR_DEBUG
|
||||
#define SOUND_DETECTOR_DEBUG 0
|
||||
#endif
|
||||
|
||||
// Envelope / detector tuning (per 16 ms block).
|
||||
static constexpr float ENV_ATTACK = 0.55f;
|
||||
static constexpr float ENV_RELEASE = 0.18f;
|
||||
static constexpr float FLOOR_FALL = 0.12f; // fast follow when quieter
|
||||
static constexpr float FLOOR_RISE = 0.0008f; // very slow drift upward
|
||||
static constexpr float FLOOR_MARGIN = 1.25f;
|
||||
static constexpr float ABS_GATE = 0.0008f;
|
||||
static constexpr float BALANCE_DEADBAND = 0.06f;
|
||||
static constexpr float LOUD_MIN_DB = -48.0f;
|
||||
static constexpr float LOUD_MAX_DB = -6.0f;
|
||||
static constexpr uint32_t HOLD_MS = 260;
|
||||
|
||||
// ES7210 SDP interface 2 / high-pass registers. The LilyGO driver leaves the
|
||||
// codec in 2-channel I2S mode and the HPF disabled; enable both here.
|
||||
static const uint8_t ES7210_SDP_INTERFACE2 = 0x12;
|
||||
static const uint8_t ES7210_ADC34_HPF2 = 0x20;
|
||||
static const uint8_t ES7210_ADC34_HPF1 = 0x21;
|
||||
static const uint8_t ES7210_ADC12_HPF1 = 0x22;
|
||||
static const uint8_t ES7210_ADC12_HPF2 = 0x23;
|
||||
|
||||
static bool es7210WriteReg(uint8_t reg, uint8_t value)
|
||||
{
|
||||
Wire.beginTransmission(ES7210_ADDR);
|
||||
Wire.write(reg);
|
||||
Wire.write(value);
|
||||
return Wire.endTransmission() == 0;
|
||||
}
|
||||
|
||||
bool SoundDetector::begin(uint32_t sampleRate)
|
||||
{
|
||||
if (running_) return true;
|
||||
sampleRate_ = sampleRate;
|
||||
|
||||
audio_hal_codec_config_t cfg = {};
|
||||
cfg.adc_input = AUDIO_HAL_ADC_INPUT_ALL;
|
||||
cfg.codec_mode = AUDIO_HAL_CODEC_MODE_ENCODE;
|
||||
cfg.i2s_iface.mode = AUDIO_HAL_MODE_SLAVE;
|
||||
cfg.i2s_iface.fmt = AUDIO_HAL_I2S_NORMAL;
|
||||
cfg.i2s_iface.samples = AUDIO_HAL_16K_SAMPLES;
|
||||
cfg.i2s_iface.bits = AUDIO_HAL_BIT_LENGTH_16BITS;
|
||||
|
||||
esp_err_t err = es7210_adc_init(&Wire, &cfg);
|
||||
err |= es7210_adc_config_i2s(cfg.codec_mode, &cfg.i2s_iface);
|
||||
|
||||
// Put all four ADCs on SDOUT1 as TDM slots. Without 0x02 on this register
|
||||
// only MIC1/MIC2 reach SDOUT1 and the right microphone is missing.
|
||||
if (!es7210WriteReg(ES7210_SDP_INTERFACE2, 0x02)) {
|
||||
Serial.println("ES7210 TDM config failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
// High-pass the ADC outputs so the detector sees audio, not DC offset.
|
||||
es7210WriteReg(ES7210_ADC12_HPF2, 0x2a);
|
||||
es7210WriteReg(ES7210_ADC12_HPF1, 0x0a);
|
||||
es7210WriteReg(ES7210_ADC34_HPF2, 0x0a);
|
||||
es7210WriteReg(ES7210_ADC34_HPF1, 0x2a);
|
||||
|
||||
// Use the same analog gain on both populated mics so the amplitude
|
||||
// difference between them is meaningful.
|
||||
err |= es7210_adc_set_gain(ES7210_INPUT_MIC1, MIC_GAIN);
|
||||
err |= es7210_adc_set_gain(ES7210_INPUT_MIC3, MIC_GAIN);
|
||||
err |= es7210_adc_ctrl_state(cfg.codec_mode, AUDIO_HAL_CTRL_START);
|
||||
if (err != ESP_OK) {
|
||||
Serial.println("ES7210 init failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
i2s_config_t i2s = {};
|
||||
i2s.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_RX);
|
||||
i2s.sample_rate = sampleRate_;
|
||||
i2s.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT;
|
||||
i2s.channel_format = I2S_CHANNEL_FMT_ALL_LEFT;
|
||||
i2s.communication_format = I2S_COMM_FORMAT_STAND_I2S;
|
||||
i2s.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1;
|
||||
i2s.dma_buf_count = 8;
|
||||
i2s.dma_buf_len = 64;
|
||||
i2s.use_apll = false;
|
||||
i2s.tx_desc_auto_clear = true;
|
||||
i2s.fixed_mclk = 0;
|
||||
i2s.mclk_multiple = I2S_MCLK_MULTIPLE_256;
|
||||
i2s.bits_per_chan = I2S_BITS_PER_CHAN_16BIT;
|
||||
i2s.chan_mask = (i2s_channel_t)(I2S_TDM_ACTIVE_CH0 | I2S_TDM_ACTIVE_CH1 |
|
||||
I2S_TDM_ACTIVE_CH2 | I2S_TDM_ACTIVE_CH3);
|
||||
i2s.total_chan = 4;
|
||||
|
||||
if (i2s_driver_install(MIC_I2S_PORT, &i2s, 0, nullptr) != ESP_OK) {
|
||||
Serial.println("Mic I2S install failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
i2s_pin_config_t pins = {};
|
||||
pins.mck_io_num = BOARD_ES7210_MCLK;
|
||||
pins.bck_io_num = BOARD_ES7210_SCK;
|
||||
pins.ws_io_num = BOARD_ES7210_LRCK;
|
||||
pins.data_in_num = BOARD_ES7210_DIN;
|
||||
pins.data_out_num = I2S_PIN_NO_CHANGE;
|
||||
i2s_set_pin(MIC_I2S_PORT, &pins);
|
||||
i2s_zero_dma_buffer(MIC_I2S_PORT);
|
||||
|
||||
running_ = true;
|
||||
if (xTaskCreatePinnedToCore(taskEntry, "sound-detector", 8192, this, 4,
|
||||
&task_, 0) != pdPASS) {
|
||||
running_ = false;
|
||||
i2s_driver_uninstall(MIC_I2S_PORT);
|
||||
Serial.println("Mic task create failed");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void SoundDetector::taskEntry(void *arg)
|
||||
{
|
||||
static_cast<SoundDetector *>(arg)->taskLoop();
|
||||
}
|
||||
|
||||
void SoundDetector::suppress(uint32_t ms)
|
||||
{
|
||||
suppressUntil_ = millis() + ms;
|
||||
}
|
||||
|
||||
void SoundDetector::taskLoop()
|
||||
{
|
||||
static int16_t block[MIC_BLOCK_FRAMES * MIC_CHANNELS];
|
||||
|
||||
for (;;) {
|
||||
size_t bytesRead = 0;
|
||||
esp_err_t err = i2s_read(MIC_I2S_PORT, block, sizeof(block), &bytesRead,
|
||||
portMAX_DELAY);
|
||||
if (err != ESP_OK || bytesRead == 0) {
|
||||
vTaskDelay(pdMS_TO_TICKS(10));
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t frames = bytesRead / (MIC_CHANNELS * sizeof(int16_t));
|
||||
if (frames == 0) continue;
|
||||
|
||||
if ((int32_t)(millis() - suppressUntil_) < 0) {
|
||||
// Still drain I2S so the DMA doesn't overflow while muted. Drop the
|
||||
// envelopes too, otherwise the device's own ping would be the first
|
||||
// thing the detector reports once the mute expires.
|
||||
envL_ = 0.0f;
|
||||
envR_ = 0.0f;
|
||||
publish(0.0f, 0.0f, 0.0f, 0.0f, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
processBlock(block, frames);
|
||||
}
|
||||
}
|
||||
|
||||
void SoundDetector::processBlock(const int16_t *frames, size_t frameCount)
|
||||
{
|
||||
int64_t sumL = 0, sumR = 0;
|
||||
int64_t sqL = 0, sqR = 0;
|
||||
|
||||
for (size_t i = 0; i < frameCount; i++) {
|
||||
int32_t l = frames[i * MIC_CHANNELS + MIC_SLOT_LEFT];
|
||||
int32_t r = frames[i * MIC_CHANNELS + MIC_SLOT_RIGHT];
|
||||
sumL += l;
|
||||
sumR += r;
|
||||
sqL += (int64_t)l * l;
|
||||
sqR += (int64_t)r * r;
|
||||
}
|
||||
|
||||
float meanL = (float)sumL / (float)frameCount;
|
||||
float meanR = (float)sumR / (float)frameCount;
|
||||
float varL = (float)sqL / (float)frameCount - meanL * meanL;
|
||||
float varR = (float)sqR / (float)frameCount - meanR * meanR;
|
||||
if (varL < 0.0f) varL = 0.0f;
|
||||
if (varR < 0.0f) varR = 0.0f;
|
||||
|
||||
// Normalise to full scale, then smooth with a fast attack / slow release.
|
||||
float rmsL = sqrtf(varL) / 32768.0f;
|
||||
float rmsR = sqrtf(varR) / 32768.0f;
|
||||
envL_ += (rmsL - envL_) * (rmsL > envL_ ? ENV_ATTACK : ENV_RELEASE);
|
||||
envR_ += (rmsR - envR_) * (rmsR > envR_ ? ENV_ATTACK : ENV_RELEASE);
|
||||
|
||||
// Per-channel noise floor: fall quickly when it gets quieter, creep up very
|
||||
// slowly so sustained noise does not desensitise the detector.
|
||||
floorL_ += (envL_ - floorL_) * (envL_ < floorL_ ? FLOOR_FALL : FLOOR_RISE);
|
||||
floorR_ += (envR_ - floorR_) * (envR_ < floorR_ ? FLOOR_FALL : FLOOR_RISE);
|
||||
if (floorL_ < 1e-5f) floorL_ = 1e-5f;
|
||||
if (floorR_ < 1e-5f) floorR_ = 1e-5f;
|
||||
|
||||
float sigL = envL_ - floorL_ * FLOOR_MARGIN;
|
||||
float sigR = envR_ - floorR_ * FLOOR_MARGIN;
|
||||
if (sigL < 0.0f) sigL = 0.0f;
|
||||
if (sigR < 0.0f) sigR = 0.0f;
|
||||
|
||||
// Adaptive gate: a sound must stand clearly above its channel's floor.
|
||||
float gateL = fmaxf(floorL_ * 2.0f, ABS_GATE);
|
||||
float gateR = fmaxf(floorR_ * 2.0f, ABS_GATE);
|
||||
bool present = (sigL > gateL) || (sigR > gateR);
|
||||
|
||||
uint32_t now = millis();
|
||||
if (present) lastActiveMs_ = now;
|
||||
bool active = present || (uint32_t)(now - lastActiveMs_) < HOLD_MS;
|
||||
|
||||
float total = sigL + sigR;
|
||||
float balance = 0.0f;
|
||||
if (total > 1e-6f) {
|
||||
balance = (sigR - sigL) / total;
|
||||
if (fabsf(balance) < BALANCE_DEADBAND) {
|
||||
balance = 0.0f;
|
||||
} else {
|
||||
// Remove the deadband and rescale the remaining range to +/-1.
|
||||
float sign = balance < 0.0f ? -1.0f : 1.0f;
|
||||
balance = (balance - sign * BALANCE_DEADBAND) / (1.0f - BALANCE_DEADBAND);
|
||||
}
|
||||
}
|
||||
|
||||
float peak = fmaxf(sigL, sigR);
|
||||
float db = 20.0f * log10f(peak + 1e-9f);
|
||||
float loudness = (db - LOUD_MIN_DB) / (LOUD_MAX_DB - LOUD_MIN_DB);
|
||||
if (loudness < 0.0f) loudness = 0.0f;
|
||||
if (loudness > 1.0f) loudness = 1.0f;
|
||||
|
||||
#if SOUND_DETECTOR_DEBUG
|
||||
static uint32_t lastPrintMs = 0;
|
||||
if (active && (uint32_t)(millis() - lastPrintMs) > 250) {
|
||||
lastPrintMs = millis();
|
||||
Serial.printf("mic L=%.4f R=%.4f bal=%+.2f loud=%.2f\n",
|
||||
sigL, sigR, balance, loudness);
|
||||
}
|
||||
#endif
|
||||
|
||||
publish(sigL, sigR, balance, loudness, active);
|
||||
}
|
||||
|
||||
void SoundDetector::publish(float left, float right, float balance, float loudness,
|
||||
bool active)
|
||||
{
|
||||
portENTER_CRITICAL(&mux_);
|
||||
reading_.left = left;
|
||||
reading_.right = right;
|
||||
reading_.balance = balance;
|
||||
reading_.loudness = loudness;
|
||||
reading_.active = active;
|
||||
portEXIT_CRITICAL(&mux_);
|
||||
}
|
||||
|
||||
SoundReading SoundDetector::read()
|
||||
{
|
||||
SoundReading r;
|
||||
portENTER_CRITICAL(&mux_);
|
||||
r = reading_;
|
||||
portEXIT_CRITICAL(&mux_);
|
||||
return r;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* @file sound_detector.h
|
||||
* @brief Dual-microphone sound localizer for the T-Deck Plus (ES7210).
|
||||
*
|
||||
* Reads the two populated MEMS microphones as a 4-slot TDM stream:
|
||||
* slot 0 = MIC1 (left), slot 2 = MIC3 (right)
|
||||
* and exposes a smoothed left/right level, a stereo balance and a loudness
|
||||
* value suitable for placing a radar blip.
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <Arduino.h>
|
||||
#include <driver/i2s.h>
|
||||
|
||||
struct SoundReading {
|
||||
float left; // smoothed level above the noise floor (0..1)
|
||||
float right; // smoothed level above the noise floor (0..1)
|
||||
float balance; // -1 = far left, 0 = centre, +1 = far right
|
||||
float loudness; // 0 = quiet, 1 = loud (dB-mapped)
|
||||
bool active; // true while a sound is present (with a short hold)
|
||||
};
|
||||
|
||||
class SoundDetector {
|
||||
public:
|
||||
// Starts the ES7210 codec and the I2S receive task. Returns false if the
|
||||
// codec or I2S driver could not be initialised.
|
||||
bool begin(uint32_t sampleRate = 16000);
|
||||
|
||||
// Latest measurement. Safe to call from the main loop.
|
||||
SoundReading read();
|
||||
|
||||
bool available() const { return running_; }
|
||||
|
||||
// Ignore microphone input for a while (used so the device's own radar
|
||||
// ping does not register as a target).
|
||||
void suppress(uint32_t ms);
|
||||
|
||||
private:
|
||||
static void taskEntry(void *arg);
|
||||
void taskLoop();
|
||||
void processBlock(const int16_t *frames, size_t frameCount);
|
||||
void publish(float left, float right, float balance, float loudness, bool active);
|
||||
|
||||
uint32_t sampleRate_ = 16000;
|
||||
TaskHandle_t task_ = nullptr;
|
||||
volatile bool running_ = false;
|
||||
volatile uint32_t suppressUntil_ = 0;
|
||||
|
||||
portMUX_TYPE mux_ = portMUX_INITIALIZER_UNLOCKED;
|
||||
SoundReading reading_ = {};
|
||||
uint32_t lastActiveMs_ = 0;
|
||||
|
||||
float envL_ = 0.0f;
|
||||
float envR_ = 0.0f;
|
||||
float floorL_ = 0.01f;
|
||||
float floorR_ = 0.01f;
|
||||
};
|
||||
Reference in New Issue
Block a user