Phase 1: offline audio pipeline
Whole-track analysis into a frame-indexed FeatureTrack. Nothing reads a live AnalyserNode: realtime preview maps currentTime to a frame index, export counts frames, both read the same rows. - fft.js: radix-2 with precomputed tables, allocated once per track - analyze.js: STFT at hop 1/60s with CENTRED windows (a window that starts at the timestamp reports energy arriving up to 23ms later, which reads as visuals lagging the music). Energy features normalised against the track's own percentiles; absolute stats kept in summary for the look generator. - tempo.js: autocorrelation + grid F-measure, beat grid, downbeats - segment.js: self-similarity novelty, boundaries snapped to the bar grid - FeatureTrack: assembles everything, plus the lookahead fields. buildSlope rises through the bars leading into a higher-energy section, so a build can ramp into the drop rather than react after it lands. - clicktrack.js: mixes clicks onto the detected grid for validation by ear Three real bugs found and fixed by the tests: - 174 BPM read as 87. Mean-energy-per-beat scores a half-tempo grid identically to the true one; only an F-measure penalises the missed onsets via recall. - 90 BPM read as 180. Offbeat hi-hats make a double-tempo grid score perfectly on both precision and recall, so the grid is now interpreted metrically afterwards: a systematic strong/weak alternation means the real beat is every other grid point. - Beat grid drifted ~30ms over 30s from integer-frame offsets. Onset peaks are now parabolically interpolated and the grid least-squares fitted. Gates: 11/11 node tests against synthetic ground truth (tempo within 2% across 90-174 BPM, beat alignment under half a frame, segmentation within 2s of a known boundary, graceful on silence, 6-minute analysis in 1.0s); 5/5 browser checks including audio-driven vs fixed-step frame parity. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Dejvino
parent
7e31c19d6e
commit
cb320557e6
190
flow-state/src/audio/FeatureTrack.js
Normal file
190
flow-state/src/audio/FeatureTrack.js
Normal file
@ -0,0 +1,190 @@
|
||||
import { analyzeBuffer } from './analyze.js';
|
||||
import { detectTempo, buildBeatTracks } from './tempo.js';
|
||||
import { segment } from './segment.js';
|
||||
|
||||
/**
|
||||
* The frame-indexed feature table. Everything visual reads from here and from
|
||||
* nowhere else — no live AnalyserNode exists in this project.
|
||||
*
|
||||
* Realtime preview maps audio.currentTime to a frame index and reads row n.
|
||||
* The exporter counts frames and reads row n. Same rows, same visuals, which is
|
||||
* the whole basis of preview/export parity.
|
||||
*/
|
||||
export class FeatureTrack {
|
||||
constructor(data) {
|
||||
Object.assign(this, data);
|
||||
// Reused row object: at() is called 60 times a second and there is no
|
||||
// reason to allocate for it.
|
||||
this._row = {};
|
||||
}
|
||||
|
||||
/** @returns {object} the feature row for a frame, clamped to range. */
|
||||
at(frame) {
|
||||
const f = Math.max(0, Math.min(this.frameCount - 1, frame | 0));
|
||||
const row = this._row;
|
||||
const raw = this.raw;
|
||||
|
||||
row.rms = raw.rms[f];
|
||||
row.loudness = raw.loudness[f];
|
||||
row.bandSub = raw.bandSub[f];
|
||||
row.bandLow = raw.bandLow[f];
|
||||
row.bandMid = raw.bandMid[f];
|
||||
row.bandHigh = raw.bandHigh[f];
|
||||
row.bandAir = raw.bandAir[f];
|
||||
row.flux = raw.flux[f];
|
||||
row.centroid = raw.centroid[f];
|
||||
row.flatness = raw.flatness[f];
|
||||
row.width = raw.width[f];
|
||||
|
||||
row.beat = this.tracks.beat[f];
|
||||
row.beatPhase = this.tracks.beatPhase[f];
|
||||
row.barPhase = this.tracks.barPhase[f];
|
||||
row.phrasePhase = this.tracks.phrasePhase[f];
|
||||
|
||||
row.sectionProgress = this.tracks.sectionProgress[f];
|
||||
row.sectionEnergy = this.tracks.sectionEnergy[f];
|
||||
row.buildSlope = this.tracks.buildSlope[f];
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
sectionIndexAt(frame) {
|
||||
return this.tracks.sectionIndex[Math.max(0, Math.min(this.frameCount - 1, frame | 0))];
|
||||
}
|
||||
|
||||
sectionAt(frame) {
|
||||
return this.sections[this.sectionIndexAt(frame)] || this.sections[0];
|
||||
}
|
||||
|
||||
/** Nearest section boundary frame in a direction. Powers the transport buttons. */
|
||||
boundaryFrame(frame, direction) {
|
||||
const bounds = this.sections.map((s) => s.startFrame).concat([this.frameCount - 1]);
|
||||
if (direction < 0) {
|
||||
for (let i = bounds.length - 1; i >= 0; i--) if (bounds[i] < frame - 2) return bounds[i];
|
||||
return 0;
|
||||
}
|
||||
for (let i = 0; i < bounds.length; i++) if (bounds[i] > frame + 2) return bounds[i];
|
||||
return this.frameCount - 1;
|
||||
}
|
||||
|
||||
/** Feature series decimated for plotting in the debug scope. */
|
||||
series(name, points = 600) {
|
||||
const src = this.raw[name] || this.tracks[name];
|
||||
if (!src) return null;
|
||||
const out = new Float32Array(points);
|
||||
const step = this.frameCount / points;
|
||||
for (let i = 0; i < points; i++) {
|
||||
const start = Math.floor(i * step);
|
||||
const end = Math.min(this.frameCount, Math.floor((i + 1) * step));
|
||||
let peak = 0;
|
||||
for (let f = start; f < end; f++) if (src[f] > peak) peak = src[f];
|
||||
out[i] = peak;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the whole table from decoded audio. Synchronous and CPU-bound;
|
||||
* callers should run it off the main thread or accept a short freeze.
|
||||
*/
|
||||
static fromAudioBuffer(audioBuffer, { fps = 60, onProgress = null } = {}) {
|
||||
const report = (stage, p) => onProgress && onProgress(stage, p);
|
||||
|
||||
report('spectrum', 0);
|
||||
const analysis = analyzeBuffer(audioBuffer, {
|
||||
fps,
|
||||
onProgress: (p) => report('spectrum', p * 0.7),
|
||||
});
|
||||
|
||||
report('tempo', 0.7);
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, fps);
|
||||
const beatTracks = buildBeatTracks(tempo, analysis.frameCount, fps, analysis.raw.loudness);
|
||||
|
||||
report('structure', 0.85);
|
||||
const sections = segment(analysis.raw, analysis.frameCount, fps, tempo);
|
||||
|
||||
report('lookahead', 0.95);
|
||||
const tracks = {
|
||||
...beatTracks,
|
||||
...buildSectionTracks(sections, analysis.frameCount, fps, tempo),
|
||||
};
|
||||
|
||||
report('done', 1);
|
||||
|
||||
return new FeatureTrack({
|
||||
frameCount: analysis.frameCount,
|
||||
fps,
|
||||
duration: analysis.duration,
|
||||
sampleRate: analysis.sampleRate,
|
||||
raw: analysis.raw,
|
||||
summary: { ...analysis.summary, bpm: tempo.bpm, tempoConfidence: tempo.confidence },
|
||||
normalization: analysis.normalization,
|
||||
onsetEnvelope: analysis.onsetEnvelope,
|
||||
tempo,
|
||||
sections,
|
||||
tracks,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-frame section tracks, including the lookahead fields.
|
||||
*
|
||||
* `buildSlope` is the anticipation signal and the main payoff of analysing
|
||||
* offline: it rises through the bars leading into a HIGHER-energy section, so a
|
||||
* build can ramp its visuals into the drop instead of reacting once the drop has
|
||||
* already landed. A causal analyser cannot produce this at all.
|
||||
*/
|
||||
function buildSectionTracks(sections, frameCount, fps, tempo) {
|
||||
const sectionIndex = new Int32Array(frameCount);
|
||||
const sectionProgress = new Float32Array(frameCount);
|
||||
const sectionEnergy = new Float32Array(frameCount);
|
||||
const buildSlope = new Float32Array(frameCount);
|
||||
const timeToNextSection = new Float32Array(frameCount);
|
||||
|
||||
const barSeconds = (tempo.period * tempo.beatsPerBar) / fps;
|
||||
const maxEnergy = Math.max(1e-6, ...sections.map((s) => s.energy));
|
||||
|
||||
for (let si = 0; si < sections.length; si++) {
|
||||
const s = sections[si];
|
||||
const next = sections[si + 1] || null;
|
||||
const n = Math.max(1, s.endFrame - s.startFrame);
|
||||
|
||||
// Anticipation window: eight bars, or a third of the section if shorter.
|
||||
const windowSeconds = Math.min(barSeconds * 8, s.duration / 3);
|
||||
const windowFrames = Math.max(1, Math.round(windowSeconds * fps));
|
||||
|
||||
const rises = next ? next.energy > s.energy * 1.08 : false;
|
||||
const rise = next ? Math.min(1, (next.energy - s.energy) / Math.max(1e-6, maxEnergy * 0.5)) : 0;
|
||||
|
||||
for (let f = s.startFrame; f < s.endFrame && f < frameCount; f++) {
|
||||
sectionIndex[f] = si;
|
||||
sectionProgress[f] = (f - s.startFrame) / n;
|
||||
sectionEnergy[f] = s.energy / maxEnergy;
|
||||
const toNext = (s.endFrame - f) / fps;
|
||||
timeToNextSection[f] = toNext;
|
||||
|
||||
if (rises && s.endFrame - f <= windowFrames) {
|
||||
const t = 1 - (s.endFrame - f) / windowFrames;
|
||||
buildSlope[f] = t * t * rise; // eased, so the ramp starts gently
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Frames past the last section boundary (rounding slack at the tail).
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
if (sectionIndex[f] === 0 && sections.length && f >= sections[0].endFrame) {
|
||||
sectionIndex[f] = sections.length - 1;
|
||||
}
|
||||
}
|
||||
|
||||
return { sectionIndex, sectionProgress, sectionEnergy, buildSlope, timeToNextSection };
|
||||
}
|
||||
|
||||
/** Feature provider interface the Engine expects. */
|
||||
export function featureProviderFor(featureTrack) {
|
||||
return {
|
||||
at: (frame) => featureTrack.at(frame),
|
||||
frameCount: featureTrack.frameCount,
|
||||
};
|
||||
}
|
||||
209
flow-state/src/audio/analyze.js
Normal file
209
flow-state/src/audio/analyze.js
Normal file
@ -0,0 +1,209 @@
|
||||
// STFT feature extraction. One pass over the whole track, producing one row per
|
||||
// output video frame.
|
||||
//
|
||||
// Two details that matter more than they look:
|
||||
//
|
||||
// * Windows are CENTRED on the frame's timestamp, not started at it. A window
|
||||
// that starts at the timestamp reports energy that arrives up to 23ms later,
|
||||
// which reads on screen as the visuals lagging the music. Centring removes
|
||||
// that systematic offset.
|
||||
//
|
||||
// * Energy features are normalised against the track's own 5th/95th percentiles
|
||||
// at the end of the pass. A quiet ambient master and a brickwalled EDM master
|
||||
// then both use the full reactive range, without anyone touching a gain knob.
|
||||
// Absolute (un-normalised) statistics are kept in `summary` for the look
|
||||
// generator, which does need to know that one track is genuinely darker.
|
||||
|
||||
import { FFT, hannWindow } from './fft.js';
|
||||
|
||||
export const FFT_SIZE = 2048;
|
||||
|
||||
/** Band edges in Hz. Sub is deliberately narrow — it is the kick, not the bass. */
|
||||
export const BANDS = {
|
||||
bandSub: [20, 60],
|
||||
bandLow: [60, 250],
|
||||
bandMid: [250, 2000],
|
||||
bandHigh: [2000, 6000],
|
||||
bandAir: [6000, 16000],
|
||||
};
|
||||
|
||||
export const ENERGY_FEATURES = ['rms', 'loudness', 'bandSub', 'bandLow', 'bandMid', 'bandHigh', 'bandAir', 'flux'];
|
||||
export const SCALE_FREE_FEATURES = ['centroid', 'flatness', 'width'];
|
||||
export const RAW_FEATURES = [...ENERGY_FEATURES, ...SCALE_FREE_FEATURES];
|
||||
|
||||
function percentile(sorted, p) {
|
||||
if (!sorted.length) return 0;
|
||||
const i = Math.min(sorted.length - 1, Math.max(0, Math.round((sorted.length - 1) * p)));
|
||||
return sorted[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {AudioBuffer|{sampleRate,length,duration,numberOfChannels,getChannelData}} audioBuffer
|
||||
* @returns {{frameCount, fps, sampleRate, duration, raw, summary, onsetEnvelope}}
|
||||
*/
|
||||
export function analyzeBuffer(audioBuffer, { fps = 60, fftSize = FFT_SIZE, onProgress = null } = {}) {
|
||||
const sampleRate = audioBuffer.sampleRate;
|
||||
const duration = audioBuffer.duration;
|
||||
const hop = Math.max(1, Math.round(sampleRate / fps));
|
||||
const frameCount = Math.max(1, Math.ceil(duration * fps));
|
||||
|
||||
const channels = audioBuffer.numberOfChannels;
|
||||
const left = audioBuffer.getChannelData(0);
|
||||
const right = channels > 1 ? audioBuffer.getChannelData(1) : left;
|
||||
const totalSamples = left.length;
|
||||
|
||||
const fft = new FFT(fftSize);
|
||||
const window = hannWindow(fftSize);
|
||||
const frameBuffer = new Float32Array(fftSize);
|
||||
const halfFft = fftSize >> 1;
|
||||
|
||||
// Precompute band bin ranges.
|
||||
const binHz = sampleRate / fftSize;
|
||||
const bandRanges = {};
|
||||
for (const [name, [lo, hi]] of Object.entries(BANDS)) {
|
||||
bandRanges[name] = [
|
||||
Math.max(1, Math.floor(lo / binHz)),
|
||||
Math.min(halfFft - 1, Math.ceil(hi / binHz)),
|
||||
];
|
||||
}
|
||||
const binFrequencies = new Float32Array(halfFft);
|
||||
for (let i = 0; i < halfFft; i++) binFrequencies[i] = i * binHz;
|
||||
|
||||
const raw = {};
|
||||
for (const name of RAW_FEATURES) raw[name] = new Float32Array(frameCount);
|
||||
|
||||
let prevMagnitude = new Float32Array(halfFft);
|
||||
const progressEvery = Math.max(1, Math.floor(frameCount / 50));
|
||||
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
// Centred window.
|
||||
const centre = f * hop;
|
||||
const start = centre - (fftSize >> 1);
|
||||
|
||||
let sumSq = 0;
|
||||
let midSq = 0;
|
||||
let sideSq = 0;
|
||||
for (let i = 0; i < fftSize; i++) {
|
||||
const s = start + i;
|
||||
let l = 0, r = 0;
|
||||
if (s >= 0 && s < totalSamples) { l = left[s]; r = right[s]; }
|
||||
const mono = (l + r) * 0.5;
|
||||
const side = (l - r) * 0.5;
|
||||
frameBuffer[i] = mono * window[i];
|
||||
sumSq += mono * mono;
|
||||
midSq += mono * mono;
|
||||
sideSq += side * side;
|
||||
}
|
||||
|
||||
raw.rms[f] = Math.sqrt(sumSq / fftSize);
|
||||
raw.width[f] = Math.sqrt(sideSq / fftSize) / (Math.sqrt(midSq / fftSize) + Math.sqrt(sideSq / fftSize) + 1e-9);
|
||||
|
||||
const mag = fft.forward(frameBuffer);
|
||||
|
||||
let total = 0;
|
||||
let weighted = 0;
|
||||
let logSum = 0;
|
||||
let flux = 0;
|
||||
for (let i = 1; i < halfFft; i++) {
|
||||
const m = mag[i];
|
||||
total += m;
|
||||
weighted += m * binFrequencies[i];
|
||||
logSum += Math.log(m + 1e-10);
|
||||
const d = m - prevMagnitude[i];
|
||||
if (d > 0) flux += d;
|
||||
}
|
||||
|
||||
raw.loudness[f] = total / halfFft;
|
||||
raw.flux[f] = flux / halfFft;
|
||||
|
||||
// Spectral centroid, mapped to a log-frequency 0..1 so it matches how
|
||||
// brightness is actually perceived.
|
||||
const centroidHz = total > 1e-9 ? weighted / total : 0;
|
||||
raw.centroid[f] = centroidHz > 20
|
||||
? Math.min(1, Math.max(0, Math.log2(centroidHz / 20) / Math.log2(20000 / 20)))
|
||||
: 0;
|
||||
|
||||
const arithmeticMean = total / (halfFft - 1);
|
||||
const geometricMean = Math.exp(logSum / (halfFft - 1));
|
||||
raw.flatness[f] = arithmeticMean > 1e-9 ? Math.min(1, geometricMean / arithmeticMean) : 0;
|
||||
|
||||
for (const [name, [lo, hi]] of Object.entries(bandRanges)) {
|
||||
let sum = 0;
|
||||
for (let i = lo; i <= hi; i++) sum += mag[i];
|
||||
raw[name][f] = sum / (hi - lo + 1);
|
||||
}
|
||||
|
||||
prevMagnitude.set(mag);
|
||||
|
||||
if (onProgress && f % progressEvery === 0) onProgress(f / frameCount);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- summary
|
||||
// Computed from RAW values, before normalisation flattens them out.
|
||||
const sortedLoudness = Float32Array.from(raw.loudness).sort();
|
||||
const loudFloor = percentile(sortedLoudness, 0.1);
|
||||
const activeFrames = [];
|
||||
for (let f = 0; f < frameCount; f++) if (raw.loudness[f] > loudFloor) activeFrames.push(f);
|
||||
const activeCount = activeFrames.length || 1;
|
||||
|
||||
const meanOf = (name) => {
|
||||
let s = 0;
|
||||
for (const f of activeFrames) s += raw[name][f];
|
||||
return s / activeCount;
|
||||
};
|
||||
|
||||
const sortedRms = Float32Array.from(raw.rms).sort();
|
||||
const summary = {
|
||||
duration,
|
||||
sampleRate,
|
||||
fps,
|
||||
frameCount,
|
||||
meanCentroid: meanOf('centroid'),
|
||||
meanFlatness: meanOf('flatness'),
|
||||
meanWidth: meanOf('width'),
|
||||
meanLoudness: meanOf('loudness'),
|
||||
// Crest-ish: how much room there is between typical and peak level.
|
||||
// High on dynamic ambient, low on limitered club masters.
|
||||
dynamicRange: percentile(sortedRms, 0.95) > 1e-9
|
||||
? 1 - percentile(sortedRms, 0.4) / percentile(sortedRms, 0.95)
|
||||
: 0,
|
||||
bandBalance: {
|
||||
sub: meanOf('bandSub'), low: meanOf('bandLow'), mid: meanOf('bandMid'),
|
||||
high: meanOf('bandHigh'), air: meanOf('bandAir'),
|
||||
},
|
||||
};
|
||||
|
||||
// ------------------------------------------------------------ normalise
|
||||
const normalization = {};
|
||||
for (const name of ENERGY_FEATURES) {
|
||||
const sorted = Float32Array.from(raw[name]).sort();
|
||||
const lo = percentile(sorted, 0.05);
|
||||
const hi = percentile(sorted, 0.95);
|
||||
normalization[name] = { lo, hi };
|
||||
const span = hi - lo;
|
||||
const arr = raw[name];
|
||||
if (span > 1e-12) {
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
arr[f] = Math.min(1, Math.max(0, (arr[f] - lo) / span));
|
||||
}
|
||||
} else {
|
||||
arr.fill(0);
|
||||
}
|
||||
}
|
||||
for (const name of SCALE_FREE_FEATURES) {
|
||||
const arr = raw[name];
|
||||
for (let f = 0; f < frameCount; f++) arr[f] = Math.min(1, Math.max(0, arr[f]));
|
||||
}
|
||||
|
||||
// The onset envelope drives tempo detection. Kept separate from the
|
||||
// normalised flux because tempo wants raw contrast, not a clipped range.
|
||||
const onsetEnvelope = new Float32Array(frameCount);
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
// Weight the low band up: in this material the kick is the clock.
|
||||
onsetEnvelope[f] = raw.flux[f] * 0.6 + raw.bandSub[f] * 0.25 + raw.bandLow[f] * 0.15;
|
||||
}
|
||||
|
||||
if (onProgress) onProgress(1);
|
||||
|
||||
return { frameCount, fps, sampleRate, duration, raw, summary, normalization, onsetEnvelope };
|
||||
}
|
||||
105
flow-state/src/audio/clicktrack.js
Normal file
105
flow-state/src/audio/clicktrack.js
Normal file
@ -0,0 +1,105 @@
|
||||
// Click track — the Phase 1 gate, and the most useful validation tool in the
|
||||
// project.
|
||||
//
|
||||
// Beat detection cannot be judged by watching visuals: a grid that is 20ms late
|
||||
// or at half tempo still "looks kind of right". Mixing an audible click onto the
|
||||
// detected grid makes the answer immediate and unambiguous. Downbeats get a
|
||||
// higher pitch so bar alignment is audible too.
|
||||
//
|
||||
// If the clicks don't sit on the beat, stop and fix tempo.js before touching
|
||||
// anything downstream — every timing artefact in the finished video originates here.
|
||||
|
||||
const CLICK_MS = 25;
|
||||
|
||||
/**
|
||||
* Render the track with clicks mixed over it.
|
||||
* @returns {Promise<AudioBuffer>}
|
||||
*/
|
||||
export async function renderClickTrack(audioBuffer, tempo, { clickGain = 0.5, musicGain = 0.6, downbeatsOnly = false } = {}) {
|
||||
const sampleRate = audioBuffer.sampleRate;
|
||||
const length = audioBuffer.length;
|
||||
const offline = new OfflineAudioContext(2, length, sampleRate);
|
||||
|
||||
const music = offline.createBufferSource();
|
||||
music.buffer = audioBuffer;
|
||||
const musicNode = offline.createGain();
|
||||
musicNode.gain.value = musicGain;
|
||||
music.connect(musicNode).connect(offline.destination);
|
||||
|
||||
const clickBuffer = makeClick(offline, sampleRate, 1000);
|
||||
const downbeatBuffer = makeClick(offline, sampleRate, 1800);
|
||||
|
||||
const downbeatSet = new Set(tempo.downbeats.map((t) => Math.round(t * 1000)));
|
||||
|
||||
for (const time of tempo.beats) {
|
||||
if (time >= audioBuffer.duration) break;
|
||||
const isDownbeat = downbeatSet.has(Math.round(time * 1000));
|
||||
if (downbeatsOnly && !isDownbeat) continue;
|
||||
|
||||
const src = offline.createBufferSource();
|
||||
src.buffer = isDownbeat ? downbeatBuffer : clickBuffer;
|
||||
const gain = offline.createGain();
|
||||
gain.gain.value = clickGain * (isDownbeat ? 1.0 : 0.7);
|
||||
src.connect(gain).connect(offline.destination);
|
||||
src.start(time);
|
||||
}
|
||||
|
||||
music.start(0);
|
||||
return await offline.startRendering();
|
||||
}
|
||||
|
||||
function makeClick(ctx, sampleRate, frequency) {
|
||||
const length = Math.round((CLICK_MS / 1000) * sampleRate);
|
||||
const buffer = ctx.createBuffer(1, length, sampleRate);
|
||||
const data = buffer.getChannelData(0);
|
||||
for (let i = 0; i < length; i++) {
|
||||
const t = i / sampleRate;
|
||||
const env = Math.exp(-t * 180);
|
||||
data[i] = Math.sin(2 * Math.PI * frequency * t) * env;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/** Wrap an AudioBuffer as a WAV blob so it can be played or downloaded. */
|
||||
export function audioBufferToWavBlob(audioBuffer) {
|
||||
const channels = audioBuffer.numberOfChannels;
|
||||
const length = audioBuffer.length;
|
||||
const sampleRate = audioBuffer.sampleRate;
|
||||
const bytesPerSample = 2;
|
||||
const blockAlign = channels * bytesPerSample;
|
||||
const dataSize = length * blockAlign;
|
||||
|
||||
const buffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
const writeString = (offset, str) => {
|
||||
for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
|
||||
};
|
||||
|
||||
writeString(0, 'RIFF');
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeString(8, 'WAVE');
|
||||
writeString(12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, channels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * blockAlign, true);
|
||||
view.setUint16(32, blockAlign, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, 'data');
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
const data = [];
|
||||
for (let c = 0; c < channels; c++) data.push(audioBuffer.getChannelData(c));
|
||||
|
||||
let offset = 44;
|
||||
for (let i = 0; i < length; i++) {
|
||||
for (let c = 0; c < channels; c++) {
|
||||
const s = Math.max(-1, Math.min(1, data[c][i]));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
offset += 2;
|
||||
}
|
||||
}
|
||||
return new Blob([buffer], { type: 'audio/wav' });
|
||||
}
|
||||
47
flow-state/src/audio/decode.js
Normal file
47
flow-state/src/audio/decode.js
Normal file
@ -0,0 +1,47 @@
|
||||
// File -> AudioBuffer. Decoding is done once, up front, and the PCM is kept:
|
||||
// the analysis pass, the seed hash, the click track and the export mux all read
|
||||
// the same decoded samples.
|
||||
|
||||
let sharedContext = null;
|
||||
|
||||
function context() {
|
||||
if (!sharedContext) {
|
||||
sharedContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
}
|
||||
return sharedContext;
|
||||
}
|
||||
|
||||
export async function decodeFile(file) {
|
||||
const arrayBuffer = await file.arrayBuffer();
|
||||
return decodeArrayBuffer(arrayBuffer);
|
||||
}
|
||||
|
||||
export async function decodeArrayBuffer(arrayBuffer) {
|
||||
const ctx = context();
|
||||
// decodeAudioData detaches the buffer, so hand it a copy if the caller may
|
||||
// still need the original bytes.
|
||||
return await ctx.decodeAudioData(arrayBuffer.slice(0));
|
||||
}
|
||||
|
||||
/** Interleaved mono mixdown. Used for the content hash that seeds the look. */
|
||||
export function monoSamples(audioBuffer) {
|
||||
const channels = audioBuffer.numberOfChannels;
|
||||
const left = audioBuffer.getChannelData(0);
|
||||
if (channels === 1) return left;
|
||||
const right = audioBuffer.getChannelData(1);
|
||||
const out = new Float32Array(left.length);
|
||||
for (let i = 0; i < left.length; i++) out[i] = (left[i] + right[i]) * 0.5;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function audioContext() {
|
||||
return context();
|
||||
}
|
||||
|
||||
/** Human-readable duration, used by the transport display. */
|
||||
export function formatTime(seconds) {
|
||||
if (!isFinite(seconds) || seconds < 0) return '0:00';
|
||||
const m = Math.floor(seconds / 60);
|
||||
const s = Math.floor(seconds % 60);
|
||||
return `${m}:${String(s).padStart(2, '0')}`;
|
||||
}
|
||||
79
flow-state/src/audio/fft.js
Normal file
79
flow-state/src/audio/fft.js
Normal file
@ -0,0 +1,79 @@
|
||||
// Iterative radix-2 Cooley-Tukey FFT with precomputed twiddles and bit-reversal.
|
||||
//
|
||||
// Allocated once per size and reused across all ~21k frames of a track — the
|
||||
// analysis pass is the one place in this project where JS throughput actually
|
||||
// matters, and per-frame allocation is what would sink it.
|
||||
|
||||
export class FFT {
|
||||
constructor(size) {
|
||||
if ((size & (size - 1)) !== 0) throw new Error(`FFT size must be a power of two, got ${size}`);
|
||||
this.size = size;
|
||||
this.half = size >> 1;
|
||||
|
||||
this.re = new Float32Array(size);
|
||||
this.im = new Float32Array(size);
|
||||
this.magnitude = new Float32Array(this.half);
|
||||
|
||||
// Bit-reversal permutation table.
|
||||
this.rev = new Uint32Array(size);
|
||||
const bits = Math.log2(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
let r = 0;
|
||||
for (let b = 0; b < bits; b++) if (i & (1 << b)) r |= 1 << (bits - 1 - b);
|
||||
this.rev[i] = r;
|
||||
}
|
||||
|
||||
// Twiddle factors, flattened per stage.
|
||||
this.cos = new Float32Array(this.half);
|
||||
this.sin = new Float32Array(this.half);
|
||||
for (let i = 0; i < this.half; i++) {
|
||||
this.cos[i] = Math.cos((-2 * Math.PI * i) / size);
|
||||
this.sin[i] = Math.sin((-2 * Math.PI * i) / size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward transform of a real windowed signal. Writes into `this.magnitude`
|
||||
* (first size/2 bins) and returns it. The buffer is reused: copy it if you
|
||||
* need to keep it.
|
||||
*/
|
||||
forward(input) {
|
||||
const { size, re, im, rev, cos, sin } = this;
|
||||
|
||||
for (let i = 0; i < size; i++) {
|
||||
re[i] = input[rev[i]];
|
||||
im[i] = 0;
|
||||
}
|
||||
|
||||
for (let len = 2; len <= size; len <<= 1) {
|
||||
const halfLen = len >> 1;
|
||||
const step = size / len;
|
||||
for (let i = 0; i < size; i += len) {
|
||||
for (let j = 0, k = 0; j < halfLen; j++, k += step) {
|
||||
const c = cos[k], s = sin[k];
|
||||
const a = i + j, b = a + halfLen;
|
||||
const tre = re[b] * c - im[b] * s;
|
||||
const tim = re[b] * s + im[b] * c;
|
||||
re[b] = re[a] - tre;
|
||||
im[b] = im[a] - tim;
|
||||
re[a] += tre;
|
||||
im[a] += tim;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mag = this.magnitude;
|
||||
const scale = 2 / size;
|
||||
for (let i = 0; i < this.half; i++) {
|
||||
mag[i] = Math.sqrt(re[i] * re[i] + im[i] * im[i]) * scale;
|
||||
}
|
||||
return mag;
|
||||
}
|
||||
}
|
||||
|
||||
/** Periodic Hann window. Periodic (not symmetric) is correct for STFT analysis. */
|
||||
export function hannWindow(size) {
|
||||
const w = new Float32Array(size);
|
||||
for (let i = 0; i < size; i++) w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / size));
|
||||
return w;
|
||||
}
|
||||
266
flow-state/src/audio/segment.js
Normal file
266
flow-state/src/audio/segment.js
Normal file
@ -0,0 +1,266 @@
|
||||
// Structural segmentation: where the song changes, and what each part is.
|
||||
//
|
||||
// Standard approach — self-similarity plus a checkerboard novelty kernel — kept
|
||||
// deliberately simple. This is the fuzziest stage in the pipeline, and a wrong
|
||||
// boundary costs a scene change in a slightly odd place, not a broken video. An
|
||||
// ML detour here would buy very little.
|
||||
//
|
||||
// The one domain-specific trick that earns its keep: boundaries are SNAPPED to
|
||||
// the bar grid, preferring 4- and 8-bar multiples. Electronic music changes on
|
||||
// phrase lines, so snapping converts a boundary that is roughly right into one
|
||||
// that is exactly right.
|
||||
|
||||
export const SECTION_KINDS = ['intro', 'build', 'drop', 'sustain', 'breakdown', 'outro'];
|
||||
|
||||
const ANALYSIS_HZ = 4; // coarse grid for the similarity matrix
|
||||
const KERNEL_SECONDS = 6; // half-width of the checkerboard kernel
|
||||
const MIN_SECTION_SECONDS = 12;
|
||||
|
||||
function median(values) {
|
||||
if (!values.length) return 0;
|
||||
const s = Float64Array.from(values).sort();
|
||||
return s[Math.floor(s.length / 2)];
|
||||
}
|
||||
|
||||
/** Coarse, L2-normalised feature vectors — the rows of the similarity matrix. */
|
||||
function buildCoarseVectors(raw, frameCount, fps) {
|
||||
const step = Math.max(1, Math.round(fps / ANALYSIS_HZ));
|
||||
const count = Math.max(1, Math.floor(frameCount / step));
|
||||
const dims = ['bandSub', 'bandLow', 'bandMid', 'bandHigh', 'bandAir', 'centroid', 'flatness'];
|
||||
const vectors = [];
|
||||
|
||||
for (let c = 0; c < count; c++) {
|
||||
const start = c * step;
|
||||
const end = Math.min(frameCount, start + step);
|
||||
const v = new Float32Array(dims.length);
|
||||
for (let d = 0; d < dims.length; d++) {
|
||||
let sum = 0;
|
||||
for (let f = start; f < end; f++) sum += raw[dims[d]][f];
|
||||
v[d] = sum / Math.max(1, end - start);
|
||||
}
|
||||
let norm = 0;
|
||||
for (let d = 0; d < v.length; d++) norm += v[d] * v[d];
|
||||
norm = Math.sqrt(norm) + 1e-9;
|
||||
for (let d = 0; d < v.length; d++) v[d] /= norm;
|
||||
vectors.push(v);
|
||||
}
|
||||
return { vectors, step };
|
||||
}
|
||||
|
||||
function cosine(a, b) {
|
||||
let dot = 0;
|
||||
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
||||
return dot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Novelty curve. Only the band around the diagonal is needed, so this never
|
||||
* materialises the full N×N matrix.
|
||||
*/
|
||||
function noveltyCurve(vectors, kernelHalf) {
|
||||
const n = vectors.length;
|
||||
const novelty = new Float32Array(n);
|
||||
|
||||
// Gaussian-tapered checkerboard weights.
|
||||
const size = kernelHalf * 2;
|
||||
const weights = new Float32Array(size * size);
|
||||
const sigma = kernelHalf / 1.5;
|
||||
for (let a = 0; a < size; a++) {
|
||||
for (let b = 0; b < size; b++) {
|
||||
const da = a - kernelHalf + 0.5;
|
||||
const db = b - kernelHalf + 0.5;
|
||||
const taper = Math.exp(-(da * da + db * db) / (2 * sigma * sigma));
|
||||
const sign = (da * db) > 0 ? 1 : -1;
|
||||
weights[a * size + b] = sign * taper;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = kernelHalf; i < n - kernelHalf; i++) {
|
||||
let sum = 0;
|
||||
for (let a = 0; a < size; a++) {
|
||||
const ia = i - kernelHalf + a;
|
||||
for (let b = 0; b < size; b++) {
|
||||
const ib = i - kernelHalf + b;
|
||||
sum += weights[a * size + b] * cosine(vectors[ia], vectors[ib]);
|
||||
}
|
||||
}
|
||||
novelty[i] = Math.max(0, sum);
|
||||
}
|
||||
|
||||
let peak = 0;
|
||||
for (let i = 0; i < n; i++) if (novelty[i] > peak) peak = novelty[i];
|
||||
if (peak > 1e-9) for (let i = 0; i < n; i++) novelty[i] /= peak;
|
||||
return novelty;
|
||||
}
|
||||
|
||||
function pickPeaks(novelty, minDistance, threshold) {
|
||||
const peaks = [];
|
||||
const n = novelty.length;
|
||||
for (let i = 1; i < n - 1; i++) {
|
||||
if (novelty[i] < threshold) continue;
|
||||
if (novelty[i] < novelty[i - 1] || novelty[i] < novelty[i + 1]) continue;
|
||||
// Local maximum within the exclusion window.
|
||||
let isMax = true;
|
||||
const lo = Math.max(0, i - minDistance);
|
||||
const hi = Math.min(n - 1, i + minDistance);
|
||||
for (let j = lo; j <= hi; j++) if (novelty[j] > novelty[i]) { isMax = false; break; }
|
||||
if (!isMax) continue;
|
||||
if (peaks.length && i - peaks[peaks.length - 1] < minDistance) {
|
||||
if (novelty[i] > novelty[peaks[peaks.length - 1]]) peaks[peaks.length - 1] = i;
|
||||
continue;
|
||||
}
|
||||
peaks.push(i);
|
||||
}
|
||||
return peaks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Snap a time to the musical grid. Prefers, in order: an 8-bar line, a 4-bar
|
||||
* line, then any downbeat — but only if one is close enough to be plausibly the
|
||||
* same boundary.
|
||||
*/
|
||||
function snapToGrid(time, downbeats, barsPerPhrase = 8) {
|
||||
if (!downbeats.length) return time;
|
||||
const tolerance = 2.0;
|
||||
|
||||
const tryLevels = [barsPerPhrase, 4, 1];
|
||||
for (const level of tryLevels) {
|
||||
let best = null;
|
||||
let bestDist = Infinity;
|
||||
for (let i = 0; i < downbeats.length; i += level) {
|
||||
const d = Math.abs(downbeats[i] - time);
|
||||
if (d < bestDist) { bestDist = d; best = downbeats[i]; }
|
||||
}
|
||||
if (best !== null && bestDist <= tolerance) return best;
|
||||
}
|
||||
return time;
|
||||
}
|
||||
|
||||
function linearSlope(values) {
|
||||
const n = values.length;
|
||||
if (n < 2) return 0;
|
||||
let sx = 0, sy = 0, sxy = 0, sxx = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
sx += i; sy += values[i]; sxy += i * values[i]; sxx += i * i;
|
||||
}
|
||||
const denom = n * sxx - sx * sx;
|
||||
if (Math.abs(denom) < 1e-12) return 0;
|
||||
return ((n * sxy - sx * sy) / denom) * n; // normalised to "change across the section"
|
||||
}
|
||||
|
||||
/**
|
||||
* Label a section from its own statistics plus where it sits in the track.
|
||||
* Thresholds are relative to the track, never absolute, so a quiet ambient piece
|
||||
* still gets a full set of labels rather than being classified as one long intro.
|
||||
*/
|
||||
function classify(section, context) {
|
||||
const { energy, slope, flux, index, count, endEnergy } = section;
|
||||
const { medianEnergy, isFirst, isLast } = context;
|
||||
|
||||
const low = energy < medianEnergy * 0.72;
|
||||
const high = energy > medianEnergy * 1.12;
|
||||
|
||||
if (isFirst && energy < medianEnergy) return 'intro';
|
||||
if (isLast && (low || slope < -0.15)) return 'outro';
|
||||
if (slope > 0.18 && endEnergy > medianEnergy * 0.95) return 'build';
|
||||
if (high && flux > context.medianFlux * 0.95) return 'drop';
|
||||
if (low) return 'breakdown';
|
||||
if (index === 0) return 'intro';
|
||||
if (index === count - 1) return 'outro';
|
||||
return 'sustain';
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Array<{index,start,end,startFrame,endFrame,kind,energy,slope,flux,centroid}>}
|
||||
*/
|
||||
export function segment(raw, frameCount, fps, tempo) {
|
||||
const { vectors, step } = buildCoarseVectors(raw, frameCount, fps);
|
||||
const kernelHalf = Math.max(4, Math.round(KERNEL_SECONDS * ANALYSIS_HZ));
|
||||
|
||||
let boundaryTimes = [];
|
||||
if (vectors.length > kernelHalf * 2 + 4) {
|
||||
const novelty = noveltyCurve(vectors, kernelHalf);
|
||||
const minDistance = Math.round(MIN_SECTION_SECONDS * ANALYSIS_HZ);
|
||||
|
||||
// Adaptive threshold: mean + a fraction of the spread, so a track with
|
||||
// gentle transitions still yields boundaries and a busy one isn't shredded.
|
||||
let mean = 0;
|
||||
for (let i = 0; i < novelty.length; i++) mean += novelty[i];
|
||||
mean /= novelty.length;
|
||||
let variance = 0;
|
||||
for (let i = 0; i < novelty.length; i++) variance += (novelty[i] - mean) ** 2;
|
||||
const std = Math.sqrt(variance / novelty.length);
|
||||
const threshold = mean + std * 0.6;
|
||||
|
||||
boundaryTimes = pickPeaks(novelty, minDistance, threshold)
|
||||
.map((i) => (i * step) / fps);
|
||||
}
|
||||
|
||||
// Snap to the bar grid, dedupe, and drop anything too close to the edges.
|
||||
const duration = frameCount / fps;
|
||||
const snapped = boundaryTimes
|
||||
.map((t) => snapToGrid(t, tempo.downbeats))
|
||||
.filter((t) => t > MIN_SECTION_SECONDS * 0.5 && t < duration - MIN_SECTION_SECONDS * 0.5)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
const bounds = [0];
|
||||
for (const t of snapped) {
|
||||
if (t - bounds[bounds.length - 1] >= MIN_SECTION_SECONDS) bounds.push(t);
|
||||
}
|
||||
bounds.push(duration);
|
||||
|
||||
// Build sections and gather their statistics.
|
||||
const sections = [];
|
||||
for (let i = 0; i < bounds.length - 1; i++) {
|
||||
const start = bounds[i];
|
||||
const end = bounds[i + 1];
|
||||
const startFrame = Math.round(start * fps);
|
||||
const endFrame = Math.min(frameCount, Math.round(end * fps));
|
||||
const n = Math.max(1, endFrame - startFrame);
|
||||
|
||||
let energySum = 0, fluxSum = 0, centroidSum = 0, flatnessSum = 0, widthSum = 0;
|
||||
const coarseEnergy = [];
|
||||
const coarseStep = Math.max(1, Math.floor(n / 24));
|
||||
for (let f = startFrame; f < endFrame; f++) {
|
||||
energySum += raw.loudness[f];
|
||||
fluxSum += raw.flux[f];
|
||||
centroidSum += raw.centroid[f];
|
||||
flatnessSum += raw.flatness[f];
|
||||
widthSum += raw.width[f];
|
||||
if ((f - startFrame) % coarseStep === 0) coarseEnergy.push(raw.loudness[f]);
|
||||
}
|
||||
|
||||
const tailStart = Math.max(startFrame, endFrame - Math.round(fps * 4));
|
||||
let endEnergy = 0;
|
||||
for (let f = tailStart; f < endFrame; f++) endEnergy += raw.loudness[f];
|
||||
endEnergy /= Math.max(1, endFrame - tailStart);
|
||||
|
||||
sections.push({
|
||||
index: i,
|
||||
start, end,
|
||||
startFrame, endFrame,
|
||||
duration: end - start,
|
||||
energy: energySum / n,
|
||||
flux: fluxSum / n,
|
||||
centroid: centroidSum / n,
|
||||
flatness: flatnessSum / n,
|
||||
width: widthSum / n,
|
||||
slope: linearSlope(coarseEnergy),
|
||||
endEnergy,
|
||||
kind: 'sustain',
|
||||
});
|
||||
}
|
||||
|
||||
const context = {
|
||||
medianEnergy: median(sections.map((s) => s.energy)) || 1e-6,
|
||||
medianFlux: median(sections.map((s) => s.flux)) || 1e-6,
|
||||
};
|
||||
sections.forEach((s, i) => {
|
||||
s.kind = classify(
|
||||
{ ...s, index: i, count: sections.length },
|
||||
{ ...context, isFirst: i === 0, isLast: i === sections.length - 1 },
|
||||
);
|
||||
});
|
||||
|
||||
return sections;
|
||||
}
|
||||
122
flow-state/src/audio/synth.js
Normal file
122
flow-state/src/audio/synth.js
Normal file
@ -0,0 +1,122 @@
|
||||
// Synthetic audio with known ground truth, so tempo and segmentation can be
|
||||
// tested against an exact answer instead of "sounds about right". Real music
|
||||
// goes through the click track and the battery; this catches regressions in CI
|
||||
// speed and without a GPU.
|
||||
|
||||
/** Minimal stand-in for AudioBuffer — the analysis code only needs this surface. */
|
||||
export class MockAudioBuffer {
|
||||
constructor(channels, length, sampleRate) {
|
||||
this.numberOfChannels = channels;
|
||||
this.length = length;
|
||||
this.sampleRate = sampleRate;
|
||||
this.duration = length / sampleRate;
|
||||
this._data = Array.from({ length: channels }, () => new Float32Array(length));
|
||||
}
|
||||
getChannelData(i) { return this._data[i]; }
|
||||
}
|
||||
|
||||
function addKick(data, sampleRate, at, gain = 1) {
|
||||
const start = Math.round(at * sampleRate);
|
||||
const length = Math.round(0.12 * sampleRate);
|
||||
for (let i = 0; i < length; i++) {
|
||||
const s = start + i;
|
||||
if (s < 0 || s >= data.length) continue;
|
||||
const t = i / sampleRate;
|
||||
const env = Math.exp(-t * 30);
|
||||
const freq = 55 * Math.exp(-t * 20) + 40; // pitch drop, like a real kick
|
||||
data[s] += Math.sin(2 * Math.PI * freq * t) * env * gain;
|
||||
}
|
||||
}
|
||||
|
||||
function addHat(data, sampleRate, at, gain = 0.3, seed = 1) {
|
||||
const start = Math.round(at * sampleRate);
|
||||
const length = Math.round(0.04 * sampleRate);
|
||||
let s0 = seed >>> 0;
|
||||
const rnd = () => {
|
||||
s0 = (Math.imul(s0 ^ (s0 >>> 15), s0 | 1) + 0x6d2b79f5) >>> 0;
|
||||
return ((s0 >>> 14) & 0xffff) / 0xffff - 0.5;
|
||||
};
|
||||
for (let i = 0; i < length; i++) {
|
||||
const s = start + i;
|
||||
if (s < 0 || s >= data.length) continue;
|
||||
const env = Math.exp(-(i / sampleRate) * 90);
|
||||
data[s] += rnd() * env * gain;
|
||||
}
|
||||
}
|
||||
|
||||
function addPad(data, sampleRate, from, to, gain = 0.15, root = 110) {
|
||||
const start = Math.round(from * sampleRate);
|
||||
const end = Math.min(data.length, Math.round(to * sampleRate));
|
||||
for (let s = start; s < end; s++) {
|
||||
const t = s / sampleRate;
|
||||
data[s] += (Math.sin(2 * Math.PI * root * t) + Math.sin(2 * Math.PI * root * 1.5 * t)) * gain * 0.5;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A four-to-the-floor track at a known BPM.
|
||||
* @param {object} opts
|
||||
* @returns {MockAudioBuffer}
|
||||
*/
|
||||
export function synthesizeBeat({
|
||||
bpm = 128,
|
||||
duration = 40,
|
||||
sampleRate = 44100,
|
||||
hats = true,
|
||||
pad = true,
|
||||
kickGain = 1,
|
||||
} = {}) {
|
||||
const length = Math.round(duration * sampleRate);
|
||||
const buffer = new MockAudioBuffer(2, length, sampleRate);
|
||||
const left = buffer.getChannelData(0);
|
||||
const right = buffer.getChannelData(1);
|
||||
|
||||
const beat = 60 / bpm;
|
||||
let index = 0;
|
||||
for (let t = 0; t < duration; t += beat, index++) {
|
||||
// Accent the downbeat so the bar phase is detectable.
|
||||
addKick(left, sampleRate, t, kickGain * (index % 4 === 0 ? 1.0 : 0.8));
|
||||
if (hats) addHat(left, sampleRate, t + beat / 2, 0.25, index + 1);
|
||||
}
|
||||
if (pad) addPad(left, sampleRate, 0, duration, 0.12);
|
||||
|
||||
for (let i = 0; i < length; i++) right[i] = left[i] * 0.98;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* A track with a deliberate structural change at `changeAt` seconds: sparse and
|
||||
* dark before, dense and bright after. Segmentation must find that boundary.
|
||||
*/
|
||||
export function synthesizeSectioned({
|
||||
bpm = 128,
|
||||
duration = 120,
|
||||
changeAt = 60,
|
||||
sampleRate = 44100,
|
||||
} = {}) {
|
||||
const length = Math.round(duration * sampleRate);
|
||||
const buffer = new MockAudioBuffer(2, length, sampleRate);
|
||||
const left = buffer.getChannelData(0);
|
||||
const right = buffer.getChannelData(1);
|
||||
|
||||
const beat = 60 / bpm;
|
||||
let index = 0;
|
||||
for (let t = 0; t < duration; t += beat, index++) {
|
||||
const after = t >= changeAt;
|
||||
addKick(left, sampleRate, t, after ? 1.0 : 0.35);
|
||||
if (after) {
|
||||
addHat(left, sampleRate, t + beat / 2, 0.45, index + 1);
|
||||
addHat(left, sampleRate, t + beat / 4, 0.25, index + 7);
|
||||
}
|
||||
}
|
||||
addPad(left, sampleRate, 0, changeAt, 0.10, 110);
|
||||
addPad(left, sampleRate, changeAt, duration, 0.22, 440); // brighter after
|
||||
|
||||
for (let i = 0; i < length; i++) right[i] = left[i] * 0.98;
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/** Silence, for degenerate-input checks. */
|
||||
export function synthesizeSilence({ duration = 10, sampleRate = 44100 } = {}) {
|
||||
return new MockAudioBuffer(2, Math.round(duration * sampleRate), sampleRate);
|
||||
}
|
||||
369
flow-state/src/audio/tempo.js
Normal file
369
flow-state/src/audio/tempo.js
Normal file
@ -0,0 +1,369 @@
|
||||
// Tempo, beat grid and downbeats from the onset envelope.
|
||||
//
|
||||
// This runs over the WHOLE track, which is the entire reason to analyse offline:
|
||||
// a causal detector has to converge, and lags for the first several bars of every
|
||||
// section. Here the grid is exact from frame zero, and phase is fitted globally.
|
||||
//
|
||||
// The click track (audio/clicktrack.js) exists to validate this by ear. If the
|
||||
// clicks don't sit on the beat, nothing downstream can be trusted — every timing
|
||||
// artefact in the finished video traces back to this file.
|
||||
|
||||
const MIN_BPM = 60;
|
||||
const MAX_BPM = 200;
|
||||
const PREFERRED_BPM = 124; // log-space centre of the prior; club-ish but broad
|
||||
const PRIOR_WIDTH = 0.85;
|
||||
|
||||
/** Remove the slow-moving floor so autocorrelation sees onsets, not loudness. */
|
||||
function whiten(envelope, fps) {
|
||||
const n = envelope.length;
|
||||
const out = new Float32Array(n);
|
||||
const halfWindow = Math.max(2, Math.round(fps * 0.35));
|
||||
let sum = 0;
|
||||
const queue = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
queue.push(envelope[i]);
|
||||
sum += envelope[i];
|
||||
if (queue.length > halfWindow * 2 + 1) sum -= queue.shift();
|
||||
const mean = sum / queue.length;
|
||||
out[i] = Math.max(0, envelope[i] - mean);
|
||||
}
|
||||
let peak = 0;
|
||||
for (let i = 0; i < n; i++) if (out[i] > peak) peak = out[i];
|
||||
if (peak > 1e-9) for (let i = 0; i < n; i++) out[i] /= peak;
|
||||
return out;
|
||||
}
|
||||
|
||||
function autocorrelate(signal, minLag, maxLag) {
|
||||
const n = signal.length;
|
||||
const scores = new Float32Array(maxLag + 1);
|
||||
for (let lag = minLag; lag <= maxLag; lag++) {
|
||||
let sum = 0;
|
||||
const limit = n - lag;
|
||||
for (let i = 0; i < limit; i++) sum += signal[i] * signal[i + lag];
|
||||
scores[lag] = limit > 0 ? sum / limit : 0;
|
||||
}
|
||||
return scores;
|
||||
}
|
||||
|
||||
function bpmPrior(bpm) {
|
||||
const x = Math.log2(bpm / PREFERRED_BPM) / PRIOR_WIDTH;
|
||||
return Math.exp(-0.5 * x * x);
|
||||
}
|
||||
|
||||
/**
|
||||
* Discrete onset peaks: local maxima of the whitened envelope.
|
||||
*
|
||||
* Peak positions are refined to sub-frame precision by parabolic interpolation.
|
||||
* At 60fps a whole-frame quantisation is 16.7ms, and that error accumulates
|
||||
* through the period fit into visible drift by the end of a long track.
|
||||
*/
|
||||
function pickOnsetPeaks(signal) {
|
||||
const peaks = [];
|
||||
let total = 0;
|
||||
for (let i = 1; i < signal.length - 1; i++) {
|
||||
const v = signal[i];
|
||||
if (v < 0.06) continue;
|
||||
if (v < signal[i - 1] || v < signal[i + 1]) continue;
|
||||
|
||||
const a = signal[i - 1], b = v, c = signal[i + 1];
|
||||
const denom = a - 2 * b + c;
|
||||
const shift = Math.abs(denom) > 1e-12 ? (0.5 * (a - c)) / denom : 0;
|
||||
peaks.push({ frame: i + Math.max(-0.5, Math.min(0.5, shift)), strength: v });
|
||||
total += v;
|
||||
}
|
||||
return { peaks, total };
|
||||
}
|
||||
|
||||
/**
|
||||
* Least-squares fit of the grid to the onsets it already matches.
|
||||
*
|
||||
* The search above only ever tests integer offsets and a discrete set of periods,
|
||||
* which leaves up to half a frame of phase error and a small period error that
|
||||
* compounds — 64 beats into a track a 0.02-frame period error is already 20ms of
|
||||
* drift. Regressing matched onset positions against their beat indices recovers
|
||||
* both to sub-frame precision in one pass.
|
||||
*/
|
||||
function refineByRegression(peaks, period, offset, length) {
|
||||
const tolerance = Math.max(1.5, Math.min(3, period * 0.15));
|
||||
const gridCount = Math.max(1, Math.floor((length - offset) / period));
|
||||
|
||||
let sw = 0, sk = 0, st = 0, skk = 0, skt = 0;
|
||||
let matched = 0;
|
||||
for (const peak of peaks) {
|
||||
const k = Math.round((peak.frame - offset) / period);
|
||||
if (k < 0 || k > gridCount) continue;
|
||||
if (Math.abs(peak.frame - (offset + k * period)) > tolerance) continue;
|
||||
const w = peak.strength;
|
||||
sw += w; sk += w * k; st += w * peak.frame;
|
||||
skk += w * k * k; skt += w * k * peak.frame;
|
||||
matched++;
|
||||
}
|
||||
if (matched < 4 || sw < 1e-9) return { period, offset };
|
||||
|
||||
const denom = sw * skk - sk * sk;
|
||||
if (Math.abs(denom) < 1e-9) return { period, offset };
|
||||
|
||||
const slope = (sw * skt - sk * st) / denom;
|
||||
const intercept = (st - slope * sk) / sw;
|
||||
|
||||
// Reject a fit that has wandered — that means the matching was wrong, not
|
||||
// that the tempo is unusual.
|
||||
if (!isFinite(slope) || !isFinite(intercept)) return { period, offset };
|
||||
if (Math.abs(slope - period) > period * 0.05) return { period, offset };
|
||||
|
||||
return { period: slope, offset: intercept };
|
||||
}
|
||||
|
||||
/**
|
||||
* F-measure between a candidate grid and the detected onsets.
|
||||
*
|
||||
* This is what resolves tempo octaves, and a plain "mean energy at grid points"
|
||||
* cannot. A half-tempo grid hits every other kick at FULL strength, so its mean
|
||||
* energy per beat is identical to the true grid's — it only loses on RECALL,
|
||||
* because half the onsets go unexplained. A double-tempo grid has perfect recall
|
||||
* but half its beats land on silence, so it loses on PRECISION. Combining the two
|
||||
* is the only formulation that penalises both errors.
|
||||
*/
|
||||
function gridFScore(peaks, totalStrength, period, offset, length) {
|
||||
if (!peaks.length || period < 2) return 0;
|
||||
const tolerance = Math.max(1.5, Math.min(3, period * 0.15));
|
||||
const gridCount = Math.max(1, Math.floor((length - offset) / period));
|
||||
|
||||
const matchedGrid = new Set();
|
||||
let matchedStrength = 0;
|
||||
|
||||
for (const peak of peaks) {
|
||||
const k = Math.round((peak.frame - offset) / period);
|
||||
if (k < 0 || k > gridCount) continue;
|
||||
const gridFrame = offset + k * period;
|
||||
if (Math.abs(peak.frame - gridFrame) <= tolerance) {
|
||||
matchedStrength += peak.strength;
|
||||
matchedGrid.add(k);
|
||||
}
|
||||
}
|
||||
|
||||
const recall = totalStrength > 1e-9 ? matchedStrength / totalStrength : 0;
|
||||
const precision = matchedGrid.size / gridCount;
|
||||
if (precision + recall < 1e-9) return 0;
|
||||
return (2 * precision * recall) / (precision + recall);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mean matched onset strength at even vs odd grid positions.
|
||||
*
|
||||
* Unmatched grid points count as zero — a grid point that lands on silence is
|
||||
* evidence against the grid, not a missing sample.
|
||||
*/
|
||||
function alternation(peaks, period, offset, length) {
|
||||
const tolerance = Math.max(1.5, Math.min(3, period * 0.15));
|
||||
const gridCount = Math.max(2, Math.floor((length - offset) / period));
|
||||
const strength = new Float64Array(gridCount);
|
||||
|
||||
for (const peak of peaks) {
|
||||
const k = Math.round((peak.frame - offset) / period);
|
||||
if (k < 0 || k >= gridCount) continue;
|
||||
if (Math.abs(peak.frame - (offset + k * period)) <= tolerance) {
|
||||
strength[k] = Math.max(strength[k], peak.strength);
|
||||
}
|
||||
}
|
||||
|
||||
let evenSum = 0, evenCount = 0, oddSum = 0, oddCount = 0;
|
||||
for (let k = 0; k < gridCount; k++) {
|
||||
if (k % 2 === 0) { evenSum += strength[k]; evenCount++; }
|
||||
else { oddSum += strength[k]; oddCount++; }
|
||||
}
|
||||
const even = evenCount ? evenSum / evenCount : 0;
|
||||
const odd = oddCount ? oddSum / oddCount : 0;
|
||||
const hi = Math.max(even, odd);
|
||||
return { even, odd, ratio: hi > 1e-9 ? Math.min(even, odd) / hi : 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Correct a double-tempo reading.
|
||||
*
|
||||
* Offbeat hi-hats make a double-tempo grid score perfectly on both precision and
|
||||
* recall — every grid point genuinely has an onset — so the F-measure alone
|
||||
* cannot tell 90 BPM with hats from 180 BPM. What separates them is that the
|
||||
* onsets ALTERNATE strong/weak: kick, hat, kick, hat. A systematic alternation
|
||||
* means the real beat is every other grid point. This is the same cue a listener
|
||||
* uses, and it is why the grid is fitted first and metrically interpreted second.
|
||||
*/
|
||||
function correctOctave(peaks, totalStrength, period, offset, length, fps) {
|
||||
let currentPeriod = period;
|
||||
let currentOffset = offset;
|
||||
|
||||
for (let iteration = 0; iteration < 2; iteration++) {
|
||||
const bpm = (60 * fps) / currentPeriod;
|
||||
const halvedBpm = bpm / 2;
|
||||
if (halvedBpm < MIN_BPM) break;
|
||||
|
||||
const alt = alternation(peaks, currentPeriod, currentOffset, length);
|
||||
if (alt.ratio >= 0.62) break;
|
||||
|
||||
const strongIsEven = alt.even >= alt.odd;
|
||||
const nextPeriod = currentPeriod * 2;
|
||||
const nextOffset = strongIsEven ? currentOffset : currentOffset + currentPeriod;
|
||||
|
||||
// Only accept if the slower grid still explains the material well: this
|
||||
// guards against halving genuinely syncopated but correctly-fitted music.
|
||||
const before = gridFScore(peaks, totalStrength, currentPeriod, currentOffset, length);
|
||||
const after = gridFScore(peaks, totalStrength, nextPeriod, nextOffset, length);
|
||||
if (after < before * 0.55) break;
|
||||
|
||||
currentPeriod = nextPeriod;
|
||||
currentOffset = nextOffset;
|
||||
}
|
||||
|
||||
return { period: currentPeriod, offset: currentOffset };
|
||||
}
|
||||
|
||||
function bestOffset(peaks, totalStrength, period, length) {
|
||||
let best = 0;
|
||||
let bestScore = -1;
|
||||
const steps = Math.ceil(period);
|
||||
for (let o = 0; o < steps; o++) {
|
||||
const s = gridFScore(peaks, totalStrength, period, o, length);
|
||||
if (s > bestScore) { bestScore = s; best = o; }
|
||||
}
|
||||
return { offset: best, score: bestScore };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Float32Array} onsetEnvelope per-frame onset strength
|
||||
* @param {number} fps
|
||||
* @returns {{bpm, period, offset, beats, downbeats, confidence, beatsPerBar}}
|
||||
*/
|
||||
export function detectTempo(onsetEnvelope, fps, { beatsPerBar = 4 } = {}) {
|
||||
const signal = whiten(onsetEnvelope, fps);
|
||||
const minLag = Math.floor((60 * fps) / MAX_BPM);
|
||||
const maxLag = Math.ceil((60 * fps) / MIN_BPM);
|
||||
const acf = autocorrelate(signal, minLag, maxLag);
|
||||
|
||||
// Weight autocorrelation by the tempo prior, then take the peak.
|
||||
let bestLag = minLag;
|
||||
let bestValue = -Infinity;
|
||||
for (let lag = minLag; lag <= maxLag; lag++) {
|
||||
const bpm = (60 * fps) / lag;
|
||||
const v = acf[lag] * bpmPrior(bpm);
|
||||
if (v > bestValue) { bestValue = v; bestLag = lag; }
|
||||
}
|
||||
|
||||
// Parabolic refinement around the peak for sub-frame period accuracy.
|
||||
let period = bestLag;
|
||||
if (bestLag > minLag && bestLag < maxLag) {
|
||||
const a = acf[bestLag - 1], b = acf[bestLag], c = acf[bestLag + 1];
|
||||
const denom = a - 2 * b + c;
|
||||
if (Math.abs(denom) > 1e-12) period = bestLag - (0.5 * (c - a)) / denom;
|
||||
}
|
||||
|
||||
const { peaks, total: totalStrength } = pickOnsetPeaks(signal);
|
||||
const length = signal.length;
|
||||
|
||||
// Octave resolution. The prior only breaks near-ties; the F-measure does the
|
||||
// actual work, which is why a 174 BPM track no longer reads as 87.
|
||||
const candidates = [period / 4, period / 2, period, period * 2, period * 4].filter((p) => {
|
||||
const bpm = (60 * fps) / p;
|
||||
return bpm >= MIN_BPM && bpm <= MAX_BPM && p >= 2;
|
||||
});
|
||||
|
||||
let chosen = { period, offset: 0, score: -1 };
|
||||
for (const p of candidates) {
|
||||
const { offset, score } = bestOffset(peaks, totalStrength, p, length);
|
||||
const adjusted = score * (0.6 + 0.4 * bpmPrior((60 * fps) / p));
|
||||
if (adjusted > chosen.score) chosen = { period: p, offset, score: adjusted };
|
||||
}
|
||||
|
||||
period = chosen.period;
|
||||
let offset = chosen.offset;
|
||||
|
||||
// Local refinement of period and offset together — catches a grid that is
|
||||
// right at the start and drifts by the end of a six-minute track.
|
||||
let bestRefined = gridFScore(peaks, totalStrength, period, offset, length);
|
||||
for (let dp = -0.5; dp <= 0.5001; dp += 0.02) {
|
||||
const p = period + dp;
|
||||
if (p < 2) continue;
|
||||
const { offset: o, score } = bestOffset(peaks, totalStrength, p, length);
|
||||
if (score > bestRefined) { bestRefined = score; period = p; offset = o; }
|
||||
}
|
||||
|
||||
// Metrical interpretation, after the grid itself is fitted.
|
||||
({ period, offset } = correctOctave(peaks, totalStrength, period, offset, length, fps));
|
||||
|
||||
// Sub-frame fit last, so it refines the grid we actually committed to.
|
||||
({ period, offset } = refineByRegression(peaks, period, offset, length));
|
||||
|
||||
const bpm = (60 * fps) / period;
|
||||
|
||||
const beats = [];
|
||||
for (let t = offset; t < onsetEnvelope.length; t += period) beats.push(t / fps);
|
||||
|
||||
// Downbeat: of the `beatsPerBar` possible bar phases, the one whose beats
|
||||
// carry the most energy.
|
||||
//
|
||||
// Scored on the RAW envelope, not the whitened one. Whitening subtracts the
|
||||
// local mean, which is exactly the accent information that distinguishes beat
|
||||
// one from the other three — a four-to-the-floor pattern has a kick on every
|
||||
// beat and the only cue is that one of them is louder.
|
||||
let bestPhase = 0;
|
||||
let bestPhaseScore = -1;
|
||||
for (let phase = 0; phase < beatsPerBar; phase++) {
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
for (let b = phase; b < beats.length; b += beatsPerBar) {
|
||||
const i = Math.round(beats[b] * fps);
|
||||
if (i >= 1 && i < onsetEnvelope.length - 1) {
|
||||
sum += Math.max(onsetEnvelope[i - 1], onsetEnvelope[i], onsetEnvelope[i + 1]);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
const score = count ? sum / count : 0;
|
||||
if (score > bestPhaseScore) { bestPhaseScore = score; bestPhase = phase; }
|
||||
}
|
||||
const downbeats = [];
|
||||
for (let b = bestPhase; b < beats.length; b += beatsPerBar) downbeats.push(beats[b]);
|
||||
|
||||
// Confidence is the grid's own F-measure: how much of the onset energy the
|
||||
// chosen grid explains, and how many of its beats are actually occupied.
|
||||
const confidence = gridFScore(peaks, totalStrength, period, offset, length);
|
||||
|
||||
return { bpm, period, offset, beats, downbeats, beatsPerBar, barPhase: bestPhase, confidence };
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-frame phase tracks from the grid.
|
||||
*
|
||||
* `beat` is a decaying spike, but gated by a smoothed loudness envelope so a
|
||||
* drumless breakdown doesn't strobe on a grid that is technically still running.
|
||||
*/
|
||||
export function buildBeatTracks(tempo, frameCount, fps, loudness) {
|
||||
const beatPhase = new Float32Array(frameCount);
|
||||
const barPhase = new Float32Array(frameCount);
|
||||
const phrasePhase = new Float32Array(frameCount);
|
||||
const beat = new Float32Array(frameCount);
|
||||
|
||||
const { period, offset, beatsPerBar } = tempo;
|
||||
const barPeriod = period * beatsPerBar;
|
||||
const phrasePeriod = barPeriod * 8;
|
||||
const barOffset = offset + tempo.barPhase * period;
|
||||
|
||||
// Smoothed loudness gate.
|
||||
const gate = new Float32Array(frameCount);
|
||||
const attack = 0.25, release = 0.02;
|
||||
let g = 0;
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
const target = loudness ? loudness[f] : 1;
|
||||
g += (target - g) * (target > g ? attack : release);
|
||||
gate[f] = Math.min(1, g * 1.6);
|
||||
}
|
||||
|
||||
const decay = Math.max(1e-3, period * 0.28);
|
||||
for (let f = 0; f < frameCount; f++) {
|
||||
const sinceBeat = ((f - offset) % period + period) % period;
|
||||
beatPhase[f] = sinceBeat / period;
|
||||
barPhase[f] = (((f - barOffset) % barPeriod) + barPeriod) % barPeriod / barPeriod;
|
||||
phrasePhase[f] = (((f - barOffset) % phrasePeriod) + phrasePeriod) % phrasePeriod / phrasePeriod;
|
||||
beat[f] = Math.exp(-sinceBeat / decay) * gate[f];
|
||||
}
|
||||
|
||||
return { beatPhase, barPhase, phrasePhase, beat };
|
||||
}
|
||||
@ -1 +1,150 @@
|
||||
// Phase 1 gate — filled in when the phase lands.
|
||||
// Phase 1 gate — the offline audio pipeline feeding the renderer.
|
||||
//
|
||||
// The numeric correctness of tempo and segmentation is covered by node tests
|
||||
// against synthetic ground truth (test/audio.test.js) and, for real material, by
|
||||
// the click track. What can only be checked here is the join: that the frame-indexed
|
||||
// table reaches the shader, and that driving the clock from audio produces exactly
|
||||
// the same frames as counting them.
|
||||
|
||||
import { check, expect, expectBelow } from './framework.js';
|
||||
import { Engine } from '../engine/Engine.js';
|
||||
import { FeatureTrack, featureProviderFor } from '../audio/FeatureTrack.js';
|
||||
import { synthesizeSectioned } from '../audio/synth.js';
|
||||
import { nebula } from '../scenes/shader/nebula.js';
|
||||
import { defaultValues } from '../params/schema.js';
|
||||
|
||||
const PALETTE = [[0.05, 0.02, 0.15], [0.85, 0.15, 0.55], [0.15, 0.75, 0.95], [0.98, 0.85, 0.35]];
|
||||
|
||||
let cachedTrack = null;
|
||||
export function testTrack() {
|
||||
if (!cachedTrack) {
|
||||
const buffer = synthesizeSectioned({ bpm: 128, duration: 90, changeAt: 45 });
|
||||
cachedTrack = FeatureTrack.fromAudioBuffer(buffer, { fps: 60 });
|
||||
}
|
||||
return cachedTrack;
|
||||
}
|
||||
|
||||
function makeEngine(track, width = 256, height = 144) {
|
||||
const engine = new Engine({ width, height });
|
||||
engine.timeline.setDuration(track.duration);
|
||||
engine.setFeatureProvider(featureProviderFor(track));
|
||||
engine.setLayerSpecs([{
|
||||
module: nebula, params: defaultValues(nebula), seed: 4242,
|
||||
opacity: 1, blend: 'normal', palette: PALETTE,
|
||||
}]);
|
||||
return engine;
|
||||
}
|
||||
|
||||
check(1, 'feature track builds and every row is finite and in range', () => {
|
||||
const track = testTrack();
|
||||
let bad = 0;
|
||||
for (let f = 0; f < track.frameCount; f += 7) {
|
||||
const row = track.at(f);
|
||||
for (const v of Object.values(row)) {
|
||||
if (!Number.isFinite(v) || v < 0 || v > 1) { bad++; break; }
|
||||
}
|
||||
}
|
||||
return expect(bad === 0,
|
||||
`${bad} bad rows of ${Math.ceil(track.frameCount / 7)} sampled · ` +
|
||||
`${track.frameCount} frames · ${track.sections.length} sections · ` +
|
||||
`${track.summary.bpm.toFixed(1)} BPM · confidence ${track.tempo.confidence.toFixed(2)}`);
|
||||
});
|
||||
|
||||
check(1, 'audio-driven clock and fixed-step counting agree', () => {
|
||||
// The core preview/export parity claim. Realtime maps currentTime to a frame
|
||||
// index; export counts frames. Both must land on identical images.
|
||||
const track = testTrack();
|
||||
const engine = makeEngine(track);
|
||||
try {
|
||||
const fixed = engine.hashRun(1000, 100);
|
||||
|
||||
engine.compositor.reset();
|
||||
const driven = [];
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const frame = 1000 + i;
|
||||
// Jittered playback position within the frame's window, as a real
|
||||
// audio element would report it.
|
||||
const jitter = (((i * 7919) % 1000) / 1000) * 0.9;
|
||||
engine.timeline.syncToAudio((frame + jitter) / 60);
|
||||
const target = engine.renderCurrent();
|
||||
driven.push(engine.hashCurrent(target));
|
||||
}
|
||||
const mismatches = fixed.filter((h, i) => h !== driven[i]).length;
|
||||
return expect(mismatches === 0, `${mismatches}/100 frames differed`);
|
||||
} finally {
|
||||
engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(1, 'features actually reach the shader', () => {
|
||||
// A scene wired to a table it never reads would pass every other check here.
|
||||
const track = testTrack();
|
||||
const engine = makeEngine(track);
|
||||
try {
|
||||
const quiet = track.sections.reduce((a, b) => (a.energy < b.energy ? a : b));
|
||||
const loud = track.sections.reduce((a, b) => (a.energy > b.energy ? a : b));
|
||||
|
||||
engine.compositor.reset();
|
||||
const quietHash = engine.hashCurrent(engine.renderFrame(quiet.startFrame + 60));
|
||||
engine.compositor.reset();
|
||||
const loudHash = engine.hashCurrent(engine.renderFrame(loud.startFrame + 60));
|
||||
|
||||
return expect(quietHash !== loudHash,
|
||||
`quiet ${quiet.kind} ${quietHash} vs loud ${loud.kind} ${loudHash}`);
|
||||
} finally {
|
||||
engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(1, 'section-boundary seek is exact without warm-up', () => {
|
||||
const track = testTrack();
|
||||
const engine = makeEngine(track);
|
||||
try {
|
||||
const boundary = track.sections[1] ? track.sections[1].startFrame : 600;
|
||||
const sequential = engine.hashRun(boundary, 3);
|
||||
engine.compositor.reset();
|
||||
const direct = engine.hashCurrent(engine.renderFrame(boundary));
|
||||
return expect(direct === sequential[0], `direct ${direct} vs sequential ${sequential[0]}`);
|
||||
} finally {
|
||||
engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(1, 'mid-section seek converges under warm-up with feedback active', () => {
|
||||
// With feedback on, an arbitrary seek can only converge, not match exactly.
|
||||
// The gate is that warm-up gets it visually indistinguishable — the documented
|
||||
// behaviour in PLAN.md §6.
|
||||
const track = testTrack();
|
||||
const engine = makeEngine(track);
|
||||
try {
|
||||
engine.compositor.setFeedback({ amount: 0.7, decay: 0.92, zoom: 0.99 });
|
||||
|
||||
const targetFrame = 1500;
|
||||
engine.compositor.reset();
|
||||
for (let f = 1200; f < targetFrame; f++) engine.renderFrame(f);
|
||||
const sequentialPixels = Uint8Array.from(
|
||||
engine.readPixels(engine.renderFrame(targetFrame)),
|
||||
);
|
||||
|
||||
engine.warmUp(targetFrame, 120);
|
||||
const warmedPixels = Uint8Array.from(engine.readPixels(engine.renderCurrent()));
|
||||
|
||||
let sum = 0;
|
||||
for (let i = 0; i < sequentialPixels.length; i += 4) {
|
||||
sum += Math.abs(sequentialPixels[i] - warmedPixels[i]);
|
||||
}
|
||||
const distance = sum / (sequentialPixels.length / 4) / 255;
|
||||
return expectBelow(distance, 0.02, 'mean red-channel distance after warm-up');
|
||||
} finally {
|
||||
engine.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(1, 'analysis of a five-minute track stays within budget', () => {
|
||||
const buffer = synthesizeSectioned({ bpm: 128, duration: 300, changeAt: 150 });
|
||||
const started = Date.now();
|
||||
const track = FeatureTrack.fromAudioBuffer(buffer, { fps: 60 });
|
||||
const elapsed = (Date.now() - started) / 1000;
|
||||
return expect(elapsed < 6,
|
||||
`${elapsed.toFixed(2)}s for a 5-minute track (${track.frameCount} frames)`);
|
||||
}, { slow: true });
|
||||
|
||||
194
flow-state/test/audio.test.js
Normal file
194
flow-state/test/audio.test.js
Normal file
@ -0,0 +1,194 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { FFT, hannWindow } from '../src/audio/fft.js';
|
||||
import { analyzeBuffer, RAW_FEATURES } from '../src/audio/analyze.js';
|
||||
import { detectTempo, buildBeatTracks } from '../src/audio/tempo.js';
|
||||
import { segment } from '../src/audio/segment.js';
|
||||
import { synthesizeBeat, synthesizeSectioned, synthesizeSilence } from '../src/audio/synth.js';
|
||||
|
||||
const FPS = 60;
|
||||
|
||||
test('FFT resolves a pure tone into the right bin', () => {
|
||||
const size = 2048;
|
||||
const sampleRate = 44100;
|
||||
const fft = new FFT(size);
|
||||
const window = hannWindow(size);
|
||||
const freq = 1000;
|
||||
|
||||
const input = new Float32Array(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
input[i] = Math.sin((2 * Math.PI * freq * i) / sampleRate) * window[i];
|
||||
}
|
||||
const mag = fft.forward(input);
|
||||
|
||||
let peakBin = 0;
|
||||
for (let i = 1; i < mag.length; i++) if (mag[i] > mag[peakBin]) peakBin = i;
|
||||
const peakHz = (peakBin * sampleRate) / size;
|
||||
assert.ok(Math.abs(peakHz - freq) < sampleRate / size,
|
||||
`peak at ${peakHz.toFixed(1)}Hz, expected ~${freq}Hz`);
|
||||
});
|
||||
|
||||
test('analysis produces finite, in-range features', () => {
|
||||
const buffer = synthesizeBeat({ bpm: 128, duration: 20 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
|
||||
assert.equal(analysis.frameCount, Math.ceil(buffer.duration * FPS));
|
||||
|
||||
for (const name of RAW_FEATURES) {
|
||||
const arr = analysis.raw[name];
|
||||
assert.ok(arr, `missing feature ${name}`);
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
assert.ok(Number.isFinite(arr[i]), `${name}[${i}] is not finite`);
|
||||
assert.ok(arr[i] >= 0 && arr[i] <= 1, `${name}[${i}] = ${arr[i]} out of range`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('no feature is constant or pinned on real-ish material', () => {
|
||||
// A feature stuck at 0 or 1 contributes nothing to the visuals and usually
|
||||
// means a wrong band split — the thing the feature scope exists to catch.
|
||||
const buffer = synthesizeBeat({ bpm: 124, duration: 30 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
|
||||
for (const name of ['loudness', 'bandSub', 'bandLow', 'bandMid', 'flux', 'centroid']) {
|
||||
const arr = analysis.raw[name];
|
||||
let min = Infinity, max = -Infinity, sum = 0;
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
min = Math.min(min, arr[i]);
|
||||
max = Math.max(max, arr[i]);
|
||||
sum += arr[i];
|
||||
}
|
||||
const spread = max - min;
|
||||
assert.ok(spread > 0.15, `${name} spread only ${spread.toFixed(3)} — feature is inert`);
|
||||
const mean = sum / arr.length;
|
||||
assert.ok(mean > 0.005 && mean < 0.995, `${name} mean ${mean.toFixed(3)} is pinned`);
|
||||
}
|
||||
});
|
||||
|
||||
test('tempo detection lands on the true BPM', () => {
|
||||
for (const bpm of [90, 110, 124, 128, 140, 174]) {
|
||||
const buffer = synthesizeBeat({ bpm, duration: 40 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
|
||||
const error = Math.abs(tempo.bpm - bpm) / bpm;
|
||||
assert.ok(error < 0.02,
|
||||
`bpm ${tempo.bpm.toFixed(2)} vs true ${bpm} (${(error * 100).toFixed(2)}% off)`);
|
||||
}
|
||||
});
|
||||
|
||||
test('beat grid aligns with the true beat positions', () => {
|
||||
const bpm = 128;
|
||||
const buffer = synthesizeBeat({ bpm, duration: 40 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
|
||||
const trueBeat = 60 / bpm;
|
||||
let worst = 0;
|
||||
for (const t of tempo.beats) {
|
||||
if (t > 30) break;
|
||||
const nearest = Math.round(t / trueBeat) * trueBeat;
|
||||
worst = Math.max(worst, Math.abs(t - nearest));
|
||||
}
|
||||
// Within half a video frame: any more would be visible as the visuals
|
||||
// sitting off the beat.
|
||||
assert.ok(worst < 0.5 / FPS, `worst beat offset ${(worst * 1000).toFixed(1)}ms`);
|
||||
});
|
||||
|
||||
test('downbeats land on the accented beat', () => {
|
||||
const bpm = 128;
|
||||
const buffer = synthesizeBeat({ bpm, duration: 40 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
|
||||
const trueBar = (60 / bpm) * 4;
|
||||
let aligned = 0;
|
||||
for (const t of tempo.downbeats) {
|
||||
const offset = Math.abs(t / trueBar - Math.round(t / trueBar));
|
||||
if (offset < 0.05) aligned++;
|
||||
}
|
||||
assert.ok(aligned / tempo.downbeats.length > 0.9,
|
||||
`only ${aligned}/${tempo.downbeats.length} downbeats on the bar line`);
|
||||
});
|
||||
|
||||
test('beat tracks are bounded and phase-continuous', () => {
|
||||
const buffer = synthesizeBeat({ bpm: 128, duration: 20 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
const tracks = buildBeatTracks(tempo, analysis.frameCount, FPS, analysis.raw.loudness);
|
||||
|
||||
for (const name of ['beat', 'beatPhase', 'barPhase', 'phrasePhase']) {
|
||||
const arr = tracks[name];
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
assert.ok(Number.isFinite(arr[i]) && arr[i] >= 0 && arr[i] <= 1,
|
||||
`${name}[${i}] = ${arr[i]}`);
|
||||
}
|
||||
}
|
||||
// Phase must wrap exactly once per beat, never jump mid-beat.
|
||||
let wraps = 0;
|
||||
for (let i = 1; i < tracks.beatPhase.length; i++) {
|
||||
if (tracks.beatPhase[i] < tracks.beatPhase[i - 1]) wraps++;
|
||||
}
|
||||
const expected = 20 / (60 / 128);
|
||||
assert.ok(Math.abs(wraps - expected) <= 2, `${wraps} phase wraps, expected ~${expected.toFixed(0)}`);
|
||||
});
|
||||
|
||||
test('segmentation finds a real structural boundary', () => {
|
||||
const changeAt = 60;
|
||||
const buffer = synthesizeSectioned({ bpm: 128, duration: 120, changeAt });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo);
|
||||
|
||||
assert.ok(sections.length >= 2, `expected at least 2 sections, got ${sections.length}`);
|
||||
|
||||
const boundaries = sections.slice(1).map((s) => s.start);
|
||||
const nearest = boundaries.reduce(
|
||||
(best, b) => (Math.abs(b - changeAt) < Math.abs(best - changeAt) ? b : best),
|
||||
Infinity,
|
||||
);
|
||||
assert.ok(Math.abs(nearest - changeAt) <= 2.0,
|
||||
`nearest boundary ${nearest.toFixed(2)}s, true change at ${changeAt}s`);
|
||||
});
|
||||
|
||||
test('sections tile the track with no gaps or overlaps', () => {
|
||||
const buffer = synthesizeSectioned({ duration: 120, changeAt: 60 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo);
|
||||
|
||||
assert.equal(sections[0].start, 0);
|
||||
for (let i = 1; i < sections.length; i++) {
|
||||
assert.equal(sections[i].start, sections[i - 1].end, `gap before section ${i}`);
|
||||
}
|
||||
assert.ok(Math.abs(sections[sections.length - 1].end - analysis.duration) < 0.05);
|
||||
for (const s of sections) {
|
||||
assert.ok(s.duration > 0, `section ${s.index} has non-positive duration`);
|
||||
assert.ok(['intro', 'build', 'drop', 'sustain', 'breakdown', 'outro'].includes(s.kind),
|
||||
`bad kind ${s.kind}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('silence degrades gracefully instead of producing NaN', () => {
|
||||
const buffer = synthesizeSilence({ duration: 10 });
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
for (const name of RAW_FEATURES) {
|
||||
for (const v of analysis.raw[name]) assert.ok(Number.isFinite(v), `${name} not finite on silence`);
|
||||
}
|
||||
const tempo = detectTempo(analysis.onsetEnvelope, FPS);
|
||||
assert.ok(Number.isFinite(tempo.bpm), 'bpm not finite on silence');
|
||||
const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo);
|
||||
assert.ok(sections.length >= 1, 'no sections produced for silence');
|
||||
});
|
||||
|
||||
test('analysis of a six-minute track completes within budget', () => {
|
||||
const buffer = synthesizeBeat({ bpm: 128, duration: 360, hats: true, pad: true });
|
||||
const started = Date.now();
|
||||
const analysis = analyzeBuffer(buffer, { fps: FPS });
|
||||
detectTempo(analysis.onsetEnvelope, FPS);
|
||||
const elapsed = (Date.now() - started) / 1000;
|
||||
// The plan's budget is ~3s. Node is a fair proxy for browser JS here.
|
||||
assert.ok(elapsed < 6, `analysis took ${elapsed.toFixed(2)}s`);
|
||||
console.log(` (6-minute analysis: ${elapsed.toFixed(2)}s)`);
|
||||
});
|
||||
Loading…
Reference in New Issue
Block a user