diff --git a/flow-state/src/audio/FeatureTrack.js b/flow-state/src/audio/FeatureTrack.js new file mode 100644 index 0000000..0c2bb40 --- /dev/null +++ b/flow-state/src/audio/FeatureTrack.js @@ -0,0 +1,190 @@ +import { analyzeBuffer } from './analyze.js'; +import { detectTempo, buildBeatTracks } from './tempo.js'; +import { segment } from './segment.js'; + +/** + * The frame-indexed feature table. Everything visual reads from here and from + * nowhere else — no live AnalyserNode exists in this project. + * + * Realtime preview maps audio.currentTime to a frame index and reads row n. + * The exporter counts frames and reads row n. Same rows, same visuals, which is + * the whole basis of preview/export parity. + */ +export class FeatureTrack { + constructor(data) { + Object.assign(this, data); + // Reused row object: at() is called 60 times a second and there is no + // reason to allocate for it. + this._row = {}; + } + + /** @returns {object} the feature row for a frame, clamped to range. */ + at(frame) { + const f = Math.max(0, Math.min(this.frameCount - 1, frame | 0)); + const row = this._row; + const raw = this.raw; + + row.rms = raw.rms[f]; + row.loudness = raw.loudness[f]; + row.bandSub = raw.bandSub[f]; + row.bandLow = raw.bandLow[f]; + row.bandMid = raw.bandMid[f]; + row.bandHigh = raw.bandHigh[f]; + row.bandAir = raw.bandAir[f]; + row.flux = raw.flux[f]; + row.centroid = raw.centroid[f]; + row.flatness = raw.flatness[f]; + row.width = raw.width[f]; + + row.beat = this.tracks.beat[f]; + row.beatPhase = this.tracks.beatPhase[f]; + row.barPhase = this.tracks.barPhase[f]; + row.phrasePhase = this.tracks.phrasePhase[f]; + + row.sectionProgress = this.tracks.sectionProgress[f]; + row.sectionEnergy = this.tracks.sectionEnergy[f]; + row.buildSlope = this.tracks.buildSlope[f]; + + return row; + } + + sectionIndexAt(frame) { + return this.tracks.sectionIndex[Math.max(0, Math.min(this.frameCount - 1, frame | 0))]; + } + + sectionAt(frame) { + return this.sections[this.sectionIndexAt(frame)] || this.sections[0]; + } + + /** Nearest section boundary frame in a direction. Powers the transport buttons. */ + boundaryFrame(frame, direction) { + const bounds = this.sections.map((s) => s.startFrame).concat([this.frameCount - 1]); + if (direction < 0) { + for (let i = bounds.length - 1; i >= 0; i--) if (bounds[i] < frame - 2) return bounds[i]; + return 0; + } + for (let i = 0; i < bounds.length; i++) if (bounds[i] > frame + 2) return bounds[i]; + return this.frameCount - 1; + } + + /** Feature series decimated for plotting in the debug scope. */ + series(name, points = 600) { + const src = this.raw[name] || this.tracks[name]; + if (!src) return null; + const out = new Float32Array(points); + const step = this.frameCount / points; + for (let i = 0; i < points; i++) { + const start = Math.floor(i * step); + const end = Math.min(this.frameCount, Math.floor((i + 1) * step)); + let peak = 0; + for (let f = start; f < end; f++) if (src[f] > peak) peak = src[f]; + out[i] = peak; + } + return out; + } + + /** + * Build the whole table from decoded audio. Synchronous and CPU-bound; + * callers should run it off the main thread or accept a short freeze. + */ + static fromAudioBuffer(audioBuffer, { fps = 60, onProgress = null } = {}) { + const report = (stage, p) => onProgress && onProgress(stage, p); + + report('spectrum', 0); + const analysis = analyzeBuffer(audioBuffer, { + fps, + onProgress: (p) => report('spectrum', p * 0.7), + }); + + report('tempo', 0.7); + const tempo = detectTempo(analysis.onsetEnvelope, fps); + const beatTracks = buildBeatTracks(tempo, analysis.frameCount, fps, analysis.raw.loudness); + + report('structure', 0.85); + const sections = segment(analysis.raw, analysis.frameCount, fps, tempo); + + report('lookahead', 0.95); + const tracks = { + ...beatTracks, + ...buildSectionTracks(sections, analysis.frameCount, fps, tempo), + }; + + report('done', 1); + + return new FeatureTrack({ + frameCount: analysis.frameCount, + fps, + duration: analysis.duration, + sampleRate: analysis.sampleRate, + raw: analysis.raw, + summary: { ...analysis.summary, bpm: tempo.bpm, tempoConfidence: tempo.confidence }, + normalization: analysis.normalization, + onsetEnvelope: analysis.onsetEnvelope, + tempo, + sections, + tracks, + }); + } +} + +/** + * Per-frame section tracks, including the lookahead fields. + * + * `buildSlope` is the anticipation signal and the main payoff of analysing + * offline: it rises through the bars leading into a HIGHER-energy section, so a + * build can ramp its visuals into the drop instead of reacting once the drop has + * already landed. A causal analyser cannot produce this at all. + */ +function buildSectionTracks(sections, frameCount, fps, tempo) { + const sectionIndex = new Int32Array(frameCount); + const sectionProgress = new Float32Array(frameCount); + const sectionEnergy = new Float32Array(frameCount); + const buildSlope = new Float32Array(frameCount); + const timeToNextSection = new Float32Array(frameCount); + + const barSeconds = (tempo.period * tempo.beatsPerBar) / fps; + const maxEnergy = Math.max(1e-6, ...sections.map((s) => s.energy)); + + for (let si = 0; si < sections.length; si++) { + const s = sections[si]; + const next = sections[si + 1] || null; + const n = Math.max(1, s.endFrame - s.startFrame); + + // Anticipation window: eight bars, or a third of the section if shorter. + const windowSeconds = Math.min(barSeconds * 8, s.duration / 3); + const windowFrames = Math.max(1, Math.round(windowSeconds * fps)); + + const rises = next ? next.energy > s.energy * 1.08 : false; + const rise = next ? Math.min(1, (next.energy - s.energy) / Math.max(1e-6, maxEnergy * 0.5)) : 0; + + for (let f = s.startFrame; f < s.endFrame && f < frameCount; f++) { + sectionIndex[f] = si; + sectionProgress[f] = (f - s.startFrame) / n; + sectionEnergy[f] = s.energy / maxEnergy; + const toNext = (s.endFrame - f) / fps; + timeToNextSection[f] = toNext; + + if (rises && s.endFrame - f <= windowFrames) { + const t = 1 - (s.endFrame - f) / windowFrames; + buildSlope[f] = t * t * rise; // eased, so the ramp starts gently + } + } + } + + // Frames past the last section boundary (rounding slack at the tail). + for (let f = 0; f < frameCount; f++) { + if (sectionIndex[f] === 0 && sections.length && f >= sections[0].endFrame) { + sectionIndex[f] = sections.length - 1; + } + } + + return { sectionIndex, sectionProgress, sectionEnergy, buildSlope, timeToNextSection }; +} + +/** Feature provider interface the Engine expects. */ +export function featureProviderFor(featureTrack) { + return { + at: (frame) => featureTrack.at(frame), + frameCount: featureTrack.frameCount, + }; +} diff --git a/flow-state/src/audio/analyze.js b/flow-state/src/audio/analyze.js new file mode 100644 index 0000000..9362e77 --- /dev/null +++ b/flow-state/src/audio/analyze.js @@ -0,0 +1,209 @@ +// STFT feature extraction. One pass over the whole track, producing one row per +// output video frame. +// +// Two details that matter more than they look: +// +// * Windows are CENTRED on the frame's timestamp, not started at it. A window +// that starts at the timestamp reports energy that arrives up to 23ms later, +// which reads on screen as the visuals lagging the music. Centring removes +// that systematic offset. +// +// * Energy features are normalised against the track's own 5th/95th percentiles +// at the end of the pass. A quiet ambient master and a brickwalled EDM master +// then both use the full reactive range, without anyone touching a gain knob. +// Absolute (un-normalised) statistics are kept in `summary` for the look +// generator, which does need to know that one track is genuinely darker. + +import { FFT, hannWindow } from './fft.js'; + +export const FFT_SIZE = 2048; + +/** Band edges in Hz. Sub is deliberately narrow — it is the kick, not the bass. */ +export const BANDS = { + bandSub: [20, 60], + bandLow: [60, 250], + bandMid: [250, 2000], + bandHigh: [2000, 6000], + bandAir: [6000, 16000], +}; + +export const ENERGY_FEATURES = ['rms', 'loudness', 'bandSub', 'bandLow', 'bandMid', 'bandHigh', 'bandAir', 'flux']; +export const SCALE_FREE_FEATURES = ['centroid', 'flatness', 'width']; +export const RAW_FEATURES = [...ENERGY_FEATURES, ...SCALE_FREE_FEATURES]; + +function percentile(sorted, p) { + if (!sorted.length) return 0; + const i = Math.min(sorted.length - 1, Math.max(0, Math.round((sorted.length - 1) * p))); + return sorted[i]; +} + +/** + * @param {AudioBuffer|{sampleRate,length,duration,numberOfChannels,getChannelData}} audioBuffer + * @returns {{frameCount, fps, sampleRate, duration, raw, summary, onsetEnvelope}} + */ +export function analyzeBuffer(audioBuffer, { fps = 60, fftSize = FFT_SIZE, onProgress = null } = {}) { + const sampleRate = audioBuffer.sampleRate; + const duration = audioBuffer.duration; + const hop = Math.max(1, Math.round(sampleRate / fps)); + const frameCount = Math.max(1, Math.ceil(duration * fps)); + + const channels = audioBuffer.numberOfChannels; + const left = audioBuffer.getChannelData(0); + const right = channels > 1 ? audioBuffer.getChannelData(1) : left; + const totalSamples = left.length; + + const fft = new FFT(fftSize); + const window = hannWindow(fftSize); + const frameBuffer = new Float32Array(fftSize); + const halfFft = fftSize >> 1; + + // Precompute band bin ranges. + const binHz = sampleRate / fftSize; + const bandRanges = {}; + for (const [name, [lo, hi]] of Object.entries(BANDS)) { + bandRanges[name] = [ + Math.max(1, Math.floor(lo / binHz)), + Math.min(halfFft - 1, Math.ceil(hi / binHz)), + ]; + } + const binFrequencies = new Float32Array(halfFft); + for (let i = 0; i < halfFft; i++) binFrequencies[i] = i * binHz; + + const raw = {}; + for (const name of RAW_FEATURES) raw[name] = new Float32Array(frameCount); + + let prevMagnitude = new Float32Array(halfFft); + const progressEvery = Math.max(1, Math.floor(frameCount / 50)); + + for (let f = 0; f < frameCount; f++) { + // Centred window. + const centre = f * hop; + const start = centre - (fftSize >> 1); + + let sumSq = 0; + let midSq = 0; + let sideSq = 0; + for (let i = 0; i < fftSize; i++) { + const s = start + i; + let l = 0, r = 0; + if (s >= 0 && s < totalSamples) { l = left[s]; r = right[s]; } + const mono = (l + r) * 0.5; + const side = (l - r) * 0.5; + frameBuffer[i] = mono * window[i]; + sumSq += mono * mono; + midSq += mono * mono; + sideSq += side * side; + } + + raw.rms[f] = Math.sqrt(sumSq / fftSize); + raw.width[f] = Math.sqrt(sideSq / fftSize) / (Math.sqrt(midSq / fftSize) + Math.sqrt(sideSq / fftSize) + 1e-9); + + const mag = fft.forward(frameBuffer); + + let total = 0; + let weighted = 0; + let logSum = 0; + let flux = 0; + for (let i = 1; i < halfFft; i++) { + const m = mag[i]; + total += m; + weighted += m * binFrequencies[i]; + logSum += Math.log(m + 1e-10); + const d = m - prevMagnitude[i]; + if (d > 0) flux += d; + } + + raw.loudness[f] = total / halfFft; + raw.flux[f] = flux / halfFft; + + // Spectral centroid, mapped to a log-frequency 0..1 so it matches how + // brightness is actually perceived. + const centroidHz = total > 1e-9 ? weighted / total : 0; + raw.centroid[f] = centroidHz > 20 + ? Math.min(1, Math.max(0, Math.log2(centroidHz / 20) / Math.log2(20000 / 20))) + : 0; + + const arithmeticMean = total / (halfFft - 1); + const geometricMean = Math.exp(logSum / (halfFft - 1)); + raw.flatness[f] = arithmeticMean > 1e-9 ? Math.min(1, geometricMean / arithmeticMean) : 0; + + for (const [name, [lo, hi]] of Object.entries(bandRanges)) { + let sum = 0; + for (let i = lo; i <= hi; i++) sum += mag[i]; + raw[name][f] = sum / (hi - lo + 1); + } + + prevMagnitude.set(mag); + + if (onProgress && f % progressEvery === 0) onProgress(f / frameCount); + } + + // ---------------------------------------------------------------- summary + // Computed from RAW values, before normalisation flattens them out. + const sortedLoudness = Float32Array.from(raw.loudness).sort(); + const loudFloor = percentile(sortedLoudness, 0.1); + const activeFrames = []; + for (let f = 0; f < frameCount; f++) if (raw.loudness[f] > loudFloor) activeFrames.push(f); + const activeCount = activeFrames.length || 1; + + const meanOf = (name) => { + let s = 0; + for (const f of activeFrames) s += raw[name][f]; + return s / activeCount; + }; + + const sortedRms = Float32Array.from(raw.rms).sort(); + const summary = { + duration, + sampleRate, + fps, + frameCount, + meanCentroid: meanOf('centroid'), + meanFlatness: meanOf('flatness'), + meanWidth: meanOf('width'), + meanLoudness: meanOf('loudness'), + // Crest-ish: how much room there is between typical and peak level. + // High on dynamic ambient, low on limitered club masters. + dynamicRange: percentile(sortedRms, 0.95) > 1e-9 + ? 1 - percentile(sortedRms, 0.4) / percentile(sortedRms, 0.95) + : 0, + bandBalance: { + sub: meanOf('bandSub'), low: meanOf('bandLow'), mid: meanOf('bandMid'), + high: meanOf('bandHigh'), air: meanOf('bandAir'), + }, + }; + + // ------------------------------------------------------------ normalise + const normalization = {}; + for (const name of ENERGY_FEATURES) { + const sorted = Float32Array.from(raw[name]).sort(); + const lo = percentile(sorted, 0.05); + const hi = percentile(sorted, 0.95); + normalization[name] = { lo, hi }; + const span = hi - lo; + const arr = raw[name]; + if (span > 1e-12) { + for (let f = 0; f < frameCount; f++) { + arr[f] = Math.min(1, Math.max(0, (arr[f] - lo) / span)); + } + } else { + arr.fill(0); + } + } + for (const name of SCALE_FREE_FEATURES) { + const arr = raw[name]; + for (let f = 0; f < frameCount; f++) arr[f] = Math.min(1, Math.max(0, arr[f])); + } + + // The onset envelope drives tempo detection. Kept separate from the + // normalised flux because tempo wants raw contrast, not a clipped range. + const onsetEnvelope = new Float32Array(frameCount); + for (let f = 0; f < frameCount; f++) { + // Weight the low band up: in this material the kick is the clock. + onsetEnvelope[f] = raw.flux[f] * 0.6 + raw.bandSub[f] * 0.25 + raw.bandLow[f] * 0.15; + } + + if (onProgress) onProgress(1); + + return { frameCount, fps, sampleRate, duration, raw, summary, normalization, onsetEnvelope }; +} diff --git a/flow-state/src/audio/clicktrack.js b/flow-state/src/audio/clicktrack.js new file mode 100644 index 0000000..2631e8a --- /dev/null +++ b/flow-state/src/audio/clicktrack.js @@ -0,0 +1,105 @@ +// Click track — the Phase 1 gate, and the most useful validation tool in the +// project. +// +// Beat detection cannot be judged by watching visuals: a grid that is 20ms late +// or at half tempo still "looks kind of right". Mixing an audible click onto the +// detected grid makes the answer immediate and unambiguous. Downbeats get a +// higher pitch so bar alignment is audible too. +// +// If the clicks don't sit on the beat, stop and fix tempo.js before touching +// anything downstream — every timing artefact in the finished video originates here. + +const CLICK_MS = 25; + +/** + * Render the track with clicks mixed over it. + * @returns {Promise} + */ +export async function renderClickTrack(audioBuffer, tempo, { clickGain = 0.5, musicGain = 0.6, downbeatsOnly = false } = {}) { + const sampleRate = audioBuffer.sampleRate; + const length = audioBuffer.length; + const offline = new OfflineAudioContext(2, length, sampleRate); + + const music = offline.createBufferSource(); + music.buffer = audioBuffer; + const musicNode = offline.createGain(); + musicNode.gain.value = musicGain; + music.connect(musicNode).connect(offline.destination); + + const clickBuffer = makeClick(offline, sampleRate, 1000); + const downbeatBuffer = makeClick(offline, sampleRate, 1800); + + const downbeatSet = new Set(tempo.downbeats.map((t) => Math.round(t * 1000))); + + for (const time of tempo.beats) { + if (time >= audioBuffer.duration) break; + const isDownbeat = downbeatSet.has(Math.round(time * 1000)); + if (downbeatsOnly && !isDownbeat) continue; + + const src = offline.createBufferSource(); + src.buffer = isDownbeat ? downbeatBuffer : clickBuffer; + const gain = offline.createGain(); + gain.gain.value = clickGain * (isDownbeat ? 1.0 : 0.7); + src.connect(gain).connect(offline.destination); + src.start(time); + } + + music.start(0); + return await offline.startRendering(); +} + +function makeClick(ctx, sampleRate, frequency) { + const length = Math.round((CLICK_MS / 1000) * sampleRate); + const buffer = ctx.createBuffer(1, length, sampleRate); + const data = buffer.getChannelData(0); + for (let i = 0; i < length; i++) { + const t = i / sampleRate; + const env = Math.exp(-t * 180); + data[i] = Math.sin(2 * Math.PI * frequency * t) * env; + } + return buffer; +} + +/** Wrap an AudioBuffer as a WAV blob so it can be played or downloaded. */ +export function audioBufferToWavBlob(audioBuffer) { + const channels = audioBuffer.numberOfChannels; + const length = audioBuffer.length; + const sampleRate = audioBuffer.sampleRate; + const bytesPerSample = 2; + const blockAlign = channels * bytesPerSample; + const dataSize = length * blockAlign; + + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + + const writeString = (offset, str) => { + for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i)); + }; + + writeString(0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); + writeString(8, 'WAVE'); + writeString(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, channels, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * blockAlign, true); + view.setUint16(32, blockAlign, true); + view.setUint16(34, 16, true); + writeString(36, 'data'); + view.setUint32(40, dataSize, true); + + const data = []; + for (let c = 0; c < channels; c++) data.push(audioBuffer.getChannelData(c)); + + let offset = 44; + for (let i = 0; i < length; i++) { + for (let c = 0; c < channels; c++) { + const s = Math.max(-1, Math.min(1, data[c][i])); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + offset += 2; + } + } + return new Blob([buffer], { type: 'audio/wav' }); +} diff --git a/flow-state/src/audio/decode.js b/flow-state/src/audio/decode.js new file mode 100644 index 0000000..56b1c4e --- /dev/null +++ b/flow-state/src/audio/decode.js @@ -0,0 +1,47 @@ +// File -> AudioBuffer. Decoding is done once, up front, and the PCM is kept: +// the analysis pass, the seed hash, the click track and the export mux all read +// the same decoded samples. + +let sharedContext = null; + +function context() { + if (!sharedContext) { + sharedContext = new (window.AudioContext || window.webkitAudioContext)(); + } + return sharedContext; +} + +export async function decodeFile(file) { + const arrayBuffer = await file.arrayBuffer(); + return decodeArrayBuffer(arrayBuffer); +} + +export async function decodeArrayBuffer(arrayBuffer) { + const ctx = context(); + // decodeAudioData detaches the buffer, so hand it a copy if the caller may + // still need the original bytes. + return await ctx.decodeAudioData(arrayBuffer.slice(0)); +} + +/** Interleaved mono mixdown. Used for the content hash that seeds the look. */ +export function monoSamples(audioBuffer) { + const channels = audioBuffer.numberOfChannels; + const left = audioBuffer.getChannelData(0); + if (channels === 1) return left; + const right = audioBuffer.getChannelData(1); + const out = new Float32Array(left.length); + for (let i = 0; i < left.length; i++) out[i] = (left[i] + right[i]) * 0.5; + return out; +} + +export function audioContext() { + return context(); +} + +/** Human-readable duration, used by the transport display. */ +export function formatTime(seconds) { + if (!isFinite(seconds) || seconds < 0) return '0:00'; + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}:${String(s).padStart(2, '0')}`; +} diff --git a/flow-state/src/audio/fft.js b/flow-state/src/audio/fft.js new file mode 100644 index 0000000..bd5dd5c --- /dev/null +++ b/flow-state/src/audio/fft.js @@ -0,0 +1,79 @@ +// Iterative radix-2 Cooley-Tukey FFT with precomputed twiddles and bit-reversal. +// +// Allocated once per size and reused across all ~21k frames of a track — the +// analysis pass is the one place in this project where JS throughput actually +// matters, and per-frame allocation is what would sink it. + +export class FFT { + constructor(size) { + if ((size & (size - 1)) !== 0) throw new Error(`FFT size must be a power of two, got ${size}`); + this.size = size; + this.half = size >> 1; + + this.re = new Float32Array(size); + this.im = new Float32Array(size); + this.magnitude = new Float32Array(this.half); + + // Bit-reversal permutation table. + this.rev = new Uint32Array(size); + const bits = Math.log2(size); + for (let i = 0; i < size; i++) { + let r = 0; + for (let b = 0; b < bits; b++) if (i & (1 << b)) r |= 1 << (bits - 1 - b); + this.rev[i] = r; + } + + // Twiddle factors, flattened per stage. + this.cos = new Float32Array(this.half); + this.sin = new Float32Array(this.half); + for (let i = 0; i < this.half; i++) { + this.cos[i] = Math.cos((-2 * Math.PI * i) / size); + this.sin[i] = Math.sin((-2 * Math.PI * i) / size); + } + } + + /** + * Forward transform of a real windowed signal. Writes into `this.magnitude` + * (first size/2 bins) and returns it. The buffer is reused: copy it if you + * need to keep it. + */ + forward(input) { + const { size, re, im, rev, cos, sin } = this; + + for (let i = 0; i < size; i++) { + re[i] = input[rev[i]]; + im[i] = 0; + } + + for (let len = 2; len <= size; len <<= 1) { + const halfLen = len >> 1; + const step = size / len; + for (let i = 0; i < size; i += len) { + for (let j = 0, k = 0; j < halfLen; j++, k += step) { + const c = cos[k], s = sin[k]; + const a = i + j, b = a + halfLen; + const tre = re[b] * c - im[b] * s; + const tim = re[b] * s + im[b] * c; + re[b] = re[a] - tre; + im[b] = im[a] - tim; + re[a] += tre; + im[a] += tim; + } + } + } + + const mag = this.magnitude; + const scale = 2 / size; + for (let i = 0; i < this.half; i++) { + mag[i] = Math.sqrt(re[i] * re[i] + im[i] * im[i]) * scale; + } + return mag; + } +} + +/** Periodic Hann window. Periodic (not symmetric) is correct for STFT analysis. */ +export function hannWindow(size) { + const w = new Float32Array(size); + for (let i = 0; i < size; i++) w[i] = 0.5 * (1 - Math.cos((2 * Math.PI * i) / size)); + return w; +} diff --git a/flow-state/src/audio/segment.js b/flow-state/src/audio/segment.js new file mode 100644 index 0000000..bb911d7 --- /dev/null +++ b/flow-state/src/audio/segment.js @@ -0,0 +1,266 @@ +// Structural segmentation: where the song changes, and what each part is. +// +// Standard approach — self-similarity plus a checkerboard novelty kernel — kept +// deliberately simple. This is the fuzziest stage in the pipeline, and a wrong +// boundary costs a scene change in a slightly odd place, not a broken video. An +// ML detour here would buy very little. +// +// The one domain-specific trick that earns its keep: boundaries are SNAPPED to +// the bar grid, preferring 4- and 8-bar multiples. Electronic music changes on +// phrase lines, so snapping converts a boundary that is roughly right into one +// that is exactly right. + +export const SECTION_KINDS = ['intro', 'build', 'drop', 'sustain', 'breakdown', 'outro']; + +const ANALYSIS_HZ = 4; // coarse grid for the similarity matrix +const KERNEL_SECONDS = 6; // half-width of the checkerboard kernel +const MIN_SECTION_SECONDS = 12; + +function median(values) { + if (!values.length) return 0; + const s = Float64Array.from(values).sort(); + return s[Math.floor(s.length / 2)]; +} + +/** Coarse, L2-normalised feature vectors — the rows of the similarity matrix. */ +function buildCoarseVectors(raw, frameCount, fps) { + const step = Math.max(1, Math.round(fps / ANALYSIS_HZ)); + const count = Math.max(1, Math.floor(frameCount / step)); + const dims = ['bandSub', 'bandLow', 'bandMid', 'bandHigh', 'bandAir', 'centroid', 'flatness']; + const vectors = []; + + for (let c = 0; c < count; c++) { + const start = c * step; + const end = Math.min(frameCount, start + step); + const v = new Float32Array(dims.length); + for (let d = 0; d < dims.length; d++) { + let sum = 0; + for (let f = start; f < end; f++) sum += raw[dims[d]][f]; + v[d] = sum / Math.max(1, end - start); + } + let norm = 0; + for (let d = 0; d < v.length; d++) norm += v[d] * v[d]; + norm = Math.sqrt(norm) + 1e-9; + for (let d = 0; d < v.length; d++) v[d] /= norm; + vectors.push(v); + } + return { vectors, step }; +} + +function cosine(a, b) { + let dot = 0; + for (let i = 0; i < a.length; i++) dot += a[i] * b[i]; + return dot; +} + +/** + * Novelty curve. Only the band around the diagonal is needed, so this never + * materialises the full N×N matrix. + */ +function noveltyCurve(vectors, kernelHalf) { + const n = vectors.length; + const novelty = new Float32Array(n); + + // Gaussian-tapered checkerboard weights. + const size = kernelHalf * 2; + const weights = new Float32Array(size * size); + const sigma = kernelHalf / 1.5; + for (let a = 0; a < size; a++) { + for (let b = 0; b < size; b++) { + const da = a - kernelHalf + 0.5; + const db = b - kernelHalf + 0.5; + const taper = Math.exp(-(da * da + db * db) / (2 * sigma * sigma)); + const sign = (da * db) > 0 ? 1 : -1; + weights[a * size + b] = sign * taper; + } + } + + for (let i = kernelHalf; i < n - kernelHalf; i++) { + let sum = 0; + for (let a = 0; a < size; a++) { + const ia = i - kernelHalf + a; + for (let b = 0; b < size; b++) { + const ib = i - kernelHalf + b; + sum += weights[a * size + b] * cosine(vectors[ia], vectors[ib]); + } + } + novelty[i] = Math.max(0, sum); + } + + let peak = 0; + for (let i = 0; i < n; i++) if (novelty[i] > peak) peak = novelty[i]; + if (peak > 1e-9) for (let i = 0; i < n; i++) novelty[i] /= peak; + return novelty; +} + +function pickPeaks(novelty, minDistance, threshold) { + const peaks = []; + const n = novelty.length; + for (let i = 1; i < n - 1; i++) { + if (novelty[i] < threshold) continue; + if (novelty[i] < novelty[i - 1] || novelty[i] < novelty[i + 1]) continue; + // Local maximum within the exclusion window. + let isMax = true; + const lo = Math.max(0, i - minDistance); + const hi = Math.min(n - 1, i + minDistance); + for (let j = lo; j <= hi; j++) if (novelty[j] > novelty[i]) { isMax = false; break; } + if (!isMax) continue; + if (peaks.length && i - peaks[peaks.length - 1] < minDistance) { + if (novelty[i] > novelty[peaks[peaks.length - 1]]) peaks[peaks.length - 1] = i; + continue; + } + peaks.push(i); + } + return peaks; +} + +/** + * Snap a time to the musical grid. Prefers, in order: an 8-bar line, a 4-bar + * line, then any downbeat — but only if one is close enough to be plausibly the + * same boundary. + */ +function snapToGrid(time, downbeats, barsPerPhrase = 8) { + if (!downbeats.length) return time; + const tolerance = 2.0; + + const tryLevels = [barsPerPhrase, 4, 1]; + for (const level of tryLevels) { + let best = null; + let bestDist = Infinity; + for (let i = 0; i < downbeats.length; i += level) { + const d = Math.abs(downbeats[i] - time); + if (d < bestDist) { bestDist = d; best = downbeats[i]; } + } + if (best !== null && bestDist <= tolerance) return best; + } + return time; +} + +function linearSlope(values) { + const n = values.length; + if (n < 2) return 0; + let sx = 0, sy = 0, sxy = 0, sxx = 0; + for (let i = 0; i < n; i++) { + sx += i; sy += values[i]; sxy += i * values[i]; sxx += i * i; + } + const denom = n * sxx - sx * sx; + if (Math.abs(denom) < 1e-12) return 0; + return ((n * sxy - sx * sy) / denom) * n; // normalised to "change across the section" +} + +/** + * Label a section from its own statistics plus where it sits in the track. + * Thresholds are relative to the track, never absolute, so a quiet ambient piece + * still gets a full set of labels rather than being classified as one long intro. + */ +function classify(section, context) { + const { energy, slope, flux, index, count, endEnergy } = section; + const { medianEnergy, isFirst, isLast } = context; + + const low = energy < medianEnergy * 0.72; + const high = energy > medianEnergy * 1.12; + + if (isFirst && energy < medianEnergy) return 'intro'; + if (isLast && (low || slope < -0.15)) return 'outro'; + if (slope > 0.18 && endEnergy > medianEnergy * 0.95) return 'build'; + if (high && flux > context.medianFlux * 0.95) return 'drop'; + if (low) return 'breakdown'; + if (index === 0) return 'intro'; + if (index === count - 1) return 'outro'; + return 'sustain'; +} + +/** + * @returns {Array<{index,start,end,startFrame,endFrame,kind,energy,slope,flux,centroid}>} + */ +export function segment(raw, frameCount, fps, tempo) { + const { vectors, step } = buildCoarseVectors(raw, frameCount, fps); + const kernelHalf = Math.max(4, Math.round(KERNEL_SECONDS * ANALYSIS_HZ)); + + let boundaryTimes = []; + if (vectors.length > kernelHalf * 2 + 4) { + const novelty = noveltyCurve(vectors, kernelHalf); + const minDistance = Math.round(MIN_SECTION_SECONDS * ANALYSIS_HZ); + + // Adaptive threshold: mean + a fraction of the spread, so a track with + // gentle transitions still yields boundaries and a busy one isn't shredded. + let mean = 0; + for (let i = 0; i < novelty.length; i++) mean += novelty[i]; + mean /= novelty.length; + let variance = 0; + for (let i = 0; i < novelty.length; i++) variance += (novelty[i] - mean) ** 2; + const std = Math.sqrt(variance / novelty.length); + const threshold = mean + std * 0.6; + + boundaryTimes = pickPeaks(novelty, minDistance, threshold) + .map((i) => (i * step) / fps); + } + + // Snap to the bar grid, dedupe, and drop anything too close to the edges. + const duration = frameCount / fps; + const snapped = boundaryTimes + .map((t) => snapToGrid(t, tempo.downbeats)) + .filter((t) => t > MIN_SECTION_SECONDS * 0.5 && t < duration - MIN_SECTION_SECONDS * 0.5) + .sort((a, b) => a - b); + + const bounds = [0]; + for (const t of snapped) { + if (t - bounds[bounds.length - 1] >= MIN_SECTION_SECONDS) bounds.push(t); + } + bounds.push(duration); + + // Build sections and gather their statistics. + const sections = []; + for (let i = 0; i < bounds.length - 1; i++) { + const start = bounds[i]; + const end = bounds[i + 1]; + const startFrame = Math.round(start * fps); + const endFrame = Math.min(frameCount, Math.round(end * fps)); + const n = Math.max(1, endFrame - startFrame); + + let energySum = 0, fluxSum = 0, centroidSum = 0, flatnessSum = 0, widthSum = 0; + const coarseEnergy = []; + const coarseStep = Math.max(1, Math.floor(n / 24)); + for (let f = startFrame; f < endFrame; f++) { + energySum += raw.loudness[f]; + fluxSum += raw.flux[f]; + centroidSum += raw.centroid[f]; + flatnessSum += raw.flatness[f]; + widthSum += raw.width[f]; + if ((f - startFrame) % coarseStep === 0) coarseEnergy.push(raw.loudness[f]); + } + + const tailStart = Math.max(startFrame, endFrame - Math.round(fps * 4)); + let endEnergy = 0; + for (let f = tailStart; f < endFrame; f++) endEnergy += raw.loudness[f]; + endEnergy /= Math.max(1, endFrame - tailStart); + + sections.push({ + index: i, + start, end, + startFrame, endFrame, + duration: end - start, + energy: energySum / n, + flux: fluxSum / n, + centroid: centroidSum / n, + flatness: flatnessSum / n, + width: widthSum / n, + slope: linearSlope(coarseEnergy), + endEnergy, + kind: 'sustain', + }); + } + + const context = { + medianEnergy: median(sections.map((s) => s.energy)) || 1e-6, + medianFlux: median(sections.map((s) => s.flux)) || 1e-6, + }; + sections.forEach((s, i) => { + s.kind = classify( + { ...s, index: i, count: sections.length }, + { ...context, isFirst: i === 0, isLast: i === sections.length - 1 }, + ); + }); + + return sections; +} diff --git a/flow-state/src/audio/synth.js b/flow-state/src/audio/synth.js new file mode 100644 index 0000000..3cb9bd3 --- /dev/null +++ b/flow-state/src/audio/synth.js @@ -0,0 +1,122 @@ +// Synthetic audio with known ground truth, so tempo and segmentation can be +// tested against an exact answer instead of "sounds about right". Real music +// goes through the click track and the battery; this catches regressions in CI +// speed and without a GPU. + +/** Minimal stand-in for AudioBuffer — the analysis code only needs this surface. */ +export class MockAudioBuffer { + constructor(channels, length, sampleRate) { + this.numberOfChannels = channels; + this.length = length; + this.sampleRate = sampleRate; + this.duration = length / sampleRate; + this._data = Array.from({ length: channels }, () => new Float32Array(length)); + } + getChannelData(i) { return this._data[i]; } +} + +function addKick(data, sampleRate, at, gain = 1) { + const start = Math.round(at * sampleRate); + const length = Math.round(0.12 * sampleRate); + for (let i = 0; i < length; i++) { + const s = start + i; + if (s < 0 || s >= data.length) continue; + const t = i / sampleRate; + const env = Math.exp(-t * 30); + const freq = 55 * Math.exp(-t * 20) + 40; // pitch drop, like a real kick + data[s] += Math.sin(2 * Math.PI * freq * t) * env * gain; + } +} + +function addHat(data, sampleRate, at, gain = 0.3, seed = 1) { + const start = Math.round(at * sampleRate); + const length = Math.round(0.04 * sampleRate); + let s0 = seed >>> 0; + const rnd = () => { + s0 = (Math.imul(s0 ^ (s0 >>> 15), s0 | 1) + 0x6d2b79f5) >>> 0; + return ((s0 >>> 14) & 0xffff) / 0xffff - 0.5; + }; + for (let i = 0; i < length; i++) { + const s = start + i; + if (s < 0 || s >= data.length) continue; + const env = Math.exp(-(i / sampleRate) * 90); + data[s] += rnd() * env * gain; + } +} + +function addPad(data, sampleRate, from, to, gain = 0.15, root = 110) { + const start = Math.round(from * sampleRate); + const end = Math.min(data.length, Math.round(to * sampleRate)); + for (let s = start; s < end; s++) { + const t = s / sampleRate; + data[s] += (Math.sin(2 * Math.PI * root * t) + Math.sin(2 * Math.PI * root * 1.5 * t)) * gain * 0.5; + } +} + +/** + * A four-to-the-floor track at a known BPM. + * @param {object} opts + * @returns {MockAudioBuffer} + */ +export function synthesizeBeat({ + bpm = 128, + duration = 40, + sampleRate = 44100, + hats = true, + pad = true, + kickGain = 1, +} = {}) { + const length = Math.round(duration * sampleRate); + const buffer = new MockAudioBuffer(2, length, sampleRate); + const left = buffer.getChannelData(0); + const right = buffer.getChannelData(1); + + const beat = 60 / bpm; + let index = 0; + for (let t = 0; t < duration; t += beat, index++) { + // Accent the downbeat so the bar phase is detectable. + addKick(left, sampleRate, t, kickGain * (index % 4 === 0 ? 1.0 : 0.8)); + if (hats) addHat(left, sampleRate, t + beat / 2, 0.25, index + 1); + } + if (pad) addPad(left, sampleRate, 0, duration, 0.12); + + for (let i = 0; i < length; i++) right[i] = left[i] * 0.98; + return buffer; +} + +/** + * A track with a deliberate structural change at `changeAt` seconds: sparse and + * dark before, dense and bright after. Segmentation must find that boundary. + */ +export function synthesizeSectioned({ + bpm = 128, + duration = 120, + changeAt = 60, + sampleRate = 44100, +} = {}) { + const length = Math.round(duration * sampleRate); + const buffer = new MockAudioBuffer(2, length, sampleRate); + const left = buffer.getChannelData(0); + const right = buffer.getChannelData(1); + + const beat = 60 / bpm; + let index = 0; + for (let t = 0; t < duration; t += beat, index++) { + const after = t >= changeAt; + addKick(left, sampleRate, t, after ? 1.0 : 0.35); + if (after) { + addHat(left, sampleRate, t + beat / 2, 0.45, index + 1); + addHat(left, sampleRate, t + beat / 4, 0.25, index + 7); + } + } + addPad(left, sampleRate, 0, changeAt, 0.10, 110); + addPad(left, sampleRate, changeAt, duration, 0.22, 440); // brighter after + + for (let i = 0; i < length; i++) right[i] = left[i] * 0.98; + return buffer; +} + +/** Silence, for degenerate-input checks. */ +export function synthesizeSilence({ duration = 10, sampleRate = 44100 } = {}) { + return new MockAudioBuffer(2, Math.round(duration * sampleRate), sampleRate); +} diff --git a/flow-state/src/audio/tempo.js b/flow-state/src/audio/tempo.js new file mode 100644 index 0000000..c67046a --- /dev/null +++ b/flow-state/src/audio/tempo.js @@ -0,0 +1,369 @@ +// Tempo, beat grid and downbeats from the onset envelope. +// +// This runs over the WHOLE track, which is the entire reason to analyse offline: +// a causal detector has to converge, and lags for the first several bars of every +// section. Here the grid is exact from frame zero, and phase is fitted globally. +// +// The click track (audio/clicktrack.js) exists to validate this by ear. If the +// clicks don't sit on the beat, nothing downstream can be trusted — every timing +// artefact in the finished video traces back to this file. + +const MIN_BPM = 60; +const MAX_BPM = 200; +const PREFERRED_BPM = 124; // log-space centre of the prior; club-ish but broad +const PRIOR_WIDTH = 0.85; + +/** Remove the slow-moving floor so autocorrelation sees onsets, not loudness. */ +function whiten(envelope, fps) { + const n = envelope.length; + const out = new Float32Array(n); + const halfWindow = Math.max(2, Math.round(fps * 0.35)); + let sum = 0; + const queue = []; + for (let i = 0; i < n; i++) { + queue.push(envelope[i]); + sum += envelope[i]; + if (queue.length > halfWindow * 2 + 1) sum -= queue.shift(); + const mean = sum / queue.length; + out[i] = Math.max(0, envelope[i] - mean); + } + let peak = 0; + for (let i = 0; i < n; i++) if (out[i] > peak) peak = out[i]; + if (peak > 1e-9) for (let i = 0; i < n; i++) out[i] /= peak; + return out; +} + +function autocorrelate(signal, minLag, maxLag) { + const n = signal.length; + const scores = new Float32Array(maxLag + 1); + for (let lag = minLag; lag <= maxLag; lag++) { + let sum = 0; + const limit = n - lag; + for (let i = 0; i < limit; i++) sum += signal[i] * signal[i + lag]; + scores[lag] = limit > 0 ? sum / limit : 0; + } + return scores; +} + +function bpmPrior(bpm) { + const x = Math.log2(bpm / PREFERRED_BPM) / PRIOR_WIDTH; + return Math.exp(-0.5 * x * x); +} + +/** + * Discrete onset peaks: local maxima of the whitened envelope. + * + * Peak positions are refined to sub-frame precision by parabolic interpolation. + * At 60fps a whole-frame quantisation is 16.7ms, and that error accumulates + * through the period fit into visible drift by the end of a long track. + */ +function pickOnsetPeaks(signal) { + const peaks = []; + let total = 0; + for (let i = 1; i < signal.length - 1; i++) { + const v = signal[i]; + if (v < 0.06) continue; + if (v < signal[i - 1] || v < signal[i + 1]) continue; + + const a = signal[i - 1], b = v, c = signal[i + 1]; + const denom = a - 2 * b + c; + const shift = Math.abs(denom) > 1e-12 ? (0.5 * (a - c)) / denom : 0; + peaks.push({ frame: i + Math.max(-0.5, Math.min(0.5, shift)), strength: v }); + total += v; + } + return { peaks, total }; +} + +/** + * Least-squares fit of the grid to the onsets it already matches. + * + * The search above only ever tests integer offsets and a discrete set of periods, + * which leaves up to half a frame of phase error and a small period error that + * compounds — 64 beats into a track a 0.02-frame period error is already 20ms of + * drift. Regressing matched onset positions against their beat indices recovers + * both to sub-frame precision in one pass. + */ +function refineByRegression(peaks, period, offset, length) { + const tolerance = Math.max(1.5, Math.min(3, period * 0.15)); + const gridCount = Math.max(1, Math.floor((length - offset) / period)); + + let sw = 0, sk = 0, st = 0, skk = 0, skt = 0; + let matched = 0; + for (const peak of peaks) { + const k = Math.round((peak.frame - offset) / period); + if (k < 0 || k > gridCount) continue; + if (Math.abs(peak.frame - (offset + k * period)) > tolerance) continue; + const w = peak.strength; + sw += w; sk += w * k; st += w * peak.frame; + skk += w * k * k; skt += w * k * peak.frame; + matched++; + } + if (matched < 4 || sw < 1e-9) return { period, offset }; + + const denom = sw * skk - sk * sk; + if (Math.abs(denom) < 1e-9) return { period, offset }; + + const slope = (sw * skt - sk * st) / denom; + const intercept = (st - slope * sk) / sw; + + // Reject a fit that has wandered — that means the matching was wrong, not + // that the tempo is unusual. + if (!isFinite(slope) || !isFinite(intercept)) return { period, offset }; + if (Math.abs(slope - period) > period * 0.05) return { period, offset }; + + return { period: slope, offset: intercept }; +} + +/** + * F-measure between a candidate grid and the detected onsets. + * + * This is what resolves tempo octaves, and a plain "mean energy at grid points" + * cannot. A half-tempo grid hits every other kick at FULL strength, so its mean + * energy per beat is identical to the true grid's — it only loses on RECALL, + * because half the onsets go unexplained. A double-tempo grid has perfect recall + * but half its beats land on silence, so it loses on PRECISION. Combining the two + * is the only formulation that penalises both errors. + */ +function gridFScore(peaks, totalStrength, period, offset, length) { + if (!peaks.length || period < 2) return 0; + const tolerance = Math.max(1.5, Math.min(3, period * 0.15)); + const gridCount = Math.max(1, Math.floor((length - offset) / period)); + + const matchedGrid = new Set(); + let matchedStrength = 0; + + for (const peak of peaks) { + const k = Math.round((peak.frame - offset) / period); + if (k < 0 || k > gridCount) continue; + const gridFrame = offset + k * period; + if (Math.abs(peak.frame - gridFrame) <= tolerance) { + matchedStrength += peak.strength; + matchedGrid.add(k); + } + } + + const recall = totalStrength > 1e-9 ? matchedStrength / totalStrength : 0; + const precision = matchedGrid.size / gridCount; + if (precision + recall < 1e-9) return 0; + return (2 * precision * recall) / (precision + recall); +} + +/** + * Mean matched onset strength at even vs odd grid positions. + * + * Unmatched grid points count as zero — a grid point that lands on silence is + * evidence against the grid, not a missing sample. + */ +function alternation(peaks, period, offset, length) { + const tolerance = Math.max(1.5, Math.min(3, period * 0.15)); + const gridCount = Math.max(2, Math.floor((length - offset) / period)); + const strength = new Float64Array(gridCount); + + for (const peak of peaks) { + const k = Math.round((peak.frame - offset) / period); + if (k < 0 || k >= gridCount) continue; + if (Math.abs(peak.frame - (offset + k * period)) <= tolerance) { + strength[k] = Math.max(strength[k], peak.strength); + } + } + + let evenSum = 0, evenCount = 0, oddSum = 0, oddCount = 0; + for (let k = 0; k < gridCount; k++) { + if (k % 2 === 0) { evenSum += strength[k]; evenCount++; } + else { oddSum += strength[k]; oddCount++; } + } + const even = evenCount ? evenSum / evenCount : 0; + const odd = oddCount ? oddSum / oddCount : 0; + const hi = Math.max(even, odd); + return { even, odd, ratio: hi > 1e-9 ? Math.min(even, odd) / hi : 1 }; +} + +/** + * Correct a double-tempo reading. + * + * Offbeat hi-hats make a double-tempo grid score perfectly on both precision and + * recall — every grid point genuinely has an onset — so the F-measure alone + * cannot tell 90 BPM with hats from 180 BPM. What separates them is that the + * onsets ALTERNATE strong/weak: kick, hat, kick, hat. A systematic alternation + * means the real beat is every other grid point. This is the same cue a listener + * uses, and it is why the grid is fitted first and metrically interpreted second. + */ +function correctOctave(peaks, totalStrength, period, offset, length, fps) { + let currentPeriod = period; + let currentOffset = offset; + + for (let iteration = 0; iteration < 2; iteration++) { + const bpm = (60 * fps) / currentPeriod; + const halvedBpm = bpm / 2; + if (halvedBpm < MIN_BPM) break; + + const alt = alternation(peaks, currentPeriod, currentOffset, length); + if (alt.ratio >= 0.62) break; + + const strongIsEven = alt.even >= alt.odd; + const nextPeriod = currentPeriod * 2; + const nextOffset = strongIsEven ? currentOffset : currentOffset + currentPeriod; + + // Only accept if the slower grid still explains the material well: this + // guards against halving genuinely syncopated but correctly-fitted music. + const before = gridFScore(peaks, totalStrength, currentPeriod, currentOffset, length); + const after = gridFScore(peaks, totalStrength, nextPeriod, nextOffset, length); + if (after < before * 0.55) break; + + currentPeriod = nextPeriod; + currentOffset = nextOffset; + } + + return { period: currentPeriod, offset: currentOffset }; +} + +function bestOffset(peaks, totalStrength, period, length) { + let best = 0; + let bestScore = -1; + const steps = Math.ceil(period); + for (let o = 0; o < steps; o++) { + const s = gridFScore(peaks, totalStrength, period, o, length); + if (s > bestScore) { bestScore = s; best = o; } + } + return { offset: best, score: bestScore }; +} + +/** + * @param {Float32Array} onsetEnvelope per-frame onset strength + * @param {number} fps + * @returns {{bpm, period, offset, beats, downbeats, confidence, beatsPerBar}} + */ +export function detectTempo(onsetEnvelope, fps, { beatsPerBar = 4 } = {}) { + const signal = whiten(onsetEnvelope, fps); + const minLag = Math.floor((60 * fps) / MAX_BPM); + const maxLag = Math.ceil((60 * fps) / MIN_BPM); + const acf = autocorrelate(signal, minLag, maxLag); + + // Weight autocorrelation by the tempo prior, then take the peak. + let bestLag = minLag; + let bestValue = -Infinity; + for (let lag = minLag; lag <= maxLag; lag++) { + const bpm = (60 * fps) / lag; + const v = acf[lag] * bpmPrior(bpm); + if (v > bestValue) { bestValue = v; bestLag = lag; } + } + + // Parabolic refinement around the peak for sub-frame period accuracy. + let period = bestLag; + if (bestLag > minLag && bestLag < maxLag) { + const a = acf[bestLag - 1], b = acf[bestLag], c = acf[bestLag + 1]; + const denom = a - 2 * b + c; + if (Math.abs(denom) > 1e-12) period = bestLag - (0.5 * (c - a)) / denom; + } + + const { peaks, total: totalStrength } = pickOnsetPeaks(signal); + const length = signal.length; + + // Octave resolution. The prior only breaks near-ties; the F-measure does the + // actual work, which is why a 174 BPM track no longer reads as 87. + const candidates = [period / 4, period / 2, period, period * 2, period * 4].filter((p) => { + const bpm = (60 * fps) / p; + return bpm >= MIN_BPM && bpm <= MAX_BPM && p >= 2; + }); + + let chosen = { period, offset: 0, score: -1 }; + for (const p of candidates) { + const { offset, score } = bestOffset(peaks, totalStrength, p, length); + const adjusted = score * (0.6 + 0.4 * bpmPrior((60 * fps) / p)); + if (adjusted > chosen.score) chosen = { period: p, offset, score: adjusted }; + } + + period = chosen.period; + let offset = chosen.offset; + + // Local refinement of period and offset together — catches a grid that is + // right at the start and drifts by the end of a six-minute track. + let bestRefined = gridFScore(peaks, totalStrength, period, offset, length); + for (let dp = -0.5; dp <= 0.5001; dp += 0.02) { + const p = period + dp; + if (p < 2) continue; + const { offset: o, score } = bestOffset(peaks, totalStrength, p, length); + if (score > bestRefined) { bestRefined = score; period = p; offset = o; } + } + + // Metrical interpretation, after the grid itself is fitted. + ({ period, offset } = correctOctave(peaks, totalStrength, period, offset, length, fps)); + + // Sub-frame fit last, so it refines the grid we actually committed to. + ({ period, offset } = refineByRegression(peaks, period, offset, length)); + + const bpm = (60 * fps) / period; + + const beats = []; + for (let t = offset; t < onsetEnvelope.length; t += period) beats.push(t / fps); + + // Downbeat: of the `beatsPerBar` possible bar phases, the one whose beats + // carry the most energy. + // + // Scored on the RAW envelope, not the whitened one. Whitening subtracts the + // local mean, which is exactly the accent information that distinguishes beat + // one from the other three — a four-to-the-floor pattern has a kick on every + // beat and the only cue is that one of them is louder. + let bestPhase = 0; + let bestPhaseScore = -1; + for (let phase = 0; phase < beatsPerBar; phase++) { + let sum = 0; + let count = 0; + for (let b = phase; b < beats.length; b += beatsPerBar) { + const i = Math.round(beats[b] * fps); + if (i >= 1 && i < onsetEnvelope.length - 1) { + sum += Math.max(onsetEnvelope[i - 1], onsetEnvelope[i], onsetEnvelope[i + 1]); + count++; + } + } + const score = count ? sum / count : 0; + if (score > bestPhaseScore) { bestPhaseScore = score; bestPhase = phase; } + } + const downbeats = []; + for (let b = bestPhase; b < beats.length; b += beatsPerBar) downbeats.push(beats[b]); + + // Confidence is the grid's own F-measure: how much of the onset energy the + // chosen grid explains, and how many of its beats are actually occupied. + const confidence = gridFScore(peaks, totalStrength, period, offset, length); + + return { bpm, period, offset, beats, downbeats, beatsPerBar, barPhase: bestPhase, confidence }; +} + +/** + * Per-frame phase tracks from the grid. + * + * `beat` is a decaying spike, but gated by a smoothed loudness envelope so a + * drumless breakdown doesn't strobe on a grid that is technically still running. + */ +export function buildBeatTracks(tempo, frameCount, fps, loudness) { + const beatPhase = new Float32Array(frameCount); + const barPhase = new Float32Array(frameCount); + const phrasePhase = new Float32Array(frameCount); + const beat = new Float32Array(frameCount); + + const { period, offset, beatsPerBar } = tempo; + const barPeriod = period * beatsPerBar; + const phrasePeriod = barPeriod * 8; + const barOffset = offset + tempo.barPhase * period; + + // Smoothed loudness gate. + const gate = new Float32Array(frameCount); + const attack = 0.25, release = 0.02; + let g = 0; + for (let f = 0; f < frameCount; f++) { + const target = loudness ? loudness[f] : 1; + g += (target - g) * (target > g ? attack : release); + gate[f] = Math.min(1, g * 1.6); + } + + const decay = Math.max(1e-3, period * 0.28); + for (let f = 0; f < frameCount; f++) { + const sinceBeat = ((f - offset) % period + period) % period; + beatPhase[f] = sinceBeat / period; + barPhase[f] = (((f - barOffset) % barPeriod) + barPeriod) % barPeriod / barPeriod; + phrasePhase[f] = (((f - barOffset) % phrasePeriod) + phrasePeriod) % phrasePeriod / phrasePeriod; + beat[f] = Math.exp(-sinceBeat / decay) * gate[f]; + } + + return { beatPhase, barPhase, phrasePhase, beat }; +} diff --git a/flow-state/src/checks/phase1.js b/flow-state/src/checks/phase1.js index 1e59de5..45f433f 100644 --- a/flow-state/src/checks/phase1.js +++ b/flow-state/src/checks/phase1.js @@ -1 +1,150 @@ -// Phase 1 gate — filled in when the phase lands. +// Phase 1 gate — the offline audio pipeline feeding the renderer. +// +// The numeric correctness of tempo and segmentation is covered by node tests +// against synthetic ground truth (test/audio.test.js) and, for real material, by +// the click track. What can only be checked here is the join: that the frame-indexed +// table reaches the shader, and that driving the clock from audio produces exactly +// the same frames as counting them. + +import { check, expect, expectBelow } from './framework.js'; +import { Engine } from '../engine/Engine.js'; +import { FeatureTrack, featureProviderFor } from '../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../audio/synth.js'; +import { nebula } from '../scenes/shader/nebula.js'; +import { defaultValues } from '../params/schema.js'; + +const PALETTE = [[0.05, 0.02, 0.15], [0.85, 0.15, 0.55], [0.15, 0.75, 0.95], [0.98, 0.85, 0.35]]; + +let cachedTrack = null; +export function testTrack() { + if (!cachedTrack) { + const buffer = synthesizeSectioned({ bpm: 128, duration: 90, changeAt: 45 }); + cachedTrack = FeatureTrack.fromAudioBuffer(buffer, { fps: 60 }); + } + return cachedTrack; +} + +function makeEngine(track, width = 256, height = 144) { + const engine = new Engine({ width, height }); + engine.timeline.setDuration(track.duration); + engine.setFeatureProvider(featureProviderFor(track)); + engine.setLayerSpecs([{ + module: nebula, params: defaultValues(nebula), seed: 4242, + opacity: 1, blend: 'normal', palette: PALETTE, + }]); + return engine; +} + +check(1, 'feature track builds and every row is finite and in range', () => { + const track = testTrack(); + let bad = 0; + for (let f = 0; f < track.frameCount; f += 7) { + const row = track.at(f); + for (const v of Object.values(row)) { + if (!Number.isFinite(v) || v < 0 || v > 1) { bad++; break; } + } + } + return expect(bad === 0, + `${bad} bad rows of ${Math.ceil(track.frameCount / 7)} sampled · ` + + `${track.frameCount} frames · ${track.sections.length} sections · ` + + `${track.summary.bpm.toFixed(1)} BPM · confidence ${track.tempo.confidence.toFixed(2)}`); +}); + +check(1, 'audio-driven clock and fixed-step counting agree', () => { + // The core preview/export parity claim. Realtime maps currentTime to a frame + // index; export counts frames. Both must land on identical images. + const track = testTrack(); + const engine = makeEngine(track); + try { + const fixed = engine.hashRun(1000, 100); + + engine.compositor.reset(); + const driven = []; + for (let i = 0; i < 100; i++) { + const frame = 1000 + i; + // Jittered playback position within the frame's window, as a real + // audio element would report it. + const jitter = (((i * 7919) % 1000) / 1000) * 0.9; + engine.timeline.syncToAudio((frame + jitter) / 60); + const target = engine.renderCurrent(); + driven.push(engine.hashCurrent(target)); + } + const mismatches = fixed.filter((h, i) => h !== driven[i]).length; + return expect(mismatches === 0, `${mismatches}/100 frames differed`); + } finally { + engine.dispose(); + } +}); + +check(1, 'features actually reach the shader', () => { + // A scene wired to a table it never reads would pass every other check here. + const track = testTrack(); + const engine = makeEngine(track); + try { + const quiet = track.sections.reduce((a, b) => (a.energy < b.energy ? a : b)); + const loud = track.sections.reduce((a, b) => (a.energy > b.energy ? a : b)); + + engine.compositor.reset(); + const quietHash = engine.hashCurrent(engine.renderFrame(quiet.startFrame + 60)); + engine.compositor.reset(); + const loudHash = engine.hashCurrent(engine.renderFrame(loud.startFrame + 60)); + + return expect(quietHash !== loudHash, + `quiet ${quiet.kind} ${quietHash} vs loud ${loud.kind} ${loudHash}`); + } finally { + engine.dispose(); + } +}); + +check(1, 'section-boundary seek is exact without warm-up', () => { + const track = testTrack(); + const engine = makeEngine(track); + try { + const boundary = track.sections[1] ? track.sections[1].startFrame : 600; + const sequential = engine.hashRun(boundary, 3); + engine.compositor.reset(); + const direct = engine.hashCurrent(engine.renderFrame(boundary)); + return expect(direct === sequential[0], `direct ${direct} vs sequential ${sequential[0]}`); + } finally { + engine.dispose(); + } +}); + +check(1, 'mid-section seek converges under warm-up with feedback active', () => { + // With feedback on, an arbitrary seek can only converge, not match exactly. + // The gate is that warm-up gets it visually indistinguishable — the documented + // behaviour in PLAN.md §6. + const track = testTrack(); + const engine = makeEngine(track); + try { + engine.compositor.setFeedback({ amount: 0.7, decay: 0.92, zoom: 0.99 }); + + const targetFrame = 1500; + engine.compositor.reset(); + for (let f = 1200; f < targetFrame; f++) engine.renderFrame(f); + const sequentialPixels = Uint8Array.from( + engine.readPixels(engine.renderFrame(targetFrame)), + ); + + engine.warmUp(targetFrame, 120); + const warmedPixels = Uint8Array.from(engine.readPixels(engine.renderCurrent())); + + let sum = 0; + for (let i = 0; i < sequentialPixels.length; i += 4) { + sum += Math.abs(sequentialPixels[i] - warmedPixels[i]); + } + const distance = sum / (sequentialPixels.length / 4) / 255; + return expectBelow(distance, 0.02, 'mean red-channel distance after warm-up'); + } finally { + engine.dispose(); + } +}); + +check(1, 'analysis of a five-minute track stays within budget', () => { + const buffer = synthesizeSectioned({ bpm: 128, duration: 300, changeAt: 150 }); + const started = Date.now(); + const track = FeatureTrack.fromAudioBuffer(buffer, { fps: 60 }); + const elapsed = (Date.now() - started) / 1000; + return expect(elapsed < 6, + `${elapsed.toFixed(2)}s for a 5-minute track (${track.frameCount} frames)`); +}, { slow: true }); diff --git a/flow-state/test/audio.test.js b/flow-state/test/audio.test.js new file mode 100644 index 0000000..812e9c9 --- /dev/null +++ b/flow-state/test/audio.test.js @@ -0,0 +1,194 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { FFT, hannWindow } from '../src/audio/fft.js'; +import { analyzeBuffer, RAW_FEATURES } from '../src/audio/analyze.js'; +import { detectTempo, buildBeatTracks } from '../src/audio/tempo.js'; +import { segment } from '../src/audio/segment.js'; +import { synthesizeBeat, synthesizeSectioned, synthesizeSilence } from '../src/audio/synth.js'; + +const FPS = 60; + +test('FFT resolves a pure tone into the right bin', () => { + const size = 2048; + const sampleRate = 44100; + const fft = new FFT(size); + const window = hannWindow(size); + const freq = 1000; + + const input = new Float32Array(size); + for (let i = 0; i < size; i++) { + input[i] = Math.sin((2 * Math.PI * freq * i) / sampleRate) * window[i]; + } + const mag = fft.forward(input); + + let peakBin = 0; + for (let i = 1; i < mag.length; i++) if (mag[i] > mag[peakBin]) peakBin = i; + const peakHz = (peakBin * sampleRate) / size; + assert.ok(Math.abs(peakHz - freq) < sampleRate / size, + `peak at ${peakHz.toFixed(1)}Hz, expected ~${freq}Hz`); +}); + +test('analysis produces finite, in-range features', () => { + const buffer = synthesizeBeat({ bpm: 128, duration: 20 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + + assert.equal(analysis.frameCount, Math.ceil(buffer.duration * FPS)); + + for (const name of RAW_FEATURES) { + const arr = analysis.raw[name]; + assert.ok(arr, `missing feature ${name}`); + for (let i = 0; i < arr.length; i++) { + assert.ok(Number.isFinite(arr[i]), `${name}[${i}] is not finite`); + assert.ok(arr[i] >= 0 && arr[i] <= 1, `${name}[${i}] = ${arr[i]} out of range`); + } + } +}); + +test('no feature is constant or pinned on real-ish material', () => { + // A feature stuck at 0 or 1 contributes nothing to the visuals and usually + // means a wrong band split — the thing the feature scope exists to catch. + const buffer = synthesizeBeat({ bpm: 124, duration: 30 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + + for (const name of ['loudness', 'bandSub', 'bandLow', 'bandMid', 'flux', 'centroid']) { + const arr = analysis.raw[name]; + let min = Infinity, max = -Infinity, sum = 0; + for (let i = 0; i < arr.length; i++) { + min = Math.min(min, arr[i]); + max = Math.max(max, arr[i]); + sum += arr[i]; + } + const spread = max - min; + assert.ok(spread > 0.15, `${name} spread only ${spread.toFixed(3)} — feature is inert`); + const mean = sum / arr.length; + assert.ok(mean > 0.005 && mean < 0.995, `${name} mean ${mean.toFixed(3)} is pinned`); + } +}); + +test('tempo detection lands on the true BPM', () => { + for (const bpm of [90, 110, 124, 128, 140, 174]) { + const buffer = synthesizeBeat({ bpm, duration: 40 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + + const error = Math.abs(tempo.bpm - bpm) / bpm; + assert.ok(error < 0.02, + `bpm ${tempo.bpm.toFixed(2)} vs true ${bpm} (${(error * 100).toFixed(2)}% off)`); + } +}); + +test('beat grid aligns with the true beat positions', () => { + const bpm = 128; + const buffer = synthesizeBeat({ bpm, duration: 40 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + + const trueBeat = 60 / bpm; + let worst = 0; + for (const t of tempo.beats) { + if (t > 30) break; + const nearest = Math.round(t / trueBeat) * trueBeat; + worst = Math.max(worst, Math.abs(t - nearest)); + } + // Within half a video frame: any more would be visible as the visuals + // sitting off the beat. + assert.ok(worst < 0.5 / FPS, `worst beat offset ${(worst * 1000).toFixed(1)}ms`); +}); + +test('downbeats land on the accented beat', () => { + const bpm = 128; + const buffer = synthesizeBeat({ bpm, duration: 40 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + + const trueBar = (60 / bpm) * 4; + let aligned = 0; + for (const t of tempo.downbeats) { + const offset = Math.abs(t / trueBar - Math.round(t / trueBar)); + if (offset < 0.05) aligned++; + } + assert.ok(aligned / tempo.downbeats.length > 0.9, + `only ${aligned}/${tempo.downbeats.length} downbeats on the bar line`); +}); + +test('beat tracks are bounded and phase-continuous', () => { + const buffer = synthesizeBeat({ bpm: 128, duration: 20 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + const tracks = buildBeatTracks(tempo, analysis.frameCount, FPS, analysis.raw.loudness); + + for (const name of ['beat', 'beatPhase', 'barPhase', 'phrasePhase']) { + const arr = tracks[name]; + for (let i = 0; i < arr.length; i++) { + assert.ok(Number.isFinite(arr[i]) && arr[i] >= 0 && arr[i] <= 1, + `${name}[${i}] = ${arr[i]}`); + } + } + // Phase must wrap exactly once per beat, never jump mid-beat. + let wraps = 0; + for (let i = 1; i < tracks.beatPhase.length; i++) { + if (tracks.beatPhase[i] < tracks.beatPhase[i - 1]) wraps++; + } + const expected = 20 / (60 / 128); + assert.ok(Math.abs(wraps - expected) <= 2, `${wraps} phase wraps, expected ~${expected.toFixed(0)}`); +}); + +test('segmentation finds a real structural boundary', () => { + const changeAt = 60; + const buffer = synthesizeSectioned({ bpm: 128, duration: 120, changeAt }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo); + + assert.ok(sections.length >= 2, `expected at least 2 sections, got ${sections.length}`); + + const boundaries = sections.slice(1).map((s) => s.start); + const nearest = boundaries.reduce( + (best, b) => (Math.abs(b - changeAt) < Math.abs(best - changeAt) ? b : best), + Infinity, + ); + assert.ok(Math.abs(nearest - changeAt) <= 2.0, + `nearest boundary ${nearest.toFixed(2)}s, true change at ${changeAt}s`); +}); + +test('sections tile the track with no gaps or overlaps', () => { + const buffer = synthesizeSectioned({ duration: 120, changeAt: 60 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo); + + assert.equal(sections[0].start, 0); + for (let i = 1; i < sections.length; i++) { + assert.equal(sections[i].start, sections[i - 1].end, `gap before section ${i}`); + } + assert.ok(Math.abs(sections[sections.length - 1].end - analysis.duration) < 0.05); + for (const s of sections) { + assert.ok(s.duration > 0, `section ${s.index} has non-positive duration`); + assert.ok(['intro', 'build', 'drop', 'sustain', 'breakdown', 'outro'].includes(s.kind), + `bad kind ${s.kind}`); + } +}); + +test('silence degrades gracefully instead of producing NaN', () => { + const buffer = synthesizeSilence({ duration: 10 }); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + for (const name of RAW_FEATURES) { + for (const v of analysis.raw[name]) assert.ok(Number.isFinite(v), `${name} not finite on silence`); + } + const tempo = detectTempo(analysis.onsetEnvelope, FPS); + assert.ok(Number.isFinite(tempo.bpm), 'bpm not finite on silence'); + const sections = segment(analysis.raw, analysis.frameCount, FPS, tempo); + assert.ok(sections.length >= 1, 'no sections produced for silence'); +}); + +test('analysis of a six-minute track completes within budget', () => { + const buffer = synthesizeBeat({ bpm: 128, duration: 360, hats: true, pad: true }); + const started = Date.now(); + const analysis = analyzeBuffer(buffer, { fps: FPS }); + detectTempo(analysis.onsetEnvelope, FPS); + const elapsed = (Date.now() - started) / 1000; + // The plan's budget is ~3s. Node is a fair proxy for browser JS here. + assert.ok(elapsed < 6, `analysis took ${elapsed.toFixed(2)}s`); + console.log(` (6-minute analysis: ${elapsed.toFixed(2)}s)`); +});