Dejvino
13b0939221
A new top-level section building the metabolism concept: the track supplies the food field, event impulses and per-section laws; the seed picks the species (a rule set, not a parameter vector). Same song + seed => bit-identical frames; different songs => different worlds. - analysis: single FFT pass -> per-frame feature table; score (onsets folded into world geometry via voice ring + pitch-class sector), wells (food placed by the harmony), graph (band correlations -> interaction matrix), laws (per-section regimes) - sim: typed-array world, 3 castes (builder/orbit, forager/commute, spark/orbit-wide) with causality rings, persistence trails, consequence field; fully deterministic - lens: software framebuffer renderer (PNG in node, ImageData in browser) + event-reactive camera (cuts on activity change, alternating shot sizes, drift-and-push) - metrics: coupling x residual fitness (not per-frame descriptors) plus a grown-structure fingerprint; report + strip tools; 15 tests green - DESIGN.md captures the concept, invariants and build order 3 reference test signals (drone/runner/dusk) grow measurably distinct worlds while seeds of one song stay in the same biome.
411 lines
15 KiB
JavaScript
411 lines
15 KiB
JavaScript
// PCM -> per-frame feature table. Whole track, before the first frame renders.
|
|
//
|
|
// Nothing here reacts in realtime. The table is the song's score as numbers:
|
|
// band energy, spectral flux, centroid, flatness, a dominant pitch, and the
|
|
// places where the track changes. Everything downstream reads this table, so
|
|
// preview and export cannot diverge.
|
|
|
|
import { Fft, hann } from './fft.js';
|
|
|
|
export const DEFAULT_FPS = 30;
|
|
const BANDS = [
|
|
[30, 160], // low — weight, pulse, foundation
|
|
[160, 2000], // mid — body, voice, harmony
|
|
[2000, 11000], // high — air, noise, detail
|
|
];
|
|
|
|
const clamp01 = (x) => (x < 0 ? 0 : x > 1 ? 1 : x);
|
|
|
|
function percentile(values, p) {
|
|
const a = Float32Array.from(values);
|
|
a.sort();
|
|
const i = Math.min(a.length - 1, Math.max(0, Math.round((a.length - 1) * p)));
|
|
return a[i];
|
|
}
|
|
|
|
function normalise(series, loP = 0.03, hiP = 0.97) {
|
|
const lo = percentile(series, loP);
|
|
const hi = percentile(series, hiP);
|
|
const k = hi - lo > 1e-9 ? 1 / (hi - lo) : 0;
|
|
const out = new Float32Array(series.length);
|
|
for (let i = 0; i < series.length; i++) out[i] = clamp01((series[i] - lo) * k);
|
|
return out;
|
|
}
|
|
|
|
/** Local-maximum picking with an adaptive threshold. Returns discrete events. */
|
|
export function pickOnsets(flux, fps, { minGap = 0.07, threshold = 1.35, window = 0.5 } = {}) {
|
|
const n = flux.length;
|
|
const w = Math.max(1, Math.round(window * fps));
|
|
const gap = Math.max(1, Math.round(minGap * fps));
|
|
const events = [];
|
|
let sum = 0;
|
|
for (let i = 0; i < n; i++) {
|
|
sum += flux[i];
|
|
if (i - w >= 0) sum -= flux[i - w];
|
|
const mean = sum / Math.min(i + 1, w);
|
|
if (i < 2 || i >= n - 1) continue;
|
|
if (flux[i] <= flux[i - 1] || flux[i] < flux[i + 1]) continue;
|
|
const floor = mean * threshold + 0.004;
|
|
if (flux[i] < floor) continue;
|
|
if (events.length && i - events[events.length - 1].frame < gap) {
|
|
if (flux[i] > events[events.length - 1].strength) {
|
|
events.pop();
|
|
} else {
|
|
continue;
|
|
}
|
|
}
|
|
events.push({ frame: i, t: i / fps, strength: clamp01((flux[i] - floor) / Math.max(1e-6, 1 - floor)) });
|
|
}
|
|
return events;
|
|
}
|
|
|
|
/** Tempo from the onset envelope's autocorrelation. */
|
|
function estimateTempo(onset, fps) {
|
|
const minLag = Math.round((fps * 60) / 200);
|
|
const maxLag = Math.round((fps * 60) / 50);
|
|
let best = 0;
|
|
let bestLag = 0;
|
|
let total = 0;
|
|
for (let i = 0; i < onset.length; i++) total += onset[i] * onset[i];
|
|
for (let lag = minLag; lag <= maxLag; lag++) {
|
|
let s = 0;
|
|
for (let i = lag; i < onset.length; i++) s += onset[i] * onset[i - lag];
|
|
s /= onset.length - lag;
|
|
if (s > best) {
|
|
best = s;
|
|
bestLag = lag;
|
|
}
|
|
}
|
|
if (!bestLag || total <= 1e-9) return { bpm: 0, beatStrength: 0 };
|
|
// Normalised autocorrelation: a steady 4/4 lands well above 0.3, a drone
|
|
// near 0.1. The sim uses strength, not bpm, to decide whether to march.
|
|
return { bpm: (60 * fps) / bestLag, beatStrength: clamp01(best / (total / onset.length)) };
|
|
}
|
|
|
|
/**
|
|
* Section boundaries: where the 4 Hz feature cloud changes its mind.
|
|
* A boundary is a peak in the distance between the mean of the eight buckets
|
|
* before it and the eight after it — cheap self-similarity, no FFT of the FFT.
|
|
*/
|
|
function findSections(frames4, fps4, duration) {
|
|
const n = frames4.length;
|
|
if (n < 8) return [{ start: 0, end: duration, index: 0 }];
|
|
const half = 8;
|
|
const novelty = new Float32Array(n);
|
|
for (let i = half; i < n - half; i++) {
|
|
let d = 0;
|
|
for (let k = 0; k < 6; k++) {
|
|
let a = 0;
|
|
let b = 0;
|
|
for (let j = 1; j <= half; j++) {
|
|
a += frames4[i - j][k];
|
|
b += frames4[i + j][k];
|
|
}
|
|
d += ((a - b) / half) ** 2;
|
|
}
|
|
novelty[i] = Math.sqrt(d);
|
|
}
|
|
const peak = Math.max(...novelty);
|
|
if (peak <= 1e-8) return [{ start: 0, end: duration, index: 0 }];
|
|
for (let i = 0; i < n; i++) novelty[i] /= peak;
|
|
|
|
const minGap = Math.round(10 * fps4);
|
|
const accepted = [];
|
|
const guard = Math.round(4 * fps4);
|
|
for (let i = guard; i < n - guard; i++) {
|
|
if (novelty[i] < 0.45) continue;
|
|
if (novelty[i] < novelty[i - 1] || novelty[i] < novelty[i + 1]) continue;
|
|
if (accepted.length && i - accepted[accepted.length - 1] < minGap) {
|
|
if (novelty[i] > novelty[accepted[accepted.length - 1]]) accepted.pop();
|
|
else continue;
|
|
}
|
|
accepted.push(i);
|
|
if (accepted.length >= 6) break;
|
|
}
|
|
let cuts = [0, ...accepted.map((i) => i / fps4), duration];
|
|
// Absorb sections shorter than four seconds: a boundary that survives for
|
|
// two seconds is a swell, not an era. The sim cannot change laws that fast.
|
|
cuts = cuts.filter((c, i) => i === 0 || i === cuts.length - 1 || c - cuts[i - 1] >= 4);
|
|
const sections = [];
|
|
for (let i = 0; i < cuts.length - 1; i++) {
|
|
if (cuts[i + 1] - cuts[i] < 1.5) continue;
|
|
sections.push({ start: cuts[i], end: cuts[i + 1], index: sections.length });
|
|
}
|
|
if (!sections.length) return [{ start: 0, end: duration, index: 0 }];
|
|
return sections;
|
|
}
|
|
|
|
function classifySections(sections, energyAt) {
|
|
const es = sections.map((s) => energyAt(s));
|
|
const sorted = [...es].sort((a, b) => a - b);
|
|
const p20 = sorted[Math.floor(sorted.length * 0.2)];
|
|
const p50 = sorted[Math.floor(sorted.length * 0.5)];
|
|
const p80 = sorted[Math.floor(sorted.length * 0.8)];
|
|
const kinds = sections.map((s, i) => {
|
|
const e = es[i];
|
|
const prev = es[i - 1] ?? e;
|
|
if (i === 0 && e <= p50) return 'intro';
|
|
if (i === sections.length - 1 && e <= p50) return 'outro';
|
|
if (e >= p80) return 'drop';
|
|
if (e <= p20) return 'breakdown';
|
|
if (e > prev + 0.04) return 'build';
|
|
return 'sustain';
|
|
});
|
|
return sections.map((s, i) => ({
|
|
...s,
|
|
kind: kinds[i],
|
|
energy: es[i],
|
|
centroid: s.centroid,
|
|
flatness: s.flatness,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* @param {Float32Array} samples mono
|
|
* @param {number} sampleRate
|
|
* @param {{fps?:number, fftSize?:number}} options
|
|
*/
|
|
export function analyse(samples, sampleRate, { fps = DEFAULT_FPS, fftSize = 2048 } = {}) {
|
|
const duration = samples.length / sampleRate;
|
|
const frames = Math.max(1, Math.floor(duration * fps));
|
|
const hop = sampleRate / fps;
|
|
const fft = new Fft(fftSize);
|
|
const win = hann(fftSize);
|
|
const re = new Float32Array(fftSize);
|
|
const im = new Float32Array(fftSize);
|
|
const mag = new Float32Array(fftSize / 2);
|
|
const prevMag = new Float32Array(fftSize / 2);
|
|
|
|
const nyquist = sampleRate / 2;
|
|
const binHz = nyquist / (fftSize / 2);
|
|
const bandBins = BANDS.map(([lo, hi]) => [
|
|
Math.max(1, Math.floor(lo / binHz)),
|
|
Math.min(fftSize / 2 - 1, Math.ceil(hi / binHz)),
|
|
]);
|
|
|
|
const energyRaw = new Float32Array(frames);
|
|
const bandsRaw = [new Float32Array(frames), new Float32Array(frames), new Float32Array(frames)];
|
|
const fluxRaw = new Float32Array(frames);
|
|
const bandFluxRaw = [new Float32Array(frames), new Float32Array(frames), new Float32Array(frames)];
|
|
const centroidRaw = new Float32Array(frames);
|
|
const flatnessRaw = new Float32Array(frames);
|
|
const dominantRaw = new Float32Array(frames);
|
|
const chroma = new Float32Array(12);
|
|
|
|
for (let f = 0; f < frames; f++) {
|
|
const start = Math.min(samples.length - fftSize, Math.floor(f * hop));
|
|
let rms = 0;
|
|
for (let i = 0; i < fftSize; i++) {
|
|
const s = start + i >= 0 ? samples[start + i] || 0 : 0;
|
|
re[i] = s * win[i];
|
|
im[i] = 0;
|
|
rms += s * s;
|
|
}
|
|
energyRaw[f] = Math.sqrt(rms / fftSize);
|
|
fft.forward(re, im);
|
|
|
|
let sumP = 0;
|
|
let sumLog = 0;
|
|
let weighted = 0;
|
|
let peakBin = 1;
|
|
let peakMag = 0;
|
|
let flux = 0;
|
|
for (let b = 1; b < fftSize / 2; b++) {
|
|
const m = Math.sqrt(re[b] * re[b] + im[b] * im[b]) / (fftSize / 2);
|
|
mag[b] = m;
|
|
const p = m * m + 1e-12;
|
|
sumP += p;
|
|
sumLog += Math.log(p);
|
|
weighted += b * binHz * p;
|
|
if (m > peakMag) {
|
|
peakMag = m;
|
|
peakBin = b;
|
|
}
|
|
const hz = b * binHz;
|
|
if (hz > 45 && hz < 2400) {
|
|
const semis = Math.round(12 * Math.log2(hz / 440));
|
|
const pc = ((semis % 12) + 12 + 9) % 12;
|
|
chroma[pc] += m * m;
|
|
}
|
|
flux += Math.max(0, m - prevMag[b]);
|
|
}
|
|
const count = fftSize / 2 - 1;
|
|
// Log-frequency centroid: 50 Hz -> 0, Nyquist -> 1. A linear Hz centroid
|
|
// sits near 0.02 for every real track and cannot tell dark from bright.
|
|
const meanHz = weighted / Math.max(sumP, 1e-12);
|
|
centroidRaw[f] = clamp01(Math.log2(Math.max(50, meanHz) / 50) / Math.log2(nyquist / 50));
|
|
flatnessRaw[f] = sumP > 1e-8 ? clamp01((Math.exp(sumLog / count) / (sumP / count)) * 20) : 0;
|
|
fluxRaw[f] = Math.min(1, flux * 4);
|
|
dominantRaw[f] = peakBin * binHz;
|
|
|
|
for (let k = 0; k < 3; k++) {
|
|
const [lo, hi] = bandBins[k];
|
|
let e = 0;
|
|
let fl = 0;
|
|
for (let b = lo; b <= hi; b++) {
|
|
const m = mag[b];
|
|
e += m * m;
|
|
fl += Math.max(0, m - prevMag[b]);
|
|
}
|
|
bandsRaw[k][f] = Math.sqrt(e / (hi - lo + 1));
|
|
bandFluxRaw[k][f] = Math.min(1, fl * 4);
|
|
}
|
|
prevMag.set(mag);
|
|
}
|
|
|
|
const energy = normalise(energyRaw);
|
|
const low = normalise(bandsRaw[0]);
|
|
const mid = normalise(bandsRaw[1]);
|
|
const high = normalise(bandsRaw[2]);
|
|
const flux = normalise(fluxRaw);
|
|
const bandFlux = bandFluxRaw.map((s) => normalise(s));
|
|
|
|
const onsets = bandFlux.map((s, k) => pickOnsets(s, fps, { threshold: k === 2 ? 1.5 : 1.3 }));
|
|
const combined = new Float32Array(frames);
|
|
for (let i = 0; i < frames; i++) combined[i] = Math.max(bandFlux[0][i], bandFlux[1][i] * 0.8, bandFlux[2][i] * 0.5);
|
|
const allOnsets = mergeOnsets(onsets, fps);
|
|
const tempo = estimateTempo(combined, fps);
|
|
|
|
// 4 Hz section view.
|
|
const fps4 = Math.min(fps, 4);
|
|
const per = fps / fps4;
|
|
const frames4 = [];
|
|
for (let i = 0; i * per < frames; i++) {
|
|
const a = Math.floor(i * per);
|
|
const b = Math.min(frames, Math.floor((i + 1) * per));
|
|
const row = [0, 0, 0, 0, 0, 0];
|
|
for (let j = a; j < b; j++) {
|
|
row[0] += energy[j];
|
|
row[1] += low[j];
|
|
row[2] += mid[j];
|
|
row[3] += high[j];
|
|
row[4] += centroidRaw[j];
|
|
row[5] += flux[j];
|
|
}
|
|
const k = 1 / Math.max(1, b - a);
|
|
frames4.push(row.map((v) => v * k));
|
|
}
|
|
const rawSections = findSections(frames4, fps4, duration);
|
|
const sections = classifySections(rawSections, (s) => {
|
|
const a = Math.floor(s.start * fps);
|
|
const b = Math.max(a + 1, Math.floor(s.end * fps));
|
|
let sum = 0;
|
|
for (let i = a; i < b; i++) sum += energy[i];
|
|
return sum / (b - a);
|
|
}).map((s) => {
|
|
const a = Math.floor(s.start * fps);
|
|
const b = Math.max(a + 1, Math.floor(s.end * fps));
|
|
let c = 0;
|
|
let fl = 0;
|
|
for (let i = a; i < b; i++) {
|
|
c += centroidRaw[i];
|
|
fl += flatnessRaw[i];
|
|
}
|
|
return { ...s, centroid: c / (b - a), flatness: fl / (b - a) };
|
|
});
|
|
|
|
const bandCorrelation = correlate3([low, mid, high]);
|
|
const dynamicRange = Math.max(0, percentile(energyRaw, 0.96) - percentile(energyRaw, 0.1)) /
|
|
Math.max(1e-9, percentile(energyRaw, 0.96));
|
|
let key = 0;
|
|
let keyEnergy = 0;
|
|
const chromaPeak = Math.max(...chroma);
|
|
for (let i = 0; i < 12; i++) {
|
|
if (chroma[i] > keyEnergy) {
|
|
keyEnergy = chroma[i];
|
|
key = i;
|
|
}
|
|
}
|
|
const chromaNorm = Array.from(chroma, (v) => (chromaPeak > 1e-9 ? v / chromaPeak : 0));
|
|
|
|
const summary = {
|
|
duration,
|
|
bpm: tempo.bpm,
|
|
beatStrength: tempo.beatStrength,
|
|
centroid: mean(centroidRaw),
|
|
flatness: mean(flatnessRaw),
|
|
dynamics: clamp01(dynamicRange),
|
|
loudness: mean(energyRaw),
|
|
onsetDensity: allOnsets.length / Math.max(1, duration),
|
|
bandBalance: [mean(low), mean(mid), mean(high)],
|
|
bandCorrelation,
|
|
chroma: chromaNorm,
|
|
key,
|
|
sections: sections.length,
|
|
kinds: sections.map((s) => s.kind),
|
|
};
|
|
|
|
return {
|
|
fps,
|
|
frames,
|
|
duration,
|
|
sampleRate,
|
|
bands: { low, mid, high },
|
|
bandFlux,
|
|
energy,
|
|
flux,
|
|
centroid: centroidRaw,
|
|
flatness: flatnessRaw,
|
|
dominantHz: dominantRaw,
|
|
onsets: { low: onsets[0], mid: onsets[1], high: onsets[2], all: allOnsets },
|
|
sections,
|
|
summary,
|
|
};
|
|
}
|
|
|
|
function mean(a) {
|
|
let s = 0;
|
|
for (let i = 0; i < a.length; i++) s += a[i];
|
|
return s / Math.max(1, a.length);
|
|
}
|
|
|
|
/**
|
|
* One event per moment. Bands fire together on a transient; three simultaneous
|
|
* events would triple the food at the same instant. Keep the strongest, at
|
|
* least 120 ms apart, and carry the voice so the world knows what spoke.
|
|
*/
|
|
function mergeOnsets(bandOnsets, fps, minGap = 0.12) {
|
|
const all = [];
|
|
for (let voice = 0; voice < bandOnsets.length; voice++) {
|
|
for (const o of bandOnsets[voice]) all.push({ ...o, voice });
|
|
}
|
|
all.sort((a, b) => a.frame - b.frame || b.strength - a.strength);
|
|
const gap = Math.max(1, Math.round(minGap * fps));
|
|
const out = [];
|
|
for (const e of all) {
|
|
const last = out[out.length - 1];
|
|
if (last && e.frame - last.frame < gap) {
|
|
if (e.strength > last.strength) out[out.length - 1] = e;
|
|
continue;
|
|
}
|
|
out.push(e);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function correlate3(series) {
|
|
const out = [[1, 0, 0], [0, 1, 0], [0, 0, 1]];
|
|
for (let i = 0; i < 3; i++) {
|
|
for (let j = i + 1; j < 3; j++) {
|
|
const a = series[i];
|
|
const b = series[j];
|
|
const ma = mean(a);
|
|
const mb = mean(b);
|
|
let num = 0;
|
|
let da = 0;
|
|
let db = 0;
|
|
for (let k = 0; k < a.length; k++) {
|
|
const x = a[k] - ma;
|
|
const y = b[k] - mb;
|
|
num += x * y;
|
|
da += x * x;
|
|
db += y * y;
|
|
}
|
|
const r = num / Math.sqrt(Math.max(1e-12, da * db));
|
|
out[i][j] = r;
|
|
out[j][i] = r;
|
|
}
|
|
}
|
|
return out;
|
|
}
|