dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clickable previews: exact app deep-links everywhere

Browse Source 
Every rendered still links to the app frame that produced it (song + seed + frame + venue/viz pins + probe warm-up, new window), via shared src/ui/preview.js. Supporting work, each measured: app honors ?frame/?venueSel/?viz/?warm= with pre-build pins (Show.applyPins) and primes deep-link loads; shared pinViz/pinVenueShell paths for gallery, filmstrip and app; probes prime and carry their warm-up; bank probes analyse the bank PCM and file decode is fixed-rate with bit-exact WAV parsing so app and probes see identical samples; materials built mid-update compile synchronously (needsCompile). Round trip verified bit-identical gallery-versus-app.
This commit is contained in:
Dejvino
2026-09-20 18:42:53 +02:00
parent cc8888077f
commit 4d05250294
18 changed files with 377 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/audio/decode.js
+90 -1
View File
@@ -16,13 +16,102 @@ export async function decodeFile(file) {
return decodeArrayBuffer(arrayBuffer);
}
/**
* The rate everything is decoded at: the synth's rate (see audio/synth.js),
* never the hardware context rate.
*
* `AudioContext.decodeAudioData` resamples to the CONTEXT rate, so the same
* file came out 44.1k in one browser and 48k in another — and, worse, the
* probe pages (filmstrip/gallery/venue-gallery) all render from 44.1k synth
* while the app decoded the bank wav at the context rate. Different spectra,
* different features, different videos: no preview deep-link could ever
* reproduce its image. Decoding in a fixed-rate OfflineAudioContext makes
* the app see the same audio the probes do. See src/ui/preview.js.
*/
export const DECODE_RATE = 44100;
export async function decodeArrayBuffer(arrayBuffer) {
const ctx = context();
// Our own files first, sample-exact: even at a 1.0 ratio Chrome's
// decodeAudioData perturbs int16→float conversion (measured 2.6e-5),
// which is enough to flip section classification on quiet tracks and
// break every preview deep-link. A manual parse keeps every bit.
const wav = decodeWav(arrayBuffer);
if (wav) return wav;
const ctx = new OfflineAudioContext(2, DECODE_RATE, DECODE_RATE);
// decodeAudioData detaches the buffer, so hand it a copy if the caller may
// still need the original bytes.
return await ctx.decodeAudioData(arrayBuffer.slice(0));
}
/**
* Exact WAV decode: 8/16/24-bit PCM and 32-bit float, any rate, any
* channels. No resampling, no conversion noise — int16 comes back as
* V / 32768, the same floats the bank PCM holds. Returns null for
* anything else (compressed formats still go through decodeAudioData).
*/
export function decodeWav(arrayBuffer) {
const bytes = new Uint8Array(arrayBuffer);
const dv = new DataView(arrayBuffer);
const str = (o, n) => {
let s = '';
for (let i = 0; i < n; i++) s += String.fromCharCode(bytes[o + i]);
return s;
};
if (bytes.length < 44 || str(0, 4) !== 'RIFF' || str(8, 4) !== 'WAVE') return null;
let fmt = null, dataOff = 0, dataLen = 0;
let o = 12;
while (o + 8 <= bytes.length) {
const id = str(o, 4);
const size = dv.getUint32(o + 4, true);
if (id === 'fmt ') {
fmt = {
audioFormat: dv.getUint16(o + 8, true),
channels: dv.getUint16(o + 10, true),
sampleRate: dv.getUint32(o + 12, true),
bits: dv.getUint16(o + 22, true),
};
} else if (id === 'data') {
dataOff = o + 8;
dataLen = Math.min(size, bytes.length - dataOff);
}
o += 8 + size + (size % 2);
}
if (!fmt || !dataLen || (fmt.audioFormat !== 1 && fmt.audioFormat !== 3)) return null;
if (fmt.channels < 1 || fmt.channels > 32 || !fmt.sampleRate) return null;
const bytesPerSample = fmt.audioFormat === 3 ? 4 : fmt.bits / 8;
if (![1, 2, 3, 4].includes(bytesPerSample) || (fmt.audioFormat === 3 && fmt.bits !== 32)) return null;
const frames = Math.floor(dataLen / (bytesPerSample * fmt.channels));
if (frames < 1) return null;
let out;
try {
out = new AudioBuffer({
length: frames, sampleRate: fmt.sampleRate, numberOfChannels: fmt.channels,
});
} catch { return null; }
for (let c = 0; c < fmt.channels; c++) {
const ch = out.getChannelData(c);
for (let i = 0; i < frames; i++) {
const at = dataOff + (i * fmt.channels + c) * bytesPerSample;
let v;
if (fmt.audioFormat === 3) {
v = dv.getFloat32(at, true);
} else if (bytesPerSample === 1) {
v = (bytes[at] - 128) / 128;
} else if (bytesPerSample === 2) {
v = dv.getInt16(at, true) / 32768;
} else if (bytesPerSample === 3) {
let s = bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16);
if (s & 0x800000) s -= 0x1000000;
v = s / 8388608;
} else {
v = dv.getInt32(at, true) / 2147483648;
}
ch[i] = v;
}
}
return out;
}
/** Interleaved mono mixdown. Used for the content hash that seeds the look. */
export function monoSamples(audioBuffer) {
const channels = audioBuffer.numberOfChannels;
flow-state/src/audio/songbank.js
+42 -15
View File
@@ -26,7 +26,30 @@
// These are test signals, not music. They exist to be measured.
import { FeatureTrack } from './FeatureTrack.js';
import { synthesizeSong } from './synth.js';
import { synthesizeSong, MockAudioBuffer } from './synth.js';
/**
* The bank's PCM contract: peak-normalise to -1 dBFS, then round to the
* 16-bit grid — bit-exactly what tools/build-song-bank.js writes into
* test/songs/*.wav (int16 LE of round(v * 32767), read back as V / 32768).
*
* Why this exists: the app decodes the wav while every probe page analyses
* the synth, and unless both see these exact samples their features differ
* (measured: 3x loudness and a whole extra drop section on drone) and no
* preview deep-link can reproduce its image. song() applies it so the bank
* IS the files; the writer shares it so the files stay the bank.
*/
export function bankPcm(samples) {
let peak = 0;
for (let i = 0; i < samples.length; i++) peak = Math.max(peak, Math.abs(samples[i]));
const gain = peak > 1e-9 ? 0.89 / peak : 1;
const out = new Float32Array(samples.length);
for (let i = 0; i < samples.length; i++) {
const v = Math.max(-1, Math.min(1, samples[i] * gain));
out[i] = Math.round(v * 32767) / 32768;
}
return out;
}
/**
* @typedef {object} SongSpec
@@ -88,20 +111,24 @@ export function song(name, { fps = 60, duration = DURATION } = {}) {
const spec = SONGS.find((s) => s.name === name);
if (!spec) throw new Error(`no song named "${name}" — have: ${SONGS.map((s) => s.name).join(', ')}`);
const entry = {
...spec,
track: FeatureTrack.fromAudioBuffer(synthesizeSong({
bpm: spec.bpm,
duration,
arrangement: spec.arrangement,
brightness: spec.brightness,
noise: spec.noise,
noiseColour: spec.noiseColour ?? null,
dynamics: spec.dynamics,
transition: spec.transition ?? 0.5,
seed: 1 + SONGS.indexOf(spec),
}), { fps }),
};
// Analysed from the bank PCM, not the raw synth: peak gain, 16-bit
// grid, mono — the exact samples test/songs/*.wav holds, so this track
// is the file the app decodes rather than a near-twin of it. See bankPcm.
const synth = synthesizeSong({
bpm: spec.bpm,
duration,
arrangement: spec.arrangement,
brightness: spec.brightness,
noise: spec.noise,
noiseColour: spec.noiseColour ?? null,
dynamics: spec.dynamics,
transition: spec.transition ?? 0.5,
seed: 1 + SONGS.indexOf(spec),
});
const mono = new MockAudioBuffer(
1, synth.length, synth.sampleRate);
mono.getChannelData(0).set(bankPcm(synth.getChannelData(0)));
const entry = { ...spec, track: FeatureTrack.fromAudioBuffer(mono, { fps }) };
cache.set(cacheKey, entry);
return entry;
}