diff --git a/flow-state/README.md b/flow-state/README.md index 83d88d5..4955b9d 100644 --- a/flow-state/README.md +++ b/flow-state/README.md @@ -59,6 +59,31 @@ npm run lint:scenes # determinism grep + scene schema/shader agreement `http://localhost:5180/checks.html` runs the GPU gates for every phase. Add `?slow=1` for the full suite, `?phase=5` for one phase. +### Seed variety test + +Every other phase asks whether one video is correct. Phase 12 asks whether two +videos are *different* — the one failure the rest of the suite cannot see, since +a generator that ignores its seed passes determinism and liveness perfectly. + +``` +http://localhost:5180/checks.html?variety=1 +``` + +Frames are reduced to a structural descriptor that is deliberately blind to +brightness, contrast, hue and rotation, and sensitive to feature scale, +orientation structure, composition, texture statistics and motion. Colour is +measured but never counted — its job is to expose the case where two seeds +differ only in palette. The score sits between two references the same +instrument produced: the **floor** is how far one video travels from itself +across its own sections, the **ceiling** is the same pipeline with every layer +recast at random. Two checks gate the instrument itself before any number from +it is trusted. + +The report also sweeps **all** visualizations in the library, every pair, and +names the structural twins — different scenes that are the same look in +different colours, which the per-scene `distinct` gate cannot catch because it +compares raw pixels. + ## Adding a scene Quick walkthrough: `HOWTO-visualizers.md`. diff --git a/flow-state/src/checks/main.js b/flow-state/src/checks/main.js index 989fdd7..95c0aa1 100644 --- a/flow-state/src/checks/main.js +++ b/flow-state/src/checks/main.js @@ -1,5 +1,6 @@ import { runAll, summarize, allChecks } from './framework.js'; import { runSceneGate } from './scene-gate.js'; +import { varietyReportLines } from './variety/print.js'; // Registering a phase's checks is a side effect of importing it. import './phase0.js'; @@ -14,6 +15,7 @@ import './phase8.js'; import './phase9.js'; import './phase10.js'; import './phase11.js'; +import './phase12.js'; const out = document.getElementById('results'); const summaryEl = document.getElementById('summary'); @@ -51,6 +53,31 @@ async function main() { return; } + // Seed variety mode: the full diagnostic table rather than a pass/fail. + // Phase 12 gates on this measurement, but a gate answers "is it bad" and + // this answers "which axis is flat", which is the question you have while + // fixing it. + // + // checks.html?variety=1 8 seeds, plus the whole library + // checks.html?variety=1&seeds=16 slower, tighter + // checks.html?variety=1&library=0 skip the library sweep (much faster) + if (params.get('variety')) { + summaryEl.textContent = 'seed variety: rendering seeds and sweeping the library…'; + const started = Date.now(); + const { lines, ok, headline } = await varietyReportLines({ + seeds: Number(params.get('seeds')) || 8, + probes: Number(params.get('probes')) || 5, + library: params.get('library') !== '0', + }); + out.innerHTML = `
${lines.join('\n')}
`; + summaryEl.textContent = `${headline} · ${((Date.now() - started) / 1000).toFixed(1)}s`; + summaryEl.className = ok ? 'ok' : 'bad'; + window.__CHECKS__ = { variety: true, ok, lines }; + window.__CHECKS_DONE__ = true; + console.log('[variety]\n' + lines.join('\n')); + return; + } + const phaseArg = params.get('phase'); const phases = phaseArg ? phaseArg.split(',').map(Number) : null; const skipSlow = params.get('slow') !== '1'; diff --git a/flow-state/src/checks/phase12.js b/flow-state/src/checks/phase12.js new file mode 100644 index 0000000..6f3b24d --- /dev/null +++ b/flow-state/src/checks/phase12.js @@ -0,0 +1,184 @@ +// Phase 12 — the SEED VARIETY TEST. +// +// Every other phase asks whether one video is correct. This one asks whether +// two videos are different, which is the failure the rest of the suite is +// structurally unable to see: a generator that ignores its seed passes +// determinism, flash safety, liveness and end-to-end rendering perfectly. +// +// The measurement lives in checks/variety/. The two checks that come first here +// are not about the generator at all — they are about the instrument. A +// structural variety score is worthless unless it can be shown to ignore the +// cheap axes (recolour, rotate) and to react to the expensive one (a different +// scene), so those are gated before any number derived from them is trusted. +// +// The library sweep in the middle is the map the rest is drawn on: the seed can +// only produce as much variety as the library holds, so "how many structurally +// distinct looks are there" is measured before "how many does a seed reach". + +import { check, expect } from './framework.js'; +import { FeatureTrack } from '../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../audio/synth.js'; +import { Show } from '../Show.js'; +import { generateLook } from '../look/LookGenerator.js'; +import { scenes } from '../scenes/registry.js'; +import { frameDescriptor, rotate90, recolour } from './variety/descriptors.js'; +import { descriptorDistance, signatureDistance } from './variety/signature.js'; +import { + measureVariety, measureSpecDiversity, signatureForScene, librarySweep, +} from './variety/report.js'; + +let cachedTrack = null; +export function varietyTrack() { + if (!cachedTrack) { + cachedTrack = FeatureTrack.fromAudioBuffer( + synthesizeSectioned({ bpm: 124, duration: 90, changeAt: 30 }), { fps: 60 }); + } + return cachedTrack; +} + +// The library sweep is the most expensive thing in the suite — every scene +// rendered — and three checks read it. Computed once per page load. +let cachedSweep = null; +export function librarySweepCached() { + if (!cachedSweep) cachedSweep = librarySweep(varietyTrack(), { probes: 3 }); + return cachedSweep; +} + +/** One frame of one seed, rendered the normal way. */ +function sampleFrame(seed = 12001, width = 160, height = 90) { + const track = varietyTrack(); + const show = new Show({ width, height }); + try { + show.useTrack(track, generateLook(track, { seed })); + const section = show.look.sections[Math.floor(show.look.sections.length / 2)]; + const frame = section.startFrame + Math.floor((section.endFrame - section.startFrame) / 2); + show.engine.compositor.reset(); + for (let f = Math.max(0, frame - 20); f < frame; f++) show.renderFrame(f); + return Uint8Array.from(show.readPixels(show.renderFrame(frame))); + } finally { + show.dispose(); + } +} + +check(12, 'seed variety · the metric ignores colour and rotation', () => { + const w = 160, h = 90; + const pixels = sampleFrame(12001, w, h); + const base = frameDescriptor(pixels, w, h); + + // Hue-rotated and brightened: the same picture in a different palette, which + // is precisely the difference we refuse to count as variety. + const graded = descriptorDistance(base, frameDescriptor(recolour(pixels), w, h)); + // Turned ninety degrees. Layout is EXCLUDED here by design — where structure + // sits in the frame is real information, and a metric blind to it could not + // see a library that centres everything. + const spun = rotate90(pixels, w, h); + const turned = descriptorDistance(base, frameDescriptor(spun.pixels, spun.width, spun.height)); + + const structural = ['scale', 'orient', 'texture']; + const worstGrade = Math.max(...structural.concat('layout').map((b) => graded[b])); + const worstTurn = Math.max(...structural.map((b) => turned[b])); + + return expect(worstGrade < 0.05 && worstTurn < 0.02 && graded.colour > 0.2, + `recolour moves structure ${worstGrade.toFixed(4)} (colour block ${graded.colour.toFixed(3)}) · ` + + `rotate moves structure ${worstTurn.toFixed(4)}`); +}); + +check(12, 'seed variety · the metric separates different scenes from the same scene', () => { + // The instrument's other half: it must react to the thing that IS a + // difference. Two renders of one scene must land far below two renders of + // two scenes, or a low variety score would just be a blind metric. + const track = varietyTrack(); + const pool = scenes.filter((m) => m.role !== 'accent' && m.kind === 'fragment'); + const a = pool[0], b = pool[Math.floor(pool.length / 2)], c = pool[pool.length - 1]; + + const sigA = signatureForScene(track, a, 777, { probes: 2 }); + const sigA2 = signatureForScene(track, a, 777, { probes: 2 }); + const sigB = signatureForScene(track, b, 777, { probes: 2 }); + const sigC = signatureForScene(track, c, 777, { probes: 2 }); + + const same = signatureDistance(sigA, sigA2).total; + const diff = Math.min( + signatureDistance(sigA, sigB).total, + signatureDistance(sigA, sigC).total, + signatureDistance(sigB, sigC).total, + ); + + return expect(same < 0.01 && diff > same * 8, + `same scene ${same.toFixed(4)} · different scenes ${diff.toFixed(4)} ` + + `(${a.name} / ${b.name} / ${c.name})`); +}, { slow: true }); + +check(12, 'seed variety · every visualization in the library is a distinct look', () => { + // Runs against ALL of them, not a sample. The existing per-scene gate does + // a version of this on raw pixels, where two scenes that are the same image + // in different colours pass comfortably; here colour is not counted, so a + // structural twin is visible as one. + // + // Twins are reported rather than failed on a fixed distance: what matters is + // that the library does not contain a CLUSTER of scenes that are one look + // wearing several names, because the casting code will happily "vary" a + // video by rotating between them. + const sweep = librarySweepCached(); + const bigTwins = sweep.twins.filter((g) => g.length >= 3); + + return expect(bigTwins.length === 0, + (bigTwins.length ? `structural twin groups: ${bigTwins.map((g) => g.join('≈')).join(' · ')} — ` : '') + + `${sweep.scenes.length} scenes · median distance ${sweep.median.toFixed(3)} · ` + + `closest ${sweep.closestPairs[0].name}≈${sweep.closestPairs[0].nearest} ` + + `at ${sweep.closestPairs[0].distance.toFixed(3)}`); +}, { slow: true }); + +check(12, 'seed variety · the generator casts a different show for a different seed', () => { + // Cheap, GPU-free, and the first thing to read when the rendered score + // drops: this says whether the generator ever DECIDED to make two different + // videos, before asking whether the pixels came out different. + const d = measureSpecDiversity(varietyTrack(), { seeds: 24 }); + const problems = []; + if (d.sceneSetDistance < 0.6) problems.push(`scene sets only ${d.sceneSetDistance.toFixed(2)} apart`); + if (d.identicalCasts > 0) problems.push(`${d.identicalCasts} seed pairs cast identically`); + if (d.libraryCoverage < 0.6) { + problems.push(`only ${(d.libraryCoverage * 100).toFixed(0)}% of the library used ` + + `(never cast: ${d.uncast.slice(0, 6).join(', ')})`); + } + if (d.director.unique < 2) problems.push('one director for every seed'); + + return expect(problems.length === 0, + problems.length ? problems.join(' · ') + : `cast distance ${d.sceneSetDistance.toFixed(2)} · ` + + `library ${(d.libraryCoverage * 100).toFixed(0)}% · ` + + `${d.director.unique} directors · ${d.paletteScheme.unique} schemes · ` + + `${d.signature.unique} signatures · ${d.grain.unique} grain modes`); +}); + +check(12, 'seed variety · different seeds render structurally different videos', () => { + // THE gate. The threshold is a target the generator does not currently meet + // — a failure here is the known open defect, not a flaky check. The detail + // line carries the full breakdown so a run can be compared against the last + // one while the number is being moved. + const r = measureVariety(varietyTrack(), { seeds: 6, refScenes: 4, probes: 4 }); + const problems = []; + + // Against the floor: two seeds must differ by more than one seed differs + // from itself across its own sections. Below this the seed is decoration. + if (r.separation < 0.35) { + problems.push(`separation ${r.separation.toFixed(2)} (floor ${r.floor.toFixed(3)}, ` + + `observed ${r.observed.toFixed(3)}, ceiling ${r.ceiling.toFixed(3)})`); + } + // No seed may be shadowed by another. A healthy mean hides pairs that are + // the same video, and a viewer only ever sees the pair. + const shadowed = r.nearest.filter((d) => d < r.floor * 0.75).length; + if (shadowed) problems.push(`${shadowed} seeds shadowed by another seed`); + + // Per block: it is not enough for the total to pass on colour-adjacent + // motion while every frame is composed the same way. + for (const [name, b] of Object.entries(r.byBlock)) { + if (name === 'colour') continue; + if (b.ratio < 0.25) problems.push(`${name} at ${(b.ratio * 100).toFixed(0)}% of achievable`); + } + + const blocks = Object.entries(r.byBlock) + .map(([n, b]) => `${n} ${(b.ratio * 100).toFixed(0)}%`).join(' · '); + return expect(problems.length === 0, + (problems.length ? problems.join(' · ') + ' — ' : '') + + `separation ${r.separation.toFixed(2)} · ${blocks}`); +}, { slow: true }); diff --git a/flow-state/src/checks/variety/descriptors.js b/flow-state/src/checks/variety/descriptors.js new file mode 100644 index 0000000..c135c29 --- /dev/null +++ b/flow-state/src/checks/variety/descriptors.js @@ -0,0 +1,363 @@ +// A frame reduced to a STRUCTURAL descriptor: what shape the image is, not what +// colour it is or which way up. +// +// The problem this exists to solve: two seeds can produce videos that a pixel +// difference calls wildly different — one is teal and rotating left, the other +// is magenta and rotating right — while a viewer calls them the same video. +// Any metric built on raw pixels rewards exactly the variation we do not care +// about. So the descriptor is built to be blind to the cheap axes and sensitive +// to the expensive ones: +// +// blind to brightness, contrast, hue, saturation, global rotation +// sensitive to feature SCALE (fine texture vs big soft blobs), ORIENTATION +// structure (grid vs radial vs stripes), LAYOUT (centred glow vs +// full-frame vs banded), and TEXTURE STATISTICS (how many +// distinct elements, how sparse, how symmetric) +// +// Colour is still measured, but it is kept in its own block and excluded from +// the structural total — so a report can say "your palettes vary, your +// structure does not", which is the distinction the whole exercise is about. +// +// Everything here is plain Float32 maths on a 96x96 luma image. No FFT: a +// Laplacian pyramid gives the radial power spectrum directly, and a magnitude- +// weighted gradient histogram gives the angular one. + +export const SIZE = 96; + +/** RGBA bytes → square Float32 luma in 0..1, resampled to SIZE x SIZE. */ +export function toLuma(pixels, width, height, size = SIZE) { + const out = new Float32Array(size * size); + for (let y = 0; y < size; y++) { + const sy = Math.min(height - 1, Math.floor((y + 0.5) * height / size)); + for (let x = 0; x < size; x++) { + const sx = Math.min(width - 1, Math.floor((x + 0.5) * width / size)); + const i = (sy * width + sx) * 4; + out[y * size + x] = + (0.2126 * pixels[i] + 0.7152 * pixels[i + 1] + 0.0722 * pixels[i + 2]) / 255; + } + } + return out; +} + +/** + * Zero mean, unit standard deviation, in place. + * + * This is where exposure and contrast leave the metric. Two renders of the same + * structure at different brightness are the same image after this; that is the + * point, and it is why the descriptor cannot be fooled by a palette reroll. + */ +export function standardize(img) { + let mean = 0; + for (let i = 0; i < img.length; i++) mean += img[i]; + mean /= img.length; + let variance = 0; + for (let i = 0; i < img.length; i++) variance += (img[i] - mean) ** 2; + const sd = Math.sqrt(variance / img.length) || 1e-6; + for (let i = 0; i < img.length; i++) img[i] = (img[i] - mean) / sd; + return img; +} + +/** 2x2 box downsample. */ +function halve(src, w, h) { + const ow = w >> 1, oh = h >> 1; + const out = new Float32Array(ow * oh); + for (let y = 0; y < oh; y++) { + for (let x = 0; x < ow; x++) { + const i = (y * 2) * w + x * 2; + out[y * ow + x] = (src[i] + src[i + 1] + src[i + w] + src[i + w + 1]) * 0.25; + } + } + return out; +} + +/** Nearest-neighbour 2x upsample — good enough as the pyramid's low-pass. */ +function double(src, w, h) { + const ow = w * 2, oh = h * 2; + const out = new Float32Array(ow * oh); + for (let y = 0; y < oh; y++) { + for (let x = 0; x < ow; x++) out[y * ow + x] = src[(y >> 1) * w + (x >> 1)]; + } + return out; +} + +/** + * Laplacian pyramid: one band-pass image per octave. + * + * Band k holds the detail that lives at roughly 2^k pixels. The energy across + * bands IS the radial power spectrum, which is the single most useful thing to + * know about an abstract image: fine grain, mid-scale filigree and slow blobs + * are structurally different pictures no matter what colour they are. + */ +export function pyramid(img, size = SIZE, levels = 5) { + const bands = []; + let cur = img, w = size, h = size; + for (let k = 0; k < levels; k++) { + const low = halve(cur, w, h); + const up = double(low, w >> 1, h >> 1); + const band = new Float32Array(cur.length); + for (let i = 0; i < cur.length; i++) band[i] = cur[i] - up[i]; + bands.push({ data: band, w, h }); + cur = low; w >>= 1; h >>= 1; + } + return bands; +} + +function rms(a) { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * a[i]; + return Math.sqrt(s / a.length); +} + +/** Scale block: the radial spectrum, normalised to its own sum so it describes + * the SHAPE of the spectrum rather than how contrasty the frame was. */ +function scaleBlock(bands) { + const v = bands.map((b) => rms(b.data)); + const total = v.reduce((a, x) => a + x, 0) || 1e-6; + return v.map((x) => x / total); +} + +const ORIENT_BINS = 12; + +/** + * Orientation block, made rotation-invariant. + * + * A magnitude-weighted histogram of gradient direction (mod pi) says whether + * the image is a grid (two peaks 90 apart), stripes (one peak), radial + * (flat-ish), or a quasicrystal (five peaks). Rotating the image circularly + * SHIFTS that histogram, so taking the magnitude of its DFT throws the shift + * away and keeps the pattern. That is exactly the "not just rotation" property + * we need: turn a scene 30 degrees and this block does not move. + */ +function orientBlock(band) { + const { data, w, h } = band; + const hist = new Float64Array(ORIENT_BINS); + for (let y = 1; y < h - 1; y++) { + for (let x = 1; x < w - 1; x++) { + const i = y * w + x; + const gx = data[i + 1] - data[i - 1]; + const gy = data[i + w] - data[i - w]; + const mag = Math.hypot(gx, gy); + if (mag < 1e-6) continue; + let a = Math.atan2(gy, gx); + if (a < 0) a += Math.PI; // direction, not sign + if (a >= Math.PI) a -= Math.PI; + hist[Math.min(ORIENT_BINS - 1, Math.floor(a / Math.PI * ORIENT_BINS))] += mag; + } + } + const dc = hist.reduce((a, x) => a + x, 0) || 1e-6; + const out = []; + for (let k = 1; k <= 6; k++) { + let re = 0, im = 0; + for (let n = 0; n < ORIENT_BINS; n++) { + const th = -2 * Math.PI * k * n / ORIENT_BINS; + re += hist[n] * Math.cos(th); + im += hist[n] * Math.sin(th); + } + out.push(Math.hypot(re, im) / dc); + } + return out; +} + +/** + * Layout block: where in the frame the structure actually is, on a 4x4 grid. + * + * Deliberately NOT rotation-invariant, and kept separate for that reason. + * "Everything the generator makes is a bright thing in the middle of a dark + * frame" is a composition failure, and it is invisible to every other block + * here — the scale and orientation spectra of two differently-composed frames + * can match perfectly. + */ +function layoutBlock(band, cells = 4) { + const { data, w, h } = band; + const grid = new Float64Array(cells * cells); + const counts = new Float64Array(cells * cells); + for (let y = 0; y < h; y++) { + const gy = Math.min(cells - 1, Math.floor(y * cells / h)); + for (let x = 0; x < w; x++) { + const gx = Math.min(cells - 1, Math.floor(x * cells / w)); + const v = data[y * w + x]; + grid[gy * cells + gx] += v * v; + counts[gy * cells + gx]++; + } + } + let total = 0; + for (let i = 0; i < grid.length; i++) { + grid[i] = Math.sqrt(grid[i] / (counts[i] || 1)); + total += grid[i]; + } + return Array.from(grid, (x) => x / (total || 1e-6)); +} + +function corr(a, b) { + let num = 0, da = 0, db = 0; + for (let i = 0; i < a.length; i++) { num += a[i] * b[i]; da += a[i] * a[i]; db += b[i] * b[i]; } + return num / (Math.sqrt(da * db) || 1e-6); +} + +/** + * Texture block: how many things are in the frame, how sparse they are, and how + * symmetric the composition is. + * + * Symmetry earns its place: mirror and 180-degree self-correlation is what + * separates a kaleidoscope from a drift, and a library that quietly funnels + * every seed into radially symmetric imagery will show up here as three numbers + * that never move, when nothing else in the descriptor notices. + */ +function textureBlock(band) { + const { data, w, h } = band; + // Zero crossings per row and per column: a proxy for element count that + // costs nothing and does not care about contrast. + let rowCross = 0, colCross = 0; + for (let y = 0; y < h; y++) { + for (let x = 1; x < w; x++) { + if ((data[y * w + x] > 0) !== (data[y * w + x - 1] > 0)) rowCross++; + } + } + for (let x = 0; x < w; x++) { + for (let y = 1; y < h; y++) { + if ((data[y * w + x] > 0) !== (data[(y - 1) * w + x] > 0)) colCross++; + } + } + const sd = rms(data) || 1e-6; + let sparse = 0, tail = 0; + for (let i = 0; i < data.length; i++) { + const m = Math.abs(data[i]); + if (m > sd) sparse++; + if (m > sd * 2.5) tail++; + } + + const mirrorH = new Float32Array(data.length); + const mirrorV = new Float32Array(data.length); + const rot = new Float32Array(data.length); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + mirrorH[y * w + x] = data[y * w + (w - 1 - x)]; + mirrorV[y * w + x] = data[(h - 1 - y) * w + x]; + rot[y * w + x] = data[(h - 1 - y) * w + (w - 1 - x)]; + } + } + + // Horizontal and vertical measurements are folded into a sum and an + // absolute difference rather than reported as a pair. A quarter turn SWAPS + // the two, and a descriptor that changed under a quarter turn would be + // measuring orientation twice — once here, unintentionally, on top of the + // orient block that handles it properly. Folded this way the anisotropy + // survives (a striped frame still reads as anisotropic) but its direction + // does not. + const cx = rowCross / (w * h), cy = colCross / (w * h); + const mh = corr(data, mirrorH), mv = corr(data, mirrorV); + return [ + cx + cy, Math.abs(cx - cy), + sparse / data.length, tail / data.length, + (mh + mv) / 2, Math.abs(mh - mv), corr(data, rot), + ]; +} + +/** + * Colour block. Reported, never counted in the structural score. + * + * Its job in the report is adversarial: it is the number that proves a high + * pixel-difference between two seeds was only ever a palette swap. + */ +function colourBlock(pixels) { + const hist = new Float64Array(6); + let sat = 0, val = 0, n = 0; + for (let i = 0; i < pixels.length; i += 4) { + const r = pixels[i] / 255, g = pixels[i + 1] / 255, b = pixels[i + 2] / 255; + const max = Math.max(r, g, b), min = Math.min(r, g, b); + const d = max - min; + const s = max <= 0 ? 0 : d / max; + sat += s; val += max; n++; + if (d < 1e-4) continue; + let hue; + if (max === r) hue = ((g - b) / d + 6) % 6; + else if (max === g) hue = (b - r) / d + 2; + else hue = (r - g) / d + 4; + hist[Math.min(5, Math.floor(hue))] += s * max; + } + const total = hist.reduce((a, x) => a + x, 0) || 1e-6; + return [...Array.from(hist, (x) => x / total), sat / n, val / n]; +} + +/** + * Full descriptor for one frame. + * + * @param {Uint8Array} pixels RGBA readback + * @returns {{scale:number[], orient:number[], layout:number[], texture:number[], colour:number[]}} + */ +export function frameDescriptor(pixels, width, height) { + const luma = standardize(toLuma(pixels, width, height)); + const bands = pyramid(luma); + // Band 1 (~4px detail) is the working band for orientation, layout and + // texture: band 0 is dominated by grain and compression-scale noise, and + // the coarse bands are too small to have a layout worth reading. + const band = bands[1]; + return { + scale: scaleBlock(bands), + orient: orientBlock(band), + layout: layoutBlock(band), + texture: textureBlock(band), + colour: colourBlock(pixels), + }; +} + +/** + * Descriptor of what MOVED between two frames. + * + * The absolute difference image, run through the same machinery. This is the + * block that catches "every scene in the library is a slow full-frame drift": + * two videos can differ in every static frame and still move identically, and + * motion is a large part of what a viewer reads as the identity of a visual. + */ +export function motionDescriptor(a, b, width, height) { + const diff = new Uint8Array(a.length); + for (let i = 0; i < a.length; i += 4) { + diff[i] = Math.abs(a[i] - b[i]); + diff[i + 1] = Math.abs(a[i + 1] - b[i + 1]); + diff[i + 2] = Math.abs(a[i + 2] - b[i + 2]); + diff[i + 3] = 255; + } + const d = frameDescriptor(diff, width, height); + let energy = 0; + for (let i = 0; i < diff.length; i += 4) energy += diff[i] + diff[i + 1] + diff[i + 2]; + energy = energy / ((diff.length / 4) * 3 * 255); + return { ...d, energy }; +} + +// --- transforms used to VALIDATE the metric ------------------------------ +// A structural metric that has never been shown to ignore colour and rotation +// is a claim, not a measurement. These let the checks prove it. + +/** Rotate an RGBA frame by 90 degrees. Returns {pixels, width, height}. */ +export function rotate90(pixels, width, height) { + const out = new Uint8Array(pixels.length); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const si = (y * width + x) * 4; + const di = (x * height + (height - 1 - y)) * 4; + out[di] = pixels[si]; out[di + 1] = pixels[si + 1]; + out[di + 2] = pixels[si + 2]; out[di + 3] = pixels[si + 3]; + } + } + return { pixels: out, width: height, height: width }; +} + +/** Rotate the hue of every pixel by `turns` and scale brightness. */ +export function recolour(pixels, turns = 0.33, gain = 1.25) { + const out = new Uint8Array(pixels.length); + const c = Math.cos(turns * 2 * Math.PI), s = Math.sin(turns * 2 * Math.PI); + // YIQ hue rotation — cheap, and it leaves luma structure exactly alone. + const m = [ + 0.299 + 0.701 * c + 0.168 * s, 0.587 - 0.587 * c + 0.330 * s, 0.114 - 0.114 * c - 0.497 * s, + 0.299 - 0.299 * c - 0.328 * s, 0.587 + 0.413 * c + 0.035 * s, 0.114 - 0.114 * c + 0.292 * s, + 0.299 - 0.300 * c + 1.250 * s, 0.587 - 0.588 * c - 1.050 * s, 0.114 + 0.886 * c - 0.203 * s, + ]; + for (let i = 0; i < pixels.length; i += 4) { + const r = pixels[i], g = pixels[i + 1], b = pixels[i + 2]; + out[i] = Math.max(0, Math.min(255, (m[0] * r + m[1] * g + m[2] * b) * gain)); + out[i + 1] = Math.max(0, Math.min(255, (m[3] * r + m[4] * g + m[5] * b) * gain)); + out[i + 2] = Math.max(0, Math.min(255, (m[6] * r + m[7] * g + m[8] * b) * gain)); + out[i + 3] = pixels[i + 3]; + } + return out; +} diff --git a/flow-state/src/checks/variety/print.js b/flow-state/src/checks/variety/print.js new file mode 100644 index 0000000..6c4802d --- /dev/null +++ b/flow-state/src/checks/variety/print.js @@ -0,0 +1,124 @@ +// The variety report as text. +// +// Kept apart from the measurement so the gate and the diagnostic read the same +// numbers. The layout is chosen so the first three lines answer "is this bad", +// and everything below answers "where did the variety go" — which is the only +// question worth printing a table for. + +import { FeatureTrack } from '../../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../../audio/synth.js'; +import { measureVariety, measureSpecDiversity, librarySweep } from './report.js'; +import { generateLook, describeLook } from '../../look/LookGenerator.js'; + +const bar = (v, width = 24) => { + const n = Math.max(0, Math.min(width, Math.round(v * width))); + return '█'.repeat(n) + '·'.repeat(width - n); +}; + +const pct = (v) => `${(v * 100).toFixed(0)}%`.padStart(4); + +export async function varietyReportLines({ seeds = 8, probes = 5, library = true } = {}) { + const track = FeatureTrack.fromAudioBuffer( + synthesizeSectioned({ bpm: 124, duration: 90, changeAt: 30 }), { fps: 60 }); + + const lines = []; + const spec = measureSpecDiversity(track, { seeds: Math.max(seeds, 24) }); + + // Yield so the "measuring…" line paints before the GPU work blocks. + await new Promise((r) => setTimeout(r, 0)); + const r = measureVariety(track, { seeds, refScenes: 5, probes }); + + const ok = r.separation >= 0.35; + const headline = `seed separation ${r.separation.toFixed(2)} ` + + `(${ok ? 'acceptable' : 'TOO LOW'}) · ${pct(r.identity)} of what the library can express`; + + lines.push('SEED VARIETY — one track, one analysis, only the seed changes'); + lines.push(''); + lines.push(` floor ${r.floor.toFixed(4)} one video against itself, across its own sections`); + lines.push(` observed ${r.observed.toFixed(4)} two seeds against each other`); + lines.push(` ceiling ${r.ceiling.toFixed(4)} same pipeline, every layer recast at random`); + lines.push(''); + lines.push(` separation ${bar(r.separation)} ${r.separation.toFixed(2)}`); + lines.push(' 0 = the seed changes nothing a viewer could name'); + lines.push(' 1 = two seeds as unalike as two randomly assembled videos'); + lines.push(''); + lines.push('WHERE THE VARIETY IS — per structural block, as a fraction of achievable'); + lines.push(''); + for (const [name, b] of Object.entries(r.byBlock)) { + const note = name === 'colour' ? ' (not counted — this is the axis that lies)' : ''; + lines.push(` ${name.padEnd(8)} ${bar(b.ratio)} ${pct(b.ratio)}` + + ` ${b.between.toFixed(3)} of ${b.ceiling.toFixed(3)}${note}`); + } + lines.push(''); + lines.push(' scale feature size — fine texture vs large soft forms'); + lines.push(' orient grid vs radial vs stripes, measured rotation-blind'); + lines.push(' layout where in the frame the structure sits'); + lines.push(' texture element count, sparsity, mirror and radial symmetry'); + lines.push(' motion what moves and where, not how much'); + lines.push(''); + + lines.push('CLOSEST SIBLING PER SEED — a seed below the floor is a duplicate video'); + lines.push(''); + r.nearest.forEach((d, i) => { + const flag = d < r.floor * 0.75 ? ' ← shadowed' : d < r.floor ? ' ← thin' : ''; + lines.push(` seed ${r.seeds[i].toString(16).padStart(8, '0')} ${bar(d / Math.max(r.ceiling, 1e-6))} ${d.toFixed(3)}${flag}`); + }); + if (r.worstPair) { + const w = r.worstPair; + const blocks = Object.entries(w.byBlock) + .map(([n, v]) => `${n} ${v.toFixed(3)}`).join(' · '); + lines.push(''); + lines.push(` closest pair: seeds ${w.a} and ${w.b} at ${w.distance.toFixed(3)} — ${blocks}`); + } + lines.push(''); + + lines.push('THE DECISIONS BEHIND IT — spec-level, no rendering'); + lines.push(' (high here + low above = the generator decides freely and the render flattens it;'); + lines.push(' low here = casting is the bottleneck, and shader work will not fix it)'); + lines.push(''); + lines.push(` scene-set distance ${bar(spec.sceneSetDistance)} ${spec.sceneSetDistance.toFixed(2)} how differently ${spec.seeds} seeds cast`); + lines.push(` library coverage ${bar(spec.libraryCoverage)} ${pct(spec.libraryCoverage)} of non-accent scenes ever chosen`); + lines.push(` identical casts ${spec.identicalCasts} seed pairs`); + for (const key of ['director', 'paletteScheme', 'signature', 'grain', 'framing', 'paletteArc', 'anchorScenes']) { + const e = spec[key]; + lines.push(` ${key.padEnd(20)} ${bar(e.normalized)} ${String(e.unique).padStart(3)} distinct (entropy ${e.entropy.toFixed(2)} bits)`); + } + lines.push(''); + + if (library) { + // The map the whole test is drawn on: a seed cannot reach more variety + // than the library holds, so a low separation is only the generator's + // fault once the library is known to hold distinct looks. + const sweep = librarySweep(track, { probes: 3 }); + lines.push('THE LIBRARY — all ' + sweep.scenes.length + ' visualizations, every pair, colour not counted'); + lines.push(''); + lines.push(` median pair distance ${sweep.median.toFixed(3)} ` + + `(seed test ceiling for reference: ${r.ceiling.toFixed(3)})`); + lines.push(` closest 5% under ${sweep.p05.toFixed(3)}`); + lines.push(''); + lines.push(` structural twins — every pair inside a group is under ${sweep.twinAt.toFixed(3)}:`); + if (sweep.twins.length === 0) { + lines.push(' no group of three or more; see the closest pairs below'); + } else { + for (const group of sweep.twins) lines.push(` ${group.join(' ≈ ')}`); + } + lines.push(''); + lines.push(' closest pairs in the library:'); + for (const p of sweep.closestPairs) { + lines.push(` ${p.distance.toFixed(3)} ${p.name.padEnd(22)} ≈ ${p.nearest} (${p.family})`); + } + lines.push(''); + if (spec.uncast.length) { + lines.push(` never cast in ${spec.seeds} seeds: ${spec.uncast.join(', ')}`); + lines.push(''); + } + } + + lines.push('SAMPLE LOOKS'); + lines.push(''); + for (const seed of r.seeds.slice(0, 6)) { + lines.push(` ${describeLook(generateLook(track, { seed }))}`); + } + + return { lines, ok, headline }; +} diff --git a/flow-state/src/checks/variety/report.js b/flow-state/src/checks/variety/report.js new file mode 100644 index 0000000..48e5aa3 --- /dev/null +++ b/flow-state/src/checks/variety/report.js @@ -0,0 +1,361 @@ +// The variety measurement: does changing the seed actually change the video? +// +// A raw distance between two seeds means nothing on its own — 0.14 is not +// interpretable. It is only a number once it sits between two references that +// the same instrument produced: +// +// FLOOR how far one video travels from ITSELF over its own length (drift). +// Two seeds that differ by less than this are, in the only sense that +// matters, the same video shown twice. +// CEILING how far two videos are when the same pipeline is run with the +// design deliberately thrown away — every layer recast at random. +// This is the most variety the library can express, so it is what the +// generator is measured against, not some abstract 1.0. +// +// separation = (between-seed - floor) / (ceiling - floor) +// +// 0 means the seed does nothing a viewer could name. 1 means two seeds are as +// unalike as two randomly assembled videos. The honest target is somewhere well below +// 1 — a generator with a house style SHOULD land under the ceiling — but it has +// to clear the floor by a wide margin, and the per-block breakdown is what says +// where the missing variety went. +// +// Alongside the pixel measurement there is a SPEC measurement, which needs no +// GPU: how much the generator's own decisions differ across seeds. Pixels +// measure the symptom, the spec measures the cause. If spec diversity is high +// and pixel variety is low, the generator is deciding freely and the renderer +// or the post chain is flattening it. If spec diversity is also low, the +// casting is the bottleneck and no amount of shader work will fix it. + +import { Show } from '../../Show.js'; +import { generateLook } from '../../look/LookGenerator.js'; +import { scenes } from '../../scenes/registry.js'; +import { defaultValues, sampleValues } from '../../params/schema.js'; +import { Rng } from '../../engine/rng.js'; +import { videoSignature, signatureDistance, STRUCTURAL } from './signature.js'; + +const RENDER = { width: 160, height: 90 }; + +/** Signature for one seed, rendered through the whole normal pipeline. */ +export function signatureForSeed(track, seed, options = {}) { + const show = new Show({ ...RENDER }); + try { + show.useTrack(track, generateLook(track, { seed: seed >>> 0 })); + return videoSignature(show, options); + } finally { + show.dispose(); + } +} + +/** + * The ceiling reference: a video with the design thrown away. + * + * Same pipeline, same shot planning, same post — but every layer is recast to a + * scene picked uniformly at random and its parameters resampled without regard + * for the section. Two of these agree about nothing, so the distance between + * them is the most this library and this renderer can express. That is the + * honest thing to measure the generator against: not 1.0, which no pipeline + * reaches, and not two default-parameter scenes either — that was the first + * attempt, and it produced videos so internally uneventful that real seeds + * scored ABOVE the supposed ceiling on three blocks out of five. + */ +export function signatureForChaos(track, seed, options = {}) { + const show = new Show({ ...RENDER }); + try { + const look = generateLook(track, { seed: seed >>> 0 }); + const rng = new Rng((seed * 2246822519) >>> 0); + const pool = scenes.filter((m) => m.role !== 'accent'); + const temperament = look.personality && look.personality.temperament; + for (const section of look.sections) { + for (const variant of section.variants) { + for (const layer of variant) { + layer.module = rng.pick(pool); + layer.params = sampleValues(layer.module, rng, section.bias, temperament); + layer.seed = rng.int(0, 0x7fffffff); + } + } + section.layers = section.variants[0]; + } + show.useTrack(track, look); + return videoSignature(show, options); + } finally { + show.dispose(); + } +} + +/** Signature for a video forced onto ONE library scene. Used to validate the metric. */ +export function signatureForScene(track, module, seed, options = {}) { + const show = new Show({ ...RENDER }); + try { + const look = generateLook(track, { seed: seed >>> 0 }); + for (const section of look.sections) { + const layers = [{ + module, + params: defaultValues(module), + seed: seed >>> 0, + blend: 'normal', + opacity: 1, + }]; + section.variants = [layers]; + section.layers = layers; + for (const shot of section.shots || []) shot.variant = 0; + } + show.useTrack(track, look); + return videoSignature(show, options); + } finally { + show.dispose(); + } +} + +/** + * Every visualization in the library, measured structurally, against every + * other one. + * + * This is the map the seed test is drawn on. A library of sixty names is not a + * library of sixty looks: two scenes built from different maths can land on the + * same image — the same feature scale, the same composition, the same kind of + * movement — and once they do, no amount of casting variety can produce a + * different-looking video by choosing between them. + * + * Note what this catches that the existing per-scene "distinct" gate cannot. + * That one compares raw pixels, so two scenes that are structurally the same + * image in different colours pass it comfortably. Here colour is not counted at + * all, so structural twins have nowhere to hide. + * + * Default parameters throughout: the question is what a scene inherently looks + * like, and sampled parameters would make the answer depend on which roll it + * got. + */ +export function librarySweep(track, { probes = 3, onProgress = null } = {}) { + const pool = scenes.filter((m) => m.role !== 'accent'); + const sigs = []; + for (let i = 0; i < pool.length; i++) { + sigs.push(signatureForScene(track, pool[i], 4242, { probes })); + if (onProgress) onProgress(i + 1, pool.length, pool[i].name); + } + + const n = pool.length; + const matrix = Array.from({ length: n }, () => new Float64Array(n)); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + const d = signatureDistance(sigs[i], sigs[j]).total; + matrix[i][j] = d; + matrix[j][i] = d; + } + } + + const all = []; + for (let i = 0; i < n; i++) for (let j = i + 1; j < n; j++) all.push(matrix[i][j]); + all.sort((a, b) => a - b); + + const nearest = pool.map((module, i) => { + let best = Infinity, at = -1; + for (let j = 0; j < n; j++) { + if (i !== j && matrix[i][j] < best) { best = matrix[i][j]; at = j; } + } + return { name: module.name, family: module.family, nearest: pool[at].name, distance: best }; + }); + + // COMPLETE-link clustering at the twin threshold: a scene joins a group only + // if it is close to every member, not merely to one of them. + // + // Single link was the first attempt and it lied. Structural distance is + // chainable — A near B, B near C, C near D — so it reported fourteen scenes + // as one look when what actually existed was a chain of overlapping pairs. + // A group here means every pair inside it is a twin, which is a claim worth + // acting on. + const twinAt = all[Math.floor(all.length * 0.02)]; // the closest 2% of pairs + const order = []; + for (let i = 0; i < n; i++) for (let j = i + 1; j < n; j++) order.push([matrix[i][j], i, j]); + order.sort((a, b) => a[0] - b[0]); + + const groupOf = new Array(n).fill(-1); + const clusters = []; + for (const [d, i, j] of order) { + if (d > twinAt) break; + const gi = groupOf[i], gj = groupOf[j]; + const fits = (member, group) => group.every((k) => matrix[member][k] <= twinAt); + if (gi < 0 && gj < 0) { + groupOf[i] = groupOf[j] = clusters.length; + clusters.push([i, j]); + } else if (gi >= 0 && gj < 0 && fits(j, clusters[gi])) { + clusters[gi].push(j); groupOf[j] = gi; + } else if (gj >= 0 && gi < 0 && fits(i, clusters[gj])) { + clusters[gj].push(i); groupOf[i] = gj; + } + } + const groups = clusters.map((c) => c.map((i) => pool[i].name)); + + return { + scenes: pool.map((m) => m.name), + matrix, + nearest, + median: all[Math.floor(all.length / 2)], + mean: mean(all), + p05: all[Math.floor(all.length * 0.05)], + twinAt, + twins: groups.filter((g) => g.length > 1).sort((a, b) => b.length - a.length), + twinPairs: order.filter(([d]) => d <= twinAt) + .map(([d, i, j]) => ({ a: pool[i].name, b: pool[j].name, distance: d })), + closestPairs: nearest.slice().sort((a, b) => a.distance - b.distance).slice(0, 8), + }; +} + +function pairwise(items, fn) { + const out = []; + for (let i = 0; i < items.length; i++) { + for (let j = i + 1; j < items.length; j++) out.push(fn(items[i], items[j], i, j)); + } + return out; +} + +const mean = (a) => (a.length ? a.reduce((x, y) => x + y, 0) / a.length : 0); + +/** + * The full measurement. + * + * @param {FeatureTrack} track one analysed track — held FIXED, so the only + * variable is the seed. Measuring across different audio would confound + * "the generator ignores its seed" with "these songs are alike". + * @param {object} options + * @returns {object} report + */ +export function measureVariety(track, { + seeds = 8, refScenes = 5, probes = 5, seed0 = 0x5eed, +} = {}) { + const seedList = []; + for (let i = 0; i < seeds; i++) seedList.push((seed0 + i * 2654435761) >>> 0); + + const sigs = seedList.map((s) => signatureForSeed(track, s, { probes })); + + // --- floor: a video against itself, later --------------------------- + const floor = mean(sigs.map((s) => s.drift)); + + // --- between-seed --------------------------------------------------- + const between = pairwise(sigs, (a, b) => signatureDistance(a, b)); + const observed = mean(between.map((d) => d.total)); + + // --- ceiling: the same pipeline with the design thrown away ---------- + const refSigs = []; + for (let i = 0; i < refScenes; i++) { + refSigs.push(signatureForChaos(track, (0xbadc0de + i * 40503) >>> 0, { probes })); + } + const ceilingPairs = pairwise(refSigs, (a, b) => signatureDistance(a, b)); + const ceiling = mean(ceilingPairs.map((d) => d.total)); + + const span = Math.max(1e-6, ceiling - floor); + const separation = (observed - floor) / span; + + // Per block, the same three numbers — this is the diagnosis. + const byBlock = {}; + for (const block of [...STRUCTURAL, 'colour']) { + const b = mean(between.map((d) => d.byBlock[block] ?? 0)); + const c = mean(ceilingPairs.map((d) => d.byBlock[block] ?? 0)); + byBlock[block] = { + between: b, + ceiling: c, + ratio: c > 1e-6 ? b / c : 0, + }; + } + + // Nearest-neighbour collapse: for each seed, how close is its closest + // sibling? A healthy generator has no seed that another seed shadows. A mean + // can look acceptable while two of eight seeds are visually the same video. + const nearest = sigs.map((_, i) => { + let best = 1; + for (let j = 0; j < sigs.length; j++) { + if (i === j) continue; + best = Math.min(best, signatureDistance(sigs[i], sigs[j]).total); + } + return best; + }); + + return { + seeds: seedList, + floor, + ceiling, + observed, + separation, + identity: ceiling > 1e-6 ? observed / ceiling : 0, + byBlock, + nearest, + worstPair: worstPairOf(seedList, between), + motion: mean(sigs.map((s) => s.motion)), + }; +} + +function worstPairOf(seedList, between) { + let idx = 0, best = Infinity, k = 0; + for (let i = 0; i < seedList.length; i++) { + for (let j = i + 1; j < seedList.length; j++) { + if (between[k].total < best) { best = between[k].total; idx = k; } + k++; + } + } + k = 0; + for (let i = 0; i < seedList.length; i++) { + for (let j = i + 1; j < seedList.length; j++) { + if (k === idx) return { a: i, b: j, distance: best, byBlock: between[k].byBlock }; + k++; + } + } + return null; +} + +// --- spec-level diversity ------------------------------------------------- +// No GPU, milliseconds to run. This is the one to run first when the score +// drops, because it says whether the generator ever MEANT to make two different +// videos. + +function entropy(values) { + const counts = new Map(); + for (const v of values) counts.set(v, (counts.get(v) || 0) + 1); + let h = 0; + for (const c of counts.values()) { + const p = c / values.length; + h -= p * Math.log2(p); + } + const max = Math.log2(Math.max(2, counts.size)); + return { unique: counts.size, entropy: h, normalized: max > 0 ? h / Math.log2(values.length) : 0 }; +} + +export function measureSpecDiversity(track, { seeds = 32, seed0 = 0x5eed } = {}) { + const looks = []; + for (let i = 0; i < seeds; i++) { + looks.push(generateLook(track, { seed: (seed0 + i * 2654435761) >>> 0 })); + } + + const sceneSets = looks.map((l) => [...new Set( + l.sections.flatMap((s) => s.variants.flatMap((v) => v.map((layer) => layer.module.name))), + )].sort()); + + const usable = scenes.filter((m) => m.role !== 'accent'); + const covered = new Set(sceneSets.flat()); + const uncast = usable.filter((m) => !covered.has(m.name)).map((m) => m.name); + + // Jaccard distance between the scene SETS of two seeds: the most direct + // possible statement of "did the generator cast a different show". + const jaccard = pairwise(sceneSets, (a, b) => { + const A = new Set(a), B = new Set(b); + let inter = 0; + for (const x of A) if (B.has(x)) inter++; + const union = A.size + B.size - inter; + return union ? 1 - inter / union : 0; + }); + + return { + seeds, + libraryCoverage: covered.size / usable.length, + uncast, + sceneSetDistance: mean(jaccard), + identicalCasts: pairwise(sceneSets, (a, b) => (a.join('|') === b.join('|') ? 1 : 0)) + .reduce((x, y) => x + y, 0), + director: entropy(looks.map((l) => l.director)), + paletteScheme: entropy(looks.map((l) => l.paletteScheme)), + signature: entropy(looks.map((l) => l.personality.signature.join('+'))), + grain: entropy(looks.map((l) => l.grain.mode)), + framing: entropy(looks.map((l) => l.framing.mode)), + paletteArc: entropy(looks.map((l) => l.paletteArc.mode)), + anchorScenes: entropy(looks.map((l) => l.sections.map((s) => s.layers[0].module.name).join('>'))), + }; +} diff --git a/flow-state/src/checks/variety/signature.js b/flow-state/src/checks/variety/signature.js new file mode 100644 index 0000000..c3bc160 --- /dev/null +++ b/flow-state/src/checks/variety/signature.js @@ -0,0 +1,195 @@ +// A whole video reduced to one signature, and the distance between two of them. +// +// A single frame is not a video. Two seeds could open on identical-looking +// frames and diverge completely by the drop, or — the failure we actually +// suspect — differ frame by frame while following the same arc from the same +// kind of image to the same kind of image. So a signature samples the track at +// several points, keeps the mean (what this video looks like) AND the spread +// (how much it changes over its own length), and carries a motion block. +// +// The spread is not decoration. It gives the metric its floor: if two different +// seeds are no further apart than one seed is from itself five minutes later, +// then seed has stopped being a meaningful input, and that comparison is the +// honest way to say so. + +import { frameDescriptor, motionDescriptor } from './descriptors.js'; + +/** Blocks that count toward the structural score. Colour is measured, not counted. */ +export const STRUCTURAL = ['scale', 'orient', 'layout', 'texture', 'motion']; +export const ALL_BLOCKS = [...STRUCTURAL, 'colour']; + +/** + * Per-block weights. + * + * Flat, on purpose. Every weighting we could justify would be a guess about + * which kind of sameness matters most, and the report breaks the score down by + * block anyway — so the diagnosis survives even if the single number is + * weighted wrong. + */ +const WEIGHTS = { scale: 1, orient: 1, layout: 1, texture: 1, motion: 1 }; + +/** + * Chi-square distance, 0..1, for the blocks that are normalised histograms. + * + * Cosine was the first choice and it was wrong here: on all-positive histograms + * every pair scores as similar, and two genuinely unrelated scenes came out at + * 0.05 — a range too compressed to tell "somewhat alike" from "identical". + * Chi-square weights a difference by how small the bins involved are, which is + * what makes a shift of energy from fine detail to coarse blobs read as the + * large change it looks like. + */ +function chiSquare(a, b) { + let sa = 0, sb = 0; + for (let i = 0; i < a.length; i++) { sa += a[i]; sb += b[i]; } + sa = sa || 1e-9; sb = sb || 1e-9; + let d = 0; + for (let i = 0; i < a.length; i++) { + const x = a[i] / sa, y = b[i] / sb; + const den = x + y; + if (den > 1e-12) d += (x - y) ** 2 / den; + } + return Math.max(0, Math.min(1, d / 2)); +} + +// Per-dimension spans for the texture block, whose entries are not a histogram: +// crossing rates, sparsity fractions, and three correlations in -1..1. +const TEXTURE_SPAN = [1, 0.5, 0.5, 0.2, 2, 2, 2]; + +function scaledL1(a, b, spans) { + let d = 0; + for (let i = 0; i < a.length; i++) d += Math.min(1, Math.abs(a[i] - b[i]) / spans[i]); + return d / a.length; +} + +function blockDistance(name, a, b) { + if (name === 'texture') return scaledL1(a, b, TEXTURE_SPAN); + if (name === 'colour') { + // Hue distribution, then saturation and brightness, weighted so a hue + // rotation reads as the large colour change it is. + return 0.7 * chiSquare(a.slice(0, 6), b.slice(0, 6)) + + 0.3 * scaledL1(a.slice(6), b.slice(6), [1, 1]); + } + return chiSquare(a, b); +} + +/** Distance between two single-frame (or motion) descriptors, per block. */ +export function descriptorDistance(a, b) { + const byBlock = {}; + for (const block of ALL_BLOCKS) { + if (!a[block] || !b[block]) continue; + byBlock[block] = blockDistance(block, a[block], b[block]); + } + return byBlock; +} + +/** + * Probe frames spread across the track. + * + * Taken at section midpoints where possible: a section boundary is where the + * look is designed to change, so sampling across boundaries is what makes the + * signature describe the video rather than one shot of it. + */ +export function probeFrames(show, count = 6) { + const sections = show.look.sections; + const frames = []; + if (sections.length >= count) { + const step = sections.length / count; + for (let i = 0; i < count; i++) { + const s = sections[Math.floor(i * step)]; + frames.push(s.startFrame + Math.floor((s.endFrame - s.startFrame) * 0.5)); + } + } else { + for (const s of sections) { + const span = s.endFrame - s.startFrame; + const per = Math.max(1, Math.round(count / sections.length)); + for (let i = 0; i < per; i++) { + frames.push(s.startFrame + Math.floor(span * (i + 1) / (per + 1))); + } + } + } + return frames.slice(0, count); +} + +/** + * Render a loaded Show and reduce it to a signature. + * + * `warmup` frames are rendered before each probe so feedback and stateful + * layers are converged — an unwarmed probe measures the trail of a black frame, + * which is a structure all seeds share and would flatten the metric on its own. + * + * @returns {{blocks: object, probes: object[], drift: number, motion: number}} + */ +export function videoSignature(show, { probes = 6, gap = 5, warmup = 20 } = {}) { + const width = show.engine.width ?? show.engine.renderer.width; + const height = show.engine.height ?? show.engine.renderer.height; + const frames = probeFrames(show, probes); + const perProbe = []; + + for (const frame of frames) { + show.engine.compositor.reset(); + const start = Math.max(0, frame - warmup); + for (let f = start; f < frame; f++) show.renderFrame(f); + + const a = Uint8Array.from(show.readPixels(show.renderFrame(frame))); + const b = Uint8Array.from(show.readPixels(show.renderFrame(frame + gap))); + const still = frameDescriptor(a, width, height); + const moved = motionDescriptor(a, b, width, height); + perProbe.push({ ...still, motion: moved.scale.concat(moved.layout), energy: moved.energy }); + } + + // Self-distance: how far this video travels from itself over its own length. + let drift = 0, pairs = 0; + for (let i = 0; i < perProbe.length; i++) { + for (let j = i + 1; j < perProbe.length; j++) { + drift += blockMean(descriptorDistance(perProbe[i], perProbe[j])); + pairs++; + } + } + + return { + frames, + probes: perProbe, + drift: pairs ? drift / pairs : 0, + motion: perProbe.reduce((a, p) => a + p.energy, 0) / perProbe.length, + }; +} + +function blockMean(byBlock) { + let sum = 0, weight = 0; + for (const block of STRUCTURAL) { + if (byBlock[block] === undefined) continue; + sum += byBlock[block] * WEIGHTS[block]; + weight += WEIGHTS[block]; + } + return weight ? sum / weight : 0; +} + +/** + * Structural distance between two video signatures, 0..1. + * + * MATCHED probes, not averaged descriptors. The track is held fixed while the + * seed varies, so probe i is the same moment of the same song in both videos, + * and comparing them is the closest thing to sitting two renders side by side. + * + * The first version of this averaged each video's probes into one descriptor + * and compared those. It made the score meaningless: averaging six moments + * washes out exactly the structure being measured, so two seeds scored as + * closer to each other than one seed scored to ITSELF five minutes later — the + * floor came out above the ceiling. Matched probes put the between-seed + * distance and the within-video drift on the same scale, which is the only + * reason the ratio between them means anything. + * + * `byBlock` is the whole point of the return value — a bad total is only + * actionable once you know which block collapsed. + */ +export function signatureDistance(a, b) { + const n = Math.min(a.probes.length, b.probes.length); + const sums = {}; + for (let i = 0; i < n; i++) { + const d = descriptorDistance(a.probes[i], b.probes[i]); + for (const [block, v] of Object.entries(d)) sums[block] = (sums[block] || 0) + v / n; + } + return { total: blockMean(sums), byBlock: sums }; +} + +export { blockMean };