diff --git a/flow-state/EPIC-3.md b/flow-state/EPIC-3.md index c736701..1fea53f 100644 --- a/flow-state/EPIC-3.md +++ b/flow-state/EPIC-3.md @@ -312,6 +312,33 @@ built from legacy scenes? Measure that on five stages before touching the other --- +## 9a. What the first slice actually measured + +Four stages shipped, and the A/B ran: same songs, same instrument, same pool +size, the only difference being whether the scenes draw the song's cast or their +own content. + + arm floor observed spread + stages, four of them 0.0801 0.1133 +0.0332 + legacy scenes, four of them 0.0830 0.1122 +0.0292 + the unrestricted generator 0.1058 0.1146 +0.0088 + +The stages-versus-legacy comparison is the internally valid one — everything +except the content sharing is held constant — and it says the inversion helps by +about 14%. Real, in the predicted direction, and much smaller than hoped. + +The third row looked at first like the headline: that a small roster, not the +content sharing, was carrying the improvement. It is not. That arm varies two +things at once — the pool is smaller AND it is the same pool for every song — +and a direct sweep of pool size alone (§ `checks.html?sweep=1`, twelve songs, +three draws each) finds differences of 0.003 to 0.007 against a run-to-run noise +of ±0.003 to ±0.005. Pool size does nothing measurable between 4 and 32. + +Two lessons worth keeping. Arms that differ in more than one way cannot be read +as if they differed in one. And this metric's noise floor at seven songs is +large enough to invent findings — anything under about 0.01 of spread needs +repeats before it is believed. + ## 10. The smallest experiment worth running first One artifact, three stages, one measurement. Do not build the whole identity layer on a diff --git a/flow-state/src/checks/main.js b/flow-state/src/checks/main.js index 47c9d8d..f254859 100644 --- a/flow-state/src/checks/main.js +++ b/flow-state/src/checks/main.js @@ -1,7 +1,7 @@ import { runAll, summarize, allChecks } from './framework.js'; import { runSceneGate } from './scene-gate.js'; import { - varietyReportLines, songVarietyReportLines, experimentReportLines, + varietyReportLines, songVarietyReportLines, experimentReportLines, poolSweepLines, } from './variety/print.js'; // Registering a phase's checks is a side effect of importing it. @@ -118,6 +118,24 @@ async function main() { return; } + // checks.html?sweep=1 — how big a track's casting pool should be. + if (params.get('sweep')) { + summaryEl.textContent = 'sweeping casting pool size across the song bank…'; + const started = Date.now(); + const { lines, ok, headline } = await poolSweepLines({ + songs: Number(params.get('count')) || 6, + sizes: params.get('sizes') ? params.get('sizes').split(',').map(Number) : null, + repeats: Number(params.get('repeats')) || 1, + }); + out.innerHTML = `
${lines.join('\n')}
`; + summaryEl.textContent = `${headline} · ${((Date.now() - started) / 1000).toFixed(1)}s`; + summaryEl.className = ok ? 'ok' : 'bad'; + window.__CHECKS__ = { sweep: true, ok, lines }; + window.__CHECKS_DONE__ = true; + console.log('[sweep]\n' + lines.join('\n')); + return; + } + const phaseArg = params.get('phase'); const phases = phaseArg ? phaseArg.split(',').map(Number) : null; const skipSlow = params.get('slow') !== '1'; diff --git a/flow-state/src/checks/variety/print.js b/flow-state/src/checks/variety/print.js index 782113a..26b1858 100644 --- a/flow-state/src/checks/variety/print.js +++ b/flow-state/src/checks/variety/print.js @@ -273,3 +273,74 @@ export async function experimentReportLines({ songs = 6, probes = 4 } = {}) { (ok ? 'the inversion helps' : 'no improvement'); return { lines, ok, headline }; } + +/** + * How big should a track's casting pool be? + * + * The Epic 3 experiment found the roster size, not the content sharing, was + * carrying most of the improvement — so the number deserves to be measured + * rather than picked. Reported as spread, because both ends of it matter: a + * one-scene pool would score perfectly on the floor and be unwatchable. + */ +export async function poolSweepLines({ songs = 6, probes = 4, sizes = null, repeats = 1 } = {}) { + const list = sizes || [3, 4, 6, 8, 12, 18, 24, 40]; + const lines = []; + lines.push('CASTING POOL SWEEP — how many scenes one track may draw on'); + lines.push(''); + lines.push(' floor one video against itself, across its own sections'); + lines.push(' observed two songs against each other'); + lines.push(' spread the gap between them — the thing worth maximising'); + lines.push(''); + lines.push(' pool floor observed spread ratio distinct scenes/video'); + lines.push(' ' + '-'.repeat(66)); + + const { generateLook } = await import('../../look/LookGenerator.js'); + const { song } = await import('../../audio/songbank.js'); + const track = song('centre').track; + + // Repeats exist because the first run of this sweep was pure noise: spread + // was non-monotonic in pool size and peaked at the LARGEST pool, which is + // the opposite of what the Epic 3 arms suggested. Each size draws a + // different random pool per song, so a single run measures which scenes + // happened to come up as much as it measures the size. Repeating with a + // different draw and reporting the range is how to tell those apart. + let best = null; + for (const size of list) { + await new Promise((r) => setTimeout(r, 0)); + const runs = []; + for (let k = 0; k < repeats; k++) { + const rk = measureSongVariety({ + songs, probes, poolSize: size, seedSalt: k * 7919, + }); + runs.push(rk); + } + const r = runs[0]; + const spreads = runs.map((x) => x.observed - x.floor); + const spread = spreads.reduce((a, b) => a + b, 0) / spreads.length; + const range = repeats > 1 + ? ` ±${((Math.max(...spreads) - Math.min(...spreads)) / 2).toFixed(4)}` : ''; + + // How many distinct scenes a video actually ends up showing, which is + // the number a viewer experiences rather than the pool it was drawn from. + let distinct = 0; + for (let s = 0; s < 4; s++) { + const look = generateLook(track, { seed: 900 + s * 7919, poolSize: size }); + distinct += new Set(look.sections.flatMap( + (sec) => sec.variants.flatMap((v) => v.map((l) => l.module.name)))).size / 4; + } + + lines.push(` ${String(size).padStart(4)} ${r.floor.toFixed(4)} ${r.observed.toFixed(4)}` + + ` ${spread >= 0 ? '+' : ''}${spread.toFixed(4)} ${(r.observed / r.floor).toFixed(3)}` + + ` ${distinct.toFixed(1)}${range}`); + if (!best || spread > best.spread) best = { size, spread }; + } + + lines.push(''); + lines.push(` widest spread at pool ${best.size} (+${best.spread.toFixed(4)})`); + lines.push(''); + lines.push(' Read the whole column, not the winner. A very small pool wins this'); + lines.push(' metric by making every video repetitive, which the metric cannot see'); + lines.push(' and a viewer cannot miss — pick the knee, not the peak.'); + + return { lines, ok: true, headline: `widest spread at pool ${best.size}` }; +} diff --git a/flow-state/src/checks/variety/report.js b/flow-state/src/checks/variety/report.js index 570ef49..f3fb91b 100644 --- a/flow-state/src/checks/variety/report.js +++ b/flow-state/src/checks/variety/report.js @@ -40,10 +40,12 @@ const RENDER = { width: 160, height: 90 }; /** Signature for one seed, rendered through the whole normal pipeline. */ export function signatureForSeed(track, seed, options = {}) { - const { pool = null, ...rest } = options; + const { pool = null, poolSize, ...rest } = options; const show = new Show({ ...RENDER }); try { - show.useTrack(track, generateLook(track, { seed: seed >>> 0, pool })); + show.useTrack(track, generateLook(track, { + seed: seed >>> 0, pool, ...(poolSize ? { poolSize } : {}), + })); return videoSignature(show, rest); } finally { show.dispose(); @@ -475,14 +477,17 @@ function spearman(xs, ys) { * @param {object} options * @returns {object} report */ -export function measureSongVariety({ songs = 6, probes = 5, refScenes = 4, pool = null } = {}) { +export function measureSongVariety({ + songs = 6, probes = 5, refScenes = 4, pool = null, poolSize = null, seedSalt = 0, +} = {}) { const bank = songBank({ count: songs }); // The seed is derived from the audio in the real pipeline, so each song must // get its own — deriving it from the name is the same relationship without // needing the samples. const sigs = bank.map((entry) => - signatureForSeed(entry.track, hashString(entry.name), { probes, pool })); + signatureForSeed(entry.track, (hashString(entry.name) + seedSalt) >>> 0, + { probes, pool, poolSize })); const floor = mean(sigs.map((s) => s.drift)); diff --git a/flow-state/src/look/LookGenerator.js b/flow-state/src/look/LookGenerator.js index 7e02072..cc78a8c 100644 --- a/flow-state/src/look/LookGenerator.js +++ b/flow-state/src/look/LookGenerator.js @@ -96,7 +96,30 @@ const clamp01 = (x) => Math.max(0, Math.min(1, x)); * scene declaring two traits is now merely less likely to be drawn than one * declaring four, instead of being ineligible for six tracks in seven. */ -function castingPool(rng, signature, size = 24) { +/** + * How many of the library's scenes one track is allowed to draw on. + * + * Swept directly — checks.html?sweep=1 — across 4, 8, 16 and 32, over twelve + * songs with three pool draws each. The answer is that it does not matter: + * + * pool 4 spread +0.0061 ±0.0035 + * pool 8 spread +0.0090 ±0.0040 + * pool 16 spread +0.0102 ±0.0046 + * pool 32 spread +0.0035 ±0.0032 + * + * The differences are the same size as the run-to-run noise. This corrects a + * claim made when the Epic 3 arms first came in: those arms appeared to show + * that a small roster was the largest available win, but they varied two things + * at once — the pool was smaller AND it was the same pool for every song — and + * the sweep isolating size finds nothing. + * + * So 8 is chosen on grounds the metric cannot see. It puts about nine distinct + * scenes in a video rather than seventeen, and a video a viewer can hold in + * their head is worth having even when the instrument is indifferent. + */ +export const POOL_SIZE = 8; + +function castingPool(rng, signature, size = POOL_SIZE) { const pool = scenes.filter((m) => m.role !== 'accent'); const remaining = pool.slice(); const weights = remaining.map((m) => signatureWeight(m, signature)); @@ -320,7 +343,8 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament) * @returns {object} LookSpec */ export function generateLook(track, { - seed = null, samples = null, overrides = null, pool: poolOverride = null, + seed = null, samples = null, overrides = null, + pool: poolOverride = null, poolSize = POOL_SIZE, } = {}) { const resolvedSeed = seed !== null ? seed >>> 0 @@ -350,7 +374,7 @@ export function generateLook(track, { // comparable to the thing it is bounding. const pool = poolOverride && poolOverride.length ? poolOverride - : castingPool(rng.fork('pool'), personality.signature); + : castingPool(rng.fork('pool'), personality.signature, poolSize); const rosterByKind = assignRostersByKind( track.sections, rng.fork('scenes'), personality.signature, director, pool); // The grain treatment: usually none, and when present described rather than