diff --git a/flow-state/EPIC-3.md b/flow-state/EPIC-3.md index c736701..1fea53f 100644 --- a/flow-state/EPIC-3.md +++ b/flow-state/EPIC-3.md @@ -312,6 +312,33 @@ built from legacy scenes? Measure that on five stages before touching the other --- +## 9a. What the first slice actually measured + +Four stages shipped, and the A/B ran: same songs, same instrument, same pool +size, the only difference being whether the scenes draw the song's cast or their +own content. + + arm floor observed spread + stages, four of them 0.0801 0.1133 +0.0332 + legacy scenes, four of them 0.0830 0.1122 +0.0292 + the unrestricted generator 0.1058 0.1146 +0.0088 + +The stages-versus-legacy comparison is the internally valid one — everything +except the content sharing is held constant — and it says the inversion helps by +about 14%. Real, in the predicted direction, and much smaller than hoped. + +The third row looked at first like the headline: that a small roster, not the +content sharing, was carrying the improvement. It is not. That arm varies two +things at once — the pool is smaller AND it is the same pool for every song — +and a direct sweep of pool size alone (§ `checks.html?sweep=1`, twelve songs, +three draws each) finds differences of 0.003 to 0.007 against a run-to-run noise +of ±0.003 to ±0.005. Pool size does nothing measurable between 4 and 32. + +Two lessons worth keeping. Arms that differ in more than one way cannot be read +as if they differed in one. And this metric's noise floor at seven songs is +large enough to invent findings — anything under about 0.01 of spread needs +repeats before it is believed. + ## 10. The smallest experiment worth running first One artifact, three stages, one measurement. Do not build the whole identity layer on a diff --git a/flow-state/src/checks/main.js b/flow-state/src/checks/main.js index 47c9d8d..f254859 100644 --- a/flow-state/src/checks/main.js +++ b/flow-state/src/checks/main.js @@ -1,7 +1,7 @@ import { runAll, summarize, allChecks } from './framework.js'; import { runSceneGate } from './scene-gate.js'; import { - varietyReportLines, songVarietyReportLines, experimentReportLines, + varietyReportLines, songVarietyReportLines, experimentReportLines, poolSweepLines, } from './variety/print.js'; // Registering a phase's checks is a side effect of importing it. @@ -118,6 +118,24 @@ async function main() { return; } + // checks.html?sweep=1 — how big a track's casting pool should be. + if (params.get('sweep')) { + summaryEl.textContent = 'sweeping casting pool size across the song bank…'; + const started = Date.now(); + const { lines, ok, headline } = await poolSweepLines({ + songs: Number(params.get('count')) || 6, + sizes: params.get('sizes') ? params.get('sizes').split(',').map(Number) : null, + repeats: Number(params.get('repeats')) || 1, + }); + out.innerHTML = `
${lines.join('\n')}`;
+ summaryEl.textContent = `${headline} · ${((Date.now() - started) / 1000).toFixed(1)}s`;
+ summaryEl.className = ok ? 'ok' : 'bad';
+ window.__CHECKS__ = { sweep: true, ok, lines };
+ window.__CHECKS_DONE__ = true;
+ console.log('[sweep]\n' + lines.join('\n'));
+ return;
+ }
+
const phaseArg = params.get('phase');
const phases = phaseArg ? phaseArg.split(',').map(Number) : null;
const skipSlow = params.get('slow') !== '1';
diff --git a/flow-state/src/checks/variety/print.js b/flow-state/src/checks/variety/print.js
index 782113a..26b1858 100644
--- a/flow-state/src/checks/variety/print.js
+++ b/flow-state/src/checks/variety/print.js
@@ -273,3 +273,74 @@ export async function experimentReportLines({ songs = 6, probes = 4 } = {}) {
(ok ? 'the inversion helps' : 'no improvement');
return { lines, ok, headline };
}
+
+/**
+ * How big should a track's casting pool be?
+ *
+ * The Epic 3 experiment found the roster size, not the content sharing, was
+ * carrying most of the improvement — so the number deserves to be measured
+ * rather than picked. Reported as spread, because both ends of it matter: a
+ * one-scene pool would score perfectly on the floor and be unwatchable.
+ */
+export async function poolSweepLines({ songs = 6, probes = 4, sizes = null, repeats = 1 } = {}) {
+ const list = sizes || [3, 4, 6, 8, 12, 18, 24, 40];
+ const lines = [];
+ lines.push('CASTING POOL SWEEP — how many scenes one track may draw on');
+ lines.push('');
+ lines.push(' floor one video against itself, across its own sections');
+ lines.push(' observed two songs against each other');
+ lines.push(' spread the gap between them — the thing worth maximising');
+ lines.push('');
+ lines.push(' pool floor observed spread ratio distinct scenes/video');
+ lines.push(' ' + '-'.repeat(66));
+
+ const { generateLook } = await import('../../look/LookGenerator.js');
+ const { song } = await import('../../audio/songbank.js');
+ const track = song('centre').track;
+
+ // Repeats exist because the first run of this sweep was pure noise: spread
+ // was non-monotonic in pool size and peaked at the LARGEST pool, which is
+ // the opposite of what the Epic 3 arms suggested. Each size draws a
+ // different random pool per song, so a single run measures which scenes
+ // happened to come up as much as it measures the size. Repeating with a
+ // different draw and reporting the range is how to tell those apart.
+ let best = null;
+ for (const size of list) {
+ await new Promise((r) => setTimeout(r, 0));
+ const runs = [];
+ for (let k = 0; k < repeats; k++) {
+ const rk = measureSongVariety({
+ songs, probes, poolSize: size, seedSalt: k * 7919,
+ });
+ runs.push(rk);
+ }
+ const r = runs[0];
+ const spreads = runs.map((x) => x.observed - x.floor);
+ const spread = spreads.reduce((a, b) => a + b, 0) / spreads.length;
+ const range = repeats > 1
+ ? ` ±${((Math.max(...spreads) - Math.min(...spreads)) / 2).toFixed(4)}` : '';
+
+ // How many distinct scenes a video actually ends up showing, which is
+ // the number a viewer experiences rather than the pool it was drawn from.
+ let distinct = 0;
+ for (let s = 0; s < 4; s++) {
+ const look = generateLook(track, { seed: 900 + s * 7919, poolSize: size });
+ distinct += new Set(look.sections.flatMap(
+ (sec) => sec.variants.flatMap((v) => v.map((l) => l.module.name)))).size / 4;
+ }
+
+ lines.push(` ${String(size).padStart(4)} ${r.floor.toFixed(4)} ${r.observed.toFixed(4)}` +
+ ` ${spread >= 0 ? '+' : ''}${spread.toFixed(4)} ${(r.observed / r.floor).toFixed(3)}` +
+ ` ${distinct.toFixed(1)}${range}`);
+ if (!best || spread > best.spread) best = { size, spread };
+ }
+
+ lines.push('');
+ lines.push(` widest spread at pool ${best.size} (+${best.spread.toFixed(4)})`);
+ lines.push('');
+ lines.push(' Read the whole column, not the winner. A very small pool wins this');
+ lines.push(' metric by making every video repetitive, which the metric cannot see');
+ lines.push(' and a viewer cannot miss — pick the knee, not the peak.');
+
+ return { lines, ok: true, headline: `widest spread at pool ${best.size}` };
+}
diff --git a/flow-state/src/checks/variety/report.js b/flow-state/src/checks/variety/report.js
index 570ef49..f3fb91b 100644
--- a/flow-state/src/checks/variety/report.js
+++ b/flow-state/src/checks/variety/report.js
@@ -40,10 +40,12 @@ const RENDER = { width: 160, height: 90 };
/** Signature for one seed, rendered through the whole normal pipeline. */
export function signatureForSeed(track, seed, options = {}) {
- const { pool = null, ...rest } = options;
+ const { pool = null, poolSize, ...rest } = options;
const show = new Show({ ...RENDER });
try {
- show.useTrack(track, generateLook(track, { seed: seed >>> 0, pool }));
+ show.useTrack(track, generateLook(track, {
+ seed: seed >>> 0, pool, ...(poolSize ? { poolSize } : {}),
+ }));
return videoSignature(show, rest);
} finally {
show.dispose();
@@ -475,14 +477,17 @@ function spearman(xs, ys) {
* @param {object} options
* @returns {object} report
*/
-export function measureSongVariety({ songs = 6, probes = 5, refScenes = 4, pool = null } = {}) {
+export function measureSongVariety({
+ songs = 6, probes = 5, refScenes = 4, pool = null, poolSize = null, seedSalt = 0,
+} = {}) {
const bank = songBank({ count: songs });
// The seed is derived from the audio in the real pipeline, so each song must
// get its own — deriving it from the name is the same relationship without
// needing the samples.
const sigs = bank.map((entry) =>
- signatureForSeed(entry.track, hashString(entry.name), { probes, pool }));
+ signatureForSeed(entry.track, (hashString(entry.name) + seedSalt) >>> 0,
+ { probes, pool, poolSize }));
const floor = mean(sigs.map((s) => s.drift));
diff --git a/flow-state/src/look/LookGenerator.js b/flow-state/src/look/LookGenerator.js
index 7e02072..cc78a8c 100644
--- a/flow-state/src/look/LookGenerator.js
+++ b/flow-state/src/look/LookGenerator.js
@@ -96,7 +96,30 @@ const clamp01 = (x) => Math.max(0, Math.min(1, x));
* scene declaring two traits is now merely less likely to be drawn than one
* declaring four, instead of being ineligible for six tracks in seven.
*/
-function castingPool(rng, signature, size = 24) {
+/**
+ * How many of the library's scenes one track is allowed to draw on.
+ *
+ * Swept directly — checks.html?sweep=1 — across 4, 8, 16 and 32, over twelve
+ * songs with three pool draws each. The answer is that it does not matter:
+ *
+ * pool 4 spread +0.0061 ±0.0035
+ * pool 8 spread +0.0090 ±0.0040
+ * pool 16 spread +0.0102 ±0.0046
+ * pool 32 spread +0.0035 ±0.0032
+ *
+ * The differences are the same size as the run-to-run noise. This corrects a
+ * claim made when the Epic 3 arms first came in: those arms appeared to show
+ * that a small roster was the largest available win, but they varied two things
+ * at once — the pool was smaller AND it was the same pool for every song — and
+ * the sweep isolating size finds nothing.
+ *
+ * So 8 is chosen on grounds the metric cannot see. It puts about nine distinct
+ * scenes in a video rather than seventeen, and a video a viewer can hold in
+ * their head is worth having even when the instrument is indifferent.
+ */
+export const POOL_SIZE = 8;
+
+function castingPool(rng, signature, size = POOL_SIZE) {
const pool = scenes.filter((m) => m.role !== 'accent');
const remaining = pool.slice();
const weights = remaining.map((m) => signatureWeight(m, signature));
@@ -320,7 +343,8 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament)
* @returns {object} LookSpec
*/
export function generateLook(track, {
- seed = null, samples = null, overrides = null, pool: poolOverride = null,
+ seed = null, samples = null, overrides = null,
+ pool: poolOverride = null, poolSize = POOL_SIZE,
} = {}) {
const resolvedSeed = seed !== null
? seed >>> 0
@@ -350,7 +374,7 @@ export function generateLook(track, {
// comparable to the thing it is bounding.
const pool = poolOverride && poolOverride.length
? poolOverride
- : castingPool(rng.fork('pool'), personality.signature);
+ : castingPool(rng.fork('pool'), personality.signature, poolSize);
const rosterByKind = assignRostersByKind(
track.sections, rng.fork('scenes'), personality.signature, director, pool);
// The grain treatment: usually none, and when present described rather than