Sweep pool size directly, and withdraw the claim that it mattered

The Epic 3 arms appeared to show that a small casting roster was the largest
available win. Swept directly across 4, 8, 16 and 32 over twelve songs with
three pool draws each, the differences are 0.003 to 0.007 against a run-to-run
noise of +/-0.003 to +/-0.005. Pool size does nothing measurable.

The arms varied two things at once — smaller pool AND the same pool for every
song — and only the stages-versus-legacy comparison inside them held everything
else constant. That one still stands at +14%.

POOL_SIZE stays at 8, now on grounds the metric cannot see: about nine distinct
scenes in a video rather than seventeen.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

flow-state/EPIC-3.md

@@ -312,6 +312,33 @@ built from legacy scenes? Measure that on five stages before touching the other
 
 ---
 
+## 9a. What the first slice actually measured
+
+Four stages shipped, and the A/B ran: same songs, same instrument, same pool
+size, the only difference being whether the scenes draw the song's cast or their
+own content.
+
+    arm                              floor   observed   spread
+    stages, four of them            0.0801     0.1133   +0.0332
+    legacy scenes, four of them     0.0830     0.1122   +0.0292
+    the unrestricted generator      0.1058     0.1146   +0.0088
+
+The stages-versus-legacy comparison is the internally valid one — everything
+except the content sharing is held constant — and it says the inversion helps by
+about 14%. Real, in the predicted direction, and much smaller than hoped.
+
+The third row looked at first like the headline: that a small roster, not the
+content sharing, was carrying the improvement. It is not. That arm varies two
+things at once — the pool is smaller AND it is the same pool for every song —
+and a direct sweep of pool size alone (§ `checks.html?sweep=1`, twelve songs,
+three draws each) finds differences of 0.003 to 0.007 against a run-to-run noise
+of ±0.003 to ±0.005. Pool size does nothing measurable between 4 and 32.
+
+Two lessons worth keeping. Arms that differ in more than one way cannot be read
+as if they differed in one. And this metric's noise floor at seven songs is
+large enough to invent findings — anything under about 0.01 of spread needs
+repeats before it is believed.
+
 ## 10. The smallest experiment worth running first
 
 One artifact, three stages, one measurement. Do not build the whole identity layer on a

flow-state/src/checks/main.js

@@ -1,7 +1,7 @@
 import { runAll, summarize, allChecks } from './framework.js';
 import { runSceneGate } from './scene-gate.js';
 import {
-    varietyReportLines, songVarietyReportLines, experimentReportLines,
+    varietyReportLines, songVarietyReportLines, experimentReportLines, poolSweepLines,
 } from './variety/print.js';
 
 // Registering a phase's checks is a side effect of importing it.
@@ -118,6 +118,24 @@ async function main() {
         return;
     }
 
+    // checks.html?sweep=1 — how big a track's casting pool should be.
+    if (params.get('sweep')) {
+        summaryEl.textContent = 'sweeping casting pool size across the song bank…';
+        const started = Date.now();
+        const { lines, ok, headline } = await poolSweepLines({
+            songs: Number(params.get('count')) || 6,
+            sizes: params.get('sizes') ? params.get('sizes').split(',').map(Number) : null,
+            repeats: Number(params.get('repeats')) || 1,
+        });
+        out.innerHTML = `<pre class="scene-gate">${lines.join('\n')}</pre>`;
+        summaryEl.textContent = `${headline} · ${((Date.now() - started) / 1000).toFixed(1)}s`;
+        summaryEl.className = ok ? 'ok' : 'bad';
+        window.__CHECKS__ = { sweep: true, ok, lines };
+        window.__CHECKS_DONE__ = true;
+        console.log('[sweep]\n' + lines.join('\n'));
+        return;
+    }
+
     const phaseArg = params.get('phase');
     const phases = phaseArg ? phaseArg.split(',').map(Number) : null;
     const skipSlow = params.get('slow') !== '1';

flow-state/src/checks/variety/print.js

@@ -273,3 +273,74 @@ export async function experimentReportLines({ songs = 6, probes = 4 } = {}) {
         (ok ? 'the inversion helps' : 'no improvement');
     return { lines, ok, headline };
 }
+
+/**
+ * How big should a track's casting pool be?
+ *
+ * The Epic 3 experiment found the roster size, not the content sharing, was
+ * carrying most of the improvement — so the number deserves to be measured
+ * rather than picked. Reported as spread, because both ends of it matter: a
+ * one-scene pool would score perfectly on the floor and be unwatchable.
+ */
+export async function poolSweepLines({ songs = 6, probes = 4, sizes = null, repeats = 1 } = {}) {
+    const list = sizes || [3, 4, 6, 8, 12, 18, 24, 40];
+    const lines = [];
+    lines.push('CASTING POOL SWEEP — how many scenes one track may draw on');
+    lines.push('');
+    lines.push('  floor     one video against itself, across its own sections');
+    lines.push('  observed  two songs against each other');
+    lines.push('  spread    the gap between them — the thing worth maximising');
+    lines.push('');
+    lines.push('  pool    floor   observed    spread   ratio   distinct scenes/video');
+    lines.push('  ' + '-'.repeat(66));
+
+    const { generateLook } = await import('../../look/LookGenerator.js');
+    const { song } = await import('../../audio/songbank.js');
+    const track = song('centre').track;
+
+    // Repeats exist because the first run of this sweep was pure noise: spread
+    // was non-monotonic in pool size and peaked at the LARGEST pool, which is
+    // the opposite of what the Epic 3 arms suggested. Each size draws a
+    // different random pool per song, so a single run measures which scenes
+    // happened to come up as much as it measures the size. Repeating with a
+    // different draw and reporting the range is how to tell those apart.
+    let best = null;
+    for (const size of list) {
+        await new Promise((r) => setTimeout(r, 0));
+        const runs = [];
+        for (let k = 0; k < repeats; k++) {
+            const rk = measureSongVariety({
+                songs, probes, poolSize: size, seedSalt: k * 7919,
+            });
+            runs.push(rk);
+        }
+        const r = runs[0];
+        const spreads = runs.map((x) => x.observed - x.floor);
+        const spread = spreads.reduce((a, b) => a + b, 0) / spreads.length;
+        const range = repeats > 1
+            ? `  ±${((Math.max(...spreads) - Math.min(...spreads)) / 2).toFixed(4)}` : '';
+
+        // How many distinct scenes a video actually ends up showing, which is
+        // the number a viewer experiences rather than the pool it was drawn from.
+        let distinct = 0;
+        for (let s = 0; s < 4; s++) {
+            const look = generateLook(track, { seed: 900 + s * 7919, poolSize: size });
+            distinct += new Set(look.sections.flatMap(
+                (sec) => sec.variants.flatMap((v) => v.map((l) => l.module.name)))).size / 4;
+        }
+
+        lines.push(`  ${String(size).padStart(4)}   ${r.floor.toFixed(4)}    ${r.observed.toFixed(4)}` +
+            `   ${spread >= 0 ? '+' : ''}${spread.toFixed(4)}   ${(r.observed / r.floor).toFixed(3)}` +
+            `        ${distinct.toFixed(1)}${range}`);
+        if (!best || spread > best.spread) best = { size, spread };
+    }
+
+    lines.push('');
+    lines.push(`  widest spread at pool ${best.size} (+${best.spread.toFixed(4)})`);
+    lines.push('');
+    lines.push('  Read the whole column, not the winner. A very small pool wins this');
+    lines.push('  metric by making every video repetitive, which the metric cannot see');
+    lines.push('  and a viewer cannot miss — pick the knee, not the peak.');
+
+    return { lines, ok: true, headline: `widest spread at pool ${best.size}` };
+}

flow-state/src/checks/variety/report.js

@@ -40,10 +40,12 @@ const RENDER = { width: 160, height: 90 };
 
 /** Signature for one seed, rendered through the whole normal pipeline. */
 export function signatureForSeed(track, seed, options = {}) {
-    const { pool = null, ...rest } = options;
+    const { pool = null, poolSize, ...rest } = options;
     const show = new Show({ ...RENDER });
     try {
-        show.useTrack(track, generateLook(track, { seed: seed >>> 0, pool }));
+        show.useTrack(track, generateLook(track, {
+            seed: seed >>> 0, pool, ...(poolSize ? { poolSize } : {}),
+        }));
         return videoSignature(show, rest);
     } finally {
         show.dispose();
@@ -475,14 +477,17 @@ function spearman(xs, ys) {
  * @param {object} options
  * @returns {object} report
  */
-export function measureSongVariety({ songs = 6, probes = 5, refScenes = 4, pool = null } = {}) {
+export function measureSongVariety({
+    songs = 6, probes = 5, refScenes = 4, pool = null, poolSize = null, seedSalt = 0,
+} = {}) {
     const bank = songBank({ count: songs });
 
     // The seed is derived from the audio in the real pipeline, so each song must
     // get its own — deriving it from the name is the same relationship without
     // needing the samples.
     const sigs = bank.map((entry) =>
-        signatureForSeed(entry.track, hashString(entry.name), { probes, pool }));
+        signatureForSeed(entry.track, (hashString(entry.name) + seedSalt) >>> 0,
+            { probes, pool, poolSize }));
 
     const floor = mean(sigs.map((s) => s.drift));
 

flow-state/src/look/LookGenerator.js

@@ -96,7 +96,30 @@ const clamp01 = (x) => Math.max(0, Math.min(1, x));
  * scene declaring two traits is now merely less likely to be drawn than one
  * declaring four, instead of being ineligible for six tracks in seven.
  */
-function castingPool(rng, signature, size = 24) {
+/**
+ * How many of the library's scenes one track is allowed to draw on.
+ *
+ * Swept directly — checks.html?sweep=1 — across 4, 8, 16 and 32, over twelve
+ * songs with three pool draws each. The answer is that it does not matter:
+ *
+ *     pool 4    spread +0.0061 ±0.0035
+ *     pool 8    spread +0.0090 ±0.0040
+ *     pool 16   spread +0.0102 ±0.0046
+ *     pool 32   spread +0.0035 ±0.0032
+ *
+ * The differences are the same size as the run-to-run noise. This corrects a
+ * claim made when the Epic 3 arms first came in: those arms appeared to show
+ * that a small roster was the largest available win, but they varied two things
+ * at once — the pool was smaller AND it was the same pool for every song — and
+ * the sweep isolating size finds nothing.
+ *
+ * So 8 is chosen on grounds the metric cannot see. It puts about nine distinct
+ * scenes in a video rather than seventeen, and a video a viewer can hold in
+ * their head is worth having even when the instrument is indifferent.
+ */
+export const POOL_SIZE = 8;
+
+function castingPool(rng, signature, size = POOL_SIZE) {
     const pool = scenes.filter((m) => m.role !== 'accent');
     const remaining = pool.slice();
     const weights = remaining.map((m) => signatureWeight(m, signature));
@@ -320,7 +343,8 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament)
  * @returns {object} LookSpec
  */
 export function generateLook(track, {
-    seed = null, samples = null, overrides = null, pool: poolOverride = null,
+    seed = null, samples = null, overrides = null,
+    pool: poolOverride = null, poolSize = POOL_SIZE,
 } = {}) {
     const resolvedSeed = seed !== null
         ? seed >>> 0
@@ -350,7 +374,7 @@ export function generateLook(track, {
     // comparable to the thing it is bounding.
     const pool = poolOverride && poolOverride.length
         ? poolOverride
-        : castingPool(rng.fork('pool'), personality.signature);
+        : castingPool(rng.fork('pool'), personality.signature, poolSize);
     const rosterByKind = assignRostersByKind(
         track.sections, rng.fork('scenes'), personality.signature, director, pool);
     // The grain treatment: usually none, and when present described rather than