music-video-gen/flow-state/src/checks/filmstrip.js

// The filmstrip: every song in the bank, sampled across its own length.
//
// The gallery answers "does this scene look the same in every song". This
// answers the question one level up and one axis over: does a SONG look the
// same as itself, half an hour of playback later.
//
// Nothing else shows that. The variety harness reduces a video to a number, the
// app shows one video at the speed of the song, and the phase gates never look
// at two moments of the same track side by side. So the failure this exists to
// catch — a video that is doing plenty and going nowhere — has until now been
// something you could only notice by watching four minutes and remembering what
// the first minute looked like.
//
// One frame every thirty seconds, straight across. A strip whose frames could
// be shuffled without anyone noticing is a video with no arc, whatever its
// drift score says. See look/Story.js for the layer meant to fix that, and
// checks/variety/signature.js directionOf for the same question as a number.
//
// Everything renders through the whole normal pipeline — Show, ArcDriver, post,
// the lot — rather than through a bare Engine as the gallery does, because the
// arc IS the subject here.

import { Show, FADE_SECONDS } from '../Show.js';
import { generateLook } from '../look/LookGenerator.js';
import { song, SONGS } from '../audio/songbank.js';
import { hashString } from '../engine/rng.js';
import { pinVenueShell } from '../venues/pin.js';
import { frameDescriptor, motionDescriptor } from './variety/descriptors.js';
import { descriptorDistance, directionOf, STRUCTURAL } from './variety/signature.js';
import { describeStory } from '../look/Story.js';
import { frameLuminance } from '../engine/hash.js';
import { agitationScore, AGITATION_CEILING } from '../engine/agitation.js';

export const STRIP = { width: 256, height: 144 };

/** Seconds between frames. Thirty is a section or two at most tempos. */
export const DEFAULT_EVERY = 30;

/**
 * How long a song is synthesised for.
 *
 * Long enough to have an arc to sample, short enough that the whole bank
 * finishes in one sitting: every extra thirty seconds is another frame for each
 * of seventeen songs, and each frame costs its warm-up. `?duration=240` for the
 * fuller version when a specific song is in question.
 */
export const DEFAULT_DURATION = 180;

/**
 * Frames of warm-up before each probe.
 *
 * Feedback and any stateful layer need to be converged or the probe measures
 * the trail of a black frame — a structure every probe shares, which would make
 * the whole strip look more alike than it is. Half of what the variety harness
 * uses, because this is 17 songs deep and the cost is linear.
 */
/** Frames of warm-up behind every probe. Carried in preview links (?warm=)
 * so the app lands on identically converged state. See src/ui/preview.js. */
export const WARMUP = 24;

export const songNames = () => SONGS.map((s) => s.name);

const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0);

function blockMean(byBlock) {
    const values = STRUCTURAL.map((b) => byBlock[b]).filter((v) => v !== undefined);
    return mean(values);
}

/**
 * Render one song's strip.
 *
 * @returns {{name, frames: Array, drift: number, direction: number, look}}
 */
export function renderSong(show, name, { duration, every, venueEnabled = null, venueSel = null }) {
    const { track } = song(name, { duration });
    // Seeded off the song's name, so a strip is stable between rebuilds and two
    // songs are not accidentally handed the same look.
    const venueFlag = venueEnabled !== null ? !!venueEnabled : (typeof location !== 'undefined' && new URLSearchParams(location.search).get('venue') === '1');
    const look = generateLook(track, { seed: hashString(name), venueEnabled: venueFlag });
    // Pin one venue in every row (same shell-swap as the venue gallery): the
    // VenueSpec population stays the census one, only the shell is forced.
    const sel = venueSel || (typeof location !== 'undefined' && new URLSearchParams(location.search).get('venueSel')) || null;
    if (sel) pinVenueShell(look, sel);
    // The file name the app shows in its title plate (OSD): without it the
    // probe renders a blank plate and no deep-link can match the app's
    // default view. See src/ui/preview.js.
    show.fileName = name;
    show.useTrack(track, look);
    // Prime once per song: programs link asynchronously and a draw against an
    // unlinked program is wrong output — invisible in a sweep, wrong in an
    // export, and fatal to preview deep-links, which replay these frames in
    // the app (primed there too). See src/ui/preview.js.
    show.prime();

    const frames = [];
    const descriptors = [];
    // Pulled clear of the opening and closing fades. A probe at t=0 reads a
    // black frame — correct output, useless sample — and the same at the very
    // end, so the first and last cells sit just inside the video proper. The
    // caption shows the time actually sampled rather than the nominal one.
    const inside = (t) => Math.min(
        Math.max(t, FADE_SECONDS + 0.5),
        Math.max(0, track.duration - FADE_SECONDS - 0.5));

    for (let nominal = 0; nominal < track.duration - 0.5; nominal += every) {
        const t = inside(nominal);
        const frame = Math.min(track.frameCount - 2, Math.round(t * track.fps));
        show.warmUp(frame, WARMUP);

        const pixels = Uint8Array.from(show.readPixels(show.renderFrame(frame)));
        // A second frame five later, so the motion block is measured rather
        // than silently scored zero — the same omission the gallery had.
        const moved = Uint8Array.from(show.readPixels(show.renderFrame(frame + 5)));

        const state = show.arc.state;
        const sectionIndex = track.sectionIndexAt(frame);
        const section = look.sections[sectionIndex] || {};
        frames.push({
            time: t,
            frame,
            pixels,
            kind: state.kind || section.kind || '',
            scene: state.sceneName || '',
            act: state.act || '',
            tension: state.tension ?? 0.5,
            journey: state.journey ?? 0,
            moment: momentAt(look.story, sectionIndex),
        });

        const still = frameDescriptor(pixels, STRIP.width, STRIP.height);
        const motion = motionDescriptor(pixels, moved, STRIP.width, STRIP.height);
        descriptors.push({ ...still, motion: motion.scale.concat(motion.layout) });
    }

    // The two numbers that say what the strip shows, on the same descriptor the
    // variety harness uses: how far this video gets from itself, and whether
    // that distance grows with time or is just churn.
    const gaps = [];
    const dists = [];
    for (let i = 0; i < descriptors.length; i++) {
        for (let j = i + 1; j < descriptors.length; j++) {
            gaps.push(frames[j].time - frames[i].time);
            dists.push(blockMean(descriptorDistance(descriptors[i], descriptors[j])));
        }
    }

    // Agitation: dense 60 fps burst in the busiest section, so the filmstrip is
    // also the agitation audit. The 30 s probes are 30 s apart and cannot see
    // beat-locked whole-screen motion — the metric needs consecutive frames.
    // A 120-frame (2 s) burst at the drop/busiest section captures the worst
    // 1 s floating window at its most energetic, mirroring gallery.js which
    // samples its 60-frame strip at the busiest context. Whole-screen
    // flash/coherent motion reserved for special cases, region motion fine.
    let score = { agitation: 0, peakFlashRate: 0, peakMotion: 0, meanMotion: 0, peakStructure: 0, meanStructure: 0 };
    let agitated = false;
    try {
        const busiest = look.sections.reduce(
            (best, s) => ((s.bias?.energy ?? 0) > (best.bias?.energy ?? 0) ? s : best),
            look.sections[0] || { startFrame: 0, endFrame: Math.min(track.frameCount, 120) },
        );
        const probe = busiest.startFrame + Math.floor((busiest.endFrame - busiest.startFrame) * 0.5);
        const STRIP_AGIT = 120; // 2 s @ 60 fps → 1 s window worst
        show.warmUp(probe, WARMUP);
        const agFrames = [];
        const agLums = [];
        for (let f = probe; f < probe + STRIP_AGIT; f++) {
            const ff = Math.min(track.frameCount - 1, f);
            const px = Uint8Array.from(show.readPixels(show.renderFrame(ff)));
            agFrames.push(px);
            agLums.push(frameLuminance(px));
        }
        score = agitationScore(agLums, agFrames, 60, { width: STRIP.width, height: STRIP.height });
        agitated = score.agitation >= AGITATION_CEILING;
    } catch {
        // leave agitation at 0 — row still renders, just grey
    }

    return {
        name,
        frames,
        look,
        drift: mean(dists),
        direction: directionOf(gaps, dists),
        // Agitation over the strip: worst 1 s floating window, max(flash/3,
        // motion/0.55) → 0..1. The filmstrip already renders the whole video,
        // so this is the natural audit for "how much of the view moves/flashes
        // together". Gallery's agitation is per-scene; this one is per-song.
        agitation: score.agitation,
        agitationFlash: score.peakFlashRate,
        agitationMotion: score.peakMotion,
        agitationMean: score.meanMotion,
        agitationStructPeak: score.peakStructure,
        agitationStruct: score.meanStructure,
        agitated,
        // How much the FIRST half of the video resembles the last — the reading
        // that separates a video which ends somewhere new from one that has
        // come back on purpose. A recap should show up here and nowhere else.
        endsApart: dists.length ? dists[dists.length - 1] : 0,
        story: look.story,
        storyLine: describeStory(look.story),
        director: look.director,
        bpm: Math.round(track.summary.bpm),
        sections: look.sections.length,
    };
}

function momentAt(story, index) {
    if (!story || !story.moments) return '';
    const m = story.moments;
    if (index === m.climax) return 'climax';
    if (index === m.turn) return 'turn';
    if (index === m.arrival) return 'arrival';
    if (index === m.resolution) return 'resolution';
    return '';
}

/**
 * Build every strip, reporting each song as it lands.
 *
 * One Show for the whole run: `useTrack` replaces the track and rebuilds the
 * arc, and constructing a Show per song would rebuild the GL context seventeen
 * times for nothing.
 */
export async function buildFilmstrip({
    duration = DEFAULT_DURATION, every = DEFAULT_EVERY, names = null, venueEnabled = null,
    venueSel = null,
    onSong = null, onSongStart = null,
} = {}) {
    const list = names && names.length ? names : songNames();
    const show = new Show({ ...STRIP });
    const rows = [];
    try {
        for (let i = 0; i < list.length; i++) {
            // Announced BEFORE the work, not only after it. Synthesising and
            // rendering one song is tens of seconds, and a page that says
            // nothing until the first one lands looks broken for exactly as
            // long as the first one takes.
            if (onSongStart) {
                onSongStart(i + 1, list.length, list[i]);
                await new Promise((r) => setTimeout(r, 0));
            }
            let row;
            try {
                row = renderSong(show, list[i], { duration, every, venueEnabled, venueSel });
            } catch (err) {
                row = { name: list[i], frames: [], shots: [], drift: 0, direction: 0, error: err.message };
            }
            rows.push(row);
            if (onSong) onSong(i + 1, list.length, row);
            // Yield, so a row paints as it lands instead of the page freezing
            // for the whole run and then showing everything at once.
            await new Promise((r) => setTimeout(r, 0));
        }
    } finally {
        show.dispose();
    }
    return rows;
}