music-video-gen/flow-state/src/checks/gallery.js

// The gallery: every visualizer, six times, on six different songs' content.
//
// The complaint this answers is one a metric could not raise — watching a few
// videos, certain scenes announce themselves. You have seen that one before.
// You cannot tell which ones from a number, because the harness measures whole
// videos and a scene that always looks like itself is averaged in with
// everything around it.
//
// So this renders each scene under six complete identities — six casts, six ink
// treatments, six lattices, six palettes, six parameter draws — and puts them
// side by side. A scene whose six thumbnails are interchangeable is a scene the
// song cannot change, and that is the definition of the problem.
//
// Then it scores them, using the same structural descriptor the variety harness
// runs on, so the gallery can be sorted worst-first. Browsing sixty-six scenes
// looking for the repetitive ones is exactly the job a sort order should do.

import { Engine } from '../engine/Engine.js';
import { blit } from './blit.js';
import { scenes } from '../scenes/registry.js';
import { sampleValues } from '../params/schema.js';
import { Rng, hashString } from '../engine/rng.js';
import { featureProviderFor } from '../audio/FeatureTrack.js';
import { songBank } from '../audio/songbank.js';
import { generateLook } from '../look/LookGenerator.js';
import { describeIdentity } from '../look/Identity.js';
import { frameDescriptor, motionDescriptor } from './variety/descriptors.js';
import { frameLuminance, frameVariance } from '../engine/hash.js';
import { descriptorDistance, STRUCTURAL } from './variety/signature.js';
import { generateActor } from '../actors/ActorGenerator.js';

const THUMB = { width: 256, height: 144 };

/*
 * There used to be a MIN_VARIETY floor here — 0.1, drawn as a red line across
 * the gallery — on the theory that a scene which looks the same in every song
 * leaks that sameness between videos.
 *
 * That was true when a section was ONE scene. It is not any more: a section is
 * a ground, a shot over it and sometimes a pass over that, so what a viewer
 * sees is a combination, and a scene that is reliably itself is a perfectly
 * good ingredient in one. Held against a floor, such scenes were failing for
 * being consistent.
 *
 * The score is still measured, still reported, and still what the gallery sorts
 * by — it is genuinely the right question to ask about a scene you are working
 * on. It is no longer a bar anything has to clear, and the interesting quantity
 * moved up a level: how unalike the scenes in one stack are. That lives in
 * scenes/metadata.json and is read by the look generator.
 */

/**
 * Six contexts, one per song: everything a scene is handed when it is cast.
 *
 * Built from real bank entries rather than invented, so what the gallery shows
 * is what the generator would actually produce — the same identities, palettes
 * and section biases, only with the scene held fixed instead of chosen.
 */
export function galleryContexts(count = 6) {
    return songBank({ count }).map((entry) => {
        const look = generateLook(entry.track, { seed: hashString(entry.name) });
        // The busiest section, because that is where a scene is asked for the
        // most and where two scenes are most likely to converge.
        const section = look.sections.reduce(
            (best, s) => (s.bias.energy > best.bias.energy ? s : best), look.sections[0]);
        return {
            name: entry.name,
            track: entry.track,
            palette: look.palette,
            personality: look.personality,
            bias: section.bias,
            frame: section.startFrame + Math.floor((section.endFrame - section.startFrame) * 0.5),
            identity: describeIdentity(look.personality.identity),
        };
    });
}

/**
 * Render one scene across every context.
 *
 * @returns {{thumbs: Uint8Array[], variety: number, byBlock: object}}
 */
export function renderScene(engine, module, contexts) {
    const thumbs = [];
    const descriptors = [];

    for (const ctx of contexts) {
        engine.timeline.setDuration(ctx.track.duration);
        engine.setFeatureProvider(featureProviderFor(ctx.track));

        const rng = new Rng(hashString(`${module.name}:${ctx.name}`));
        const params = sampleValues(module, rng, ctx.bias, ctx.personality.temperament);
        const actorRng = new Rng(rng.fork(`actor:${module.actor || 'model'}`).seed);
        const summary = ctx.track.summary;
        const actorSpec = module.kind === 'model' && module.actor
            ? generateActor({ summary, rng: actorRng, archetype: module.actor, personality: ctx.personality, identity: ctx.personality.identity })
            : null;

        engine.setLayerSpecs([{
            module,
            params,
            seed: rng.int(0, 0x7fffffff),
            opacity: 1,
            blend: 'normal',
            palette: ctx.palette,
            personality: ctx.personality,
            actorSpec,
        }]);
        engine.compositor.reset();
        // A few frames of warm-up so anything with state is past its first frame.
        for (let f = ctx.frame - 6; f < ctx.frame; f++) engine.renderFrame(f);
        const pixels = Uint8Array.from(engine.readPixels(engine.renderFrame(ctx.frame)));
        // A second frame, so MOTION is measured rather than silently scored zero.
        //
        // The descriptor has five structural blocks and this only ever built
        // four of them, so `motion` came back 0.000 for every scene in the
        // library and dragged the mean down by a fifth across the board. It
        // looked like a property of the scenes; it was a missing render.
        const moved = Uint8Array.from(engine.readPixels(engine.renderFrame(ctx.frame + 5)));

        thumbs.push(pixels);
        const still = frameDescriptor(pixels, THUMB.width, THUMB.height);
        const motion = motionDescriptor(pixels, moved, THUMB.width, THUMB.height);
        descriptors.push({ ...still, motion: motion.scale.concat(motion.layout) });
    }

    // How different this scene's six outputs are from each other, on the same
    // structural descriptor the variety harness uses — colour excluded, because
    // six palettes would otherwise make every scene look varied.
    const byBlock = {};
    let total = 0, pairs = 0;
    for (let i = 0; i < descriptors.length; i++) {
        for (let j = i + 1; j < descriptors.length; j++) {
            const d = descriptorDistance(descriptors[i], descriptors[j]);
            for (const b of [...STRUCTURAL, 'colour']) byBlock[b] = (byBlock[b] || 0) + (d[b] || 0);
            total += STRUCTURAL.reduce((a, b) => a + (d[b] || 0), 0) / STRUCTURAL.length;
            pairs++;
        }
    }
    for (const b of Object.keys(byBlock)) byBlock[b] /= pairs || 1;

    // How much of the frame this scene paints, averaged over the six. Free —
    // the pixels are already here — and it is the number that explains the
    // ranking: a scene covering 2% of the frame is a few bright things on
    // black, which scores well for variety and is thin to watch on its own.
    // Whether that is a problem depends on whether it is ever layered.
    let covered = 0;
    for (const px of thumbs) {
        let lit = 0;
        for (let i = 0; i < px.length; i += 4) {
            if (px[i] + px[i + 1] + px[i + 2] > 90) lit++;
        }
        covered += lit / (px.length / 4) / thumbs.length;
    }

    // A dead shader scores zero on every block, which is indistinguishable from
    // a very boring scene if you only read the number — and it happened: the
    // subject helpers were declared above the ink they call, the whole preamble
    // failed to compile, and every scene rendered black while the gallery
    // reported a variety of exactly 0.000. Say which it is.
    const lum = frameLuminance(thumbs[0]);
    const variance = frameVariance(thumbs[0]);
    const dead = lum < 0.002 || variance < 0.001;

    return {
        thumbs,
        // The six per-context descriptors, so a caller can average them into a
        // profile rather than re-render the library to get one. See
        // checks/metadata.js — this function is the only place the scenes are
        // rendered under real identities, and everything measured about a scene
        // comes out of here.
        descriptors,
        variety: pairs ? total / pairs : 0,
        coverage: covered,
        byBlock,
        error: dead
            ? `renders nothing — luminance ${lum.toFixed(4)}, variance ${variance.toFixed(4)}. ` +
              'A failed shader compile scores 0.000 on every block; check the console for GLSL errors.'
            : undefined,
    };
}

/**
 * Build the whole gallery, reporting progress as it goes.
 *
 * @param {(done:number, of:number, name:string, row:object) => void} onScene
 */
export async function buildGallery({ contexts, onScene, only = null } = {}) {
    const list = only
        ? scenes.filter((m) => only.includes(m.name))
        : scenes.filter((m) => m.kind === 'fragment' || m.kind === 'model');

    const engine = new Engine({ ...THUMB });
    const rows = [];
    try {
        for (let i = 0; i < list.length; i++) {
            const module = list[i];
            let row;
            try {
                row = { module, ...renderScene(engine, module, contexts) };
            } catch (err) {
                row = { module, thumbs: [], variety: 0, byBlock: {}, error: err.message };
            }
            rows.push(row);
            if (onScene) onScene(i + 1, list.length, module.name, row);
            // Yield so the page can paint each row as it lands rather than
            // freezing for a minute and then showing everything at once.
            await new Promise((r) => setTimeout(r, 0));
        }
    } finally {
        engine.dispose();
    }
    return rows;
}

export { blit, THUMB };