// The gallery: every visualizer, six times, on six different songs' content. // // The complaint this answers is one a metric could not raise — watching a few // videos, certain scenes announce themselves. You have seen that one before. // You cannot tell which ones from a number, because the harness measures whole // videos and a scene that always looks like itself is averaged in with // everything around it. // // So this renders each scene under six complete identities — six casts, six ink // treatments, six lattices, six palettes, six parameter draws — and puts them // side by side. A scene whose six thumbnails are interchangeable is a scene the // song cannot change, and that is the definition of the problem. // // Then it scores them, using the same structural descriptor the variety harness // runs on, so the gallery can be sorted worst-first. Browsing sixty-six scenes // looking for the repetitive ones is exactly the job a sort order should do. import { Engine } from '../engine/Engine.js'; import { blit } from './blit.js'; import { scenes } from '../scenes/registry.js'; import { sampleValues } from '../params/schema.js'; import { Rng, hashString } from '../engine/rng.js'; import { featureProviderFor } from '../audio/FeatureTrack.js'; import { songBank } from '../audio/songbank.js'; import { generateLook } from '../look/LookGenerator.js'; import { describeIdentity } from '../look/Identity.js'; import { frameDescriptor, motionDescriptor } from './variety/descriptors.js'; import { frameLuminance, frameVariance } from '../engine/hash.js'; import { descriptorDistance, STRUCTURAL } from './variety/signature.js'; import { generateActor } from '../actors/ActorGenerator.js'; const THUMB = { width: 256, height: 144 }; /* * There used to be a MIN_VARIETY floor here — 0.1, drawn as a red line across * the gallery — on the theory that a scene which looks the same in every song * leaks that sameness between videos. * * That was true when a section was ONE scene. It is not any more: a section is * a ground, a shot over it and sometimes a pass over that, so what a viewer * sees is a combination, and a scene that is reliably itself is a perfectly * good ingredient in one. Held against a floor, such scenes were failing for * being consistent. * * The score is still measured, still reported, and still what the gallery sorts * by — it is genuinely the right question to ask about a scene you are working * on. It is no longer a bar anything has to clear, and the interesting quantity * moved up a level: how unalike the scenes in one stack are. That lives in * scenes/metadata.json and is read by the look generator. */ /** * Six contexts, one per song: everything a scene is handed when it is cast. * * Built from real bank entries rather than invented, so what the gallery shows * is what the generator would actually produce — the same identities, palettes * and section biases, only with the scene held fixed instead of chosen. */ export function galleryContexts(count = 6) { return songBank({ count }).map((entry) => { const look = generateLook(entry.track, { seed: hashString(entry.name) }); // The busiest section, because that is where a scene is asked for the // most and where two scenes are most likely to converge. const section = look.sections.reduce( (best, s) => (s.bias.energy > best.bias.energy ? s : best), look.sections[0]); return { name: entry.name, track: entry.track, palette: look.palette, personality: look.personality, bias: section.bias, frame: section.startFrame + Math.floor((section.endFrame - section.startFrame) * 0.5), identity: describeIdentity(look.personality.identity), }; }); } /** * Render one scene across every context. * * @returns {{thumbs: Uint8Array[], variety: number, byBlock: object}} */ export function renderScene(engine, module, contexts) { const thumbs = []; const descriptors = []; for (const ctx of contexts) { engine.timeline.setDuration(ctx.track.duration); engine.setFeatureProvider(featureProviderFor(ctx.track)); const rng = new Rng(hashString(`${module.name}:${ctx.name}`)); const params = sampleValues(module, rng, ctx.bias, ctx.personality.temperament); const actorRng = new Rng(rng.fork(`actor:${module.actor || 'model'}`).seed); const summary = ctx.track.summary; const actorSpec = module.kind === 'model' && module.actor ? generateActor({ summary, rng: actorRng, archetype: module.actor, personality: ctx.personality, identity: ctx.personality.identity }) : null; engine.setLayerSpecs([{ module, params, seed: rng.int(0, 0x7fffffff), opacity: 1, blend: 'normal', palette: ctx.palette, personality: ctx.personality, actorSpec, }]); engine.compositor.reset(); // A few frames of warm-up so anything with state is past its first frame. for (let f = ctx.frame - 6; f < ctx.frame; f++) engine.renderFrame(f); const pixels = Uint8Array.from(engine.readPixels(engine.renderFrame(ctx.frame))); // A second frame, so MOTION is measured rather than silently scored zero. // // The descriptor has five structural blocks and this only ever built // four of them, so `motion` came back 0.000 for every scene in the // library and dragged the mean down by a fifth across the board. It // looked like a property of the scenes; it was a missing render. const moved = Uint8Array.from(engine.readPixels(engine.renderFrame(ctx.frame + 5))); thumbs.push(pixels); const still = frameDescriptor(pixels, THUMB.width, THUMB.height); const motion = motionDescriptor(pixels, moved, THUMB.width, THUMB.height); descriptors.push({ ...still, motion: motion.scale.concat(motion.layout) }); } // How different this scene's six outputs are from each other, on the same // structural descriptor the variety harness uses — colour excluded, because // six palettes would otherwise make every scene look varied. const byBlock = {}; let total = 0, pairs = 0; for (let i = 0; i < descriptors.length; i++) { for (let j = i + 1; j < descriptors.length; j++) { const d = descriptorDistance(descriptors[i], descriptors[j]); for (const b of [...STRUCTURAL, 'colour']) byBlock[b] = (byBlock[b] || 0) + (d[b] || 0); total += STRUCTURAL.reduce((a, b) => a + (d[b] || 0), 0) / STRUCTURAL.length; pairs++; } } for (const b of Object.keys(byBlock)) byBlock[b] /= pairs || 1; // How much of the frame this scene paints, averaged over the six. Free — // the pixels are already here — and it is the number that explains the // ranking: a scene covering 2% of the frame is a few bright things on // black, which scores well for variety and is thin to watch on its own. // Whether that is a problem depends on whether it is ever layered. let covered = 0; for (const px of thumbs) { let lit = 0; for (let i = 0; i < px.length; i += 4) { if (px[i] + px[i + 1] + px[i + 2] > 90) lit++; } covered += lit / (px.length / 4) / thumbs.length; } // A dead shader scores zero on every block, which is indistinguishable from // a very boring scene if you only read the number — and it happened: the // subject helpers were declared above the ink they call, the whole preamble // failed to compile, and every scene rendered black while the gallery // reported a variety of exactly 0.000. Say which it is. const lum = frameLuminance(thumbs[0]); const variance = frameVariance(thumbs[0]); const dead = lum < 0.002 || variance < 0.001; return { thumbs, // The six per-context descriptors, so a caller can average them into a // profile rather than re-render the library to get one. See // checks/metadata.js — this function is the only place the scenes are // rendered under real identities, and everything measured about a scene // comes out of here. descriptors, variety: pairs ? total / pairs : 0, coverage: covered, byBlock, error: dead ? `renders nothing — luminance ${lum.toFixed(4)}, variance ${variance.toFixed(4)}. ` + 'A failed shader compile scores 0.000 on every block; check the console for GLSL errors.' : undefined, }; } /** * Build the whole gallery, reporting progress as it goes. * * @param {(done:number, of:number, name:string, row:object) => void} onScene */ export async function buildGallery({ contexts, onScene, only = null } = {}) { const list = only ? scenes.filter((m) => only.includes(m.name)) : scenes.filter((m) => m.kind === 'fragment' || m.kind === 'model'); const engine = new Engine({ ...THUMB }); const rows = []; try { for (let i = 0; i < list.length; i++) { const module = list[i]; let row; try { row = { module, ...renderScene(engine, module, contexts) }; } catch (err) { row = { module, thumbs: [], variety: 0, byBlock: {}, error: err.message }; } rows.push(row); if (onScene) onScene(i + 1, list.length, module.name, row); // Yield so the page can paint each row as it lands rather than // freezing for a minute and then showing everything at once. await new Promise((r) => setTimeout(r, 0)); } } finally { engine.dispose(); } return rows; } export { blit, THUMB };