dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wip: instrument guards

Browse Source
This commit is contained in:
Dejvino
2026-08-17 20:10:15 +02:00
parent 2bbf5bcf42
commit a946f0e105
5 changed files with 264 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/look/LookGenerator.js
+96 -33
View File
@@ -9,7 +9,9 @@ import { AudioPalette, generateUsablePalette } from './palette.js';
import { scenes, scenesInFamily, FAMILIES } from '../scenes/registry.js';
import { sampleValues, defaultValues } from '../params/schema.js';
import { planShots } from './shots.js';
import { generatePersonality, sceneHonours, describePersonality } from './Personality.js';
import {
generatePersonality, sceneHonours, signatureWeight, describePersonality,
} from './Personality.js';
import { deriveGrain, describeGrain, applyGrainToPost } from './grain.js';
import { pickDirector, directorByName } from './directors.js';
import { derivePaletteArc, describePaletteArc } from './paletteArc.js';
@@ -33,7 +35,7 @@ const KIND_ENERGY = {
* and `energy` up; a breakdown pulls them down. Seed variation still dominates,
* so two tracks with the same structure do not converge on the same look.
*/
function biasFor(section, summary) {
function biasFor(section, summary, motion = null) {
const kindEnergy = KIND_ENERGY[section.kind] ?? 0.5;
const measured = Math.min(1, section.energy / Math.max(1e-6, summary.meanLoudness * 1.6));
const energy = kindEnergy * 0.6 + measured * 0.4;
@@ -45,14 +47,23 @@ function biasFor(section, summary) {
// biased to 0.9 motion and scenes that skittered over it.
const tempo = clamp01((summary.bpm - 60) / 120);
// The track's motion CHARACTER, on top of its tempo. Tempo alone compresses
// — 124 and 138bpm are the same number to a viewer — and motion was the
// weakest axis in every measurement because it was the only lever. Stillness
// is allowed to halve the animation rate or half again raise it, which is a
// difference anyone can see, and it is a property of the track rather than
// of the section. See look/Personality.js.
const still = motion ? motion.stillness : 0.5;
const churn = motion ? motion.churn : 0.25;
return {
energy,
density: Math.min(1, energy * 0.7 + section.flux * 1.2),
motion: clamp01(0.12 + tempo * 0.55 + energy * 0.28),
motion: clamp01(0.12 + tempo * 0.4 + energy * 0.2 + (1 - still) * 0.35),
// Applied on top of every `rate: true` param, so absolute animation
// speed scales with the song rather than only its sampled position in
// a range. Bounded well short of a stop or a blur. See params/schema.js.
rateScale: 0.45 + tempo * 0.95,
rateScale: (0.45 + tempo * 0.95) * (1.35 - still * 0.75) * (1 + churn * 0.25),
};
}
@@ -66,26 +77,67 @@ const clamp01 = (x) => Math.max(0, Math.min(1, x));
* on is not a worse choice, it is the shot that was clearly filmed somewhere
* else. See look/Personality.js.
*/
function candidatesForKind(kind, used, signature = [], director) {
/**
* The scenes THIS TRACK is allowed to cast from — a seeded subset of the
* library, not the whole thing.
*
* There is a real tension here, and the first attempt at fixing the signature
* gate walked straight into it. The hard trait filter was doing two jobs at
* once: it was collapsing the library onto eleven over-declared scenes, which
* was the bug, and it was also giving each track a DIFFERENT pool to cast from,
* which was load-bearing. Replacing it with a soft weight fixed the collapse and
* removed the differentiation — every track then drew from the same weighted
* library, and measured song separation went from 0.03 to -0.15. Two songs came
* out more alike than before.
*
* So the differentiation is kept and its bias removed. Every track gets its own
* pool of about a third of the library, sampled without replacement, weighted by
* the signature so the track still has a point of view. What changes is that a
* scene declaring two traits is now merely less likely to be drawn than one
* declaring four, instead of being ineligible for six tracks in seven.
*/
function castingPool(rng, signature, size = 24) {
const pool = scenes.filter((m) => m.role !== 'accent');
const remaining = pool.slice();
const weights = remaining.map((m) => signatureWeight(m, signature));
const picked = [];
const target = Math.min(size, remaining.length);
while (picked.length < target && remaining.length) {
const chosen = rng.pickWeighted(remaining, weights);
const at = remaining.indexOf(chosen);
remaining.splice(at, 1);
weights.splice(at, 1);
picked.push(chosen);
}
return picked;
}
function candidatesForKind(kind, used, signature = [], director, pool = null) {
const families = director.families[kind] || Object.keys(FAMILIES);
const allowed = pool ? new Set(pool.map((m) => m.name)) : null;
const candidates = [];
for (const family of families) {
const inFamily = scenesInFamily(family)
.filter((m) => m.role !== 'accent' && sceneHonours(m, signature));
.filter((m) => m.role !== 'accent' && (!allowed || allowed.has(m.name)));
// Weight by family preference order, and push down anything already
// used so a five-section track doesn't show one scene five times.
const weight = families.length - families.indexOf(family);
for (const scene of inFamily) {
candidates.push({ scene, weight: weight * (used.has(scene.name) ? 0.15 : 1) });
candidates.push({
scene,
// The signature is a lean now, not a wall. See
// Personality.signatureWeight for why it had to stop being one.
weight: weight * signatureWeight(scene, signature)
* (used.has(scene.name) ? 0.15 : 1),
});
}
}
if (!candidates.length) {
// Every family for this kind was emptied by the signature filter. Widen
// to the whole library, still honouring the signature; only if that is
// empty too does the personality lose and the video keep its scenes.
const anywhere = scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature));
const pool = anywhere.length ? anywhere : scenes.filter((m) => m.role !== 'accent');
return pool.map((scene) => ({ scene, weight: 1 }));
// This track's pool holds nothing in the families the director wants for
// this kind. Widen to the pool, then to the library — the track keeps
// its scenes either way.
const fallback = (pool && pool.length) ? pool : scenes.filter((m) => m.role !== 'accent');
return fallback.map((scene) => ({ scene, weight: signatureWeight(scene, signature) }));
}
return candidates;
}
@@ -116,7 +168,7 @@ function rosterSizeFor(kind) {
* Variation between two sections of the same kind comes from their parameter
* sets, from where their shots fall, and from the arc driver's drift.
*/
function assignRostersByKind(sections, rng, signature = [], director) {
function assignRostersByKind(sections, rng, signature = [], director, pool = null) {
const byKind = new Map();
const used = new Set();
@@ -131,7 +183,7 @@ function assignRostersByKind(sections, rng, signature = [], director) {
const size = rosterSizeFor(kind);
for (let slot = 0; slot < size; slot++) {
const pool = candidatesForKind(kind, used, signature, director)
const options = candidatesForKind(kind, used, signature, director, pool)
.filter((c) => !roster.includes(c.scene))
.map((c) => ({
scene: c.scene,
@@ -140,9 +192,10 @@ function assignRostersByKind(sections, rng, signature = [], director) {
// whole visual language.
weight: c.weight * (roster.length && c.scene.family === roster[0].family ? 3 : 1),
}));
if (!pool.length) break;
if (!options.length) break;
const chosen = rng.pickWeighted(pool.map((c) => c.scene), pool.map((c) => c.weight));
const chosen = rng.pickWeighted(
options.map((c) => c.scene), options.map((c) => c.weight));
roster.push(chosen);
used.add(chosen.name);
}
@@ -266,7 +319,9 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament)
* @param {object} options
* @returns {object} LookSpec
*/
export function generateLook(track, { seed = null, samples = null, overrides = null } = {}) {
export function generateLook(track, {
seed = null, samples = null, overrides = null, pool: poolOverride = null,
} = {}) {
const resolvedSeed = seed !== null
? seed >>> 0
: samples ? hashSamples(samples) : 0x9e3779b9;
@@ -280,14 +335,24 @@ export function generateLook(track, { seed = null, samples = null, overrides = n
// The production design, decided before a single scene is cast — casting
// depends on it. See look/Personality.js.
const personality = generatePersonality(summary, rng.fork('personality'), (signature) =>
scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature)).length);
scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature)).length,
track.sections.length);
// The track's point of view about what a song looks like. Cast before any
// scene is, because it decides which scenes are even candidates.
const director = pickDirector(summary, rng.fork('director'));
// This track's cast, drawn before any section is assigned. See castingPool.
//
// `poolOverride` exists for the variety harness, which needs reference
// videos that share no scenes with each other but are otherwise built by
// exactly this code — a reference assembled by any other path stops being
// comparable to the thing it is bounding.
const pool = poolOverride && poolOverride.length
? poolOverride
: castingPool(rng.fork('pool'), personality.signature);
const rosterByKind = assignRostersByKind(
track.sections, rng.fork('scenes'), personality.signature, director);
track.sections, rng.fork('scenes'), personality.signature, director, pool);
// The grain treatment: usually none, and when present described rather than
// dialled. See look/grain.js.
const grain = deriveGrain(summary, rng.fork('grain'));
@@ -303,19 +368,17 @@ export function generateLook(track, { seed = null, samples = null, overrides = n
// Accents honour the signature too where they can. If none can, the track
// goes without depth layers rather than putting an off-design element into
// every stack.
const accentRoster = scenes.filter((m) => m.role === 'accent'
&& sceneHonours(m, personality.signature));
const accentRoster = scenes.filter((m) => m.role === 'accent');
// Scenes eligible to be composited OVER a background. Same casting rule as
// everything else — an overlay is on screen as much as the shot under it,
// so an off-design one would be just as visible.
const overlayRoster = scenes.filter((m) => m.role !== 'accent'
&& sceneHonours(m, personality.signature));
const overlayRoster = pool.length >= 4 ? pool : scenes.filter((m) => m.role !== 'accent');
const sections = track.sections.map((section) => {
const roster = rosterByKind.get(section.kind) || [scenes[0]];
const sectionRng = rng.fork(`section:${section.index}:${roster[0].name}`);
const bias = biasFor(section, summary);
const bias = biasFor(section, summary, personality.motion);
const variants = roster.map((module, v) => buildStack(
module, accentRoster, overlayRoster, bias,
@@ -370,25 +433,25 @@ export function rerollSection(look, track, sectionIndex, salt = 0) {
const rng = new Rng((look.seed ^ (sectionIndex * 0x9e3779b9) ^ (salt * 0x85ebca6b)) >>> 0);
const signature = (look.personality && look.personality.signature) || [];
const families = directorByName(look.director).families[section.kind] || Object.keys(FAMILIES);
// A reroll re-draws this section's cast from the same kind of pool the track
// was built with, weighted by the signature rather than filtered by it.
let candidates = families.flatMap((f) => scenesInFamily(f))
.filter((m) => m.role !== 'accent' && sceneHonours(m, signature));
if (!candidates.length) {
candidates = scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature));
}
.filter((m) => m.role !== 'accent');
if (!candidates.length) candidates = scenes.filter((m) => m.role !== 'accent');
// Re-roll the whole roster, not just the anchor: the section's shots cut
// between all of them, so replacing one would leave the section half old.
const size = Math.min(rosterSizeFor(section.kind), Math.max(1, candidates.length));
const roster = [];
while (roster.length < size) {
const pool = candidates.filter((m) => !roster.includes(m));
if (!pool.length) break;
roster.push(rng.pick(pool));
const options = candidates.filter((m) => !roster.includes(m));
if (!options.length) break;
roster.push(rng.pickWeighted(options, options.map((m) => signatureWeight(m, signature))));
}
if (!roster.length) roster.push(scenes[0]);
const accentRoster = scenes.filter((m) => m.role === 'accent');
const overlayRoster = scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature));
const overlayRoster = castingPool(rng.fork('pool'), signature);
section.variants = roster.map((module, v) => buildStack(
module, accentRoster, overlayRoster, section.bias, rng.fork(`variant:${v}`),
look.personality && look.personality.temperament,
flow-state/src/look/Personality.js
+123 -19
View File
@@ -40,6 +40,8 @@
export const TRAITS = ['shape', 'camera', 'space', 'style'];
const clamp01 = (x) => Math.max(0, Math.min(1, x));
/**
* Traits eligible to be a track's signature, and how often.
*
@@ -51,6 +53,38 @@ export const TRAITS = ['shape', 'camera', 'space', 'style'];
*/
const SIGNATURE_WEIGHTS = { shape: 4, space: 3, camera: 2, style: 2 };
/**
* How the audio tilts the choice of signature.
*
* The signature decides which scenes a track can even cast, so if it is picked
* from the seed alone then the single most consequential decision in the whole
* generator has no relationship to the music. Measured, that is exactly what
* happened: across seven songs the correlation between how different two tracks
* SOUND and how different their videos LOOK was -0.03. The videos differed; the
* differences just had nothing to do with the songs.
*
* A tilt, not a rule. Every trait stays reachable for every track — a mapping
* rigid enough to predict is the failure this layer exists to avoid — but a
* track built on transients leans toward shape, a spacious one toward space, a
* fast one toward camera and a noisy one toward style.
*/
function signatureTilt(summary, sections) {
const bright = summary.meanCentroid ?? 0.5;
const noisy = Math.min(1, (summary.meanFlatness ?? 0.2) * 3);
const fast = clamp01(((summary.bpm ?? 120) - 80) / 80);
const dynamic = clamp01(summary.dynamicRange ?? 0.5);
// A track that keeps changing what it is doing has structure to draw on;
// one that states an idea and holds it has a place instead.
const busy = clamp01((sections - 2) / 5);
return {
shape: 0.5 + busy * 1.2 + (1 - noisy) * 0.5,
space: 0.5 + dynamic * 1.1 + (1 - busy) * 0.6,
camera: 0.5 + fast * 1.2,
style: 0.5 + noisy * 1.3 + bright * 0.4,
};
}
/** Minimum scenes that must survive the signature filter for it to be usable. */
export const MIN_ELIGIBLE_SCENES = 6;
@@ -68,18 +102,34 @@ export const MIN_ELIGIBLE_SCENES = 6;
* How many scenes would survive a given signature. Injected rather than
* imported so this module never has to know the registry exists.
*/
export function generatePersonality(summary, rng, countEligible = null) {
export function generatePersonality(summary, rng, countEligible = null, sections = 4) {
const bright = summary.meanCentroid;
const noisy = Math.min(1, summary.meanFlatness * 3);
const fast = Math.min(1, Math.max(0, (summary.bpm - 80) / 80));
const loud = Math.min(1, (summary.dynamicRange ?? 0.5) + (summary.meanLoudness ?? 0.3));
const dynamic = clamp01(summary.dynamicRange ?? 0.5);
// Audio sets the CENTRE of each distribution and the seed picks within it.
//
// The alternative — deriving values from the audio outright — buys coupling
// by destroying seed variety, since two seeds on one song would then agree
// about everything. Centring keeps both: two songs land in different regions
// of the space, two seeds land in different places inside one region. The
// window is deliberately wide enough that the mapping cannot be read off a
// finished video.
const around = (centre, window, lo, hi) =>
Math.max(lo, Math.min(hi, centre + rng.range(-window, window)));
const shape = {
// 0 sides means round. Everything else is a polygon the whole track
// shares — the single most recognisable thing here.
sides: rng.pickWeighted([0, 3, 4, 5, 6, 8], [3, 2, 3, 2, 3, 1]),
roundness: rng.range(0.05, 0.5),
elongation: rng.range(0.85, 1.45),
// shares — the single most recognisable thing here. Noise and transients
// earn corners; tonal, smooth material stays round.
sides: rng.pickWeighted(
[0, 3, 4, 5, 6, 8],
[3 + (1 - noisy) * 4, 1 + noisy * 2, 2 + noisy * 2,
1 + noisy * 1.5, 2 + noisy * 2, 0.5 + noisy * 1.5]),
roundness: around(0.28 - noisy * 0.15, 0.18, 0.05, 0.5),
elongation: around(1.15, 0.3, 0.85, 1.45),
tilt: rng.range(0, Math.PI),
};
@@ -87,19 +137,21 @@ export function generatePersonality(summary, rng, countEligible = null) {
driftAngle: rng.range(0, Math.PI * 2),
// A slow track should not be filmed from a moving car.
driftRate: rng.range(0.01, 0.06) * (0.6 + fast * 0.8),
sway: rng.range(0.0, 0.06),
swayRate: rng.range(0.05, 0.22),
spin: rng.range(-0.05, 0.05),
sway: around(0.01 + fast * 0.035, 0.02, 0, 0.06),
swayRate: around(0.08 + fast * 0.1, 0.06, 0.05, 0.22),
spin: around((rng.bool() ? 1 : -1) * fast * 0.03, 0.025, -0.05, 0.05),
// Breathing is locked to the bar, so it is the one camera move that
// reads as musical rather than as drifting.
breathe: rng.range(0.0, 0.05),
// reads as musical rather than as drifting. A dynamic track breathes.
breathe: around(dynamic * 0.03, 0.02, 0, 0.05),
};
const space = {
horizon: rng.range(0.32, 0.62),
depth: rng.range(0.2, 0.9),
// A bright track sits high in its frame and a dark one sits low; a
// dynamic one has depth to fall away into.
horizon: around(0.36 + bright * 0.2, 0.1, 0.32, 0.62),
depth: around(0.25 + dynamic * 0.5, 0.25, 0.2, 0.9),
washAngle: rng.range(0, Math.PI * 2),
wash: rng.range(0.1, 0.5),
wash: around(0.18 + (1 - bright) * 0.2, 0.15, 0.1, 0.5),
};
const style = {
@@ -115,7 +167,24 @@ export function generatePersonality(summary, rng, countEligible = null) {
// Fold counts stay low and are usually off. Symmetry is the fastest way
// to make a library look like one series and also the fastest way to
// make every track look like a screensaver.
symmetry: rng.pickWeighted([1, 1, 2, 3, 4, 6], [6, 4, 2, 2, 2, 1]),
symmetry: rng.pickWeighted([1, 1, 2, 3, 4, 6],
[6, 4, 1 + (1 - noisy) * 2, 1 + (1 - noisy) * 2, 1 + (1 - noisy) * 2, 0.5 + (1 - noisy)]),
};
// How this track MOVES, as a character rather than a rate.
//
// Motion was the weakest block in every measurement — 47% of achievable
// across seeds — because the only lever on it was a tempo-derived rate
// multiplier, and tempo compresses. Two tracks at 124 and 138bpm got
// essentially the same movement. Stillness is the missing axis: some music
// wants an image that hangs almost motionless and some wants one that never
// settles, and that is not the same question as how fast it animates.
const motion = {
// 0 = hangs, 1 = never settles. Dynamic, spacious material earns the
// stillness; dense, fast material does not get it.
stillness: clamp01(around(0.5 + dynamic * 0.35 - fast * 0.45, 0.3, 0, 1)),
// Whether the movement is steady or agitated, independent of its speed.
churn: clamp01(around(0.25 + noisy * 0.5, 0.3, 0, 1)),
};
// How hard this track pushes every scene it casts. Deliberately wide, and
@@ -137,9 +206,9 @@ export function generatePersonality(summary, rng, countEligible = null) {
extremity: rng.range(0.45, 1.0),
};
const signature = pickSignature(rng, countEligible);
const signature = pickSignature(rng, countEligible, signatureTilt(summary, sections));
return { signature, shape, camera, space, style, temperament };
return { signature, shape, camera, space, style, motion, temperament };
}
/**
@@ -150,10 +219,11 @@ export function generatePersonality(summary, rng, countEligible = null) {
* rosters from, fall back to the stronger of the two rather than shipping a
* track whose every section is forced onto the same two scenes.
*/
function pickSignature(rng, countEligible) {
const primary = rng.pickWeighted(TRAITS, TRAITS.map((t) => SIGNATURE_WEIGHTS[t]));
function pickSignature(rng, countEligible, tilt = null) {
const weightOf = (t) => SIGNATURE_WEIGHTS[t] * (tilt ? tilt[t] : 1);
const primary = rng.pickWeighted(TRAITS, TRAITS.map(weightOf));
const rest = TRAITS.filter((t) => t !== primary);
const secondary = rng.pickWeighted(rest, rest.map((t) => SIGNATURE_WEIGHTS[t]));
const secondary = rng.pickWeighted(rest, rest.map(weightOf));
const pair = [primary, secondary];
if (!countEligible || countEligible(pair) >= MIN_ELIGIBLE_SCENES) return pair;
@@ -169,6 +239,35 @@ export function sceneHonours(module, signature) {
return signature.every((t) => traits.includes(t));
}
/**
* How well a scene fits the signature, 0..1 — and the casting WEIGHT that
* follows from it.
*
* `sceneHonours` was a hard filter, and as a filter it was the single largest
* cause of sameness in the generator. Measured across the library: a scene
* declaring all four traits is eligible for every track and opens half of all
* videos, while a scene declaring two is eligible for one track in fourteen.
* Eleven scenes out of sixty-one carried nearly every video, five of them from
* the same family, and that was the house style — not a decision anyone made,
* just an artefact of which scenes happened to declare four traits.
*
* A lean rather than a wall. Honouring the whole signature is worth six times
* the weight of honouring none of it, which is more than enough for the track to
* read as one production, while leaving the rest of the library reachable
* instead of disqualified.
*/
export function signatureAffinity(module, signature) {
if (!signature || !signature.length) return 1;
const traits = module.traits || [];
let hit = 0;
for (const t of signature) if (traits.includes(t)) hit++;
return hit / signature.length;
}
export function signatureWeight(module, signature) {
return 1 + signatureAffinity(module, signature) * 5;
}
/**
* Flatten to the uniform values the shader contract expects.
*
@@ -245,6 +344,11 @@ export function describePersonality(personality) {
SHAPE_NAMES[shape.sides] || `${shape.sides}-sided`,
];
if (style.symmetry > 1) parts.push(`${style.symmetry}-fold`);
if (personality.motion) {
const m = personality.motion;
parts.push(`${m.stillness > 0.6 ? 'still' : m.stillness < 0.3 ? 'restless' : 'moving'}` +
`${m.churn > 0.6 ? '+churn' : ''}`);
}
if (personality.temperament) {
const t = personality.temperament;
parts.push(`${t.intensity >= 0 ? 'hot' : 'cool'} ${t.extremity.toFixed(2)} bold`);