dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Epic 2.5.1: basis for framing layer

Browse Source
This commit is contained in:
Dejvino 2026-08-06 10:29:32 +02:00
parent 189328587d
commit 19bfff8651
9 changed files with 346 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
flow-state/EPIC-2.md
View File

@ -168,6 +168,23 @@ This is deliberately last: it is the largest change, it interacts with resolutio
independence (§1 of PLAN.md) and with the feedback buffer, and the four items above will have
changed what it needs to do.
**Status after the first pass: shipped, deliberately simple.**
A shot now carries a size and a recentre, applied inside `sigCamera` in scene coordinates —
rendered close rather than magnified, which is what keeps it resolution-independent and free at
4K. `look/framing.js` owns the vocabulary: three shot sizes with measured headroom (a close-up
past ~2.2 is a blurry wide, a wide past ~0.55 is a speck), and a per-track style (`locked` /
`gentle` / `edited`) that decides how far sizes travel and how often a cut also changes the shot.
The arc driver plans one framing per cue, walking the cuts in order so the whole video gets a
consistent hand and a seek finds the same framing as playback. The gates hold it honest: cuts
actually change size, sizes stay in headroom, and two drivers over one look agree on every shot.
What this pass does not do, and the reason it is "simple": the framing is constant within a shot.
A zoom that moves during a shot is a separate device and would fight the drift LFO and the slow
axis, both of which already own continuous motion. Scenes built outside the shader contract (the
single 3D layer) are not framed yet. Both are the obvious next steps and neither is required for
a working project.
---
## 4. Validation

128
flow-state/src/checks/phase11.js
View File

@ -24,6 +24,7 @@ import { featureProviderFor } from '../audio/FeatureTrack.js';
import { sampleValues, clampValue } from '../params/schema.js';
import { Rng } from '../engine/rng.js';
import { frameDistance, frameLuminance } from '../engine/hash.js';
import { SHOT_SIZES, SHOT_SIZE_NAMES, describeFraming } from '../look/framing.js';
/** Tracks at several tempos: shot length is measured in bars, so tempo is the axis. */
let cached = null;
@ -617,6 +618,133 @@ check(11, 'a declared slow axis actually changes the scene', () => {
problems.length ? problems.join(' · ') : `${detail.join(', ')}`);
}, { slow: true });
// --- framing ---------------------------------------------------------------
// EPIC-2.md §3.5. Every scene was a locked-off, full-frame wide, so a cut changed
// the subject and never the framing. A shot now carries a scale and a recentre,
// applied in scene coordinates inside sigCamera — see look/framing.js. These
// gates hold the layer honest: it must actually change, stay within headroom, and
// be the same on a seek as on playback.
check(11, 'a shot cut actually changes the framing', () => {
// The complaint as a number: before this, every shot in every video was the
// same full-frame wide, and a cut moved the image and not the camera. The
// gate is on the population — most tracks should be framed at all, and of
// those, the cuts must really change the shot size rather than cosmetically
// vary a number no eye can see.
const tracks = tempoBattery();
const modes = [];
let nonLocked = 0;
let withSizeChange = 0;
let transitions = 0;
let changed = 0;
// Two seed families so the draw is not at the mercy of one: the whole gate
// ran once against a single family whose first few seeds happened to land
// heavy on `locked`, and a sample that small has no business holding a
// population claim. Locked is ~15% of tracks by design; 24 seeds a family
// put it far from that.
for (const { track: t } of tracks) {
for (const family of [11100, 12200]) {
for (let s = 0; s < 12; s++) {
const look = generateLook(t, { seed: family + s * 7919 });
const style = look.framing;
modes.push(style.mode);
if (style.mode === 'locked') continue;
nonLocked++;
const arc = new ArcDriver(look, t);
try {
for (let i = 1; i < arc.cues.length; i++) {
const a = arc.cues[i - 1].framing;
const b = arc.cues[i].framing;
if (!a || !b) continue;
transitions++;
if (a.size !== b.size) changed++;
}
const sizes = new Set(arc.cues.map((c) => c.framing && c.framing.size));
if (sizes.size > 1) withSizeChange++;
} finally {
arc.dispose();
}
}
}
}
const lockedShare = modes.filter((m) => m === 'locked').length / modes.length;
const changeRate = transitions ? changed / transitions : 0;
return expect(
nonLocked > 0 && withSizeChange >= nonLocked * 0.6
&& lockedShare < 0.5 && changeRate >= 0.15,
`${modes.length} tracks · locked ${(lockedShare * 100).toFixed(0)}% · ` +
`${withSizeChange}/${nonLocked} framed tracks show 2+ sizes · ` +
`${changed}/${transitions} cuts change size (${(changeRate * 100).toFixed(0)}%)`);
});
check(11, 'framing stays inside the library\'s headroom', () => {
// A size is only worth using if the scene still has detail at that size.
// Past ~2.2 the library runs out and a close-up is a blurry wide; below
// ~0.55 the subject is a speck. Those bounds are the measured edges of what
// is watchable, so every framed shot has to respect them.
const problems = [];
let framed = 0;
for (const { track: t } of tempoBattery()) {
for (let s = 0; s < 6; s++) {
const look = generateLook(t, { seed: 12200 + s * 6841 });
if (look.framing.mode === 'locked') continue;
const arc = new ArcDriver(look, t);
try {
for (const cue of arc.cues) {
const f = cue.framing;
if (!f) continue;
framed++;
if (f.scale < 0.55 || f.scale > 2.2) {
problems.push(`§${cue.sectionIndex}#${cue.shotIndex}: scale ${f.scale.toFixed(2)}`);
}
if (Math.abs(f.shift[0]) > 0.3 || Math.abs(f.shift[1]) > 0.3) {
problems.push(`§${cue.sectionIndex}#${cue.shotIndex}: shift ${f.shift.map((v) => v.toFixed(2))}`);
}
}
} finally {
arc.dispose();
}
}
}
return expect(problems.length === 0 && framed > 0,
problems.length ? problems.slice(0, 3).join(' · ')
: `${framed} framed shots all within scale headroom and off-centre`);
});
check(11, 'framing is identical on a seek and on playback', () => {
// Framing is planned once per cue, but a reroll re-plans it. The gate that
// matters is the same one every other layer honours: the same frame has to
// carry the same framing however you reach it. Two drivers over the same
// look must agree on every shot.
const t = tempoBattery()[1].track;
const look = generateLook(t, { seed: 31337 });
const a = new ArcDriver(look, t);
const b = new ArcDriver(look, t);
const problems = [];
try {
for (let i = 0; i < a.cues.length; i++) {
const fa = a.cues[i].framing;
const fb = b.cues[i].framing;
if (fa && fb && (fa.scale !== fb.scale
|| fa.shift[0] !== fb.shift[0] || fa.shift[1] !== fb.shift[1])) {
problems.push(`cue ${i}: ${JSON.stringify(fa)} vs ${JSON.stringify(fb)}`);
}
}
} finally {
a.dispose(); b.dispose();
}
return expect(problems.length === 0,
problems.length ? problems.slice(0, 3).join(' · ')
: `${a.cues.length} cues carry the same framing in both drivers`);
});
check(11, 'the axis measurement would notice if the axis stopped working', () => {
// EPIC-2.md §4 names this failure mode by name: a gate that measures the
// wrong thing. This one has already happened once here — the first version

18
flow-state/src/engine/Layer.js
View File

@ -99,6 +99,14 @@ export function setFrameUniforms(layer, renderer, target, ctx) {
else u[name].value = v;
}
// Framing is per shot and wins over the personality's neutral defaults —
// it is the operator's hand on a shot that is already set up, not a trait
// of the track. See look/framing.js.
if (layer.framing) {
u.u_sigFrameScale.value = layer.framing.scale;
u.u_sigFrameShift.value.set(layer.framing.shift[0], layer.framing.shift[1]);
}
u.u_prev.value = prevTexture || null;
u.u_hasPrev.value = prevTexture ? 1 : 0;
}
@ -136,6 +144,16 @@ export class Layer {
return this;
}
/**
* This shot's FRAMING — a per-shot scale/recentre pushed by the arc driver,
* applied inside sigCamera. A layer that is never framed renders at the
* neutral (full-frame) scale, so nothing predating framing changes.
*/
setFraming(framing) {
this.framing = framing || null;
return this;
}
/** Resolve base params + reactive modulation into the values used this frame. */
resolveParams(features) {
const defs = this.module.params || {};

4
flow-state/src/engine/OSD.js
View File

@ -163,7 +163,7 @@ void main() {
) * 0.08;
float breath = 1.0 + u_sigBreathe * sin(u_barPhase * 6.28318530718);
vec2 corner = vec2(-u_aspect + 0.2, -1.0 + 0.1) + cam;
vec2 corner = vec2(-u_aspect + 0.3, -1.0 + 0.1) + cam;
float blockH = 0.09 * breath;
float blockW = blockH * u_titleAspect;
vec2 blockBL = corner;
@ -181,7 +181,7 @@ void main() {
// --- shape: the signature form stamped as a monogram left of the title ---
float emD = blockH * 0.95;
vec2 emC = vec2(blockBL.x - blockH * 0.35, blockC.y);
vec2 emC = vec2(blockBL.x - blockH * 1.15, blockC.y);
float d = sigShape((p - emC) / max(emD * 0.5, 1e-3)) * (emD * 0.5);
float emFill = smoothstep(u_sigSoft * emD * 0.15, -u_sigSoft * emD * 0.15, d);
float emEdge = sigEdge(d);

18
flow-state/src/engine/shader-contract.js
View File

@ -73,6 +73,20 @@ export const SIGNATURE_UNIFORMS = {
u_sigSoft: 'float', // edge softness
u_sigTexture: 'float', // surface grain
u_sigFold: 'float', // kaleidoscopic folds, 1 = none
// FRAMING. Not a personality trait — this one is per SHOT, pushed by the
// arc driver rather than derived from the track. See look/framing.js.
//
// Every scene in the library is a locked-off, full-frame wide, and always
// has been. That is one shot type, held for the length of a song, and it is
// the reason cutting between two scenes changes the subject but never the
// FRAMING — which is at least half of how a real edit holds attention.
//
// Applied inside sigCamera, in scene coordinates, so a close-up is rendered
// close rather than being a magnified 720p image. That distinction is the
// whole reason this is a coordinate transform and not a post pass.
u_sigFrameScale: 'float', // >1 pushes in, <1 pulls back
u_sigFrameShift: 'vec2', // recentre, in scene units
};
export const FRAME_UNIFORMS = [
@ -210,6 +224,10 @@ float sigForm(vec2 p, vec2 centre, float size) {
*/
vec2 sigCamera(vec2 p) {
float t = u_time;
// Framing first: everything below is the operator's hand on a shot that has
// already been set up, so it composes on top of the framing rather than
// fighting it.
p = p / max(u_sigFrameScale, 0.05) + u_sigFrameShift;
p = rot(u_sigSpin * t) * p;
p *= 1.0 - u_sigBreathe * sin(u_barPhase * 6.28318530718);
p += vec2(sin(t * u_sigSwayRate), cos(t * u_sigSwayRate * 0.83)) * u_sigSway;

37
flow-state/src/look/ArcDriver.js
View File

@ -3,6 +3,7 @@ import { Rng } from '../engine/rng.js';
import { clampValue } from '../params/schema.js';
import { paletteShiftAt } from './paletteArc.js';
import { shiftPalette } from './palette.js';
import { frameShot, neutralFraming } from './framing.js';
/**
* Drives the look across the song.
@ -39,9 +40,35 @@ export class ArcDriver {
this.driftPlans = new Map();
this.activeLayers = [];
this.cues = this._buildCues();
this.framingStyle = (look.framing && look.framing.mode !== 'locked')
? look.framing : null;
this._planFraming();
this.state = { sectionIndex: 0, shotIndex: 0, crossfade: 0, incoming: null };
}
/**
* Give every cue the framing its shot will be played with.
*
* Framing is the one device that lives at the SHOT level — a wide answered
* by a close is the whole point of it, and both belong to the same scene.
* Planned here, once, walking the cues in order, so the whole video gets a
* consistent hand and a seek always finds the same framing as playback.
* A track that decided to stay locked-off (or a hand-built look with no
* framing) gets the neutral, unframed read everywhere.
*/
_planFraming() {
if (!this.framingStyle) return;
const rng = new Rng((this.look.seed ^ 0x517cc1b7) >>> 0);
let previous = null;
for (const cue of this.cues) {
const section = this.look.sections[cue.sectionIndex];
const energy = (section.bias && section.bias.energy) || 0;
const framing = frameShot(this.framingStyle, previous, energy, rng);
cue.framing = framing;
previous = framing;
}
}
dispose() {
for (const layer of this.layerCache.values()) layer.dispose();
this.layerCache.clear();
@ -386,6 +413,12 @@ export class ArcDriver {
const t = fading ? framesIntoCue / cue.fadeFrames : 1;
const eased = t * t * (3 - 2 * t);
// The shot being played INTO carries its own framing; the shot fading
// out keeps the framing it was filmed with, so a cut changes the size
// exactly when the cut changes the image rather than half a beat after.
const framing = cue.framing || neutralFraming();
const outgoingFraming = previous ? (previous.framing || neutralFraming()) : framing;
const layers = [];
if (fading) {
@ -409,6 +442,7 @@ export class ArcDriver {
layer.blend = slot === 0 ? 'normal' : spec.blend;
layer.setPalette(palette);
layer.setPersonality(this.look.personality);
layer.setFraming(outgoingFraming);
layers.push(layer);
}
}
@ -421,6 +455,7 @@ export class ArcDriver {
layer.blend = slot === 0 ? 'normal' : spec.blend;
layer.setPalette(palette);
layer.setPersonality(this.look.personality);
layer.setFraming(framing);
layers.push(layer);
}
@ -462,6 +497,7 @@ export class ArcDriver {
}
// A reroll re-plans the section's shots, so the cue list is stale too.
this.cues = this._buildCues();
this._planFraming();
this._slopeCache = null;
}
@ -469,6 +505,7 @@ export class ArcDriver {
this.dispose();
this.driftPlans.clear();
this.cues = this._buildCues();
this._planFraming();
this._slopeCache = null;
}

6
flow-state/src/look/LookGenerator.js
View File

@ -13,6 +13,7 @@ import { generatePersonality, sceneHonours, describePersonality } from './Person
import { deriveGrain, describeGrain, applyGrainToPost } from './grain.js';
import { pickDirector, directorByName } from './directors.js';
import { derivePaletteArc, describePaletteArc } from './paletteArc.js';
import { deriveFramingStyle, describeFraming } from './framing.js';
// Which families suit which section kind now comes from the track's DIRECTOR
// (look/directors.js) rather than from a constant here. The coupling it
@ -292,6 +293,8 @@ export function generateLook(track, { seed = null, samples = null, overrides = n
const grain = deriveGrain(summary, rng.fork('grain'));
// How the palette MOVES over the track. See look/paletteArc.js.
const paletteArc = derivePaletteArc(summary, rng.fork('paletteArc'));
// Whether shots change SIZE at the cut, and how boldly. See look/framing.js.
const framing = deriveFramingStyle(summary, rng.fork('framing'));
const { post, feedback } = derivePost(summary, rng.fork('post'), grain);
// Scenes that declare role 'accent' composite over a background rather than
@ -348,6 +351,7 @@ export function generateLook(track, { seed = null, samples = null, overrides = n
paletteScheme: paletteSource.lastScheme,
director: director.name,
paletteArc,
framing,
grain,
post,
feedback,
@ -427,7 +431,7 @@ export function describeLook(look) {
const kinds = look.sections.map((s) => `${s.kind}:${s.layers[0].module.name}`);
return `seed ${look.seed.toString(16)} · ${look.director} · ${look.paletteScheme} · ` +
`${describePersonality(look.personality)} · ${describeGrain(look.grain)} · ` +
`${describePaletteArc(look.paletteArc)} · ` +
`${describePaletteArc(look.paletteArc)} · ${describeFraming(look.framing)} · ` +
`${[...new Set(kinds)].join(', ')}`;
}

7
flow-state/src/look/Personality.js
View File

@ -205,6 +205,11 @@ export function signatureUniforms(personality, module = null) {
u_sigSoft: style.softness,
u_sigTexture: style.texture * textureAffinity,
u_sigFold: style.symmetry,
// Framing is per shot, not per track: the arc driver overwrites these
// every frame. Neutral here so a layer built without one is unframed.
u_sigFrameScale: 1,
u_sigFrameShift: [0, 0],
};
}
@ -225,6 +230,8 @@ export const NEUTRAL_UNIFORMS = {
u_sigSoft: 0.5,
u_sigTexture: 0,
u_sigFold: 1,
u_sigFrameScale: 1,
u_sigFrameShift: [0, 0],
};
const SHAPE_NAMES = { 0: 'round', 3: 'triangular', 4: 'square', 5: 'pentagonal', 6: 'hexagonal', 8: 'octagonal' };

114
flow-state/src/look/framing.js Normal file
View File

@ -0,0 +1,114 @@
// How a shot is FRAMED, as opposed to what it contains.
//
// Every scene in the library is a locked-off, full-frame wide, and always has
// been. That is one shot type, held for the length of a song. Cutting between
// two scenes therefore changes the subject and never the framing — and framing
// is at least half of how a real edit holds attention. A wide answered by a
// close reads as two shots of one thing; two wides read as two things.
//
// So a shot now carries a scale and a recentre, applied in SCENE coordinates
// inside sigCamera. That distinction matters: a close-up is rendered close
// rather than being a magnified 720p frame, which is why this is a coordinate
// transform and not a post pass. It is also why it costs nothing at 4K.
//
// Framing is per shot and constant within it. A zoom that moves during a shot
// is a different device — one that would fight the drift LFO and the slow axis,
// both of which already own continuous motion.
/**
* The shot sizes, as multipliers on the scene's coordinate scale.
*
* Bounded much more tightly than a real camera would be. Past about 2.2 most
* scenes in this library run out of detail and a close-up is just a blurry
* wide; below about 0.55 the subject is a speck in an empty frame. Both were
* measured by pushing until the image stopped being worth looking at.
*/
export const SHOT_SIZES = {
wide: { scale: 0.62, drift: 0.06 },
normal: { scale: 1.0, drift: 0.05 },
close: { scale: 1.7, drift: 0.10 },
};
export const SHOT_SIZE_NAMES = Object.keys(SHOT_SIZES);
/**
* Whether this track uses framing at all, and how boldly.
*
* A track that never changes size is a legitimate look — locked-off is a style,
* and it is the one the whole library was built in — so it stays reachable.
* What is not acceptable is it being the only option, which is what it was.
*/
export function deriveFramingStyle(summary, rng) {
const mode = rng.pickWeighted(['locked', 'gentle', 'edited'], [1, 2.5, 3]);
return {
mode,
// How far from `normal` this track is willing to go.
range: mode === 'locked' ? 0 : mode === 'gentle' ? 0.45 : 1,
// Chance a cut also changes the shot size, rather than only the image.
changeChance: mode === 'locked' ? 0 : mode === 'gentle' ? 0.35 : 0.6,
};
}
/**
* Choose the framing for one shot.
*
* `previous` is the framing of the shot before it, and it is the whole point:
* a size only means something relative to the size before it. The rule is that
* a change of size must be a real change — a wide answered by a slightly less
* wide is not a cut, it is a mistake — so sizes step rather than slide.
*
* @param {object} style from deriveFramingStyle
* @param {object|null} previous the previous shot's framing
* @param {number} energy section energy, 0..1
* @param {Rng} rng
*/
export function frameShot(style, previous, energy, rng) {
if (style.mode === 'locked') return neutralFraming();
const keep = previous && !rng.bool(style.changeChance);
if (keep) return { ...previous };
// Loud material earns the close-ups; quiet material earns the wides. This
// is a lean rather than a rule, so an intro can still land on a close and
// read as intimate instead of empty.
const weights = [
1 + (1 - energy) * 2.5, // wide
2, // normal
1 + energy * 2.5, // close
];
let size = rng.pickWeighted(SHOT_SIZE_NAMES, weights);
// Never repeat the previous size when we have decided to change: repeating
// it is what "no change" already means.
if (previous && size === previous.size) {
const others = SHOT_SIZE_NAMES.filter((n) => n !== size);
size = rng.pick(others);
}
const spec = SHOT_SIZES[size];
// Scale toward 1 for a timid track, so `gentle` is genuinely gentle rather
// than the same sizes drawn less often.
const scale = 1 + (spec.scale - 1) * style.range;
// Recentring is what stops a close-up being a centre crop of the wide. Held
// small: the scenes are centred compositions and pushing far off centre
// finds their empty corners.
const angle = rng.range(0, Math.PI * 2);
const amount = spec.drift * style.range * rng.range(0.3, 1);
return {
size,
scale,
shift: [Math.cos(angle) * amount, Math.sin(angle) * amount],
};
}
export function neutralFraming() {
return { size: 'normal', scale: 1, shift: [0, 0] };
}
/** One line for the HUD and check output. */
export function describeFraming(style) {
if (!style || style.mode === 'locked') return 'framing: locked off';
return `framing: ${style.mode}`;
}