dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Phase 4: arc driver ("C" brain)

Browse Source 
Three timescales now stack: per-frame reactivity, per-section seeded LFO
drift, and whole-song scene changes with lookahead. Layer instances are
cached per section and reused across crossfades — rebuilding them per frame
would recompile shaders every transition.

Crossfades run forward from a boundary: the outgoing scene holds while the
incoming one fades in over it.

Three real bugs, each found by a check that had to be rewritten first:

1. A pop exactly at every transition. buildSlope is discontinuous by
   construction (~1 before a boundary, 0 after), and the outgoing layer is
   still on screen when it flips — collapsing its lookahead ramp in one
   frame. It now holds the slope it had entering the boundary.

2. FeatureTrack.at() returns a REUSED row object, and _boundarySlope()
   called at() again mid-render, rewriting the features the layer was about
   to read. Symptom: a frame correct on every repeat and wrong the first
   time — invisible to fresh-vs-fresh comparison, and wrong in every export,
   since export renders each frame exactly once. Now indexes the typed array
   directly, with the aliasing hazard documented on at(), and a new check
   covers the whole bug class.

3. Warm-up converged to 1%, leaving a visible 0.015 difference at heavy
   feedback settings. Now targets 0.1%.

Two checks were themselves wrong and were rebuilt: a raw delta threshold
and an outlier-vs-local-median test both flag beat flashes as pops, and a
control window taken from a different scene reads an ordinary busy scene as
a 9x spike. The working formulation A/Bs each boundary against the interior
of the two scenes adjacent to it.

PLAN.md §6 corrected: boundary seeks are NOT exact for free. Layer state is
re-seeded there but the feedback buffer is global and carries across.
Clearing it at boundaries would buy exactness for a visible flash at every
transition; warm-up is the better trade and applies everywhere.

Gate 9/9.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-08-05 11:30:50 +02:00
co-authored by Claude Opus 5
parent 022c267888
commit 5f25437b89
7 changed files with 812 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/look/ArcDriver.js
+252
View File
@@ -0,0 +1,252 @@
import { createLayer } from '../engine/Layer.js';
import { Rng } from '../engine/rng.js';
import { clampValue } from '../params/schema.js';
/**
* Drives the look across the song.
*
* Three timescales are stacked here, and it takes all three to keep six minutes
* from reading as a loop:
*
* per frame — reactive mappings (handled in Layer, from the feature row)
* per section — seeded LFO drift, so nothing sits still during a long sustain
* whole song — scene changes at real boundaries, plus lookahead ramps that
* build INTO a drop rather than reacting after it lands
*
* Layer instances are created once per section and reused. Rebuilding them per
* frame would recompile shaders and is the obvious way to make this unusably slow.
*/
export class ArcDriver {
constructor(look, track, { crossfadeBars = 1, driftAmount = 0.09 } = {}) {
this.look = look;
this.track = track;
this.driftAmount = driftAmount;
const barSeconds = (track.tempo.period * track.tempo.beatsPerBar) / track.fps;
this.crossfadeFrames = Math.max(12, Math.round(barSeconds * crossfadeBars * track.fps));
this.layerCache = new Map();
this.driftPlans = new Map();
this.activeLayers = [];
this.state = { sectionIndex: 0, crossfade: 0, incoming: null };
}
dispose() {
for (const layer of this.layerCache.values()) layer.dispose();
this.layerCache.clear();
}
/** One Layer per (section, layer) slot, built lazily and kept. */
_layerFor(sectionIndex, slot = 0) {
const key = `${sectionIndex}:${slot}`;
let layer = this.layerCache.get(key);
if (!layer) {
const spec = this.look.sections[sectionIndex].layers[slot];
layer = createLayer(spec.module, {
params: spec.params,
seed: spec.seed,
opacity: spec.opacity,
blend: spec.blend,
});
layer.setPalette(this.look.palette);
this.layerCache.set(key, layer);
}
return layer;
}
/**
* Per-param LFO plan for a section: amplitude, period and phase, all seeded.
* Slow enough to read as evolution rather than wobble — 20 to 70 seconds.
*/
_driftPlan(sectionIndex, slot = 0) {
const key = `${sectionIndex}:${slot}`;
let plan = this.driftPlans.get(key);
if (plan) return plan;
const spec = this.look.sections[sectionIndex].layers[slot];
const rng = new Rng(spec.seed ^ 0x5bf03635);
plan = [];
for (const [name, def] of Object.entries(spec.module.params || {})) {
if (def.type === 'palette' || def.type === 'bool' || def.fixed) continue;
if (def.noDrift) continue;
const [lo, hi] = def.range || [0, 1];
plan.push({
name,
def,
amplitude: (hi - lo) * this.driftAmount * rng.range(0.4, 1.3),
period: rng.range(20, 70),
phase: rng.next(),
});
}
this.driftPlans.set(key, plan);
return plan;
}
/**
* Base params for a section at a given time: the look's sampled values, plus
* drift, plus the lookahead ramp toward whatever comes next.
*/
_paramsAt(sectionIndex, slot, time, features) {
const spec = this.look.sections[sectionIndex].layers[slot];
const out = { ...spec.params };
for (const item of this._driftPlan(sectionIndex, slot)) {
const base = out[item.name];
if (typeof base !== 'number') continue;
const wave = Math.sin(2 * Math.PI * (time / item.period + item.phase));
out[item.name] = clampValue(item.def, base + wave * item.amplitude);
}
// --- lookahead ------------------------------------------------------
// buildSlope rises through the bars before a higher-energy section. This
// is the payoff of analysing offline: the visuals arrive at the drop
// already at tension instead of catching up afterwards.
const slope = features ? features.buildSlope || 0 : 0;
if (slope > 0.001) {
const next = this.look.sections[sectionIndex + 1];
if (next && next.layers[slot] && next.layers[slot].module === spec.module) {
// Same scene either side: ramp the actual target values.
const target = next.layers[slot].params;
for (const [name, def] of Object.entries(spec.module.params || {})) {
if (def.type === 'palette' || typeof out[name] !== 'number') continue;
if (typeof target[name] !== 'number') continue;
out[name] = clampValue(def, out[name] + (target[name] - out[name]) * slope);
}
} else {
// Different scene: push the intensity-ish params toward the top
// of their range so the build still reads as a build.
for (const [name, def] of Object.entries(spec.module.params || {})) {
if (typeof out[name] !== 'number') continue;
if (def.bias !== 'energy' && def.bias !== 'density') continue;
const hi = (def.range || [0, 1])[1];
out[name] = clampValue(def, out[name] + (hi - out[name]) * slope * 0.5);
}
}
}
return out;
}
/**
* The buildSlope value on the frame before a boundary. Read from the table
* rather than remembered, so a seek and playback agree.
*
* Indexes the typed array DIRECTLY rather than calling track.at(). at()
* returns a single reused row object, so calling it here — mid-render, while
* the caller is still holding the row for the current frame — silently
* rewrites the features the layer is about to read. That produced a render
* that was correct on every repeat but wrong the first time through, which is
* exactly the kind of fault the determinism checks exist to surface.
*/
_boundarySlope(sectionIndex) {
if (!this._slopeCache) this._slopeCache = new Map();
if (this._slopeCache.has(sectionIndex)) return this._slopeCache.get(sectionIndex);
const section = this.look.sections[sectionIndex];
const frame = Math.max(0, section.startFrame - 1);
const value = this.track.tracks.buildSlope[frame] || 0;
this._slopeCache.set(sectionIndex, value);
return value;
}
/**
* Compute the active layer stack for a frame.
*
* The crossfade runs FORWARD from a boundary: the outgoing scene holds at
* full opacity while the incoming one fades in over it. That keeps the
* boundary frame itself a clean state, which is what makes a boundary seek
* exact without warm-up.
*/
update(frame, features) {
const track = this.track;
const time = frame / track.fps;
const sectionIndex = track.sectionIndexAt(frame);
const section = this.look.sections[sectionIndex];
if (!section) return this.activeLayers;
const framesIntoSection = frame - section.startFrame;
const fading = sectionIndex > 0 && framesIntoSection < this.crossfadeFrames;
const t = fading ? framesIntoSection / this.crossfadeFrames : 1;
const eased = t * t * (3 - 2 * t);
const layers = [];
if (fading) {
const previousIndex = sectionIndex - 1;
const outgoing = this._layerFor(previousIndex);
// buildSlope is discontinuous at a boundary by construction: it ramps
// to ~1 through the bars before the change and is 0 immediately after.
// The outgoing layer is still on screen when that happens, so feeding
// it the new section's features collapses its lookahead ramp in a
// single frame — a visible pop precisely at the transition. Hold the
// slope it had going into the boundary; it finished its build, and it
// stays there while it fades out.
outgoing.setParams(this._paramsAt(previousIndex, 0, time, {
...features,
buildSlope: this._boundarySlope(sectionIndex),
}));
outgoing.opacity = 1;
outgoing.blend = 'normal';
outgoing.setPalette(this.look.palette);
layers.push(outgoing);
}
const current = this._layerFor(sectionIndex);
current.setParams(this._paramsAt(sectionIndex, 0, time, features));
current.opacity = fading ? eased : 1;
current.blend = 'normal';
current.setPalette(this.look.palette);
layers.push(current);
// Extra composited layers declared on the section (Phase 5 stacks).
for (let slot = 1; slot < section.layers.length; slot++) {
const spec = section.layers[slot];
const layer = this._layerFor(sectionIndex, slot);
layer.setParams(this._paramsAt(sectionIndex, slot, time, features));
layer.opacity = spec.opacity * (fading ? eased : 1);
layer.blend = spec.blend;
layer.setPalette(this.look.palette);
layers.push(layer);
}
this.state = {
sectionIndex,
kind: section.kind,
crossfade: fading ? eased : 0,
sceneName: section.layers[0].module.name,
buildSlope: features ? features.buildSlope || 0 : 0,
};
this.activeLayers = layers;
return layers;
}
/** Layers changed identity — the compositor needs the new list. */
layersChanged(previous) {
if (!previous || previous.length !== this.activeLayers.length) return true;
return this.activeLayers.some((l, i) => l !== previous[i]);
}
/** Invalidate caches for one section after an edit or reroll. */
invalidateSection(sectionIndex) {
for (const key of [...this.layerCache.keys()]) {
if (key.startsWith(`${sectionIndex}:`)) {
this.layerCache.get(key).dispose();
this.layerCache.delete(key);
this.driftPlans.delete(key);
}
}
}
invalidateAll() {
this.dispose();
this.driftPlans.clear();
}
/** Push a palette change through without rebuilding layers. */
setPalette(palette) {
this.look.palette = palette;
for (const layer of this.layerCache.values()) layer.setPalette(palette);
}
}