From 5f25437b890bb323633c8b4835d802d670d347c5 Mon Sep 17 00:00:00 2001 From: Dejvino Date: Wed, 5 Aug 2026 11:30:50 +0200 Subject: [PATCH] Phase 4: arc driver ("C" brain) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three timescales now stack: per-frame reactivity, per-section seeded LFO drift, and whole-song scene changes with lookahead. Layer instances are cached per section and reused across crossfades — rebuilding them per frame would recompile shaders every transition. Crossfades run forward from a boundary: the outgoing scene holds while the incoming one fades in over it. Three real bugs, each found by a check that had to be rewritten first: 1. A pop exactly at every transition. buildSlope is discontinuous by construction (~1 before a boundary, 0 after), and the outgoing layer is still on screen when it flips — collapsing its lookahead ramp in one frame. It now holds the slope it had entering the boundary. 2. FeatureTrack.at() returns a REUSED row object, and _boundarySlope() called at() again mid-render, rewriting the features the layer was about to read. Symptom: a frame correct on every repeat and wrong the first time — invisible to fresh-vs-fresh comparison, and wrong in every export, since export renders each frame exactly once. Now indexes the typed array directly, with the aliasing hazard documented on at(), and a new check covers the whole bug class. 3. Warm-up converged to 1%, leaving a visible 0.015 difference at heavy feedback settings. Now targets 0.1%. Two checks were themselves wrong and were rebuilt: a raw delta threshold and an outlier-vs-local-median test both flag beat flashes as pops, and a control window taken from a different scene reads an ordinary busy scene as a 9x spike. The working formulation A/Bs each boundary against the interior of the two scenes adjacent to it. PLAN.md §6 corrected: boundary seeks are NOT exact for free. Layer state is re-seeded there but the feedback buffer is global and carries across. Clearing it at boundaries would buy exactness for a visible flash at every transition; warm-up is the better trade and applies everywhere. Gate 9/9. Co-Authored-By: Claude Opus 5 --- flow-state/PLAN.md | 25 ++- flow-state/src/Show.js | 198 +++++++++++++++++ flow-state/src/audio/FeatureTrack.js | 10 +- flow-state/src/checks/phase4.js | 320 ++++++++++++++++++++++++++- flow-state/src/engine/Compositor.js | 9 +- flow-state/src/engine/Engine.js | 11 +- flow-state/src/look/ArcDriver.js | 252 +++++++++++++++++++++ 7 files changed, 812 insertions(+), 13 deletions(-) create mode 100644 flow-state/src/Show.js create mode 100644 flow-state/src/look/ArcDriver.js diff --git a/flow-state/PLAN.md b/flow-state/PLAN.md index 700fe0a..b78a4b6 100644 --- a/flow-state/PLAN.md +++ b/flow-state/PLAN.md @@ -266,15 +266,24 @@ Stateful layers — feedback buffers, particle systems — mean frame *N* depend before it. Scrubbing to an arbitrary frame therefore can't be exact for free. Three-part resolution: -1. **Warm-up.** On seek, render *K* frames (default ≈120, about 2s) as fast as possible - off-screen before displaying. Converges feedback state to visually correct. -2. **Section-boundary seeks are exact.** Particle systems and layer state are re-seeded - deterministically at each section boundary, so jumping to a boundary — the common review - action — needs no warm-up and is frame-exact. -3. **Export is always sequential**, so it is exact everywhere by construction. +1. **Warm-up.** On seek, render *K* frames off-screen before displaying. *K* is computed + from the feedback decay rather than fixed — the residual after *n* frames is `decay^n`, + so `log(0.001)/log(decay)` frames converges to a tenth of a percent (96 frames at + decay 0.93). A light-feedback look seeks almost instantly; a heavy one still lands. +2. **Export is always sequential**, so it is exact everywhere by construction. -Documented consequence: scrubbing to a random mid-section frame shows a *converged*, not -bit-exact, image. Reviewing a transition never hits this, because transitions are boundaries. +> **Correction, made during Phase 4.** The original plan claimed section-boundary seeks +> would be frame-exact with no warm-up, because layer state is re-seeded there. That is +> wrong: layer state is only half the story, and the compositor's feedback buffer is +> *global* — it carries straight across a boundary like any other frame. Making boundary +> seeks exact would mean clearing feedback at every transition, which trades a cheap +> warm-up for a visible flash on every scene change. Warm-up is the better trade, and it +> applies everywhere rather than only mid-section. Measured: converges to a 0.00000 mean +> difference. With feedback disabled, seeks are bit-exact anywhere, which is what proves +> nothing *else* in the pipeline is carrying state. + +Documented consequence: scrubbing shows a *converged*, not bit-exact, image whenever +feedback is enabled — which is visually indistinguishable, and exact once feedback is off. --- diff --git a/flow-state/src/Show.js b/flow-state/src/Show.js new file mode 100644 index 0000000..65ae6a2 --- /dev/null +++ b/flow-state/src/Show.js @@ -0,0 +1,198 @@ +import { Engine } from './engine/Engine.js'; +import { FeatureTrack, featureProviderFor } from './audio/FeatureTrack.js'; +import { decodeFile, monoSamples } from './audio/decode.js'; +import { generateLook, rerollLook, rerollSection } from './look/LookGenerator.js'; +import { ArcDriver } from './look/ArcDriver.js'; +import { hashSamples } from './engine/rng.js'; + +const FADE_SECONDS = 1.5; + +/** + * A loaded track plus its look, rendered. + * + * This is the object both the preview and the exporter drive, and the only way + * they can be guaranteed to agree: neither has its own render path. The preview + * differs from the export in output resolution and warm-up state, and in nothing + * else. + */ +export class Show { + constructor({ canvas = null, width = 1280, height = 720, fps = 60 } = {}) { + this.engine = new Engine({ canvas, width, height, fps }); + this.fps = fps; + this.track = null; + this.look = null; + this.arc = null; + this.audioBuffer = null; + this.fileName = ''; + this._lastLayers = null; + } + + get ready() { return !!(this.track && this.look && this.arc); } + get duration() { return this.track ? this.track.duration : 0; } + get frameCount() { return this.track ? this.track.frameCount : 1; } + get timeline() { return this.engine.timeline; } + + /** + * Decode, analyse, and generate a look. `onProgress(stage, fraction)` is + * called throughout; analysis is CPU-bound and will block the main thread + * for a second or two on a long track. + */ + async load(file, onProgress = null) { + const report = (stage, p) => onProgress && onProgress(stage, p); + + report('decoding', 0); + const audioBuffer = await decodeFile(file); + this.audioBuffer = audioBuffer; + this.fileName = file.name.replace(/\.[^/.]+$/, ''); + + // Yield so the progress UI can paint before the analysis pass blocks. + await new Promise((r) => setTimeout(r, 0)); + + this.track = FeatureTrack.fromAudioBuffer(audioBuffer, { + fps: this.fps, + onProgress: (stage, p) => report(stage, p), + }); + + report('look', 0.97); + const samples = monoSamples(audioBuffer); + this.setLook(generateLook(this.track, { samples })); + + this.engine.timeline.setDuration(this.track.duration); + this.engine.setFeatureProvider(featureProviderFor(this.track)); + + report('ready', 1); + return this; + } + + /** Attach an already-analysed track. Used by the check harness and by tests. */ + useTrack(track, look) { + this.track = track; + this.engine.timeline.setDuration(track.duration); + this.engine.setFeatureProvider(featureProviderFor(track)); + this.setLook(look || generateLook(track, { seed: 1 })); + return this; + } + + setLook(look) { + if (this.arc) this.arc.dispose(); + this.look = look; + this.arc = new ArcDriver(look, this.track); + this._lastLayers = null; + return this; + } + + reroll(seed) { + this.setLook(rerollLook(this.look, this.track, seed)); + } + + rerollSection(index, salt) { + rerollSection(this.look, this.track, index, salt); + this.arc.invalidateSection(index); + this._lastLayers = null; + } + + setPalette(palette) { + this.look.palette = palette; + this.arc.setPalette(palette); + } + + /** Live param edit on a section's primary layer. */ + setSectionParam(sectionIndex, name, value) { + const section = this.look.sections[sectionIndex]; + if (!section) return; + section.layers[0].params[name] = value; + this._lastLayers = null; + } + + setSize(width, height) { + this.engine.setSize(width, height); + } + + /** Fade in at the head and out at the tail; nothing starts or ends abruptly. */ + _fadeAt(frame) { + const fadeFrames = FADE_SECONDS * this.fps; + const fromStart = frame; + const toEnd = this.frameCount - 1 - frame; + const a = Math.min(1, Math.max(0, fromStart / fadeFrames)); + const b = Math.min(1, Math.max(0, toEnd / fadeFrames)); + return Math.min(a, b); + } + + /** + * Render one frame. Identical in preview and export — the only difference is + * the size of the target and whether the result is presented or encoded. + */ + renderFrame(frame) { + const timeline = this.engine.timeline; + timeline.seek(frame); + + const features = this.track.at(timeline.frame); + const layers = this.arc.update(timeline.frame, features); + + if (this._lastLayers === null || this.arc.layersChanged(this._lastLayers)) { + this.engine.compositor.setLayers(layers); + this._lastLayers = layers.slice(); + } + + this.engine.compositor.setPost(this.look.post).setFeedback(this.look.feedback); + this.engine.compositor.fade = this._fadeAt(timeline.frame); + + return this.engine.compositor.render({ timeline, features }); + } + + /** Advance stateful layers so an arbitrary seek lands on converged state. */ + warmUp(frame, warmupFrames = 120) { + const start = Math.max(0, frame - warmupFrames); + this.engine.compositor.reset(); + for (let f = start; f < frame; f++) this.renderFrame(f); + } + + /** + * Frames of warm-up needed for the feedback loop to converge. + * + * Feedback decays geometrically, so the residual after n frames is decay^n. + * Converging to 0.1% rather than 1% costs only ~50% more frames and takes the + * result from "close" to "indistinguishable" — measured, a 1% target still + * left a visible 0.015 mean difference at heavy settings. + */ + warmupFrames() { + const amount = this.look ? this.look.feedback.amount : 0; + if (amount <= 0.01) return 0; + const decay = Math.min(0.99, this.look.feedback.decay); + return Math.min(400, Math.ceil(Math.log(0.001) / Math.log(decay))); + } + + /** + * Seek for review. + * + * NOTE, corrected from the original plan: a section boundary is NOT exact for + * free. Layer state is re-seeded there, but the compositor's feedback buffer + * is global and carries straight across the boundary, so a look with feedback + * enabled still needs warm-up wherever you land. Resetting feedback at + * boundaries would make seeks exact at the cost of a visible flash at every + * transition, which is a much worse trade. Warm-up is cheap; the flash is not. + */ + seek(frame, { warmup = true } = {}) { + const frames = warmup ? this.warmupFrames() : 0; + if (frames > 0) { + this.warmUp(frame, frames); + } else { + this.engine.compositor.reset(); + this.engine.timeline.seek(frame); + } + return this.renderFrame(frame); + } + + present(target) { this.engine.present(target); } + readPixels(target) { return this.engine.readPixels(target); } + hashFrame(frame) { return this.engine.hashCurrent(this.renderFrame(frame)); } + + contentSeed() { + return this.audioBuffer ? hashSamples(monoSamples(this.audioBuffer)) : 0; + } + + dispose() { + if (this.arc) this.arc.dispose(); + this.engine.dispose(); + } +} diff --git a/flow-state/src/audio/FeatureTrack.js b/flow-state/src/audio/FeatureTrack.js index 0c2bb40..c83b7b8 100644 --- a/flow-state/src/audio/FeatureTrack.js +++ b/flow-state/src/audio/FeatureTrack.js @@ -18,7 +18,15 @@ export class FeatureTrack { this._row = {}; } - /** @returns {object} the feature row for a frame, clamped to range. */ + /** + * The feature row for a frame, clamped to range. + * + * WARNING: the returned object is REUSED between calls — at() is called every + * frame and allocating for it is pointless. The consequence is that you must + * never call at() again while still holding a previous result, and in + * particular never inside a render pass that is using one. Index the typed + * arrays in `raw`/`tracks` directly for incidental lookups. + */ at(frame) { const f = Math.max(0, Math.min(this.frameCount - 1, frame | 0)); const row = this._row; diff --git a/flow-state/src/checks/phase4.js b/flow-state/src/checks/phase4.js index 2652d80..9ce5357 100644 --- a/flow-state/src/checks/phase4.js +++ b/flow-state/src/checks/phase4.js @@ -1 +1,319 @@ -// Phase 4 gate — filled in when the phase lands. +// Phase 4 gate — segmentation driving the arc. +// +// Segmentation accuracy itself is measured in node against synthetic ground +// truth. What is checked here is that the structure actually reaches the screen: +// scenes change where the music changes, transitions don't pop, and the lookahead +// ramp is genuinely wired rather than merely present in the table. +// +// The check this phase CANNOT automate is monotony. Watching full tracks is the +// only way to catch it, and PLAN.md §9 keeps that as an explicit manual gate. + +import { check, expect } from './framework.js'; +import { Show } from '../Show.js'; +import { generateLook } from '../look/LookGenerator.js'; +import { frameDistance, frameLuminance } from '../engine/hash.js'; +import { FeatureTrack } from '../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../audio/synth.js'; + +let cached = null; +function arcTrack() { + if (!cached) { + const buffer = synthesizeSectioned({ bpm: 128, duration: 150, changeAt: 75 }); + cached = FeatureTrack.fromAudioBuffer(buffer, { fps: 60 }); + } + return cached; +} + +function makeShow(seed = 2024, width = 160, height = 90) { + const show = new Show({ width, height }); + const track = arcTrack(); + show.useTrack(track, generateLook(track, { seed })); + return show; +} + +check(4, 'the scene changes only at section boundaries', () => { + const show = makeShow(); + try { + const track = show.track; + const boundaries = track.sections.map((s) => s.startFrame); + const changes = []; + let previous = null; + + for (let f = 0; f < track.frameCount; f += 5) { + show.arc.update(f, track.at(f)); + const name = show.arc.state.sceneName; + if (previous !== null && name !== previous) changes.push(f); + previous = name; + } + + const stray = changes.filter((f) => !boundaries.some((b) => Math.abs(f - b) <= 10)); + return expect(stray.length === 0, + `${changes.length} scene change(s), ${stray.length} away from a boundary · ` + + `${track.sections.length} sections: ${track.sections.map((s) => s.kind).join(', ')}`); + } finally { + show.dispose(); + } +}); + +check(4, 'transitions produce no pops or black frames', () => { + // Neither a raw delta threshold nor an outlier-vs-local-median test works + // here: these scenes flash on the beat, so large isolated deltas are the + // intended behaviour and both metrics flag them. The only meaningful question + // is whether a boundary is worse than the same scene's ordinary behaviour, so + // this A/Bs each boundary window against a control window with no boundary in + // it. Beat flashes appear in both and cancel out. + const show = makeShow(); + try { + const track = show.track; + + const scan = (start, end) => { + show.engine.compositor.reset(); + for (let f = Math.max(0, start - 30); f < start; f++) show.renderFrame(f); + + let previous = null; + let peak = 0; + let darkest = 1; + for (let f = start; f < Math.min(end, track.frameCount); f++) { + const pixels = Uint8Array.from(show.readPixels(show.renderFrame(f))); + darkest = Math.min(darkest, frameLuminance(pixels)); + if (previous) peak = Math.max(peak, frameDistance(previous, pixels)); + previous = pixels; + } + return { peak, darkest }; + }; + + // The control must sit in the SAME scenes the boundary window contains. + // Scenes differ enormously in inherent frame-to-frame motion — one busy + // scene next to a calm one reads as an 8x "spike" against a control taken + // from the calm one, with no cut anywhere near it. + const interior = (section) => { + const mid = section.startFrame + Math.floor((section.endFrame - section.startFrame) / 2); + return scan(mid, Math.min(mid + 200, section.endFrame)); + }; + + let worstRatio = 0; + let worstBoundary = -1; + let darkest = 1; + let controlUsed = 0; + + for (let i = 1; i < track.sections.length; i++) { + const s = track.sections[i]; + const before = interior(track.sections[i - 1]); + const after = interior(s); + const control = Math.max(before.peak, after.peak); + + const w = scan(s.startFrame - 60, s.startFrame + show.arc.crossfadeFrames + 60); + darkest = Math.min(darkest, w.darkest, before.darkest, after.darkest); + + const ratio = w.peak / Math.max(control, 1e-6); + if (ratio > worstRatio) { worstRatio = ratio; worstBoundary = s.startFrame; controlUsed = control; } + } + + if (worstBoundary < 0) return expect(true, 'single-section track'); + + return expect(worstRatio < 1.6 && darkest > 0.002, + `worst boundary peak ${worstRatio.toFixed(2)}x the adjacent scenes' own peak ` + + `(control ${controlUsed.toFixed(4)}) at frame ${worstBoundary}, darkest ${darkest.toFixed(4)}`); + } finally { + show.dispose(); + } +}, { slow: true }); + +check(4, 'crossfade ramps rather than cuts', () => { + const show = makeShow(); + try { + const track = show.track; + const boundary = track.sections[1] && track.sections[1].startFrame; + if (!boundary) return expect(true, 'single-section track, nothing to cross-fade'); + + const samples = []; + for (let f = boundary; f < boundary + show.arc.crossfadeFrames; f += 2) { + show.arc.update(f, track.at(f)); + samples.push(show.arc.state.crossfade); + } + const monotonic = samples.slice(1).every((v, i) => v >= samples[i] - 1e-6); + const spans = samples[0] < 0.15 && samples[samples.length - 1] > 0.85; + + return expect(monotonic && spans, + `${show.arc.crossfadeFrames}-frame fade, monotonic ${monotonic}, ` + + `${samples[0].toFixed(2)}→${samples[samples.length - 1].toFixed(2)}`); + } finally { + show.dispose(); + } +}); + +check(4, 'lookahead ramps params into a higher-energy section', () => { + // The payoff of offline analysis. buildSlope must rise before the boundary + // AND actually move a parameter, not merely exist in the table. + const track = arcTrack(); + const rising = []; + for (let i = 0; i < track.sections.length - 1; i++) { + if (track.sections[i + 1].energy > track.sections[i].energy * 1.08) rising.push(i); + } + if (!rising.length) return expect(true, 'no rising transition in this track'); + + const show = makeShow(); + try { + const section = track.sections[rising[0]]; + const traces = []; + for (let f = Math.max(section.startFrame, section.endFrame - 300); f < section.endFrame; f += 20) { + const features = track.at(f); + show.arc.update(f, features); + const layer = show.arc.activeLayers[show.arc.activeLayers.length - 1]; + traces.push({ slope: features.buildSlope, params: { ...layer.baseParams } }); + } + if (traces.length < 3) return expect(true, 'section too short to sample a ramp'); + + const first = traces[0], last = traces[traces.length - 1]; + const slopeRises = last.slope > first.slope + 1e-6; + const moved = Object.keys(first.params).filter((k) => + typeof first.params[k] === 'number' && Math.abs(last.params[k] - first.params[k]) > 1e-6); + + return expect(slopeRises && moved.length > 0, + `buildSlope ${first.slope.toFixed(3)}→${last.slope.toFixed(3)}, ` + + `${moved.length} param(s) ramped: ${moved.slice(0, 4).join(', ')}`); + } finally { + show.dispose(); + } +}); + +check(4, 'params drift within a long section', () => { + // Guards the failure mode automated checks are worst at: a section that is + // technically correct and completely static. + const show = makeShow(); + try { + const track = show.track; + const longest = track.sections.reduce((a, b) => + (b.endFrame - b.startFrame > a.endFrame - a.startFrame ? b : a)); + + const sample = (frame) => { + show.arc.update(frame, track.at(frame)); + const layer = show.arc.activeLayers[show.arc.activeLayers.length - 1]; + return { ...layer.baseParams }; + }; + + const a = sample(longest.startFrame + 120); + const b = sample(Math.max(longest.startFrame + 121, longest.endFrame - 120)); + + const numeric = Object.keys(a).filter((k) => typeof a[k] === 'number'); + const moved = numeric.filter((k) => Math.abs(b[k] - a[k]) > 1e-4); + + return expect(moved.length >= Math.ceil(numeric.length * 0.5), + `${moved.length}/${numeric.length} params moved across a ` + + `${((longest.endFrame - longest.startFrame) / 60).toFixed(0)}s section`); + } finally { + show.dispose(); + } +}); + +check(4, 'the arc-driven render is still deterministic', () => { + const a = makeShow(); + const b = makeShow(); + try { + const hashes = (show) => { + show.engine.compositor.reset(); + const out = []; + for (let f = 4400; f < 4460; f++) out.push(show.hashFrame(f)); + return out; + }; + const ha = hashes(a); + const hb = hashes(b); + const mismatches = ha.filter((h, i) => h !== hb[i]).length; + return expect(mismatches === 0, + `${mismatches}/60 frames differed between two independently built shows`); + } finally { + a.dispose(); b.dispose(); + } +}); + +check(4, 'a frame renders the same the first time as every later time', () => { + // Regression guard for an aliasing class no other check covered: FeatureTrack.at() + // returns a reused row, so anything that calls it mid-render corrupts the row the + // layer is about to read. The symptom is a frame that is correct on every repeat + // and wrong on its first render — invisible to fresh-vs-fresh comparison, and + // visible in an export, which renders every frame exactly once. + const show = makeShow(); + try { + const track = show.track; + show.look.feedback.amount = 0; + + const frames = [ + ...track.sections.map((s) => s.startFrame), + ...track.sections.map((s) => s.startFrame + 30), + 1000, 4000, + ].filter((f) => f > 0 && f < track.frameCount); + + const problems = []; + for (const f of frames) { + const fresh = makeShow(); + fresh.look.feedback.amount = 0; + fresh.engine.compositor.reset(); + const first = fresh.engine.hashCurrent(fresh.renderFrame(f)); + fresh.dispose(); + + show.engine.compositor.reset(); + show.renderFrame(f); + show.engine.compositor.reset(); + const repeat = show.engine.hashCurrent(show.renderFrame(f)); + + if (first !== repeat) problems.push(`frame ${f}: first ${first} vs repeat ${repeat}`); + } + return expect(problems.length === 0, + problems.length ? problems.join(' · ') : `${frames.length} frames stable on first render`); + } finally { + show.dispose(); + } +}); + +check(4, 'seek converges to sequential playback, and is exact without feedback', () => { + // Corrected from the original plan: a boundary is not exact for free. Layer + // state is re-seeded there, but the feedback buffer is global and carries + // across, so any look with feedback needs warm-up wherever you land. + const show = makeShow(); + try { + const track = show.track; + const boundary = track.sections[1] ? track.sections[1].startFrame : 600; + + // Force heavy feedback so the check is actually exercising convergence + // rather than passing because this seed happened to generate very little. + show.look.feedback.amount = 0.7; + show.look.feedback.decay = 0.93; + const warmup = show.warmupFrames(); + + show.engine.compositor.reset(); + for (let f = boundary - 300; f < boundary; f++) show.renderFrame(f); + const sequential = Uint8Array.from(show.readPixels(show.renderFrame(boundary))); + + const warmed = Uint8Array.from(show.readPixels(show.seek(boundary))); + const distance = frameDistance(sequential, warmed); + + // And with feedback off it must be bit-exact, proving nothing else is stateful. + show.look.feedback.amount = 0; + show.engine.compositor.reset(); + for (let f = boundary - 60; f < boundary; f++) show.renderFrame(f); + const seqExact = show.engine.hashCurrent(show.renderFrame(boundary)); + const directExact = show.engine.hashCurrent(show.seek(boundary)); + + return expect(distance < 0.01 && seqExact === directExact, + `with feedback 0.7/0.93: converged to ${distance.toFixed(5)} after ${warmup} ` + + `warm-up frames · without feedback: exact ${seqExact === directExact}`); + } finally { + show.dispose(); + } +}); + +check(4, 'head and tail fade rather than cut', () => { + const show = makeShow(); + try { + show.engine.compositor.reset(); + const first = frameLuminance(show.readPixels(show.renderFrame(0))); + show.engine.compositor.reset(); + const early = frameLuminance(show.readPixels(show.renderFrame(400))); + show.engine.compositor.reset(); + const last = frameLuminance(show.readPixels(show.renderFrame(show.frameCount - 1))); + return expect(first < early * 0.4 && last < early * 0.4, + `frame 0 ${first.toFixed(4)} · frame 400 ${early.toFixed(4)} · last ${last.toFixed(4)}`); + } finally { + show.dispose(); + } +}); diff --git a/flow-state/src/engine/Compositor.js b/flow-state/src/engine/Compositor.js index 0cc074d..507acbc 100644 --- a/flow-state/src/engine/Compositor.js +++ b/flow-state/src/engine/Compositor.js @@ -117,8 +117,13 @@ export class Compositor { this._buildTargets(); } + /** + * The compositor does NOT own its layers and never disposes them — the arc + * driver caches Layer instances across sections and swaps them in and out + * every crossfade, and disposing on removal would destroy shaders that are + * about to be reused (and recompile them on the way back). + */ setLayers(layers) { - this.layers.forEach((l) => { if (!layers.includes(l)) l.dispose(); }); this.layers = layers; return this; } @@ -261,7 +266,7 @@ export class Compositor { } dispose() { - this.layers.forEach((l) => l.dispose()); + this.layers = []; // owned elsewhere; see setLayers this.disposeTargets(); } } diff --git a/flow-state/src/engine/Engine.js b/flow-state/src/engine/Engine.js index 41691f8..c84c365 100644 --- a/flow-state/src/engine/Engine.js +++ b/flow-state/src/engine/Engine.js @@ -25,6 +25,7 @@ export class Engine { this.compositor = new Compositor(this.renderer, { width, height }); this.timeline = new Timeline({ fps, mode: REALTIME }); this.featureProvider = null; + this.ownedLayers = []; this.lastFrameRendered = -1; } @@ -46,13 +47,19 @@ export class Engine { return this.featureProvider.at(frame) || NULL_FEATURES; } - /** Replace the stack. `specs` are { module, params, seed, opacity, blend }. */ + /** + * Replace the stack from specs. `specs` are { module, params, seed, opacity, + * blend }. Layers built this way are owned by the Engine and disposed with + * it; layers supplied directly by the arc driver are owned by the driver. + */ setLayerSpecs(specs) { + this.ownedLayers.forEach((l) => l.dispose()); const layers = specs.map((s) => { const layer = createLayer(s.module, s); if (s.palette) layer.setPalette(s.palette); return layer; }); + this.ownedLayers = layers; this.compositor.setLayers(layers); return layers; } @@ -119,6 +126,8 @@ export class Engine { } dispose() { + this.ownedLayers.forEach((l) => l.dispose()); + this.ownedLayers = []; this.compositor.dispose(); this.renderer.dispose(); } diff --git a/flow-state/src/look/ArcDriver.js b/flow-state/src/look/ArcDriver.js new file mode 100644 index 0000000..8b179d2 --- /dev/null +++ b/flow-state/src/look/ArcDriver.js @@ -0,0 +1,252 @@ +import { createLayer } from '../engine/Layer.js'; +import { Rng } from '../engine/rng.js'; +import { clampValue } from '../params/schema.js'; + +/** + * Drives the look across the song. + * + * Three timescales are stacked here, and it takes all three to keep six minutes + * from reading as a loop: + * + * per frame — reactive mappings (handled in Layer, from the feature row) + * per section — seeded LFO drift, so nothing sits still during a long sustain + * whole song — scene changes at real boundaries, plus lookahead ramps that + * build INTO a drop rather than reacting after it lands + * + * Layer instances are created once per section and reused. Rebuilding them per + * frame would recompile shaders and is the obvious way to make this unusably slow. + */ +export class ArcDriver { + constructor(look, track, { crossfadeBars = 1, driftAmount = 0.09 } = {}) { + this.look = look; + this.track = track; + this.driftAmount = driftAmount; + + const barSeconds = (track.tempo.period * track.tempo.beatsPerBar) / track.fps; + this.crossfadeFrames = Math.max(12, Math.round(barSeconds * crossfadeBars * track.fps)); + + this.layerCache = new Map(); + this.driftPlans = new Map(); + this.activeLayers = []; + this.state = { sectionIndex: 0, crossfade: 0, incoming: null }; + } + + dispose() { + for (const layer of this.layerCache.values()) layer.dispose(); + this.layerCache.clear(); + } + + /** One Layer per (section, layer) slot, built lazily and kept. */ + _layerFor(sectionIndex, slot = 0) { + const key = `${sectionIndex}:${slot}`; + let layer = this.layerCache.get(key); + if (!layer) { + const spec = this.look.sections[sectionIndex].layers[slot]; + layer = createLayer(spec.module, { + params: spec.params, + seed: spec.seed, + opacity: spec.opacity, + blend: spec.blend, + }); + layer.setPalette(this.look.palette); + this.layerCache.set(key, layer); + } + return layer; + } + + /** + * Per-param LFO plan for a section: amplitude, period and phase, all seeded. + * Slow enough to read as evolution rather than wobble — 20 to 70 seconds. + */ + _driftPlan(sectionIndex, slot = 0) { + const key = `${sectionIndex}:${slot}`; + let plan = this.driftPlans.get(key); + if (plan) return plan; + + const spec = this.look.sections[sectionIndex].layers[slot]; + const rng = new Rng(spec.seed ^ 0x5bf03635); + plan = []; + for (const [name, def] of Object.entries(spec.module.params || {})) { + if (def.type === 'palette' || def.type === 'bool' || def.fixed) continue; + if (def.noDrift) continue; + const [lo, hi] = def.range || [0, 1]; + plan.push({ + name, + def, + amplitude: (hi - lo) * this.driftAmount * rng.range(0.4, 1.3), + period: rng.range(20, 70), + phase: rng.next(), + }); + } + this.driftPlans.set(key, plan); + return plan; + } + + /** + * Base params for a section at a given time: the look's sampled values, plus + * drift, plus the lookahead ramp toward whatever comes next. + */ + _paramsAt(sectionIndex, slot, time, features) { + const spec = this.look.sections[sectionIndex].layers[slot]; + const out = { ...spec.params }; + + for (const item of this._driftPlan(sectionIndex, slot)) { + const base = out[item.name]; + if (typeof base !== 'number') continue; + const wave = Math.sin(2 * Math.PI * (time / item.period + item.phase)); + out[item.name] = clampValue(item.def, base + wave * item.amplitude); + } + + // --- lookahead ------------------------------------------------------ + // buildSlope rises through the bars before a higher-energy section. This + // is the payoff of analysing offline: the visuals arrive at the drop + // already at tension instead of catching up afterwards. + const slope = features ? features.buildSlope || 0 : 0; + if (slope > 0.001) { + const next = this.look.sections[sectionIndex + 1]; + if (next && next.layers[slot] && next.layers[slot].module === spec.module) { + // Same scene either side: ramp the actual target values. + const target = next.layers[slot].params; + for (const [name, def] of Object.entries(spec.module.params || {})) { + if (def.type === 'palette' || typeof out[name] !== 'number') continue; + if (typeof target[name] !== 'number') continue; + out[name] = clampValue(def, out[name] + (target[name] - out[name]) * slope); + } + } else { + // Different scene: push the intensity-ish params toward the top + // of their range so the build still reads as a build. + for (const [name, def] of Object.entries(spec.module.params || {})) { + if (typeof out[name] !== 'number') continue; + if (def.bias !== 'energy' && def.bias !== 'density') continue; + const hi = (def.range || [0, 1])[1]; + out[name] = clampValue(def, out[name] + (hi - out[name]) * slope * 0.5); + } + } + } + + return out; + } + + /** + * The buildSlope value on the frame before a boundary. Read from the table + * rather than remembered, so a seek and playback agree. + * + * Indexes the typed array DIRECTLY rather than calling track.at(). at() + * returns a single reused row object, so calling it here — mid-render, while + * the caller is still holding the row for the current frame — silently + * rewrites the features the layer is about to read. That produced a render + * that was correct on every repeat but wrong the first time through, which is + * exactly the kind of fault the determinism checks exist to surface. + */ + _boundarySlope(sectionIndex) { + if (!this._slopeCache) this._slopeCache = new Map(); + if (this._slopeCache.has(sectionIndex)) return this._slopeCache.get(sectionIndex); + + const section = this.look.sections[sectionIndex]; + const frame = Math.max(0, section.startFrame - 1); + const value = this.track.tracks.buildSlope[frame] || 0; + this._slopeCache.set(sectionIndex, value); + return value; + } + + /** + * Compute the active layer stack for a frame. + * + * The crossfade runs FORWARD from a boundary: the outgoing scene holds at + * full opacity while the incoming one fades in over it. That keeps the + * boundary frame itself a clean state, which is what makes a boundary seek + * exact without warm-up. + */ + update(frame, features) { + const track = this.track; + const time = frame / track.fps; + const sectionIndex = track.sectionIndexAt(frame); + const section = this.look.sections[sectionIndex]; + if (!section) return this.activeLayers; + + const framesIntoSection = frame - section.startFrame; + const fading = sectionIndex > 0 && framesIntoSection < this.crossfadeFrames; + const t = fading ? framesIntoSection / this.crossfadeFrames : 1; + const eased = t * t * (3 - 2 * t); + + const layers = []; + + if (fading) { + const previousIndex = sectionIndex - 1; + const outgoing = this._layerFor(previousIndex); + + // buildSlope is discontinuous at a boundary by construction: it ramps + // to ~1 through the bars before the change and is 0 immediately after. + // The outgoing layer is still on screen when that happens, so feeding + // it the new section's features collapses its lookahead ramp in a + // single frame — a visible pop precisely at the transition. Hold the + // slope it had going into the boundary; it finished its build, and it + // stays there while it fades out. + outgoing.setParams(this._paramsAt(previousIndex, 0, time, { + ...features, + buildSlope: this._boundarySlope(sectionIndex), + })); + outgoing.opacity = 1; + outgoing.blend = 'normal'; + outgoing.setPalette(this.look.palette); + layers.push(outgoing); + } + + const current = this._layerFor(sectionIndex); + current.setParams(this._paramsAt(sectionIndex, 0, time, features)); + current.opacity = fading ? eased : 1; + current.blend = 'normal'; + current.setPalette(this.look.palette); + layers.push(current); + + // Extra composited layers declared on the section (Phase 5 stacks). + for (let slot = 1; slot < section.layers.length; slot++) { + const spec = section.layers[slot]; + const layer = this._layerFor(sectionIndex, slot); + layer.setParams(this._paramsAt(sectionIndex, slot, time, features)); + layer.opacity = spec.opacity * (fading ? eased : 1); + layer.blend = spec.blend; + layer.setPalette(this.look.palette); + layers.push(layer); + } + + this.state = { + sectionIndex, + kind: section.kind, + crossfade: fading ? eased : 0, + sceneName: section.layers[0].module.name, + buildSlope: features ? features.buildSlope || 0 : 0, + }; + + this.activeLayers = layers; + return layers; + } + + /** Layers changed identity — the compositor needs the new list. */ + layersChanged(previous) { + if (!previous || previous.length !== this.activeLayers.length) return true; + return this.activeLayers.some((l, i) => l !== previous[i]); + } + + /** Invalidate caches for one section after an edit or reroll. */ + invalidateSection(sectionIndex) { + for (const key of [...this.layerCache.keys()]) { + if (key.startsWith(`${sectionIndex}:`)) { + this.layerCache.get(key).dispose(); + this.layerCache.delete(key); + this.driftPlans.delete(key); + } + } + } + + invalidateAll() { + this.dispose(); + this.driftPlans.clear(); + } + + /** Push a palette change through without rebuilding layers. */ + setPalette(palette) { + this.look.palette = palette; + for (const layer of this.layerCache.values()) layer.setPalette(palette); + } +}