Every song in the bank, one frame every 30 seconds, left to right.
+ The gallery asks whether a scene looks the same in every song; this
+ asks whether a song looks the same as itself four minutes later.
+ Scored on direction — whether the distance between two
+ frames grows with the time between them — so the videos that are busy
+ and going nowhere sort to the top.
Drop a track and watch it. test/songs/*.wav holds the
diff --git a/flow-state/filmstrip.html b/flow-state/filmstrip.html
new file mode 100644
index 0000000..89c78ba
--- /dev/null
+++ b/flow-state/filmstrip.html
@@ -0,0 +1,240 @@
+
+
+
+ Every song in the bank, one frame every 30 seconds, + left to right. The gallery asks whether a scene looks the same in every song; + this asks whether a song looks the same as itself four minutes later. + A strip whose frames could be shuffled without anyone noticing is a video with + no arc, however busy it is. + drift is how far the video gets from itself and + direction is whether that distance grows with time — churn + scores the first and not the second. The climax frame is outlined. + ← all debug tools +
+ + + + + + + + diff --git a/flow-state/src/Show.js b/flow-state/src/Show.js index 0027802..87d07aa 100644 --- a/flow-state/src/Show.js +++ b/flow-state/src/Show.js @@ -7,7 +7,12 @@ import { ArcDriver } from './look/ArcDriver.js'; import { grainEnvelope } from './look/grain.js'; import { hashSamples } from './engine/rng.js'; -const FADE_SECONDS = 1.5; +/** + * The opening and closing fade to black. Exported because anything SAMPLING a + * video has to know about it: a probe at t=0 reads a black frame, which is + * correct output and a useless sample. See checks/filmstrip.js. + */ +export const FADE_SECONDS = 1.5; /** * A loaded track plus its look, rendered. diff --git a/flow-state/src/audio/segment.js b/flow-state/src/audio/segment.js index bb911d7..a5e8d92 100644 --- a/flow-state/src/audio/segment.js +++ b/flow-state/src/audio/segment.js @@ -14,7 +14,8 @@ export const SECTION_KINDS = ['intro', 'build', 'drop', 'sustain', 'breakdown', const ANALYSIS_HZ = 4; // coarse grid for the similarity matrix const KERNEL_SECONDS = 6; // half-width of the checkerboard kernel -const MIN_SECTION_SECONDS = 12; +const MIN_SECTION_SECONDS = 4; +const DROP_HEAD_BARS = 16; // how long a drop stays a drop before it is just the track function median(values) { if (!values.length) return 0; @@ -170,6 +171,31 @@ function classify(section, context) { return 'sustain'; } +/** + * A drop is an EVENT — the arrival — and what follows is the track's main body, + * however loud that stays. classify() has no memory of what came before, so a + * long loud stretch labels every one of its sections 'drop' and the video holds + * peak intensity for minutes on end. + * + * Collapse each run of drops down to its head: the opening sections keep the + * label until a phrase's worth of time is spent, and the rest become 'sustain'. + * Runs are read off the ORIGINAL labels so the demotion never cascades, and any + * other kind in between — a breakdown, a build — re-arms the next drop. + */ +function collapseDropRuns(sections, secondsPerBar) { + const budget = DROP_HEAD_BARS * secondsPerBar; + const original = sections.map((s) => s.kind); + let spent = 0; + + for (let i = 0; i < sections.length; i++) { + if (original[i] !== 'drop') { spent = 0; continue; } + const continues = i > 0 && original[i - 1] === 'drop'; + if (!continues) spent = 0; + else if (spent >= budget) sections[i].kind = 'sustain'; + spent += sections[i].duration; + } +} + /** * @returns {Array<{index,start,end,startFrame,endFrame,kind,energy,slope,flux,centroid}>} */ @@ -262,5 +288,8 @@ export function segment(raw, frameCount, fps, tempo) { ); }); + const bpm = Number.isFinite(tempo.bpm) && tempo.bpm > 1 ? tempo.bpm : 120; + collapseDropRuns(sections, (4 * 60) / bpm); + return sections; } diff --git a/flow-state/src/checks/filmstrip.js b/flow-state/src/checks/filmstrip.js new file mode 100644 index 0000000..2c72243 --- /dev/null +++ b/flow-state/src/checks/filmstrip.js @@ -0,0 +1,187 @@ +// The filmstrip: every song in the bank, sampled across its own length. +// +// The gallery answers "does this scene look the same in every song". This +// answers the question one level up and one axis over: does a SONG look the +// same as itself, half an hour of playback later. +// +// Nothing else shows that. The variety harness reduces a video to a number, the +// app shows one video at the speed of the song, and the phase gates never look +// at two moments of the same track side by side. So the failure this exists to +// catch — a video that is doing plenty and going nowhere — has until now been +// something you could only notice by watching four minutes and remembering what +// the first minute looked like. +// +// One frame every thirty seconds, straight across. A strip whose frames could +// be shuffled without anyone noticing is a video with no arc, whatever its +// drift score says. See look/Story.js for the layer meant to fix that, and +// checks/variety/signature.js directionOf for the same question as a number. +// +// Everything renders through the whole normal pipeline — Show, ArcDriver, post, +// the lot — rather than through a bare Engine as the gallery does, because the +// arc IS the subject here. + +import { Show, FADE_SECONDS } from '../Show.js'; +import { generateLook } from '../look/LookGenerator.js'; +import { song, SONGS } from '../audio/songbank.js'; +import { hashString } from '../engine/rng.js'; +import { frameDescriptor, motionDescriptor } from './variety/descriptors.js'; +import { descriptorDistance, directionOf, STRUCTURAL } from './variety/signature.js'; +import { describeStory } from '../look/Story.js'; + +export const STRIP = { width: 256, height: 144 }; + +/** Seconds between frames. Thirty is a section or two at most tempos. */ +export const DEFAULT_EVERY = 30; + +/** + * How long a song is synthesised for. + * + * Long enough to have an arc to sample, short enough that the whole bank + * finishes in one sitting: every extra thirty seconds is another frame for each + * of seventeen songs, and each frame costs its warm-up. `?duration=240` for the + * fuller version when a specific song is in question. + */ +export const DEFAULT_DURATION = 180; + +/** + * Frames of warm-up before each probe. + * + * Feedback and any stateful layer need to be converged or the probe measures + * the trail of a black frame — a structure every probe shares, which would make + * the whole strip look more alike than it is. Half of what the variety harness + * uses, because this is 17 songs deep and the cost is linear. + */ +const WARMUP = 24; + +export const songNames = () => SONGS.map((s) => s.name); + +const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0); + +function blockMean(byBlock) { + const values = STRUCTURAL.map((b) => byBlock[b]).filter((v) => v !== undefined); + return mean(values); +} + +/** + * Render one song's strip. + * + * @returns {{name, frames: Array, drift: number, direction: number, look}} + */ +export function renderSong(show, name, { duration, every }) { + const { track } = song(name, { duration }); + // Seeded off the song's name, so a strip is stable between rebuilds and two + // songs are not accidentally handed the same look. + const look = generateLook(track, { seed: hashString(name) }); + show.useTrack(track, look); + + const frames = []; + const descriptors = []; + // Pulled clear of the opening and closing fades. A probe at t=0 reads a + // black frame — correct output, useless sample — and the same at the very + // end, so the first and last cells sit just inside the video proper. The + // caption shows the time actually sampled rather than the nominal one. + const inside = (t) => Math.min( + Math.max(t, FADE_SECONDS + 0.5), + Math.max(0, track.duration - FADE_SECONDS - 0.5)); + + for (let nominal = 0; nominal < track.duration - 0.5; nominal += every) { + const t = inside(nominal); + const frame = Math.min(track.frameCount - 2, Math.round(t * track.fps)); + show.warmUp(frame, WARMUP); + + const pixels = Uint8Array.from(show.readPixels(show.renderFrame(frame))); + // A second frame five later, so the motion block is measured rather + // than silently scored zero — the same omission the gallery had. + const moved = Uint8Array.from(show.readPixels(show.renderFrame(frame + 5))); + + const state = show.arc.state; + const sectionIndex = track.sectionIndexAt(frame); + const section = look.sections[sectionIndex] || {}; + frames.push({ + time: t, + frame, + pixels, + kind: state.kind || section.kind || '', + scene: state.sceneName || '', + act: state.act || '', + tension: state.tension ?? 0.5, + journey: state.journey ?? 0, + moment: momentAt(look.story, sectionIndex), + }); + + const still = frameDescriptor(pixels, STRIP.width, STRIP.height); + const motion = motionDescriptor(pixels, moved, STRIP.width, STRIP.height); + descriptors.push({ ...still, motion: motion.scale.concat(motion.layout) }); + } + + // The two numbers that say what the strip shows, on the same descriptor the + // variety harness uses: how far this video gets from itself, and whether + // that distance grows with time or is just churn. + const gaps = []; + const dists = []; + for (let i = 0; i < descriptors.length; i++) { + for (let j = i + 1; j < descriptors.length; j++) { + gaps.push(frames[j].time - frames[i].time); + dists.push(blockMean(descriptorDistance(descriptors[i], descriptors[j]))); + } + } + + return { + name, + frames, + drift: mean(dists), + direction: directionOf(gaps, dists), + // How much the FIRST half of the video resembles the last — the reading + // that separates a video which ends somewhere new from one that has + // come back on purpose. A recap should show up here and nowhere else. + endsApart: dists.length ? dists[dists.length - 1] : 0, + story: look.story, + storyLine: describeStory(look.story), + director: look.director, + bpm: Math.round(track.summary.bpm), + sections: look.sections.length, + }; +} + +function momentAt(story, index) { + if (!story || !story.moments) return ''; + const m = story.moments; + if (index === m.climax) return 'climax'; + if (index === m.turn) return 'turn'; + if (index === m.arrival) return 'arrival'; + if (index === m.resolution) return 'resolution'; + return ''; +} + +/** + * Build every strip, reporting each song as it lands. + * + * One Show for the whole run: `useTrack` replaces the track and rebuilds the + * arc, and constructing a Show per song would rebuild the GL context seventeen + * times for nothing. + */ +export async function buildFilmstrip({ + duration = DEFAULT_DURATION, every = DEFAULT_EVERY, names = null, onSong = null, +} = {}) { + const list = names && names.length ? names : songNames(); + const show = new Show({ ...STRIP }); + const rows = []; + try { + for (let i = 0; i < list.length; i++) { + let row; + try { + row = renderSong(show, list[i], { duration, every }); + } catch (err) { + row = { name: list[i], frames: [], drift: 0, direction: 0, error: err.message }; + } + rows.push(row); + if (onSong) onSong(i + 1, list.length, row); + // Yield, so a row paints as it lands instead of the page freezing + // for the whole run and then showing everything at once. + await new Promise((r) => setTimeout(r, 0)); + } + } finally { + show.dispose(); + } + return rows; +} diff --git a/flow-state/src/checks/gallery-cache.js b/flow-state/src/checks/gallery-cache.js index 295c39c..1fa9353 100644 --- a/flow-state/src/checks/gallery-cache.js +++ b/flow-state/src/checks/gallery-cache.js @@ -82,7 +82,21 @@ export async function loadGallery(fingerprint) { } /** - * Store a build, and drop every other one. + * Which page a key belongs to. + * + * The store holds more than one page's builds now — the gallery keys on the + * bare source fingerprint, the filmstrip prefixes its sampling parameters — and + * "drop every other build" has to mean every other build OF THIS PAGE. Without + * the namespace the two evict each other on every save, and each page rebuilds + * for three minutes every time you visit the other one. + */ +const namespaceOf = (key) => { + const k = String(key); + return k.includes(':') ? k.slice(0, k.indexOf(':')) : 'gallery'; +}; + +/** + * Store a build, and drop every other one from the same page. * * Only the current source is ever wanted, and keeping stale builds around is how * a cache quietly grows to hundreds of megabytes of images nobody will look at. @@ -91,8 +105,11 @@ export async function saveGallery(fingerprint, payload) { try { const db = await openDb(); const keys = await tx(db, 'readonly', (store) => store.getAllKeys()); + const mine = namespaceOf(fingerprint); await tx(db, 'readwrite', (store) => { - for (const key of keys) if (key !== fingerprint) store.delete(key); + for (const key of keys) { + if (key !== fingerprint && namespaceOf(key) === mine) store.delete(key); + } return store.put(payload, fingerprint); }); db.close(); diff --git a/flow-state/src/checks/main.js b/flow-state/src/checks/main.js index 6811457..f9fd86b 100644 --- a/flow-state/src/checks/main.js +++ b/flow-state/src/checks/main.js @@ -19,6 +19,7 @@ import './phase9.js'; import './phase10.js'; import './phase11.js'; import './phase12.js'; +import './phase13.js'; const out = document.getElementById('results'); const summaryEl = document.getElementById('summary'); diff --git a/flow-state/src/checks/phase10.js b/flow-state/src/checks/phase10.js index ceec818..b144287 100644 --- a/flow-state/src/checks/phase10.js +++ b/flow-state/src/checks/phase10.js @@ -187,7 +187,7 @@ check(10, 'overlays happen sometimes and not always', () => { const look = generateLook(track, { seed: 900 + s * 5231 }); for (const stack of stacksOf(look)) { stacks++; - const overlay = stack.slice(1).find((l) => l.module.role !== 'accent'); + const overlay = stack[1]; if (overlay) { withOverlay++; blends.add(overlay.blend); @@ -215,7 +215,6 @@ check(10, 'an overlay never hides the shot underneath it', () => { for (let s = 0; s < 4; s++) { for (const stack of stacksOf(generateLook(track, { seed: 1300 + s * 8641 }))) { for (const layer of stack.slice(1)) { - if (layer.module.role === 'accent') continue; if (layer.opacity > 0.6) { problems.push(`${layer.module.name} at ${layer.opacity.toFixed(2)}`); } diff --git a/flow-state/src/checks/phase11.js b/flow-state/src/checks/phase11.js index dea6bd0..b663ae5 100644 --- a/flow-state/src/checks/phase11.js +++ b/flow-state/src/checks/phase11.js @@ -21,10 +21,11 @@ import { shiftPalette, paletteContrast } from '../look/palette.js'; import { ArcDriver } from '../look/ArcDriver.js'; import { Engine } from '../engine/Engine.js'; import { featureProviderFor } from '../audio/FeatureTrack.js'; -import { sampleValues, clampValue } from '../params/schema.js'; +import { sampleValues, clampValue, canBackground } from '../params/schema.js'; import { Rng } from '../engine/rng.js'; import { frameDistance, frameLuminance } from '../engine/hash.js'; import { SHOT_SIZES, SHOT_SIZE_NAMES, describeFraming } from '../look/framing.js'; +import { reachFor, CURVE_NAMES } from '../look/Camera.js'; /** Tracks at several tempos: shot length is measured in bars, so tempo is the axis. */ let cached = null; @@ -188,11 +189,11 @@ check(11, 'every scene in the library is reachable', () => { } } // Accents are cast by role rather than by family and are always eligible. - const missing = scenes.filter((m) => m.role !== 'accent' && !reachable.has(m.name)); + const missing = scenes.filter((m) => canBackground(m) && !reachable.has(m.name)); return expect(missing.length === 0, missing.length ? `unreachable: ${missing.map((m) => m.name).join(', ')}` - : `all ${reachable.size} non-accent scenes reachable across ${DIRECTORS.length} directors`); + : `all ${reachable.size} castable scenes reachable across ${DIRECTORS.length} directors`); }); check(11, 'no director starves a section kind', () => { @@ -204,7 +205,7 @@ check(11, 'no director starves a section kind', () => { for (const [kind, families] of Object.entries(d.families)) { const pool = new Set(); for (const f of families) { - for (const m of scenesInFamily(f)) if (m.role !== 'accent') pool.add(m.name); + for (const m of scenesInFamily(f)) if (canBackground(m)) pool.add(m.name); } if (pool.size < 12) problems.push(`${d.name}/${kind}: only ${pool.size}`); } @@ -301,7 +302,7 @@ check(11, 'a track shows more of the library than it used to', () => { (section.variants || [section.layers]).forEach((v) => cast.add(v[0].module.name))); } }); - const pool = scenes.filter((m) => m.role !== 'accent').length; + const pool = scenes.filter(canBackground).length; return expect(cast.size >= pool * 0.4, `${cast.size}/${pool} scenes cast across 12 tracks (floor ${Math.ceil(pool * 0.4)})`); }); @@ -721,8 +722,21 @@ check(11, 'framing stays inside the library\'s headroom', () => { if (f.scale < 0.55 || f.scale > 2.2) { problems.push(`§${cue.sectionIndex}#${cue.shotIndex}: scale ${f.scale.toFixed(2)}`); } - if (Math.abs(f.shift[0]) > 0.3 || Math.abs(f.shift[1]) > 0.3) { - problems.push(`§${cue.sectionIndex}#${cue.shotIndex}: shift ${f.shift.map((v) => v.toFixed(2))}`); + // The gaze is live now — it is not on cue.framing, it is + // wherever the camera has travelled to by a given frame — + // so the bound has to be measured at frames rather than + // read off the plan. Its ceiling is the reach the shot's + // own size allows: see Camera.reachFor, which is why a + // close-up is permitted further off centre than a wide. + const ceiling = look.camera + ? reachFor(f.scale, look.camera) + 1e-6 : 0.31; + for (const fr of [cue.startFrame, (cue.startFrame + cue.endFrame) >> 1, + cue.endFrame - 1]) { + const shift = arc._framingAt(cue.index, fr).shift; + if (Math.hypot(shift[0], shift[1]) > ceiling) { + problems.push(`§${cue.sectionIndex}#${cue.shotIndex}@${fr}: ` + + `|shift| ${Math.hypot(shift[0], shift[1]).toFixed(3)} > ${ceiling.toFixed(3)}`); + } } } } finally { @@ -751,9 +765,20 @@ check(11, 'framing is identical on a seek and on playback', () => { for (let i = 0; i < a.cues.length; i++) { const fa = a.cues[i].framing; const fb = b.cues[i].framing; - if (fa && fb && (fa.scale !== fb.scale - || fa.shift[0] !== fb.shift[0] || fa.shift[1] !== fb.shift[1])) { - problems.push(`cue ${i}: ${JSON.stringify(fa)} vs ${JSON.stringify(fb)}`); + if (fa && fb && fa.scale !== fb.scale) { + problems.push(`cue ${i}: scale ${fa.scale} vs ${fb.scale}`); + } + // The gaze is a position in time, so agreeing on the plan is not + // enough — two drivers have to agree on where the camera IS at a + // frame. This is the property a seek depends on. + const cue = a.cues[i]; + for (const fr of [cue.startFrame, (cue.startFrame + cue.endFrame) >> 1, + cue.endFrame - 1]) { + const sa = a._framingAt(i, fr).shift; + const sb = b._framingAt(i, fr).shift; + if (sa[0] !== sb[0] || sa[1] !== sb[1]) { + problems.push(`cue ${i}@${fr}: gaze ${sa} vs ${sb}`); + } } } } finally { @@ -765,6 +790,105 @@ check(11, 'framing is identical on a seek and on playback', () => { : `${a.cues.length} cues carry the same framing in both drivers`); }); +/** + * Everything the camera exists to do, measured on one sweep of the battery. + * + * Stated as numbers because the device this replaces LOOKED right in the source + * and did nothing on screen: a recentre of a median 0.029 of a half-frame, at a + * fresh random angle every shot so successive offsets cancelled. Nothing in the + * old gates caught that, because the only bound was a ceiling — and a device + * doing nothing passes a ceiling comfortably. + * + * So each of these has a FLOOR. That is the lesson from the old one. + */ +function gazeStats() { + const offsets = []; const jumps = []; const travels = []; + const cameras = new Set(); const curves = new Set(); + + for (const { track: t } of tempoBattery()) { + for (let s = 0; s < 6; s++) { + const look = generateLook(t, { seed: 21400 + s * 5077 }); + if (!look.camera) continue; + cameras.add(look.camera.name); + const arc = new ArcDriver(look, t); + try { + let prevEnd = null; + arc.cues.forEach((cue, i) => { + curves.add(arc.gaze[i].curve); + const start = arc._framingAt(i, cue.startFrame).shift; + const end = arc._framingAt(i, cue.endFrame - 1).shift; + offsets.push(Math.hypot(end[0], end[1])); + travels.push(Math.hypot(end[0] - start[0], end[1] - start[1])); + if (prevEnd) { + jumps.push(Math.hypot(start[0] - prevEnd[0], start[1] - prevEnd[1])); + } + prevEnd = end; + }); + } finally { + arc.dispose(); + } + } + } + const median = (a) => { + const v = a.slice().sort((x, y) => x - y); + return v.length ? v[v.length >> 1] : 0; + }; + return { + offsets, jumps, travels, cameras, curves, + medOffset: median(offsets), medJump: median(jumps), medTravel: median(travels), + }; +} + +check(11, 'the camera moves far enough at a cut to be seen', () => { + // The floor is the whole point. The device this replaces had a median jump + // of 0.014 of a half-frame — present in every video, visible in none. + // + // Measured over the REFRAMES rather than over every cut. A match cut is a + // deliberate zero and there are enough of them to drag a plain median down; + // averaging the cuts that chose not to move together with the ones that did + // would let the reframes shrink to nothing without the number noticing, + // which is the exact failure this whole gate exists to catch. + const r = gazeStats(); + const reframes = r.jumps.filter((j) => j >= 0.01); + const sorted = reframes.slice().sort((a, b) => a - b); + const med = sorted.length ? sorted[sorted.length >> 1] : 0; + return expect(med > 0.09 && reframes.length > 0, + `median reframe ${med.toFixed(3)} of a half-frame across ${reframes.length} reframes ` + + `· median offset ${r.medOffset.toFixed(3)} ` + + `(the old device measured 0.014 jump / 0.029 offset)`); +}); + +check(11, 'the camera moves DURING a shot, not only at cuts', () => { + // Framing used to be constant within a shot by design. The recentre is not + // framing — a camera that only ever steps is a slideshow of stills. + const r = gazeStats(); + const moving = r.travels.filter((t) => t > 0.02).length; + const share = moving / Math.max(1, r.travels.length); + return expect(r.medTravel > 0.02 && share > 0.5, + `${moving}/${r.travels.length} shots travel (${(share * 100).toFixed(0)}%) · ` + + `median travel ${r.medTravel.toFixed(3)}`); +}); + +check(11, 'the camera cuts through as well as jumping', () => { + // A camera that relocates at EVERY cut is as much a single rule as one that + // never does. The match cut — where the gaze walks through the change and + // two scenes read as one place — has to survive as a real minority. + const r = gazeStats(); + const matched = r.jumps.filter((j) => j < 0.01).length; + const share = matched / Math.max(1, r.jumps.length); + return expect(share > 0.04 && share < 0.6, + `${matched}/${r.jumps.length} cuts are match cuts (${(share * 100).toFixed(0)}%)`); +}); + +check(11, 'the library uses more than one camera and more than one curve', () => { + // Same argument directors.js makes about families: one camera applied to + // every track is how a library ends up with one look. + const r = gazeStats(); + return expect(r.cameras.size >= 3 && r.curves.size >= 3, + `${r.cameras.size} cameras (${[...r.cameras].join(', ')}) · ` + + `${r.curves.size}/${CURVE_NAMES.length} curves used`); +}); + check(11, 'the axis measurement would notice if the axis stopped working', () => { // EPIC-2.md §4 names this failure mode by name: a gate that measures the // wrong thing. This one has already happened once here — the first version diff --git a/flow-state/src/checks/phase12.js b/flow-state/src/checks/phase12.js index 6b7a867..69f48f5 100644 --- a/flow-state/src/checks/phase12.js +++ b/flow-state/src/checks/phase12.js @@ -20,7 +20,7 @@ import { song } from '../audio/songbank.js'; import { Show } from '../Show.js'; import { generateLook } from '../look/LookGenerator.js'; import { scenes } from '../scenes/registry.js'; -import { surfaceOf } from '../params/schema.js'; +import { surfaceOf, canBackground } from '../params/schema.js'; import { Engine } from '../engine/Engine.js'; import { defaultValues } from '../params/schema.js'; import { featureProviderFor } from '../audio/FeatureTrack.js'; @@ -88,7 +88,7 @@ check(12, 'seed variety · the metric separates different scenes from the same s // difference. Two renders of one scene must land far below two renders of // two scenes, or a low variety score would just be a blind metric. const track = varietyTrack(); - const pool = scenes.filter((m) => m.role !== 'accent' && m.kind === 'fragment'); + const pool = scenes.filter((m) => canBackground(m) && m.kind === 'fragment'); const a = pool[0], b = pool[Math.floor(pool.length / 2)], c = pool[pool.length - 1]; const sigA = signatureForScene(track, a, 777, { probes: 2 }); diff --git a/flow-state/src/checks/phase13.js b/flow-state/src/checks/phase13.js new file mode 100644 index 0000000..da83ab2 --- /dev/null +++ b/flow-state/src/checks/phase13.js @@ -0,0 +1,246 @@ +// Phase 13 gate — the story. +// +// Phase 8 gave a track cuts, Phase 9 an identity, Phase 10 a hand on the dials. +// All three are properties of a video at a MOMENT. This phase is about the one +// property that only exists over its whole length: that it is going somewhere, +// and that where it is going follows the song rather than the seed. +// +// Everything here runs on the analysis and the look, without a GPU. The +// rendered half of the question — does the arc reach the IMAGE — is the +// `direction` statistic in the variety report (checks/variety/signature.js), +// because answering it requires probing real frames. +// +// The three things this gate is actually protecting, in order of how bad the +// failure would be: +// +// 1. the story never overrides the song. A plot that declares a climax where +// the track is quiet is worse than no plot, and a tension that lifts a +// breakdown into a drop breaks the quiet-kind coupling every earlier phase +// depends on. +// 2. the same kind twice is not the same twice. This is the whole feature. +// 3. it stays pure and continuous in the frame, or the export stops matching +// the preview and the boundary pops. + +import { check, expect } from './framework.js'; +import { generateLook } from '../look/LookGenerator.js'; +import { FeatureTrack } from '../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../audio/synth.js'; +import { song } from '../audio/songbank.js'; +import { storyStateAt, STORY_VARS, PLOT_NAMES } from '../look/Story.js'; + +/** + * Bank songs rather than the two-section synthetics the other phases use. + * + * A story is a property of a SEQUENCE of sections, so a track that segments into + * two has nothing for this gate to look at — measured, the synthetic battery + * states no kind twice in any of its four tracks, which is precisely the case + * the central check here exists to measure. Four minutes each, across the bank's + * range, so every track has a real arrangement under it. + */ +const NAMES = ['ember', 'centre', 'elegy', 'lattice']; +let cachedBattery = null; +function battery() { + if (!cachedBattery) { + cachedBattery = NAMES.map((name) => ({ name, track: song(name, { duration: 220 }).track })); + } + return cachedBattery; +} + +const looks = (seed0) => battery().map(({ track, name }, i) => ({ + name, track, look: generateLook(track, { seed: (seed0 + i * 7919) >>> 0 }), +})); + +check(13, 'the climax is where the song is loudest', () => { + // The one non-negotiable. Every moment in a story is READ off the section + // energies rather than placed by the seed, so this is a check that the + // derivation still says what it claims and has not drifted into being + // decorative — a story that puts its peak somewhere the audio does not is + // the failure mode that would make the whole layer worth deleting. + const bad = []; + for (const { name, track, look } of looks(0x5709)) { + const energies = track.sections.map((s) => s.energy || 0); + const peak = Math.max(...energies); + const at = look.story.moments.climax; + if (energies[at] < peak - 1e-9) bad.push(`${name}: climax §${at} at ${energies[at].toFixed(3)} vs peak ${peak.toFixed(3)}`); + } + return expect(bad.length === 0, + bad.length ? bad.join(' · ') : `${battery().length} tracks · climax on the loudest section in each`); +}); + +check(13, 'the story does not lift a quiet section into a loud one', () => { + // Tension moves the parameter bias, and it is bounded so that it can only + // decide WHICH drop this is, never whether a breakdown is one. If this ever + // fails, the family coupling in directors.js is next: an intro at drop + // energy is exactly the strobing opening that coupling exists to prevent. + const QUIET = new Set(['intro', 'breakdown', 'outro']); + let worstQuiet = 0; + let loudest = 0; + let checked = 0; + for (const { look } of looks(0x1a71)) { + for (const section of look.sections) { + checked++; + if (QUIET.has(section.kind)) worstQuiet = Math.max(worstQuiet, section.bias.energy); + if (section.kind === 'drop') loudest = Math.max(loudest, section.bias.energy); + } + } + return expect(worstQuiet < 0.62, + `${checked} sections · loudest quiet-kind bias ${worstQuiet.toFixed(3)} ` + + `(limit 0.62) · loudest drop bias ${loudest.toFixed(3)}`); +}); + +check(13, 'the second time a kind happens is not the first time again', () => { + // The feature, as a number. Take every track that states a kind twice and + // require the two occurrences to differ in something a viewer could name: + // which scene opens them, or how hard their parameters are pushed. + // + // Both halves count, because either alone is achievable and neither alone + // is the point — a different scene at the same intensity is a shuffle, and + // the same scene at a different intensity is a fade. + const seen = []; + for (const { name, look } of looks(0x2b0c)) { + const byKind = new Map(); + for (const section of look.sections) { + if (!byKind.has(section.kind)) byKind.set(section.kind, []); + byKind.get(section.kind).push(section); + } + for (const [kind, list] of byKind) { + if (list.length < 2) continue; + const first = list[0]; + const last = list[list.length - 1]; + const sceneChanged = first.layers[0].module.name !== last.layers[0].module.name; + const pushed = Math.abs(last.story.tension - first.story.tension); + seen.push({ name, kind, sceneChanged, pushed }); + } + } + if (!seen.length) { + return expect(false, 'no track in the battery states a kind twice — the check measured nothing'); + } + const moved = seen.filter((s) => s.sceneChanged || s.pushed > 0.08); + return expect(moved.length === seen.length, + `${seen.length} repeated kinds · ${moved.length} differ · ` + + seen.map((s) => `${s.name}/${s.kind}: ${s.sceneChanged ? 'recast' : 'same scene'} ` + + `Δtension ${s.pushed.toFixed(2)}`).join(' · ')); +}); + +check(13, 'the journey travels, and travels in stages', () => { + // Two failures at once. A journey that does not move is the old blind slow + // axis with extra machinery; a journey that slides continuously is a slow + // zoom, which is an effect rather than a narrative. So it has to cover most + // of its range AND spend most of the video not moving at all. + const rows = []; + for (const { name, track, look } of looks(0x3f11)) { + let lo = 1, hi = 0, moving = 0, frames = 0, maxStep = 0; + let previous = null; + for (let f = 0; f < track.frameCount; f += 5) { + const j = storyStateAt(look.story, f).journey; + lo = Math.min(lo, j); hi = Math.max(hi, j); + if (previous !== null) { + const step = Math.abs(j - previous); + maxStep = Math.max(maxStep, step); + if (step > 1e-4) moving++; + } + previous = j; + frames++; + } + rows.push({ name, travel: hi - lo, held: 1 - moving / Math.max(1, frames - 1), maxStep }); + } + const worstTravel = Math.min(...rows.map((r) => r.travel)); + const worstHeld = Math.min(...rows.map((r) => r.held)); + const worstStep = Math.max(...rows.map((r) => r.maxStep)); + return expect(worstTravel > 0.5 && worstHeld > 0.5 && worstStep < 0.1, + rows.map((r) => `${r.name}: travel ${r.travel.toFixed(2)} held ${(r.held * 100).toFixed(0)}% ` + + `worst step ${r.maxStep.toFixed(3)}`).join(' · ')); +}); + +check(13, 'the story is a pure function of the frame', () => { + // Same rule as everything else on the render path: a seek must land on the + // story position playback would have reached, or the export stops matching + // the preview. Sampled forwards, then backwards, then at random. + const { track, look } = looks(0x4c33)[1]; + const forward = []; + for (let f = 0; f < track.frameCount; f += 37) forward.push(storyStateAt(look.story, f)); + + let mismatch = null; + for (let i = forward.length - 1; i >= 0 && !mismatch; i--) { + const again = storyStateAt(look.story, i * 37); + for (const key of [...STORY_VARS, 'journey']) { + if (again[key] !== forward[i][key]) { + mismatch = `frame ${i * 37} ${key}: ${again[key]} vs ${forward[i][key]}`; + break; + } + } + } + return expect(!mismatch, mismatch || `${forward.length} frames sampled in both directions, identical`); +}); + +check(13, 'two tracks do not tell the same story', () => { + // A story layer that gave every video the same arc would be the exact + // failure directors.js was written to fix, one level up. Population check, + // like every variety measurement here: what matters is the spread, not any + // single track's plot. + const plots = new Map(); + let total = 0; + for (let s = 0; s < 6; s++) { + for (const { look } of looks(0x7000 + s * 104729)) { + plots.set(look.story.plot, (plots.get(look.story.plot) || 0) + 1); + total++; + } + } + const reached = plots.size; + const commonest = Math.max(...plots.values()) / total; + return expect(reached >= 3 && commonest < 0.6, + `${total} looks · ${reached}/${PLOT_NAMES.length} plots reached · ` + + `commonest ${(commonest * 100).toFixed(0)}% · ` + + [...plots].map(([k, v]) => `${k} ${v}`).join(', ')); +}); + +check(13, 'a recapitulation actually recapitulates', () => { + // When the story says the outro answers the intro, it has to be the same + // scene — and the sections still have to be told apart by everything else, + // or a "return" is just the video repeating itself. + const rows = []; + for (let s = 0; s < 8; s++) { + for (const { look } of looks(0x9100 + s * 15485863)) { + const intro = look.sections.find((x) => x.kind === 'intro'); + const outro = look.sections.find((x) => x.kind === 'outro'); + if (!look.story.recap || !intro || !outro) continue; + // Not journey alone. The plot most likely to ask for a recap is + // 'return', whose whole shape is an arch that comes BACK — so its + // outro sits near the intro on the journey by design, and it is the + // residue on everything else that makes the ending a return rather + // than a rewind. + rows.push({ + same: intro.layers[0].module.name === outro.layers[0].module.name, + moved: Math.max(...['journey', 'reveal', 'tension'].map((k) => + Math.abs(outro.story[k] - intro.story[k]))), + }); + } + } + if (!rows.length) return expect(true, 'no recap track in this sample — nothing to check'); + const kept = rows.filter((r) => r.same).length; + // 0.15 rather than something ambitious, and measured rather than guessed: + // across the sample the median recap arrives a full 1.0 from where it + // started and the closest — a 'return' on a four-section track, where the + // arch has the least room — lands at 0.18. What this rules out is an outro + // that is bit-identical to the intro, which is the failure worth gating. + const travelled = rows.filter((r) => r.moved > 0.15).length; + return expect(kept === rows.length && travelled === rows.length, + `${rows.length} recap tracks · ${kept} re-cast the opening scene · ` + + `${travelled} arrive at it changed · ` + + `closest ${Math.min(...rows.map((r) => r.moved)).toFixed(2)}`); +}); + +check(13, 'a track with no structure gets no story', () => { + // The degradation path. Two sections is not enough to carry an arc, and + // forcing one produces a video that lurches rather than one that + // progresses — so the whole layer fades toward neutral instead. + const track = FeatureTrack.fromAudioBuffer( + synthesizeSectioned({ bpm: 120, duration: 40, changeAt: 20 }), { fps: 60 }); + const look = generateLook(track, { seed: 0xd15a }); + const n = look.sections.length; + const worst = Math.max(...look.sections.map((s) => + Math.max(...STORY_VARS.map((k) => Math.abs(s.story[k] - 0.5))))); + return expect(n > 3 || worst < 0.35, + `${n} sections · strength ${look.story.strength.toFixed(2)} · ` + + `furthest any variable travels from neutral ${worst.toFixed(2)}`); +}); diff --git a/flow-state/src/checks/phase2.js b/flow-state/src/checks/phase2.js index 7037ae7..4b72cab 100644 --- a/flow-state/src/checks/phase2.js +++ b/flow-state/src/checks/phase2.js @@ -9,7 +9,7 @@ import { check, expect } from './framework.js'; import { Engine } from '../engine/Engine.js'; import { scenes, FAMILIES } from '../scenes/registry.js'; -import { defaultValues, sweepValues, validateModule } from '../params/schema.js'; +import { defaultValues, sweepValues, validateModule, canBackground } from '../params/schema.js'; import { serializeParams, deserializeParams } from '../params/serialize.js'; import { ParamPanel } from '../ui/ParamPanel.js'; import { frameLuminance, frameVariance } from '../engine/hash.js'; @@ -87,9 +87,11 @@ check(2, 'every scene compiles and renders', () => { const pixels = engine.readPixels(engine.renderFrame(1200)); const lum = frameLuminance(pixels); const variance = frameVariance(pixels); - // Accent scenes composite over a background; most of their frame is - // legitimately black, so only variance is meaningful for them. - if (module.role !== 'accent' && !(lum > 0.001)) problems.push(`${module.name}: black frame`); + // A scene that cannot stand alone composites over a background; + // most of its frame is legitimately black, so only variance is + // meaningful for it. Note this is NOT every composable scene — a + // sparse scene that can still carry a section owes us a picture. + if (canBackground(module) && !(lum > 0.001)) problems.push(`${module.name}: black frame`); if (variance < 0.002) problems.push(`${module.name}: flat (var ${variance.toFixed(4)})`); } catch (err) { problems.push(`${module.name}: ${err.message}`); @@ -119,9 +121,9 @@ check(2, 'param range sweep produces no dead or blown frames', () => { const lum = frameLuminance(pixels); const variance = frameVariance(pixels); const label = `${module.name}.${name}=${JSON.stringify(value)}`; - const accent = module.role === 'accent'; - // An accent at brightness 0 really is black, and that is a - // legitimate value — judge those on variance alone. + const accent = !canBackground(module); + // An overlay-only scene at brightness 0 really is black, and + // that is a legitimate value — judge those on variance alone. if (lum > 0.985) problems.push(`${label} blown (lum ${lum.toFixed(3)})`); if (!accent && lum < 0.0008) problems.push(`${label} black (lum ${lum.toFixed(5)})`); if (!accent && variance < 0.0015) problems.push(`${label} flat (var ${variance.toFixed(4)})`); diff --git a/flow-state/src/checks/phase7.js b/flow-state/src/checks/phase7.js index b55ab65..f925643 100644 --- a/flow-state/src/checks/phase7.js +++ b/flow-state/src/checks/phase7.js @@ -10,7 +10,7 @@ import { check, expect, expectBelow } from './framework.js'; import { Engine } from '../engine/Engine.js'; import { Show } from '../Show.js'; import { scenes, FAMILIES, scenesInFamily } from '../scenes/registry.js'; -import { defaultValues, sampleValues } from '../params/schema.js'; +import { defaultValues, sampleValues, canBackground } from '../params/schema.js'; import { Rng } from '../engine/rng.js'; import { FeatureTrack, featureProviderFor } from '../audio/FeatureTrack.js'; import { synthesizeSectioned } from '../audio/synth.js'; @@ -107,7 +107,7 @@ check(7, 'every scene stays live across seeds and section energies', () => { rendered++; const lum = frameLuminance(pixels); const variance = frameVariance(pixels); - const accent = module.role === 'accent'; + const accent = !canBackground(module); const dead = accent ? variance < 0.0008 : (lum < 0.0008 || lum > 0.99 || variance < 0.0015); if (dead) { diff --git a/flow-state/src/checks/phase9.js b/flow-state/src/checks/phase9.js index 9381a41..77c7048 100644 --- a/flow-state/src/checks/phase9.js +++ b/flow-state/src/checks/phase9.js @@ -13,7 +13,7 @@ import { check, expect } from './framework.js'; import { Engine } from '../engine/Engine.js'; import { scenes } from '../scenes/registry.js'; -import { defaultValues } from '../params/schema.js'; +import { defaultValues, canBackground } from '../params/schema.js'; import { TRAITS, generatePersonality, sceneHonours, MIN_ELIGIBLE_SCENES } from '../look/Personality.js'; import { generateLook } from '../look/LookGenerator.js'; import { Rng } from '../engine/rng.js'; @@ -111,7 +111,7 @@ check(9, 'every trait has enough scenes to build a track from', () => { // The casting rule only works if the library can staff it. A trait declared // by three scenes cannot carry a track — the rosters would collapse and every // section would show the same two images, which is Phase 8 undone. - const counts = TRAITS.map((t) => [t, scenes.filter((m) => m.role !== 'accent' + const counts = TRAITS.map((t) => [t, scenes.filter((m) => canBackground(m) && sceneHonours(m, [t])).length]); const thin = counts.filter(([, n]) => n < MIN_ELIGIBLE_SCENES); return expect(thin.length === 0, diff --git a/flow-state/src/checks/variety/print.js b/flow-state/src/checks/variety/print.js index c3f714c..105621b 100644 --- a/flow-state/src/checks/variety/print.js +++ b/flow-state/src/checks/variety/print.js @@ -54,6 +54,18 @@ export async function varietyReportLines({ seeds = 8, probes = 5, library = true lines.push(' 0 = the seed changes nothing a viewer could name'); lines.push(' 1 = two seeds as unalike as two randomly assembled videos'); lines.push(''); + lines.push('DIRECTION — of the floor above, how much is the video GOING somewhere'); + lines.push(''); + lines.push(` direction ${bar((r.direction + 1) / 2)} ${r.direction.toFixed(2)}` + + ` (arcless reference ${r.directionFloor.toFixed(2)})`); + lines.push(' rank correlation between how far apart two probes are in'); + lines.push(' time and how far apart they look. A video with an arc is'); + lines.push(' most unlike itself at its two ends; one that rotates a'); + lines.push(' roster is as unlike itself after ten seconds as after four'); + lines.push(' minutes. Both score the same floor. Read them together:'); + lines.push(' a floor that rises WITH direction is a story, and a floor'); + lines.push(' that rises without one is a shuffle.'); + lines.push(''); lines.push('WHERE THE VARIETY IS — per structural block, as a fraction of achievable'); lines.push(''); for (const [name, b] of Object.entries(r.byBlock)) { @@ -89,7 +101,7 @@ export async function varietyReportLines({ seeds = 8, probes = 5, library = true lines.push(' low here = casting is the bottleneck, and shader work will not fix it)'); lines.push(''); lines.push(` scene-set distance ${bar(spec.sceneSetDistance)} ${spec.sceneSetDistance.toFixed(2)} how differently ${spec.seeds} seeds cast`); - lines.push(` library coverage ${bar(spec.libraryCoverage)} ${pct(spec.libraryCoverage)} of non-accent scenes ever chosen`); + lines.push(` library coverage ${bar(spec.libraryCoverage)} ${pct(spec.libraryCoverage)} of castable scenes ever chosen`); lines.push(` identical casts ${spec.identicalCasts} seed pairs`); for (const key of ['director', 'paletteScheme', 'signature', 'grain', 'framing', 'paletteArc', 'anchorScenes']) { const e = spec[key]; diff --git a/flow-state/src/checks/variety/report.js b/flow-state/src/checks/variety/report.js index 1462447..13d6d80 100644 --- a/flow-state/src/checks/variety/report.js +++ b/flow-state/src/checks/variety/report.js @@ -30,7 +30,7 @@ import { Show } from '../../Show.js'; import { generateLook } from '../../look/LookGenerator.js'; import { scenes } from '../../scenes/registry.js'; -import { defaultValues, sampleValues } from '../../params/schema.js'; +import { defaultValues, sampleValues, canBackground } from '../../params/schema.js'; import { Rng } from '../../engine/rng.js'; import { videoSignature, signatureDistance, STRUCTURAL } from './signature.js'; import { songBank } from '../../audio/songbank.js'; @@ -69,7 +69,7 @@ export function signatureForChaos(track, seed, options = {}) { try { const look = generateLook(track, { seed: seed >>> 0 }); const rng = new Rng((seed * 2246822519) >>> 0); - const pool = scenes.filter((m) => m.role !== 'accent'); + const pool = scenes.filter(canBackground); const temperament = look.personality && look.personality.temperament; for (const section of look.sections) { for (const variant of section.variants) { @@ -144,7 +144,7 @@ export function signatureForScene(track, module, seed, options = {}) { */ export function ceilingSignatures(track, { count = 4, probes = 4, seed = 0xbadc0de } = {}) { const rng = new Rng(seed >>> 0); - const pool = rng.shuffle(scenes.filter((m) => m.role !== 'accent')); + const pool = rng.shuffle(scenes.filter(canBackground)); const slice = Math.max(4, Math.floor(pool.length / count)); const out = []; @@ -190,7 +190,7 @@ export function ceilingSignatures(track, { count = 4, probes = 4, seed = 0xbadc0 * got. */ export function librarySweep(track, { probes = 3, onProgress = null } = {}) { - const pool = scenes.filter((m) => m.role !== 'accent'); + const pool = scenes.filter(canBackground); const sigs = []; for (let i = 0; i < pool.length; i++) { sigs.push(signatureForScene(track, pool[i], 4242, { probes })); @@ -302,6 +302,11 @@ export function measureVariety(track, { const refSigs = ceilingSignatures(track, { count: refScenes, probes }); const ceilingPairs = pairwise(refSigs, (a, b) => signatureDistance(a, b)); const ceiling = mean(ceilingPairs.map((d) => d.total)); + // The same statistic on the single-scene references — a video with no story + // in it at all. It is the zero this measurement is read against, rather than + // a theoretical 0: probes are not evenly spaced and a slow scene drifts on + // its own, so an arcless video does not score exactly nothing. + const directionFloor = mean(refSigs.map((s) => s.direction ?? 0)); // If the reference is not above the floor it is not a ceiling, and the // ratio built on it is meaningless rather than large. Say so instead of @@ -336,6 +341,11 @@ export function measureVariety(track, { return { seeds: seedList, floor, + // Of that floor, how much is a video GOING somewhere rather than merely + // changing. The floor alone cannot tell the two apart, and a video with + // a story raises it on purpose. See variety/signature.js directionOf. + direction: mean(sigs.map((s) => s.direction ?? 0)), + directionFloor, ceiling, observed, separation, @@ -389,16 +399,16 @@ export function measureSpecDiversity(track, { seeds = 32, seed0 = 0x5eed } = {}) looks.push(generateLook(track, { seed: (seed0 + i * 2654435761) >>> 0 })); } - // Accents excluded on both sides of the ratio. They were counted in the - // numerator and not the denominator, which reported 102% coverage once the - // casting pool started reaching them. + // Overlay-only scenes excluded on both sides of the ratio. They were + // counted in the numerator and not the denominator, which reported 102% + // coverage once the casting pool started reaching them. const sceneSets = looks.map((l) => [...new Set( l.sections.flatMap((s) => s.variants.flatMap( - (v) => v.filter((layer) => layer.module.role !== 'accent') + (v) => v.filter((layer) => canBackground(layer.module)) .map((layer) => layer.module.name))), )].sort()); - const usable = scenes.filter((m) => m.role !== 'accent'); + const usable = scenes.filter(canBackground); const covered = new Set(sceneSets.flat()); const uncast = usable.filter((m) => !covered.has(m.name)).map((m) => m.name); @@ -627,7 +637,7 @@ export function measureDecomposition({ songs = 6, probes = 3, stageNames = null // that happened to be written first. const stages = stageNames ? stageNames.map((n) => scenes.find((m) => m.name === n)).filter(Boolean) - : scenes.filter((m) => (m.consumes || []).includes('cast') && m.role !== 'accent'); + : scenes.filter((m) => (m.consumes || []).includes('cast') && canBackground(m)); // Each song's identity, lifted off its own look so it can be transplanted. const looks = bank.map((e) => generateLook(e.track, { seed: hashString(e.name) })); diff --git a/flow-state/src/checks/variety/signature.js b/flow-state/src/checks/variety/signature.js index 9adb2ff..e6f24d0 100644 --- a/flow-state/src/checks/variety/signature.js +++ b/flow-state/src/checks/variety/signature.js @@ -171,11 +171,19 @@ export function videoSignature(show, { probes = 6, gap = 5, warmup = 20 } = {}) } // Self-distance: how far this video travels from itself over its own length. + // + // Note what this number cannot tell you, and what `direction` below is for: + // it is the same whether the video went somewhere or merely kept changing. let drift = 0, pairs = 0; + const gaps = []; + const dists = []; for (let i = 0; i < perProbe.length; i++) { for (let j = i + 1; j < perProbe.length; j++) { - drift += blockMean(descriptorDistance(perProbe[i], perProbe[j])); + const d = blockMean(descriptorDistance(perProbe[i], perProbe[j])); + drift += d; pairs++; + gaps.push(Math.abs(frames[j].frame - frames[i].frame)); + dists.push(d); } } @@ -183,10 +191,60 @@ export function videoSignature(show, { probes = 6, gap = 5, warmup = 20 } = {}) frames, probes: perProbe, drift: pairs ? drift / pairs : 0, + // Does the video's self-distance grow with TIME? See directionOf. + direction: directionOf(gaps, dists), motion: perProbe.reduce((a, p) => a + p.energy, 0) / perProbe.length, }; } +/** + * DIRECTION: whether a video is travelling or merely wandering. + * + * `drift` measures how far a video gets from itself and cannot distinguish the + * two, which matters because they are opposite outcomes. A generator that + * shuffles unrelated images scores exactly like one that tells a story, and the + * story is the one anyone wants — so a video with a narrative arc will RAISE the + * floor the seed-variety test wants low, and without this statistic that reads + * as a regression. + * + * The split is rank correlation between how far apart two probes are in TIME and + * how far apart they are structurally. A video with an arc is most unlike itself + * at its two ends: distance grows with separation, ρ approaches 1. A video that + * rotates through a roster is as unlike itself at ten seconds as at four + * minutes: ρ sits at 0. Both can have identical drift. + * + * Spearman rather than Pearson: only the ORDER is meaningful. Nothing here + * claims the arc is linear, and it should not be — see look/Story.js on why the + * curves are staged. + */ +export function directionOf(gaps, dists) { + const n = gaps.length; + if (n < 3) return 0; + const rank = (xs) => { + const order = xs.map((v, i) => [v, i]).sort((a, b) => a[0] - b[0]); + const r = new Array(n); + for (let i = 0; i < n;) { + let j = i; + while (j + 1 < n && order[j + 1][0] === order[i][0]) j++; + const tied = (i + j) / 2 + 1; // mean rank across a tie group + for (let k = i; k <= j; k++) r[order[k][1]] = tied; + i = j + 1; + } + return r; + }; + const a = rank(gaps); + const b = rank(dists); + const mean = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length; + const ma = mean(a), mb = mean(b); + let num = 0, da = 0, db = 0; + for (let i = 0; i < n; i++) { + num += (a[i] - ma) * (b[i] - mb); + da += (a[i] - ma) ** 2; + db += (b[i] - mb) ** 2; + } + return da > 0 && db > 0 ? num / Math.sqrt(da * db) : 0; +} + function blockMean(byBlock) { let sum = 0, weight = 0; for (const block of STRUCTURAL) { diff --git a/flow-state/src/look/ArcDriver.js b/flow-state/src/look/ArcDriver.js index 1257be3..aeacd26 100644 --- a/flow-state/src/look/ArcDriver.js +++ b/flow-state/src/look/ArcDriver.js @@ -4,6 +4,8 @@ import { clampValue } from '../params/schema.js'; import { paletteShiftAt } from './paletteArc.js'; import { shiftPalette } from './palette.js'; import { frameShot, neutralFraming } from './framing.js'; +import { planGaze, gazeAt } from './Camera.js'; +import { storyStateAt, NEUTRAL_STATE } from './Story.js'; /** * Drives the look across the song. @@ -43,6 +45,7 @@ export class ArcDriver { this.framingStyle = (look.framing && look.framing.mode !== 'locked') ? look.framing : null; this._planFraming(); + this._planGaze(); this.state = { sectionIndex: 0, shotIndex: 0, crossfade: 0, incoming: null }; } @@ -63,12 +66,60 @@ export class ArcDriver { for (const cue of this.cues) { const section = this.look.sections[cue.sectionIndex]; const energy = (section.bias && section.bias.energy) || 0; - const framing = frameShot(this.framingStyle, previous, energy, rng); + // Shot size is where the story's `closeness` lands: a video that is + // approaching its subject does it at the cuts, because that is the + // only place a size is allowed to change. See look/framing.js. + const closeness = (section.story || NEUTRAL_STATE).closeness; + const framing = frameShot(this.framingStyle, previous, energy, rng, closeness); cue.framing = framing; previous = framing; } } + /** + * Plan where the camera looks, across the whole video. + * + * After framing, because the reach available to a move depends on the shot + * size it is made at — a close-up is inside the composition and can travel + * across it; a wide already sees the whole thing. See Camera.reachFor. + * + * Unlike framing this does NOT stop at a locked-off track: locked is a + * decision about SIZE, and a video that never changes size can still be one + * whose attention moves. Only a look with no camera at all — hand-built, or + * a check constructing sections directly — goes without. + */ + _planGaze() { + if (!this.look.camera) return; + const rng = new Rng((this.look.seed ^ 0x2f9c1d4b) >>> 0); + this.gaze = planGaze(this.cues, this.look.sections, this.look.camera, rng); + } + + /** Where the camera is looking at `frame`, for the cue at `cueIndex`. */ + _gazeAt(cueIndex, frame) { + if (!this.gaze) return null; + const cue = this.cues[cueIndex]; + const move = this.gaze[cueIndex]; + if (!cue || !move) return null; + return gazeAt(move, frame - cue.startFrame); + } + + /** + * The framing a cue is played with at `frame` — its size, plus wherever the + * gaze has travelled to by now. + * + * Size comes off the plan and never changes within the shot; the shift is + * live. Returning a fresh object each call is deliberate: Layer copies the + * values into uniforms immediately, and a shared mutable framing would make + * the outgoing half of a crossfade read the incoming half's position. + */ + _framingAt(cueIndex, frame) { + const cue = this.cues[cueIndex]; + const base = (cue && cue.framing) || neutralFraming(); + const shift = this._gazeAt(cueIndex, frame); + if (!shift) return base; + return { size: base.size, scale: base.scale, shift }; + } + dispose() { for (const layer of this.layerCache.values()) layer.dispose(); this.layerCache.clear(); @@ -100,6 +151,10 @@ export class ArcDriver { startFrame: shot.startFrame, endFrame: shot.endFrame, atSectionStart, + // Carried onto the cue as well as folded into fadeFrames: + // the camera reads it, because a straight cut earns a + // bigger reframe than a dissolve. See Camera.jumpFor. + hardCut: !!shot.hardCut && !atSectionStart, fadeFrames: shot.hardCut && !atSectionStart ? Math.max(2, Math.round(this.track.fps * 0.06)) : this._dissolveFrames(energy, span), @@ -245,6 +300,15 @@ export class ArcDriver { // a random draw finds the second kind almost every time. const declared = eligible.filter(([, def]) => def.slowAxis); + // Which WAY the video travels is the track's decision, not the scene's. + // + // The sign used to be an independent coin flip per scene, so a five + // minute video routinely had one scene growing denser while the next one + // thinned out — movement with no direction, which is the difference + // between a video that goes somewhere and one that merely changes. + // Magnitude stays per scene; the sign is shared. See look/Story.js. + const sign = this.look.story ? this.look.story.axisSign : (rng.bool() ? 1 : -1); + const axis = []; if (declared.length) { for (const [name, def] of declared) { @@ -256,7 +320,7 @@ export class ArcDriver { // Most of the range. This param was chosen because moving it // is what the scene looks like changing, so a timid walk // wastes the one lever that works. - travel: (hi - lo) * rng.range(0.45, 0.7) * (rng.bool() ? 1 : -1), + travel: (hi - lo) * rng.range(0.45, 0.7) * sign, }); } } else { @@ -272,7 +336,7 @@ export class ArcDriver { name, def, declared: false, - travel: (hi - lo) * rng.range(0.25, 0.5) * (rng.bool() ? 1 : -1), + travel: (hi - lo) * rng.range(0.25, 0.5) * sign, }); } } @@ -284,16 +348,28 @@ export class ArcDriver { * Base params for a section at a given time: the look's sampled values, plus * the slow axis, plus drift, plus the lookahead ramp toward what comes next. */ - _paramsAt(cue, slot, time, features) { + _paramsAt(cue, slot, time, features, story = null) { const spec = this._specFor(cue.sectionIndex, cue.variant, slot); const out = { ...spec.params }; // --- slow axis ------------------------------------------------------ - // Eased rather than linear, so the travel is slowest at the head and - // tail. A video should not open mid-move. - const duration = Math.max(1e-6, this.track.duration); - const p = Math.max(0, Math.min(1, time / duration)); - const journey = p * p * (3 - 2 * p); + // How far along the journey this frame is. The story owns this: its + // curve holds inside a section and moves at the boundary, so the axis + // travels in STAGES rather than sliding continuously for five minutes — + // a scene of a story rather than a slow zoom. See look/Story.js. + // + // With no story it falls back to the eased progress ramp this was + // before, which is also what Story.js emits for a track too short to + // carry one: slowest at the head and tail, because a video should not + // open mid-move. + let journey; + if (story) { + journey = story.journey; + } else { + const duration = Math.max(1e-6, this.track.duration); + const p = Math.max(0, Math.min(1, time / duration)); + journey = p * p * (3 - 2 * p); + } for (const item of this._slowAxisFor(spec.module)) { const base = out[item.name]; if (typeof base !== 'number') continue; @@ -381,7 +457,7 @@ export class ArcDriver { * the frame-exactness guarantee — a seeked frame gets bit-identical colours * to a played one rather than merely similar ones. */ - _paletteAt(frame, features) { + _paletteAt(frame, features, story) { const arc = this.look.paletteArc; if (!arc || arc.mode === 'static') return this.look.palette; @@ -389,6 +465,7 @@ export class ArcDriver { progress: frame / Math.max(1, this.track.frameCount - 1), sectionKind: this.track.sectionAt(frame).kind, features, + story, }); const key = `${shift.hue.toFixed(3)}|${shift.chroma.toFixed(3)}|${shift.lightness.toFixed(3)}`; @@ -399,9 +476,81 @@ export class ArcDriver { return this._palette; } + /** + * The track's personality, with as much of its identity SHOWN as the story + * has reached. + * + * This is where a story reaches Epic 3's content registers, and it needed no + * new uniform to do it: `setPersonality` is already called on every layer + * every frame, and the cast/ink/lattice uniforms are derived from the object + * it is handed. Scaling the features that make the song's forms specific — + * the notches, the hole through the middle, the outline, the size hierarchy + * — means the cast literally ARRIVES over the video instead of being fully + * stated in the first shot and merely repeated after that. + * + * Only the specificity moves, never the identity itself: the protagonist has + * the same number of sides at thirty seconds as at four minutes. A form that + * changed its shape would be a different character rather than the same one + * seen more clearly. + * + * Memoised on the rounded reveal, exactly as _paletteAt memoises on the + * rounded shift and for the same two reasons: consecutive frames want the + * same value, and rounding is what keeps a seeked frame bit-identical to a + * played one rather than merely close. + */ + _personalityAt(story) { + const base = this.look.personality; + if (!base || !base.identity || !story) return base; + + const reveal = Math.max(0, Math.min(1, story.reveal)); + const key = Math.round(reveal * 50); + if (!this._personalityCache) this._personalityCache = new Map(); + const cached = this._personalityCache.get(key); + if (cached) return cached; + + // Never all the way to nothing. A cast erased to plain circles is a + // different track's cast, not this one's withheld — the video still has + // to look like itself in its first thirty seconds. + const shown = 0.35 + (key / 50) * 0.65; + const id = base.identity; + const member = (m) => ({ + ...m, + notchDepth: m.notchDepth * shown, + hollow: m.hollow * shown, + }); + + const moved = { + ...base, + identity: { + ...id, + cast: { protagonist: member(id.cast.protagonist), chorus: member(id.cast.chorus) }, + ink: { + ...id.ink, + outline: id.ink.outline * shown, + // Posterisation is a value structure rather than an amount, + // so it arrives whole at a threshold instead of fading in. + posterize: shown > 0.6 ? id.ink.posterize : 0, + }, + lattice: { ...id.lattice, scaleSpread: id.lattice.scaleSpread * shown }, + }, + }; + this._personalityCache.set(key, moved); + return moved; + } + update(frame, features) { const time = frame / this.track.fps; - const palette = this._paletteAt(frame, features); + // Where the video is in its story. One lookup per frame, handed to + // everything below rather than recomputed — and a pure function of the + // frame, so a seek lands on the same story position as playback. + // A look with no story at all — hand-built by a check, or generated + // before this existed — passes null rather than the neutral state, so + // everything below takes its own pre-story path. The neutral state's + // `journey` is 0.5, and handing that to the slow axis would park it at + // the middle of its travel for the whole video rather than ramping. + const story = this.look.story ? storyStateAt(this.look.story, frame) : null; + const palette = this._paletteAt(frame, features, story); + const personality = this._personalityAt(story); const cueIndex = this._cueIndexAt(frame); const cue = this.cues[cueIndex]; if (!cue) return this.activeLayers; @@ -416,8 +565,15 @@ export class ArcDriver { // The shot being played INTO carries its own framing; the shot fading // out keeps the framing it was filmed with, so a cut changes the size // exactly when the cut changes the image rather than half a beat after. - const framing = cue.framing || neutralFraming(); - const outgoingFraming = previous ? (previous.framing || neutralFraming()) : framing; + // + // The outgoing shot is evaluated at the SAME frame, on its own move — + // it is still on screen, and freezing its gaze at the cut would stop + // the old image dead half a second before it disappears. A camera that + // was travelling when the edit arrived keeps travelling as it fades. + const framing = this._framingAt(cueIndex, frame); + const outgoingFraming = previous + ? this._framingAt(cueIndex - 1, frame) + : framing; const layers = []; @@ -437,11 +593,11 @@ export class ArcDriver { for (let slot = 0; slot < this._stackSize(previous); slot++) { const spec = this._specFor(previous.sectionIndex, previous.variant, slot); const layer = this._layerFor(previous.sectionIndex, previous.variant, slot); - layer.setParams(this._paramsAt(previous, slot, time, outgoingFeatures)); + layer.setParams(this._paramsAt(previous, slot, time, outgoingFeatures, story)); layer.opacity = slot === 0 ? 1 : spec.opacity; layer.blend = slot === 0 ? 'normal' : spec.blend; layer.setPalette(palette); - layer.setPersonality(this.look.personality); + layer.setPersonality(personality); layer.setFraming(outgoingFraming); layers.push(layer); } @@ -450,11 +606,11 @@ export class ArcDriver { for (let slot = 0; slot < this._stackSize(cue); slot++) { const spec = this._specFor(cue.sectionIndex, cue.variant, slot); const layer = this._layerFor(cue.sectionIndex, cue.variant, slot); - layer.setParams(this._paramsAt(cue, slot, time, features)); + layer.setParams(this._paramsAt(cue, slot, time, features, story)); layer.opacity = (slot === 0 ? 1 : spec.opacity) * (fading ? eased : 1); layer.blend = slot === 0 ? 'normal' : spec.blend; layer.setPalette(palette); - layer.setPersonality(this.look.personality); + layer.setPersonality(personality); layer.setFraming(framing); layers.push(layer); } @@ -468,6 +624,12 @@ export class ArcDriver { crossfade: fading ? eased : 0, sceneName: this._specFor(cue.sectionIndex, cue.variant, 0).module.name, buildSlope: features ? features.buildSlope || 0 : 0, + // Where the story is, for the HUD and the checks. A video that is + // supposed to be going somewhere should be able to say where. + act: story.act, + tension: story.tension, + reveal: story.reveal, + journey: story.journey, }; this.activeLayers = layers; @@ -498,6 +660,7 @@ export class ArcDriver { // A reroll re-plans the section's shots, so the cue list is stale too. this.cues = this._buildCues(); this._planFraming(); + this._planGaze(); this._slopeCache = null; } @@ -506,6 +669,7 @@ export class ArcDriver { this.driftPlans.clear(); this.cues = this._buildCues(); this._planFraming(); + this._planGaze(); this._slopeCache = null; } diff --git a/flow-state/src/look/Camera.js b/flow-state/src/look/Camera.js new file mode 100644 index 0000000..07cdb59 --- /dev/null +++ b/flow-state/src/look/Camera.js @@ -0,0 +1,396 @@ +// The GAZE: where in the scene the frame is looking, and how it gets there. +// +// This is the director's camera department. `Story.js` says what the video is +// doing — tension, closeness, order, which act a section is in — in terms that +// are deliberately imagery-free. Something has to turn that into a picture, and +// until now every consumer did its own ad-hoc translation: shots.js reads +// tension for the cut rate, framing.js reads closeness for the shot size, +// LookGenerator reads population for the overlay chance. Nobody owned the +// camera, and it showed. +// +// WHAT WAS WRONG +// +// The recentre existed — `framing.shift`, applied in the shader epilogue as +// `p / scale + shift` — and it was inert. Measured across 121 cues: +// +// median |shift| from centre 0.029 (2.9% of a half-frame) +// median |jump| at a cut 0.014 (1.4%) +// largest jump seen 0.120 +// +// Three causes, and the amplitude was only one of them: +// +// 1. `amount = spec.drift * style.range * rng.range(0.3, 1)` capped the offset +// at 0.10, because a comment worried that pushing off centre would find the +// scenes' empty corners. +// 2. The direction was `rng.range(0, 2π)` — a fresh uniform angle every shot. +// No axis, no continuity, no intent. That is why the median JUMP is smaller +// than the median OFFSET: consecutive shots mostly cancelled each other. +// 3. Nothing about the song reached it at all. Shot SIZE got the story's +// `closeness`; the recentre got a per-track constant and a die roll. +// +// WHAT THIS DOES INSTEAD +// +// A gaze is a path, planned once over the whole cue list, so the video's +// attention travels rather than jittering. Per cue it carries a move: +// +// from → to the two points, in scene units +// delay how long the shot holds before it starts moving +// travel how long the move takes +// curve how it accelerates +// +// The story decides all four. A climax jumps far and arrives hard; a resolution +// drifts a short way back toward centre and never quite stops; a low-`order` +// section throws the gaze off its axis. Between two points the gaze is a pure +// function of (plan, cue, frames-into-cue), the same rule the rest of the render +// path follows — a seek lands on the frame playback would have shown. +// +// NOTE ON THE PER-SHOT RULE. framing.js says framing is constant within a shot, +// and that a move inside a shot "would fight the drift LFO and the slow axis, +// both of which already own continuous motion". That was right about SIZE and +// wrong about the recentre: a zoom that creeps during a shot is an effect, but a +// camera that settles onto its subject is how shots have always worked. Size +// still steps at the cut and only at the cut. The gaze moves. + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); +const lerp = (a, b, t) => a + (b - a) * t; + +/** + * How the gaze accelerates between two points. + * + * These are the reason a move reads as a decision rather than as a tween. A + * `snap` and a `glide` cover the same distance in the same time and say + * completely different things about the section they are in. + */ +export const CURVES = { + // Hard out of the gate, decelerating into the target. The edit-room move: + // it feels like the camera was already going when the cut happened. + snap: (t) => 1 - (1 - t) ** 3, + // Slow away, slow in. The default, and what a calm section wants. + glide: (t) => t * t * (3 - 2 * t), + // Constant speed, and it does NOT arrive — see `travel` below, which is + // allowed to exceed the shot. A drifting camera that gets cut away from + // mid-move is the most alive of these. + drift: (t) => t, + // Overshoots and comes back. Used sparingly: it is the only curve here that + // is legible as a flourish, so it belongs at moments and not everywhere. + // + // The standard back-ease-out, written out rather than improvised. The + // improvised version evaluated to 2.0 at t=0 instead of 0 — every `settle` + // shot therefore STARTED at twice its target, well outside the reach the + // plan had clamped it to, and the headroom gate caught offsets of 0.61 + // against a 0.385 ceiling. An easing curve has two values that are not + // negotiable, f(0)=0 and f(1)=1, and this one had neither. + settle: (t) => { + const c1 = 1.70158; + const u = t - 1; + return 1 + (c1 + 1) * u * u * u + c1 * u * u; + }, +}; + +export const CURVE_NAMES = Object.keys(CURVES); + +/** + * A director's point of view about its camera. + * + * Same rationale as the family table in directors.js: a fixed rule applied to + * every track is how a library ends up with one camera, and the camera is + * exactly the register a viewer reads as "who shot this". + * + * reach multiplier on how far the gaze is allowed from centre + * pace multiplier on travel time — low is restless, high is patient + * curves weights over CURVE_NAMES, in order + * axial how much the gaze prefers to move along one axis rather than + * anywhere. High reads as composed; low reads as searching. + */ +export const CAMERAS = { + // Patient and composed. Long moves, mostly horizontal, rarely hurried. + contemplative: { reach: 0.85, pace: 1.45, curves: [1, 4, 3, 1], axial: 0.75 }, + // Cuts with the camera already moving. The closest to an edited music video. + kinetic: { reach: 1.20, pace: 0.60, curves: [5, 2, 1, 1], axial: 0.35 }, + // Locked-off until it is not. Holds, then commits to one large move. + deliberate: { reach: 1.10, pace: 0.85, curves: [3, 3, 1, 2], axial: 0.85 }, + // Never settles. Long drifts that get cut away from mid-travel. + roaming: { reach: 0.95, pace: 1.70, curves: [1, 2, 5, 1], axial: 0.25 }, + // Small, exact, and it always arrives. The one that stays near centre. + precise: { reach: 0.70, pace: 1.00, curves: [2, 5, 1, 2], axial: 0.90 }, +}; + +export const CAMERA_NAMES = Object.keys(CAMERAS); + +/** + * The furthest the gaze may sit from centre, in scene units. + * + * The old comment's worry was real but it was stated as a constant when it is a + * function of the shot size. The visible half-frame at scale s is 1/s scene + * units, so a close-up is looking at a small piece of the composition and can + * move a long way across it before reaching anywhere empty; a wide is already + * seeing everything there is and moving off centre only finds the edges. + * + * The scale term is the ADDITION rather than the whole thing, which the first + * version got wrong: `(0.12 + 0.56 * headroom)` gave a close-up 0.35 and every + * other shot 0.12, and since headroom is zero at any scale ≤ 1 that meant the + * normal and wide shots — most of the video — were still capped at barely more + * than the old inert 0.10. The base has to be worth seeing on its own. + * + * Now: `normal` and `wide` allow 0.20–0.34 depending on the camera, `close` + * 0.32–0.45. Against a measured median offset of 0.029 and a hard old ceiling + * of 0.10, a typical move is roughly six times what it was. + * + * The absolute cap is what keeps this inside the answer to "how far should it + * roam" — moderate, with close-ups furthest. + */ +export const MAX_REACH = 0.45; + +export function reachFor(scale, camera) { + const headroom = Math.max(0, 1 - 1 / Math.max(scale, 0.05)); + return Math.min(MAX_REACH, (0.28 + 0.42 * headroom) * camera.reach); +} + +/** Pick a camera for a track. The director leans, the seed decides. */ +export function deriveCamera(director, summary, rng) { + const preferred = (director && director.camera) || null; + const weights = CAMERA_NAMES.map((name) => (name === preferred ? 4 : 1)); + const name = rng.pickWeighted(CAMERA_NAMES, weights); + return { + name, + ...CAMERAS[name], + // The track's own axis. A video whose gaze moves along one line reads as + // composed even when the line is arbitrary — what reads as sloppy is a + // different direction every time, which is precisely what the uniform + // random angle was doing. + axis: rng.range(0, Math.PI * 2), + }; +} + +/** + * How far this cut should jump, 0..1 of the available reach. + * + * This is the whole "driven by the song" requirement in one function, so the + * mapping is stated rather than buried: + * + * tension the main term. A wound-up section reframes hard. + * act the climax gets the biggest move in the video, and the + * resolution gets the smallest — a video that keeps flinging its + * camera after the peak has nothing left to say with it. + * hardCut a straight cut earns a bigger reframe than a dissolve. The two + * devices are already gated on energy together (shots.js), so this + * compounds deliberately. + * order low order widens the spread, so a section coming apart is also + * less predictable about where it looks. + */ +function jumpFor(story, hardCut, rng) { + const tension = story ? story.tension : 0.5; + const order = story ? story.order : 0.5; + + let base = 0.25 + tension * 0.55; + if (story) { + if (story.act === 'climax') base = Math.max(base, 0.85); + else if (story.act === 'turn') base = Math.max(base, 0.6); + else if (story.act === 'resolution') base = Math.min(base, 0.3); + else if (story.act === 'setup') base = Math.min(base, 0.45); + } + if (hardCut) base = Math.min(1, base * 1.25); + + // Disorder widens the draw rather than raising it: a broken section is less + // predictable, not uniformly bigger. + const spread = 0.2 + (1 - order) * 0.55; + return clamp01(base * rng.range(1 - spread, 1 + spread * 0.6)); +} + +/** + * Where the gaze goes next. + * + * Direction is the track's axis, plus a wander that `order` controls and the + * camera's `axial` bounds. The one hard rule is that a move must not simply + * undo the last one — reversing along the same line is how the old uniform + * angle produced offsets that cancelled, and it is why nothing appeared to + * move even at the amplitudes it did reach. + */ +function targetFor(from, distance, camera, story, rng, previousDir) { + const order = story ? story.order : 0.5; + const wander = (1 - camera.axial) * (0.35 + (1 - order) * 0.65); + + // Both ends of the axis are legitimate; which one is a coin flip biased + // away from wherever we already are, so the gaze crosses the frame rather + // than orbiting one side of it. + const along = camera.axis + (rng.bool() ? 0 : Math.PI); + let dir = along + rng.range(-Math.PI, Math.PI) * wander; + + if (previousDir !== null) { + // Within 35° of a straight reversal, nudge it off. A reversal is a + // legitimate move; an exact retrace is the thing that reads as jitter. + const delta = Math.abs(normalizeAngle(dir - (previousDir + Math.PI))); + if (delta < 0.6) dir += (delta < 0.3 ? 1 : -1) * 0.9; + } + + return { + point: [from[0] + Math.cos(dir) * distance, from[1] + Math.sin(dir) * distance], + dir, + }; +} + +function normalizeAngle(a) { + let x = a % (Math.PI * 2); + if (x > Math.PI) x -= Math.PI * 2; + if (x < -Math.PI) x += Math.PI * 2; + return x; +} + +/** + * How long the move takes, and how long the shot waits first. + * + * The "different speeds in different sections" requirement. A loud section + * moves fast and is done; a quiet one takes most of the shot to arrive. Travel + * is allowed to exceed the shot length — that is not a bug, it is what makes a + * `drift` read as a camera that was going somewhere when the edit cut away. + */ +function timingFor(spanFrames, story, energy, camera, curve, rng) { + const tension = story ? story.tension : 0.5; + const urgency = clamp01(energy * 0.6 + tension * 0.4); + + // Fraction of the shot spent moving. Fast material arrives in the first + // third; slow material is still arriving at the cut. + let travel = lerp(1.15, 0.28, urgency) * camera.pace * rng.range(0.8, 1.25); + // A drift is defined by not arriving, so it always outruns its shot. + if (curve === 'drift') travel = Math.max(travel, 1.2); + + // How long it holds first. A `deliberate` camera earns its name here: the + // hold is what makes the move that follows read as a decision. + const delay = lerp(0.22, 0.02, urgency) * rng.range(0.4, 1.3); + + const usable = Math.max(1, spanFrames); + return { + delayFrames: Math.round(clamp01(delay) * usable), + travelFrames: Math.max(1, Math.round(travel * usable)), + }; +} + +/** + * Whether the cut RELOCATES the camera or the camera walks through it. + * + * Both are real edits and they say opposite things. A cut that lands on a new + * part of the scene is a reframe — the loud one, the one you notice. A cut the + * camera walks through is a match cut, and it is what makes two scenes read as + * one continuous place, which is the effect worth keeping. + * + * So this is a decision per cut, not a mode. Hard cuts relocate, dissolves + * mostly do not, and tension raises the odds everywhere: a wound-up section + * jumps around inside itself, a resolution stops doing that. + */ +function relocatesAt(cue, story, rng) { + if (!cue.hardCut && cue.atSectionStart) return true; // a new section is a new place + const tension = story ? story.tension : 0.5; + let chance = 0.12 + tension * 0.45; + if (cue.hardCut) chance += 0.3; + if (story) { + if (story.act === 'climax') chance += 0.2; + else if (story.act === 'resolution') chance *= 0.4; + } + return rng.bool(clamp01(chance)); +} + +/** + * Plan the gaze across a whole video. + * + * Walks the cues in order. Each move starts from where the gaze ACTUALLY was + * when the cut arrived — not from the target of the previous move, which may + * never have been reached: a `drift` is defined by outrunning its shot, and + * resuming from its unreached target is a silent teleport at every cut. That + * was measurable as a median jump of 0.000 with a p90 of 0.274, which is a + * camera that mostly does nothing and occasionally lurches. + * + * @param {Array} cues ArcDriver cues, each already carrying `framing` + * @param {Array} sections look.sections, for bias and story state + * @param {object} camera from deriveCamera + * @param {Rng} rng + * @returns {Array} one move per cue, index-aligned + */ +export function planGaze(cues, sections, camera, rng) { + const moves = []; + let at = [0, 0]; + let previousDir = null; + + for (const cue of cues) { + const section = sections[cue.sectionIndex] || {}; + const story = section.story || null; + const energy = (section.bias && section.bias.energy) || 0; + const scale = (cue.framing && cue.framing.scale) || 1; + const reach = reachFor(scale, camera); + const span = Math.max(1, cue.endFrame - cue.startFrame); + + const curve = rng.pickWeighted(CURVE_NAMES, camera.curves); + + // THE CUT. Either the camera is somewhere new when the image changes, + // or it walks through the change and the two shots read as one place. + // + // Carried-over positions are re-clamped, because reach belongs to the + // SHOT: walking a close-up's 0.45 offset into a wide would start that + // wide further off centre than a wide is ever allowed to be. The small + // snap this causes lands exactly on a cut, which is where the eye is + // least able to see it. + let from = clampToReach(at, reach); + if (moves.length && relocatesAt(cue, story, rng)) { + const hop = jumpFor(story, !!cue.hardCut, rng) * reach; + const jumped = targetFor(from, hop, camera, story, rng, previousDir); + from = clampToReach(jumped.point, reach); + previousDir = jumped.dir; + } + + // THE MOVE. Where it travels during the shot, from wherever the cut + // left it. Deliberately smaller than the cut's hop — the reframe is the + // statement and the move is the camera living inside it. + const distance = jumpFor(story, false, rng) * reach * 0.7; + const { point, dir } = targetFor(from, distance, camera, story, rng, previousDir); + + // Clamp the TARGET to the reach disc rather than the step, so a move + // that would leave the frame is shortened instead of being redirected — + // redirecting is what makes a bounded random walk orbit its boundary. + const to = clampToReach(point, reach); + const { delayFrames, travelFrames } = timingFor(span, story, energy, camera, curve, rng); + + const move = { from, to, curve, delayFrames, travelFrames, reach }; + moves.push(move); + + // Where the gaze actually ends up when this shot is cut away from. + at = gazeAt(move, span); + previousDir = dir; + } + + return moves; +} + +function clampToReach(p, reach) { + const d = Math.hypot(p[0], p[1]); + if (d <= reach || d === 0) return p; + const k = reach / d; + return [p[0] * k, p[1] * k]; +} + +/** + * Where the gaze is, `frames` into a cue. + * + * Pure in (move, frames): this is what keeps a seek frame-identical to + * playback, and it is why the plan holds points and durations rather than a + * running position. + */ +export function gazeAt(move, frames) { + if (!move) return [0, 0]; + const t = clamp01((frames - move.delayFrames) / move.travelFrames); + const eased = CURVES[move.curve] ? CURVES[move.curve](t) : t; + const p = [ + lerp(move.from[0], move.to[0], eased), + lerp(move.from[1], move.to[1], eased), + ]; + // The reach bound is enforced HERE and not only on the endpoints, because a + // curve is allowed to leave the segment between them: `settle` overshoots + // its target by about 10% on purpose. Clamping the plan is not the same as + // clamping the path, and only the path is what reaches the screen. + return move.reach ? clampToReach(p, move.reach) : p; +} + +/** One line for the HUD, the look panel and check output. */ +export function describeCamera(camera) { + if (!camera) return 'camera: locked'; + return `camera: ${camera.name} (reach ${camera.reach.toFixed(2)}, ` + + `pace ${camera.pace.toFixed(2)})`; +} diff --git a/flow-state/src/look/LookGenerator.js b/flow-state/src/look/LookGenerator.js index 7956e4e..e2cdcd7 100644 --- a/flow-state/src/look/LookGenerator.js +++ b/flow-state/src/look/LookGenerator.js @@ -7,7 +7,7 @@ import { Rng, hashSamples } from '../engine/rng.js'; import { AudioPalette, generateUsablePalette } from './palette.js'; import { scenes, scenesInFamily, FAMILIES } from '../scenes/registry.js'; -import { sampleValues, defaultValues, surfaceOf } from '../params/schema.js'; +import { sampleValues, defaultValues, surfaceOf, canBackground } from '../params/schema.js'; import { planShots } from './shots.js'; import { generatePersonality, sceneHonours, signatureWeight, describePersonality, @@ -16,6 +16,8 @@ import { deriveGrain, describeGrain, applyGrainToPost } from './grain.js'; import { pickDirector, directorByName } from './directors.js'; import { derivePaletteArc, describePaletteArc } from './paletteArc.js'; import { deriveFramingStyle, describeFraming } from './framing.js'; +import { deriveCamera, describeCamera } from './Camera.js'; +import { deriveStory, storyForSection, NEUTRAL_STATE } from './Story.js'; // Which families suit which section kind now comes from the track's DIRECTOR // (look/directors.js) rather than from a constant here. The coupling it @@ -35,10 +37,18 @@ const KIND_ENERGY = { * and `energy` up; a breakdown pulls them down. Seed variation still dominates, * so two tracks with the same structure do not converge on the same look. */ -function biasFor(section, summary, motion = null) { +function biasFor(section, summary, motion = null, story = null) { const kindEnergy = KIND_ENERGY[section.kind] ?? 0.5; const measured = Math.min(1, section.energy / Math.max(1e-6, summary.meanLoudness * 1.6)); - const energy = kindEnergy * 0.6 + measured * 0.4; + // Where this section sits in the story moves the bias, and is deliberately + // the smallest term in it. The kind decides what a section IS — the spread + // between an intro and a drop is 0.7 of the range — and the story decides + // which drop this is, worth a tenth of that. Bounded rather than trusted: + // a breakdown at maximum tension is still, unambiguously, a breakdown, and + // the quiet-kind coupling in directors.js depends on it staying that way. + const tension = story ? story.tension : 0.5; + const population = story ? story.population : 0.5; + const energy = clamp01(kindEnergy * 0.6 + measured * 0.4 + (tension - 0.5) * 0.16); // 60bpm → 0, 180bpm → 1. Tempo, not energy, is what a viewer reads as // "this is moving too fast for the song": a slow track can have a huge drop @@ -58,7 +68,7 @@ function biasFor(section, summary, motion = null) { return { energy, - density: Math.min(1, energy * 0.7 + section.flux * 1.2), + density: clamp01(energy * 0.7 + section.flux * 1.2 + (population - 0.5) * 0.2), motion: clamp01(0.12 + tempo * 0.4 + energy * 0.2 + (1 - still) * 0.35), // Applied on top of every `rate: true` param, so absolute animation // speed scales with the song rather than only its sampled position in @@ -69,6 +79,26 @@ function biasFor(section, summary, motion = null) { const clamp01 = (x) => Math.max(0, Math.min(1, x)); +/** + * The track's temperament, moved to where this section sits in the story. + * + * Temperament is the hand on every parameter dial and it was constant for the + * whole video, which is why two occurrences of a kind sampled around the same + * point however far apart they were. Scaling `extremity` by tension is the + * ratchet: the same scene, sampled nearer the ends of its own ranges the later + * it appears. Bounded by the range extremity is drawn from — this moves where a + * track sits inside its own character, it does not give it a different one. + */ +function temperamentFor(temperament, state) { + if (!temperament || !state) return temperament; + const tension = state.tension; + return { + ...temperament, + intensity: Math.max(-1, Math.min(1, temperament.intensity + (tension - 0.5) * 0.5)), + extremity: clamp01(temperament.extremity * (0.82 + tension * 0.4)), + }; +} + /** * Scenes eligible for a section kind, weighted by how well the family fits. * @@ -120,7 +150,7 @@ const clamp01 = (x) => Math.max(0, Math.min(1, x)); export const POOL_SIZE = 8; function castingPool(rng, signature, size = POOL_SIZE) { - const pool = scenes.filter((m) => m.role !== 'accent'); + const pool = scenes.filter(canBackground); const remaining = pool.slice(); const weights = remaining.map((m) => signatureWeight(m, signature)); const picked = []; @@ -141,7 +171,7 @@ function candidatesForKind(kind, used, signature = [], director, pool = null) { const candidates = []; for (const family of families) { const inFamily = scenesInFamily(family) - .filter((m) => m.role !== 'accent' && (!allowed || allowed.has(m.name))); + .filter((m) => canBackground(m) && (!allowed || allowed.has(m.name))); // Weight by family preference order, and push down anything already // used so a five-section track doesn't show one scene five times. const weight = families.length - families.indexOf(family); @@ -159,7 +189,7 @@ function candidatesForKind(kind, used, signature = [], director, pool = null) { // This track's pool holds nothing in the families the director wants for // this kind. Widen to the pool, then to the library — the track keeps // its scenes either way. - const fallback = (pool && pool.length) ? pool : scenes.filter((m) => m.role !== 'accent'); + const fallback = (pool && pool.length) ? pool : scenes.filter(canBackground); return fallback.map((scene) => ({ scene, weight: signatureWeight(scene, signature) })); } return candidates; @@ -169,7 +199,7 @@ function candidatesForKind(kind, used, signature = [], director, pool = null) { * How many stage visuals a kind rotates between. Busy material takes more. * * Sized against the library rather than picked out of the air: a kind draws - * from three families, which is seven to nine non-accent scenes, so a roster of + * from three families, which is seven to nine castable scenes, so a roster of * four still leaves the weighting room to avoid what other kinds already took. * Variants a section never reaches cost nothing — layers are built per cue, so * only the ones its shots actually show are ever compiled. @@ -228,6 +258,50 @@ function assignRostersByKind(sections, rng, signature = [], director, pool = nul return byKind; } +/** + * The RECAPITULATION: the outro re-casts what the intro opened on. + * + * The oldest device in the form and the cheapest one available here — the scene + * is already in the roster and already compiled, and all that changes is which + * member of it anchors. What makes it read as a return rather than as a repeat + * is that the outro plays it with the parameters the story has arrived at: + * the same scene, four minutes further along its slow axis, at the story's + * closing tension. See look/Story.js. + */ +function applyRecap(rosterByKind) { + const intro = rosterByKind.get('intro'); + const outro = rosterByKind.get('outro'); + if (!intro || !outro || !intro.length || !outro.length) return; + const opener = intro[0]; + // Keep the outro's own roster behind the recapped anchor, minus a duplicate: + // the section still cuts away from it, it just opens and closes there. + const rest = outro.filter((m) => m.name !== opener.name); + rosterByKind.set('outro', [opener, ...rest]); +} + +/** + * Which member of the kind's roster anchors THIS section. + * + * `roster[0]` opened every section of its kind, so a track's biggest visual was + * spent in the first fifteen seconds of the first drop and then spent again, + * identically, at every drop after it. Reserving it makes the anchor something + * the video arrives at: earlier occurrences open on a companion, and the + * anchor's own section is the one the story calls the climax. + * + * The roster itself does not change — the section still cuts between all of it, + * which is what keeps the kind's identity — only which member it opens on. + */ +function anchorOrder(roster, state) { + if (roster.length < 2 || !state) return roster; + // The climax, the resolution and any single occurrence get the real anchor. + const earned = state.act === 'climax' || state.act === 'resolution' + || state.ordinalOf < 2 || state.reveal > 0.66; + if (earned) return roster; + + const companion = 1 + (state.ordinal % (roster.length - 1)); + return [roster[companion], ...roster.filter((_, i) => i !== companion)]; +} + /** * Post-processing and feedback derived from track character. * Ambient material gets more feedback and bloom; dense club material gets @@ -267,23 +341,26 @@ function derivePost(summary, rng, grain) { } /** - * One layer stack: a background scene, sometimes a second scene composited over - * it, sometimes an accent on top of that. - * - * Three deliberately different jobs: + * One layer stack: a background scene, and sometimes one or two composable + * scenes composited over it. * * background — the shot. Always present, always opaque. - * overlay — a SECOND full scene at partial opacity. Not always: this is the + * overlay — a composable scene at partial opacity. Not always: this is the * variation valve, and a stack that always doubled up would read * as permanently cluttered rather than as occasionally layered. * Drawn from a different family so the two images argue instead * of blurring, and kept off scenes that are already busy. - * accent — the depth pass. Mostly-empty by design (role: 'accent'), - * additive, low opacity. * - * Quiet material mostly goes without either — an intro is supposed to be sparse. + * There used to be a third slot, `accent`, reserved for scenes declaring + * `role: 'accent'`. Exactly one scene ever declared it, and the overlay path + * above required a `composable` label no scene carried — so the reserved slot + * was the only layering that ever happened, and every layered stack in every + * song was the same particle field. One path, one roster: what goes on top is + * whatever is composable, which is now a third of the library. + * + * Quiet material mostly goes without any — an intro is supposed to be sparse. */ -function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament) { +function buildStack(module, overlayRoster, bias, rng, temperament, story = null) { const layers = [{ module, params: sampleValues(module, rng, bias, temperament), @@ -298,9 +375,21 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament) // corruption passes is noise, not depth. // Layering is much more likely now that what goes on top is guaranteed to // leave the shot underneath visible. + // `population` is how crowded the story wants this point in the video to + // be, and layering is the only lever on that which does not need the scene's + // cooperation: a lone form in an empty frame and the same form under two + // more passes are the sparse and crowded ends of one video. + const crowd = story ? (story.population - 0.5) * 0.5 : 0; + // The base rate was tuned when this branch was dead and layering only ever + // came from the reserved accent slot. With a third of the library eligible + // it lands on half of all stacks, which is the "permanently cluttered" the + // comment above warns about — and it costs seed separation, because a video + // where everything is doubled up looks like every other video where + // everything is doubled up. const overlayChance = module.family === 'glitch' ? 0.1 - : 0.3 + bias.energy * 0.4 + (temperament ? Math.max(0, temperament.detail) * 0.2 : 0); + : 0.2 + bias.energy * 0.35 + crowd + + (temperament ? Math.max(0, temperament.detail) * 0.2 : 0); // Only COMPOSABLE scenes go on top. A second canvas over the first is two // pictures fighting rather than one picture with depth, and it is what the @@ -309,17 +398,34 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament) // The other half of the trade: a composable scene alone is a few bright // things on black, which scores well for variety and is thin to watch. // Layering is what turns both halves into one image. - const overlays = overlayRoster.filter((m) => m.name !== module.name + let available = overlayRoster.filter((m) => m.name !== module.name && surfaceOf(m) === 'composable'); - if (overlays.length && rng.bool(Math.min(0.8, overlayChance))) { - const overlay = rng.pick(overlays); + + // Two passes at the same slot rather than two differently-named slots. The + // second is rarer and only on loud, crowded material — that is where the + // old accent pass used to land, and it is the difference between a shot with + // something over it and a shot with a texture and a shimmer over it. + const chances = [ + Math.min(0.65, overlayChance), + Math.min(0.25, overlayChance * bias.energy * 0.5), + ]; + for (const [pass, chance] of chances.entries()) { + if (!available.length || !rng.bool(chance)) break; + // Prefer a different family so the images argue instead of blurring. + const offFamily = available.filter((m) => m.family !== module.family); + const overlay = rng.pick(offFamily.length ? offFamily : available); + available = available.filter((m) => m.name !== overlay.name + && m.family !== overlay.family); // Screen and add keep the background readable underneath; softlight and // overlay tint it instead. All four preserve the shot; 'normal' would // simply replace it, which is what the shot cut is for. const blend = rng.pickWeighted(['screen', 'add', 'softlight', 'overlay'], [3, 2, 2, 1]); + // A second pass sits lighter than the first, so what accumulates is + // depth rather than a third opaque picture. + const fade = pass === 0 ? 1 : 0.6; layers.push({ module: overlay, - params: sampleValues(overlay, rng.fork(`overlay:${overlay.name}`), { + params: sampleValues(overlay, rng.fork(`overlay:${pass}:${overlay.name}`), { // An overlay reads as texture over the shot, so it is sampled // sparser and calmer than it would be as a background. ...bias, @@ -328,20 +434,7 @@ function buildStack(module, accentRoster, overlayRoster, bias, rng, temperament) }, temperament), seed: rng.int(0, 0x7fffffff), blend, - opacity: blend === 'add' ? rng.range(0.18, 0.42) : rng.range(0.25, 0.55), - }); - } - - // --- accent --------------------------------------------------------- - if (accentRoster.length && rng.bool(bias.energy * 0.8)) { - const eligible = accentRoster.filter((m) => m.family !== module.family); - const accent = rng.pick(eligible.length ? eligible : accentRoster); - layers.push({ - module: accent, - params: sampleValues(accent, rng.fork('accent'), bias, temperament), - seed: rng.int(0, 0x7fffffff), - blend: rng.pickWeighted(['add', 'screen'], [2, 1]), - opacity: rng.range(0.18, 0.5), + opacity: (blend === 'add' ? rng.range(0.18, 0.42) : rng.range(0.25, 0.55)) * fade, }); } return layers; @@ -369,7 +462,7 @@ export function generateLook(track, { // The production design, decided before a single scene is cast — casting // depends on it. See look/Personality.js. const personality = generatePersonality(summary, rng.fork('personality'), (signature) => - scenes.filter((m) => m.role !== 'accent' && sceneHonours(m, signature)).length, + scenes.filter((m) => canBackground(m) && sceneHonours(m, signature)).length, track.sections.length); // The track's point of view about what a song looks like. Cast before any @@ -385,8 +478,14 @@ export function generateLook(track, { const pool = poolOverride && poolOverride.length ? poolOverride : castingPool(rng.fork('pool'), personality.signature, poolSize); + // What HAPPENS over the track, as opposed to what it is made of. Derived + // before casting because it decides which member of a roster anchors which + // section, and whether the outro answers the intro. See look/Story.js. + const story = deriveStory(track, summary, rng.fork('story')); + const rosterByKind = assignRostersByKind( track.sections, rng.fork('scenes'), personality.signature, director, pool); + if (story.recap) applyRecap(rosterByKind); // The grain treatment: usually none, and when present described rather than // dialled. See look/grain.js. const grain = deriveGrain(summary, rng.fork('grain')); @@ -394,38 +493,50 @@ export function generateLook(track, { const paletteArc = derivePaletteArc(summary, rng.fork('paletteArc')); // Whether shots change SIZE at the cut, and how boldly. See look/framing.js. const framing = deriveFramingStyle(summary, rng.fork('framing')); + // WHERE the camera looks, and how it travels there. The director leans + // toward one camera the way it leans toward one family per kind, and the + // seed decides — see look/Camera.js. + const camera = deriveCamera(director, summary, rng.fork('camera')); const { post, feedback } = derivePost(summary, rng.fork('post'), grain); - // Scenes that declare role 'accent' composite over a background rather than - // being one — most of their frame is empty by design. They are never chosen - // as a section's primary scene. - // Accents honour the signature too where they can. If none can, the track - // goes without depth layers rather than putting an off-design element into - // every stack. - const accentRoster = scenes.filter((m) => m.role === 'accent'); - // Scenes eligible to be composited OVER a background. Same casting rule as // everything else — an overlay is on screen as much as the shot under it, // so an off-design one would be just as visible. - const overlayRoster = pool.length >= 4 ? pool : scenes.filter((m) => m.role !== 'accent'); + // + // Widened past the casting pool with the scenes that exist only to sit on + // top: those are never drawn as a section's primary scene, so the pool — + // which is built out of background candidates — would never contain them. + const overlayOnly = scenes.filter((m) => !canBackground(m)); + const overlayRoster = (pool.length >= 4 ? pool : scenes.filter(canBackground)) + .concat(overlayOnly); const sections = track.sections.map((section) => { - const roster = rosterByKind.get(section.kind) || [scenes[0]]; + const state = storyForSection(story, section.index); + const kindRoster = rosterByKind.get(section.kind) || [scenes[0]]; + // The kind's roster, opened on the member this point in the story has + // earned. Same set, different anchor. See anchorOrder. + const roster = anchorOrder(kindRoster, state); const sectionRng = rng.fork(`section:${section.index}:${roster[0].name}`); - const bias = biasFor(section, summary, personality.motion); + const bias = biasFor(section, summary, personality.motion, state); + // The RATCHET: how hard the track pushes its dials is a property of the + // track (Personality.temperament) scaled by where in the story it is, + // so the last occurrence of a kind samples further out than the first. + const temperament = temperamentFor(personality.temperament, state); const variants = roster.map((module, v) => buildStack( - module, accentRoster, overlayRoster, bias, - sectionRng.fork(`variant:${section.index}:${v}`), personality.temperament, + module, overlayRoster, bias, + sectionRng.fork(`variant:${section.index}:${v}`), temperament, state, )); const shots = planShots( - section, track, bias, variants.length, sectionRng.fork(`shots:${section.index}`), + section, track, bias, variants.length, + sectionRng.fork(`shots:${section.index}`), state, ); return { index: section.index, kind: section.kind, + story: state, startFrame: section.startFrame, endFrame: section.endFrame, start: section.start, @@ -447,8 +558,10 @@ export function generateLook(track, { personality, paletteScheme: paletteSource.lastScheme, director: director.name, + story, paletteArc, framing, + camera, grain, post, feedback, @@ -470,8 +583,8 @@ export function rerollSection(look, track, sectionIndex, salt = 0) { // A reroll re-draws this section's cast from the same kind of pool the track // was built with, weighted by the signature rather than filtered by it. let candidates = families.flatMap((f) => scenesInFamily(f)) - .filter((m) => m.role !== 'accent'); - if (!candidates.length) candidates = scenes.filter((m) => m.role !== 'accent'); + .filter(canBackground); + if (!candidates.length) candidates = scenes.filter(canBackground); // Re-roll the whole roster, not just the anchor: the section's shots cut // between all of them, so replacing one would leave the section half old. @@ -484,14 +597,17 @@ export function rerollSection(look, track, sectionIndex, salt = 0) { } if (!roster.length) roster.push(scenes[0]); - const accentRoster = scenes.filter((m) => m.role === 'accent'); - const overlayRoster = castingPool(rng.fork('pool'), signature); + const overlayRoster = castingPool(rng.fork('pool'), signature) + .concat(scenes.filter((m) => !canBackground(m))); + // A reroll changes what this section is made of. Where it sits in the story + // is a property of the song, so it survives untouched. + const state = section.story || NEUTRAL_STATE; section.variants = roster.map((module, v) => buildStack( - module, accentRoster, overlayRoster, section.bias, rng.fork(`variant:${v}`), - look.personality && look.personality.temperament, + module, overlayRoster, section.bias, rng.fork(`variant:${v}`), + temperamentFor(look.personality && look.personality.temperament, state), state, )); section.shots = planShots( - section, track, section.bias, section.variants.length, rng.fork('shots'), + section, track, section.bias, section.variants.length, rng.fork('shots'), state, ); section.layers = section.variants[0]; return look; @@ -529,6 +645,7 @@ export function describeLook(look) { return `seed ${look.seed.toString(16)} · ${look.director} · ${look.paletteScheme} · ` + `${describePersonality(look.personality)} · ${describeGrain(look.grain)} · ` + `${describePaletteArc(look.paletteArc)} · ${describeFraming(look.framing)} · ` + + `${describeCamera(look.camera)} · ` + `${[...new Set(kinds)].join(', ')}`; } diff --git a/flow-state/src/look/Story.js b/flow-state/src/look/Story.js new file mode 100644 index 0000000..0a07c4c --- /dev/null +++ b/flow-state/src/look/Story.js @@ -0,0 +1,404 @@ +// The song's STORY: where a section sits in the video, as opposed to what kind +// of section it is. +// +// Everything that decides what this generator puts on screen is keyed on +// section KIND — the roster (LookGenerator.assignRostersByKind), the parameter +// bias (KIND_ENERGY), the cutting rhythm (shots.rhythmFor), the colour offset +// (paletteArc.kindHue). Kinds recur. So the fourth drop is cast from the same +// roster, biased to the same energy, cut at the same rate and tinted the same +// hue as the first one, and nothing in the video can tell you which of them you +// are watching. +// +// The one exception was ArcDriver's slow axis, which travels one way across the +// whole track — and it is blind: it does not know where the drop is, and its +// direction is a coin flip per scene, so two scenes in one video routinely +// travel against each other. +// +// That is a song structure without a story: recurrence without consequence. +// This module adds the missing coordinate. +// +// POSITION which occurrence of its kind a section is, and which act it is in. +// `ordinal` alone — "the 3rd of 4 drops" — unlocks most of what +// follows, and nothing downstream had it. +// MOMENTS the four frames the video turns on, FOUND in the audio rather +// than placed by the seed. A plot that declares a climax where the +// track is quiet is worse than no plot at all. +// PLOT what the track does with them, as one of a few coherent narrative +// shapes, chosen the way look/directors.js chooses a director. +// +// The plot is expressed as five variables that everything downstream reads. +// They are STAGED, not ramped: they hold flat inside a section and move at its +// boundary. A story advances in scenes; a smooth ramp over five minutes is a +// slow zoom, which is an effect rather than a narrative. +// +// Everything here is a pure function of (story, frame). No state, no random +// source past derivation — the same rule the rest of the render path follows, +// and what keeps a seek frame-identical to playback. + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); +const smooth = (x) => { const t = clamp01(x); return t * t * (3 - 2 * t); }; +const lerp = (a, b, t) => a + (b - a) * t; + +/** The variables a plot moves. Neutral is 0.5 for all of them. */ +export const STORY_VARS = ['tension', 'reveal', 'closeness', 'population', 'order']; + +/** + * What the story reads as when there is none — a look built by hand, or a check + * that constructs sections directly. Everything downstream must render exactly + * what it rendered before this module existed when it sees these. + */ +export const NEUTRAL_STATE = { + tension: 0.5, reveal: 0.5, closeness: 0.5, population: 0.5, order: 0.5, + journey: 0.5, act: 'development', +}; + +/** + * The narrative shapes. + * + * One coherent point of view about what happens over a song, the same way a + * director is one point of view about what a song looks like. Weighted, with + * the audio tilting the odds and never deciding — a fixed mapping from measured + * features to narrative is how a library ends up with one story per genre. + * + * `curve` receives a section's narrative position and the track's seeded curve + * shape, and returns the five variables plus the journey the slow axis follows. + * + * ctx: + * p 0..1 through the section list + * rise 0..1 approach to the climax (1 from the climax onward) + * after 0..1 through the aftermath (0 up to the climax) + * ord 0..1 which occurrence of this kind it is + * energy 0..1 the section's measured energy, against the track's loudest + */ +export const PLOTS = [ + { + name: 'emergence', + weight: 3, + // Almost nothing, then something. The reveal rises and stays up: what + // was uncovered does not go back in the box. + curve: (ctx, sh) => { + const r = smooth(ctx.rise * 0.85 + ctx.p * 0.15); + return { + tension: 0.2 + 0.7 * r, + reveal: lerp(0.12, 1, r), + population: 0.25 + 0.6 * r, + closeness: 0.35 + 0.4 * r * sh.closeDir, + order: 0.6 - 0.1 * r, + journey: r, + }; + }, + }, + { + name: 'escalation', + weight: 3, + // Each recurrence goes further than the last. A ratchet rather than a + // curve — this is the plot that needs `ordinal`, and the one a viewer + // names as "it kept going somewhere". + curve: (ctx, sh) => { + const r = clamp01(ctx.rise * 0.5 + ctx.ord * 0.5); + return { + tension: 0.25 + 0.75 * r, + reveal: 0.3 + 0.6 * r, + population: 0.3 + 0.6 * r, + closeness: 0.35 + 0.45 * r * sh.closeDir, + order: 0.7 - 0.35 * r, + journey: clamp01(r * 0.8 + ctx.p * 0.2), + }; + }, + }, + { + name: 'collapse', + weight: 2, + // Order into entropy. The climax is the thing coming apart rather than + // the thing at its peak, so `order` is monotone down and everything else + // follows the energy. + curve: (ctx, sh) => { + const r = smooth(ctx.rise); + return { + tension: 0.3 + 0.6 * r, + reveal: 0.35 + 0.5 * ctx.p, + population: clamp01(0.35 + 0.5 * r - 0.35 * ctx.after), + closeness: 0.3 + 0.45 * ctx.p * sh.closeDir, + order: 0.9 - 0.8 * smooth(ctx.p * 0.7 + ctx.rise * 0.3), + journey: ctx.p, + }; + }, + }, + { + name: 'return', + weight: 2, + // ABA. Everything arches up to the climax and comes back down — but not + // all the way, and not to the same place: the small residue on `after` + // is what makes it a return rather than a loop. The outro re-casts the + // intro's scene; see LookGenerator and `recap`. + curve: (ctx, sh) => { + const arch = clamp01(smooth(ctx.rise) * (1 - smooth(ctx.after))); + return { + tension: 0.2 + 0.7 * arch, + reveal: clamp01(0.25 + 0.6 * arch + 0.18 * ctx.after), + population: 0.3 + 0.55 * arch, + closeness: 0.35 + 0.45 * arch * sh.closeDir, + order: 0.65 - 0.2 * arch, + journey: clamp01(arch * 0.85 + ctx.p * 0.15), + }; + }, + }, + { + name: 'unveiling', + weight: 2, + // The song's protagonist is withheld and then it is all there is. + // Population falls as reveal rises for that reason: a frame full of + // chorus is what was hiding it. + curve: (ctx, sh) => { + const shown = smooth((ctx.rise - sh.holdOut) / Math.max(0.15, 1 - sh.holdOut)); + return { + tension: 0.25 + 0.6 * smooth(ctx.rise), + reveal: 0.1 + 0.9 * shown, + population: 0.6 - 0.35 * shown, + closeness: 0.3 + 0.6 * shown * sh.closeDir, + order: 0.55 + 0.15 * shown, + journey: clamp01(shown * 0.75 + ctx.p * 0.25), + }; + }, + }, +]; + +export const PLOT_NAMES = PLOTS.map((p) => p.name); + +export function plotByName(name) { + return PLOTS.find((p) => p.name === name) || PLOTS[0]; +} + +/** + * Cast the plot. + * + * The tilts are deliberately loose: a track that keeps changing what it is + * doing has recurrences to ratchet, a dynamic one has somewhere to come back + * from, a noisy one comes apart more readily than it builds. Every plot stays + * reachable for every track. + */ +function pickPlot(summary, sectionCount, rng) { + const noisy = Math.min(1, (summary.meanFlatness ?? 0.2) * 3); + const dynamic = clamp01(summary.dynamicRange ?? 0.5); + const busy = clamp01((sectionCount - 2) / 5); + + const weights = PLOTS.map((plot) => { + let w = plot.weight; + if (plot.name === 'escalation') w *= 0.5 + busy * 1.8; + if (plot.name === 'collapse') w *= 0.5 + noisy * 1.8; + if (plot.name === 'return') w *= 0.5 + dynamic * 1.4; + if (plot.name === 'emergence') w *= 0.6 + (1 - busy) * 1.2; + if (plot.name === 'unveiling') w *= 0.6 + (1 - noisy) * 1.0; + return w; + }); + return rng.pickWeighted(PLOTS, weights); +} + +/** + * The frames the story turns on. + * + * Every one of these is read off the section energies `segment.js` already + * produced. Nothing here may invent a moment the audio does not have: the story + * follows the song, and the one failure mode worse than no story is a story + * that declares its climax where the track is quiet. + * + * A short or flat track collapses several of these onto the same section, which + * is correct rather than degenerate — a song with no structure gets no story. + */ +function findMoments(sections) { + const n = sections.length; + const energies = sections.map((s) => s.energy || 0); + const peak = Math.max(...energies, 1e-6); + + // The climax is the loudest section, ties broken toward the later one: when + // a track states the same peak twice, the second one is the one that means + // something, because the first has already happened. + let climax = 0; + for (let i = 0; i < n; i++) if (energies[i] >= energies[climax]) climax = i; + + // The arrival is the video's first "here it is" — the first section that + // clears most of the way to the peak. + let arrival = null; + for (let i = 0; i < n; i++) { + if (energies[i] >= peak * 0.6) { arrival = i; break; } + } + if (arrival === null) arrival = Math.min(climax, n - 1); + + // The turn is the largest fall, and it has to happen after something has + // arrived — a quiet opening followed by a quieter one is not a reversal. + let turn = null; + let worst = 0; + for (let i = arrival + 1; i < n; i++) { + const fall = energies[i - 1] - energies[i]; + if (fall > worst) { worst = fall; turn = i; } + } + + // The resolution is where the track stops trying to top itself. + let resolution = null; + for (let i = climax + 1; i < n; i++) { + if (energies[i] < energies[climax] * 0.85) { resolution = i; break; } + } + if (resolution === null && climax < n - 1) resolution = n - 1; + + return { arrival, turn, climax, resolution }; +} + +function actFor(index, moments) { + if (index === moments.climax) return 'climax'; + if (moments.resolution !== null && index >= moments.resolution) return 'resolution'; + if (moments.turn !== null && index === moments.turn) return 'turn'; + if (moments.arrival !== null && index < moments.arrival) return 'setup'; + return 'development'; +} + +/** + * Derive the story for a track. + * + * @param {FeatureTrack} track + * @param {object} summary track.summary + * @param {Rng} rng + * @returns {object} plain data — no functions, so a look stays serialisable + */ +export function deriveStory(track, summary, rng) { + const sections = track.sections || []; + const n = sections.length; + const plot = pickPlot(summary, n, rng); + + // The track's own version of its plot. Two tracks telling the same story + // still have to differ, for the reason directors.js gives about house + // styles — so the shape of the curves is seeded even when the plot is not. + const shape = { + // Whether this video moves toward its subject or pulls away from it. + // Toward, usually: a video that ends further away than it started is a + // real choice and a rarer one. + closeDir: rng.bool(0.75) ? 1 : -0.6, + // How long `unveiling` keeps its protagonist back. + holdOut: rng.range(0.45, 0.8), + // How hard this track commits to its plot at all. + bite: rng.range(0.7, 1.0), + }; + + const moments = findMoments(sections); + + // How much story a track has room for. Under about four sections the curves + // have nowhere to travel, and forcing them produces a video that lurches + // rather than one that progresses, so the whole layer fades toward neutral. + const strength = clamp01((n - 1) / 3) * shape.bite; + + const peak = Math.max(...sections.map((s) => s.energy || 0), 1e-6); + const climax = moments.climax; + const seenKind = new Map(); + const countKind = new Map(); + for (const s of sections) countKind.set(s.kind, (countKind.get(s.kind) || 0) + 1); + + const states = sections.map((section, i) => { + const ordinal = seenKind.get(section.kind) || 0; + seenKind.set(section.kind, ordinal + 1); + const ordinalOf = countKind.get(section.kind) || 1; + + const ctx = { + p: n > 1 ? i / (n - 1) : 0.5, + rise: climax > 0 ? clamp01(i / climax) : 1, + after: i > climax && n - 1 > climax ? clamp01((i - climax) / (n - 1 - climax)) : 0, + ord: ordinalOf > 1 ? ordinal / (ordinalOf - 1) : (n > 1 ? i / (n - 1) : 0.5), + energy: clamp01((section.energy || 0) / peak), + }; + + const raw = plot.curve(ctx, shape); + + // The plot proposes and the song disposes. Blending the section's + // measured energy back in is what stops a story-driven tension from + // overriding a quiet section — the narrative may say "further than + // before", it may not say "loud" where the track is not. + raw.tension = raw.tension * 0.75 + ctx.energy * 0.25; + + const state = { index: i, kind: section.kind, ordinal, ordinalOf, act: actFor(i, moments) }; + for (const key of STORY_VARS) { + state[key] = clamp01(lerp(0.5, clamp01(raw[key]), strength)); + } + // The journey keeps its full travel — it is the axis the whole video + // moves along, and halving it on a four-section track is the same as + // not having it. Its NEUTRAL is the plain progress ramp the slow axis + // used before this module existed, so a thin track degrades to exactly + // the old behaviour rather than to a flat line. + state.journey = clamp01(lerp(ctx.p, clamp01(raw.journey), strength)); + return state; + }); + + // How long a story variable takes to arrive at its new value. Roughly a + // bar: long enough that nothing pops mid-crossfade (the failure buildSlope + // already taught us about at a boundary), short enough that against a + // five-minute video it still reads as a step rather than a ramp. + const barSeconds = track.tempo + ? (track.tempo.period * track.tempo.beatsPerBar) / track.fps : 2; + const blendFrames = Math.max(6, Math.round(barSeconds * track.fps)); + + return { + plot: plot.name, + shape, + moments, + strength, + blendFrames, + // All of a video's scenes should travel the same WAY. Magnitude stays + // per scene, but the sign is the track's, because two scenes drifting + // against each other is what made the slow axis read as wobble rather + // than as a direction. + axisSign: rng.bool() ? 1 : -1, + // The oldest device available and the cheapest one here: the outro + // re-casts the intro's scene, played with the parameters the story has + // arrived at rather than the ones it opened with. + recap: plot.name === 'return' || rng.bool(0.15), + sections: states, + frames: sections.map((s) => ({ startFrame: s.startFrame, endFrame: s.endFrame })), + }; +} + +/** The story state of one section, or the neutral read. */ +export function storyForSection(story, index) { + if (!story || !story.sections || !story.sections[index]) return { ...NEUTRAL_STATE }; + return story.sections[index]; +} + +/** + * The story state at a frame. + * + * Held flat inside a section and blended over `blendFrames` after each boundary. + * Continuous in frame on purpose: the outgoing layer of a crossfade is still on + * screen when the boundary passes, and stepping its inputs there is a visible + * pop at exactly the moment the edit is trying to hide. + */ +export function storyStateAt(story, frame) { + if (!story || !story.sections || !story.sections.length) return { ...NEUTRAL_STATE }; + + const frames = story.frames; + let lo = 0; + let hi = frames.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (frames[mid].startFrame <= frame) lo = mid; else hi = mid - 1; + } + + const current = story.sections[lo]; + if (lo === 0) return current; + + const into = frame - frames[lo].startFrame; + if (into >= story.blendFrames) return current; + + const previous = story.sections[lo - 1]; + const t = smooth(into / story.blendFrames); + const out = { index: current.index, kind: current.kind, act: current.act, + ordinal: current.ordinal, ordinalOf: current.ordinalOf }; + for (const key of STORY_VARS) out[key] = lerp(previous[key], current[key], t); + out.journey = lerp(previous.journey, current.journey, t); + return out; +} + +/** One line for the HUD, the look panel and check output. */ +export function describeStory(story) { + if (!story) return 'story: none'; + const m = story.moments; + const at = (i) => (i === null || i === undefined ? '–' : `§${i}`); + return `story: ${story.plot}${story.recap ? ' + recap' : ''} · ` + + `arrival ${at(m.arrival)} turn ${at(m.turn)} climax ${at(m.climax)} ` + + `resolution ${at(m.resolution)}`; +} diff --git a/flow-state/src/look/directors.js b/flow-state/src/look/directors.js index 8b94107..1c623cd 100644 --- a/flow-state/src/look/directors.js +++ b/flow-state/src/look/directors.js @@ -45,6 +45,10 @@ const QUIET_KINDS = ['intro', 'breakdown', 'outro']; export const DIRECTORS = [ { name: 'ambient', + // Patient camera to match: long moves, mostly along one line. See + // look/Camera.js — a director's point of view now includes how it + // shoots, not only what it points at. + camera: 'contemplative', // The original table. A drop resolves into geometry; everything quiet is // minimal. Still the most broadly applicable, so it keeps the most weight. weight: 3, @@ -59,6 +63,8 @@ export const DIRECTORS = [ }, { name: 'brutalist', + // Holds, then commits to one large move. Architecture is looked AT. + camera: 'deliberate', // Everything is architecture. Quiet means empty rather than soft, so it // leads on minimal and reaches for organic last. weight: 2, @@ -73,6 +79,8 @@ export const DIRECTORS = [ }, { name: 'organicist', + // Never settles, because nothing here is ever finished settling. + camera: 'roaming', // Nothing is ever built; things grow and dissolve. Deliberately never // reaches for glitch — a point of view is defined by what it refuses. weight: 2, @@ -87,6 +95,8 @@ export const DIRECTORS = [ }, { name: 'corrupt', + // Cuts with the camera already moving. + camera: 'kinetic', // The signal is damaged and the damage is the subject — everywhere the // damage is allowed to be. Its quiet sections lead on flow, so the calm // reads as signal drifting rather than as rest. @@ -102,6 +112,9 @@ export const DIRECTORS = [ }, { name: 'geometer', + // Small, exact, always arrives — the pattern is the subject and the + // camera does not editorialise about it. + camera: 'precise', // Pattern first, everywhere, at every energy. The drop is not an // explosion, it is the pattern at its densest. weight: 2, diff --git a/flow-state/src/look/framing.js b/flow-state/src/look/framing.js index fe81751..a79dd80 100644 --- a/flow-state/src/look/framing.js +++ b/flow-state/src/look/framing.js @@ -11,9 +11,15 @@ // rather than being a magnified 720p frame, which is why this is a coordinate // transform and not a post pass. It is also why it costs nothing at 4K. // -// Framing is per shot and constant within it. A zoom that moves during a shot +// Shot SIZE is per shot and constant within it. A zoom that moves during a shot // is a different device — one that would fight the drift LFO and the slow axis, // both of which already own continuous motion. +// +// The RECENTRE is not: where the camera is looking moves during a shot, and +// that lives in look/Camera.js. This module used to own both and treated them +// the same way, which is how the recentre ended up as a per-shot constant of +// about 3% of a half-frame at a random angle — a device that was present in +// every frame of every video and visible in none of them. /** * The shot sizes, as multipliers on the scene's coordinate scale. @@ -24,9 +30,9 @@ * measured by pushing until the image stopped being worth looking at. */ export const SHOT_SIZES = { - wide: { scale: 0.62, drift: 0.06 }, - normal: { scale: 1.0, drift: 0.05 }, - close: { scale: 1.7, drift: 0.10 }, + wide: { scale: 0.62 }, + normal: { scale: 1.0 }, + close: { scale: 1.7 }, }; export const SHOT_SIZE_NAMES = Object.keys(SHOT_SIZES); @@ -61,8 +67,10 @@ export function deriveFramingStyle(summary, rng) { * @param {object|null} previous the previous shot's framing * @param {number} energy section energy, 0..1 * @param {Rng} rng + * @param {number} closeness 0..1 from the story — how near this point in the + * video wants to be to its subject */ -export function frameShot(style, previous, energy, rng) { +export function frameShot(style, previous, energy, rng, closeness = 0.5) { if (style.mode === 'locked') return neutralFraming(); const keep = previous && !rng.bool(style.changeChance); @@ -71,10 +79,18 @@ export function frameShot(style, previous, energy, rng) { // Loud material earns the close-ups; quiet material earns the wides. This // is a lean rather than a rule, so an intro can still land on a close and // read as intimate instead of empty. + // + // The story tilts the same draw across the video: a track that has been + // approaching its subject for four minutes should not answer its climax + // with a wide just because the dice said so. Still a tilt — both sizes stay + // reachable everywhere, because a story told by never cutting wide again is + // one shot type held for five minutes, which is what framing was added to + // stop. + const near = Math.max(0, Math.min(1, closeness)); const weights = [ - 1 + (1 - energy) * 2.5, // wide - 2, // normal - 1 + energy * 2.5, // close + (1 + (1 - energy) * 2.5) * (1.4 - near * 0.9), // wide + 2, // normal + (1 + energy * 2.5) * (0.6 + near * 0.9), // close ]; let size = rng.pickWeighted(SHOT_SIZE_NAMES, weights); @@ -90,17 +106,17 @@ export function frameShot(style, previous, energy, rng) { // than the same sizes drawn less often. const scale = 1 + (spec.scale - 1) * style.range; - // Recentring is what stops a close-up being a centre crop of the wide. Held - // small: the scenes are centred compositions and pushing far off centre - // finds their empty corners. - const angle = rng.range(0, Math.PI * 2); - const amount = spec.drift * style.range * rng.range(0.3, 1); - - return { - size, - scale, - shift: [Math.cos(angle) * amount, Math.sin(angle) * amount], - }; + // Recentring used to be decided here, as `spec.drift * style.range` at a + // uniform random angle — capped at 0.10 of a half-frame, with a fresh + // direction every shot. Measured over 121 cues it moved the frame by a + // median of 0.029 and successive shots mostly cancelled, so the device was + // inert. It is now the camera's, planned as a continuous path across the + // whole video and driven by the story. See look/Camera.js. + // + // `shift` stays on the returned framing so a look built without a camera — + // a hand-made one, or a check constructing framings directly — still has + // the field every consumer already reads. + return { size, scale, shift: [0, 0] }; } export function neutralFraming() { diff --git a/flow-state/src/look/paletteArc.js b/flow-state/src/look/paletteArc.js index f1ca877..329d9ce 100644 --- a/flow-state/src/look/paletteArc.js +++ b/flow-state/src/look/paletteArc.js @@ -24,7 +24,7 @@ export const MAX_HUE_ROTATION = 0.6; // radians, ~34 degrees export const MAX_CHROMA_SCALE = 0.35; // ±35% saturation export const MAX_LIGHT_SHIFT = 0.07; // OKLCH lightness -export const ARC_MODES = ['static', 'drift', 'sections', 'lift']; +export const ARC_MODES = ['static', 'drift', 'sections', 'lift', 'narrative']; /** * How this track's colour moves. @@ -34,7 +34,12 @@ export const ARC_MODES = ['static', 'drift', 'sections', 'lift']; * track whose colour holds is a legitimate choice and one in six or so gets it. */ export function derivePaletteArc(summary, rng) { - const mode = rng.pickWeighted(ARC_MODES, [1.5, 3, 3, 2.5]); + // 'narrative' is the same movement as 'drift', keyed on where the story is + // rather than on how much of the file has elapsed. It is weighted highest + // because it is the only mode whose colour change lands WITH something: a + // clock has no reason to turn the picture warmer at three minutes, and the + // arrival of a climax does. + const mode = rng.pickWeighted(ARC_MODES, [1.5, 2.5, 3, 2.5, 3.5]); const dir = rng.bool() ? 1 : -1; return { @@ -72,8 +77,11 @@ export function derivePaletteArc(summary, rng) { * @param {number} ctx.progress 0..1 through the track * @param {string} ctx.sectionKind kind of the section this frame is in * @param {object} ctx.features FeatureTrack row + * @param {object} ctx.story story state for this frame, see look/Story.js */ -export function paletteShiftAt(arc, { progress = 0, sectionKind = '', features = null } = {}) { +export function paletteShiftAt(arc, { + progress = 0, sectionKind = '', features = null, story = null, +} = {}) { if (!arc) return { hue: 0, chroma: 1, lightness: 0 }; // The slow underlying travel, present in every mode. Eased rather than @@ -97,6 +105,20 @@ export function paletteShiftAt(arc, { progress = 0, sectionKind = '', features = chroma = 1 + arc.chromaLift * (drive * 2 - 1); lightness = arc.lightLift * (drive - 0.35); hue += arc.kindHue[sectionKind] * 0.4 || 0; + } else if (arc.mode === 'narrative') { + // Colour follows the STORY rather than the clock. The journey carries + // the hue, tension opens and closes the saturation, and what has been + // revealed lifts the value — so the frame at the climax is a colour the + // opening implied and the resolution comes back off it. + // + // A track with no story degrades to 'drift': journey is the plain + // progress ramp when Story.js has nothing to work with, and the two + // expressions are then identical. + const s = story || {}; + const journey = s.journey ?? eased; + hue = arc.hueTravel * journey; + chroma = 1 + arc.chromaLift * ((s.tension ?? 0.5) * 2 - 1); + lightness = arc.lightLift * ((s.reveal ?? 0.5) - 0.4); } return { @@ -113,5 +135,9 @@ export function describePaletteArc(arc) { if (!arc || arc.mode === 'static') return 'colour: held'; if (arc.mode === 'drift') return `colour: drift ${(arc.hueTravel * 57.3).toFixed(0)}°`; if (arc.mode === 'lift') return `colour: lift ±${(arc.chromaLift * 100).toFixed(0)}% sat`; + if (arc.mode === 'narrative') { + return `colour: narrative ${(arc.hueTravel * 57.3).toFixed(0)}° / ` + + `±${(arc.chromaLift * 100).toFixed(0)}% sat`; + } return `colour: per-section ±${(Math.max(...Object.values(arc.kindHue).map(Math.abs)) * 57.3).toFixed(0)}°`; } diff --git a/flow-state/src/look/shots.js b/flow-state/src/look/shots.js index 8743c9d..53b3df2 100644 --- a/flow-state/src/look/shots.js +++ b/flow-state/src/look/shots.js @@ -51,7 +51,17 @@ export const HARD_CUT_ENERGY = 0.66; * Every entry is a power-of-two bar count, so a cut is always on a phrase line * of some depth even before it is snapped to a downbeat. */ -function rhythmFor(energy, rng) { +function rhythmFor(energy, rng, story = null) { + // Where the section sits in the story shifts which band it cuts in. A song + // does not only get louder toward its climax, it gets more urgent, and edit + // rate is the one register that says urgency without changing the image at + // all. The resolution goes the other way and holds — the last thing a video + // should do is keep cutting at the pace of the thing that just ended. + if (story) { + const urgency = (story.tension - 0.5) * 0.22; + energy = Math.max(0, Math.min(1, energy + urgency)); + if (story.act === 'resolution') energy = Math.min(energy, 0.44); + } if (energy > 0.72) { // Loud material: quick cuts, but still answered by a longer hold. return rng.pick([[4, 4, 8], [8, 4, 4], [4, 4, 4, 8], [8, 8, 4, 4], [4, 8, 4, 4]]); @@ -116,14 +126,15 @@ function snapCut(from, ideal, downbeats, tolerance) { * @param {object} bias the section's bias, for energy * @param {number} variantCount how many stage visuals the section has * @param {Rng} rng + * @param {object|null} story this section's story state, see look/Story.js * @returns {Array<{index,startFrame,endFrame,variant,hardCut}>} */ -export function planShots(section, track, bias, variantCount, rng) { +export function planShots(section, track, bias, variantCount, rng, story = null) { const fps = track.fps; const duration = Math.max(0, section.end - section.start); const barSeconds = (track.tempo.period * track.tempo.beatsPerBar) / fps; - const lengths = fitPattern(rhythmFor(bias.energy, rng), barSeconds); + const lengths = fitPattern(rhythmFor(bias.energy, rng, story), barSeconds); const shortest = Math.min(...lengths); // A section with only one visual to show has nothing to cut to. diff --git a/flow-state/src/main.js b/flow-state/src/main.js index fb1693a..0cf2e1f 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -287,7 +287,7 @@ function renderPanel() { if (section.layers.length > 1) { const note = document.createElement('div'); note.className = 'pp-reactive'; - note.innerHTML = '