import * as THREE from 'three'; import { VERTEX_SHADER, buildFragmentShader, AUDIO_UNIFORMS, SIGNATURE_UNIFORMS, IDENTITY_UNIFORMS, IDENTITY_ARRAY_UNIFORMS, } from './shader-contract.js'; import { signatureUniforms, NEUTRAL_UNIFORMS } from '../look/Personality.js'; import { clampValue, isDiscreteParam } from '../params/schema.js'; import { surfaceOf } from '../scenes/surface.js'; import { CANVAS_BEAT_CAP, CANVAS_FLUX_CAP, CANVAS_BEAT_SOFTEN_SPIKE, } from '../look/comfort.js'; export const BLEND_MODES = ['normal', 'add', 'screen', 'multiply', 'overlay', 'softlight']; /** * Map a raw feature value through a response curve. `spike` squares the input so * beat-driven params punch rather than wobble; `smooth` rounds the shoulders so * slow features don't step. */ function applyResponse(value, response) { switch (response) { case 'spike': return value * value; case 'smooth': return value * value * (3 - 2 * value); case 'inverse': return 1 - value; default: return value; } } /** * The standard uniform dictionary every fragment layer is compiled against. * Shared by ShaderLayer and the OSD plate so a text layer honours exactly the * same contract — palette, audio features and personality — as a scene. */ export function buildShaderUniforms(module, baseParams, seed) { const uniforms = { u_resolution: { value: new THREE.Vector2(1, 1) }, u_aspect: { value: 1 }, u_pixelScale: { value: 1 }, u_time: { value: 0 }, u_frame: { value: 0 }, u_progress: { value: 0 }, u_seed: { value: (seed >>> 0) % 100000 / 1000 }, u_opacity: { value: 1 }, u_colors: { value: Array.from({ length: 8 }, () => new THREE.Vector3(1, 1, 1)) }, u_colorCount: { value: 1 }, u_prev: { value: null }, u_hasPrev: { value: 0 }, }; for (const name of AUDIO_UNIFORMS) uniforms[name] = { value: 0 }; for (const [name, type] of Object.entries({ ...SIGNATURE_UNIFORMS, ...IDENTITY_UNIFORMS })) { const v = NEUTRAL_UNIFORMS[name]; uniforms[name] = { value: type === 'vec2' ? new THREE.Vector2(v[0], v[1]) : v }; } // The assembly rides as an array of rows — see IDENTITY_ARRAY_UNIFORMS. The // vectors are allocated once and written in place per frame, like u_colors. for (const [name, def] of Object.entries(IDENTITY_ARRAY_UNIFORMS)) { uniforms[name] = { value: Array.from({ length: def.length }, () => new THREE.Vector4()), }; } for (const [name, def] of Object.entries(module.params || {})) { if (!def.uniform || def.type === 'palette') continue; const v = baseParams[name]; uniforms[def.uniform] = { value: def.type === 'vec2' ? new THREE.Vector2(v ? v[0] : 0, v ? v[1] : 0) : def.type === 'bool' ? (v ? 1 : 0) : (v || 0), }; } return uniforms; } /** * Push the per-frame values into a fragment layer's uniforms: the frame, the * audio features, the palette and the personality. Shared by scene layers and * the OSD plate so they all see the same inputs — the same render path, so * what a scene honours a plate honours too. */ export function setFrameUniforms(layer, renderer, target, ctx) { const u = layer.uniforms; const { timeline, features, prevTexture } = ctx; const w = target ? target.width : renderer.width; const h = target ? target.height : renderer.height; u.u_resolution.value.set(w, h); u.u_aspect.value = w / h; u.u_pixelScale.value = h / 1080; // reference height; keeps 720p ≡ 4K u.u_time.value = timeline.time; u.u_frame.value = timeline.frame; u.u_progress.value = timeline.progress; u.u_opacity.value = layer.opacity; if (features) { for (const name of AUDIO_UNIFORMS) { const key = name.slice(2); // u_bandLow -> bandLow const v = features[key]; u[name].value = v === undefined ? 0 : v; } } const colors = layer.palette || []; u.u_colorCount.value = Math.max(1, Math.min(8, colors.length)); for (let i = 0; i < 8; i++) { const c = colors[i % Math.max(1, colors.length)]; if (c) u.u_colors.value[i].set(c[0], c[1], c[2]); } const signature = signatureUniforms(layer.personality, layer.module); for (const [name, type] of Object.entries({ ...SIGNATURE_UNIFORMS, ...IDENTITY_UNIFORMS })) { const v = signature[name]; if (type === 'vec2') u[name].value.set(v[0], v[1]); else u[name].value = v; } for (const [name, def] of Object.entries(IDENTITY_ARRAY_UNIFORMS)) { const rows = signature[name] || []; for (let i = 0; i < def.length; i++) { const r = rows[i]; u[name].value[i].set(r ? r[0] : 0, r ? r[1] : 0, r ? r[2] : 0, r ? r[3] : 0); } } // Framing is per shot and wins over the personality's neutral defaults — // it is the operator's hand on a shot that is already set up, not a trait // of the track. See look/framing.js. if (layer.framing) { u.u_sigFrameScale.value = layer.framing.scale; u.u_sigFrameShift.value.set(layer.framing.shift[0], layer.framing.shift[1]); } u.u_prev.value = prevTexture || null; u.u_hasPrev.value = prevTexture ? 1 : 0; } /** Common surface for shader layers and 3D layers, so Compositor holds one type. */ export class Layer { constructor({ module, params = {}, seed = 1, opacity = 1, blend = 'normal' }) { this.module = module; this.baseParams = { ...params }; this.params = { ...params }; this.seed = seed >>> 0; this.opacity = opacity; this.blend = blend; this.palette = []; this.personality = null; this.venueState = null; } /** * The track's production design. Constant for the whole video — see * look/Personality.js — and pushed in the same way the palette is, so a * layer never reaches for global state. */ setPersonality(personality) { this.personality = personality; return this; } setParams(params) { this.baseParams = { ...this.baseParams, ...params }; return this; } setPalette(colors) { this.palette = colors; return this; } /** * This shot's FRAMING — a per-shot scale/recentre pushed by the arc driver. * * A fragment layer has it applied for it, in the shader epilogue, to the * coordinate every scene is handed. A 3D layer receives it in `update` and * honours it as a camera move, because only it knows what its camera means. * A layer that is never framed renders at the neutral full-frame scale, so * nothing predating framing changes. */ setFraming(framing) { this.framing = framing || null; return this; } /** Resolve base params + reactive modulation into the values used this frame. */ resolveParams(features) { const defs = this.module.params || {}; const reactive = this.module.reactive || {}; const out = this.params; for (const name of Object.keys(defs)) out[name] = this.baseParams[name]; if (!features) return out; // A — whole-screen guard: canvas scenes (≥50% coverage) must not swerve // or jump with the beat. Every mid-tempo kick (≈2 Hz at 124 BPM) then // moves the ENTIRE view in sync — the nausea source on // ?song=centre&seed=3837488838. Only smaller regions/elements should // react wildly; they light few tiles. This is the immediate, central cap: // beat → ≤0.15 and spike→smooth, flux → ≤0.25. Composable scenes are // untouched. Tunable in look/comfort.js; stale metadata detected via // checks/metadata.js SOURCES. See engine/agitation.js for the tile metric. const isCanvas = surfaceOf(this.module) === 'canvas'; for (const [name, r] of Object.entries(reactive)) { const def = defs[name]; if (!def || def.type === 'palette') continue; // Rate params multiply absolute time; modulating them jumps the phase // by elapsed * delta. See params/schema.js RATE_FLAG. if (def.rate) continue; // Continuity rule (see params/schema.js isDiscreteParam): a reactive // count pops members on every beat. Discrete params hold per cue. if (isDiscreteParam(def)) continue; const raw = features[r.feature]; if (raw === undefined) continue; let amount = r.amount; let response = r.response; if (isCanvas) { if (r.feature === 'beat') { amount = Math.min(amount, CANVAS_BEAT_CAP); if (CANVAS_BEAT_SOFTEN_SPIKE && response === 'spike') response = 'smooth'; } else if (r.feature === 'flux') { amount = Math.min(amount, CANVAS_FLUX_CAP); } } const shaped = applyResponse(Math.max(0, Math.min(1, raw)), response); const [lo, hi] = def.range || [0, 1]; const span = hi - lo; const base = out[name] !== undefined ? out[name] : lo; if (def.type === 'vec2') { out[name] = [ clampValue(def, [base[0] + shaped * amount * span, 0])[0], clampValue(def, [0, base[1] + shaped * amount * span])[1], ]; } else if (def.type === 'bool') { out[name] = base; } else { out[name] = clampValue(def, base + shaped * amount * span); } } return out; } render() { throw new Error('Layer.render not implemented'); } dispose() {} } /** A fullscreen fragment-shader scene. The common case. */ export class ShaderLayer extends Layer { constructor(options) { super(options); this.uniforms = buildShaderUniforms(this.module, this.baseParams, this.seed); this.material = new THREE.ShaderMaterial({ vertexShader: VERTEX_SHADER, fragmentShader: buildFragmentShader(this.module), uniforms: this.uniforms, depthTest: false, depthWrite: false, }); } render(renderer, target, ctx) { const { features } = ctx; setFrameUniforms(this, renderer, target, ctx); const resolved = this.resolveParams(features); for (const [name, def] of Object.entries(this.module.params || {})) { if (!def.uniform || def.type === 'palette') continue; const target_u = this.uniforms[def.uniform]; const v = resolved[name]; if (v === undefined) continue; if (def.type === 'vec2') target_u.value.set(v[0], v[1]); else if (def.type === 'bool') target_u.value = v ? 1 : 0; else target_u.value = v; } renderer.blit(this.material, target); } dispose() { this.material.dispose(); } } /** * A layer backed by a real three.js scene — particles, geometry, camera motion. * The module supplies build/update hooks; everything determinism-related (seeded * rng, fixed dt, explicit re-seed at section boundaries) is handled here so 3D * modules can't accidentally reintroduce wall-clock or Math.random. */ export class SceneLayer extends Layer { constructor(options) { super(options); this.scene = new THREE.Scene(); this.camera = new THREE.PerspectiveCamera(60, 16 / 9, 0.1, 200); this.camera.position.set(0, 0, 5); this.instance = this.module.build({ scene: this.scene, camera: this.camera, seed: this.seed, params: this.baseParams, THREE, }); } render(renderer, target, ctx) { const { timeline, features } = ctx; const w = target ? target.width : renderer.width; const h = target ? target.height : renderer.height; if (this.camera.aspect !== w / h) { this.camera.aspect = w / h; this.camera.updateProjectionMatrix(); } const resolved = this.resolveParams(features); this.module.update({ instance: this.instance, scene: this.scene, camera: this.camera, timeline, features: features || {}, params: resolved, palette: this.palette, personality: this.personality, // A 3D layer has a literal camera, so framing reaches it as a // camera move rather than as a coordinate transform. Fragment // scenes get it applied for them in the shader epilogue; this one // has to honour it itself, because only it knows what its camera // means. See look/framing.js. framing: this.framing, opacity: this.opacity, THREE, }); renderer.renderScene(this.scene, this.camera, target, true); } dispose() { this.scene.traverse((obj) => { if (obj.geometry) obj.geometry.dispose(); if (obj.material) { const mats = Array.isArray(obj.material) ? obj.material : [obj.material]; mats.forEach((m) => m.dispose()); } }); } } /** * A 3D model layer — the mesh twin of the shader impostor. * * Like SceneLayer it owns a THREE.Scene and receives build/update hooks, but * its geometry comes from the song's ActorSpec (the mesh assembly), not from * a hand-written point cloud. Determinism rule is the same: no integration, * only analytic f(time, index, seed). See processors/meshes.js and * src/actors/ActorGenerator.js. * * The camera is borrowed from Compositor.sharedCamera when one exists, so * multiple ModelLayers share one perspective and one depth buffer — that is * what makes a ground mesh occlude a subject mesh from another layer. * Falls back to its own camera when no shared rig is present (tests, solo * preview), so existing SceneLayer behaviour is unchanged. */ export class ModelLayer extends Layer { constructor(options) { super(options); this.scene = new THREE.Scene(); // Model layers are sparse: a hero plus a few satellites on a ground. // Mark the scene for a transparent clear so Renderer clears to alpha 0 // and Compositor's lumakey/normal blend shows the composition underneath // where there is no mesh — same role as a 'composable' shader layer. this.scene.userData.transparentBackground = true; // Keep the WebGL clear colour transparent as well; renderScene reads the // scene flag but the fallback alpha is set here for safety. this.scene.background = null; this.camera = new THREE.PerspectiveCamera(60, 16 / 9, 0.1, 200); this.camera.position.set(0, 0, 5); this.actorSpec = options.actorSpec || null; this.venueSpec = options.venueSpec || null; // Shared rig injected by Compositor at render time when available. this.sharedCamera = null; this.instance = this.module.build({ scene: this.scene, camera: this.camera, seed: this.seed, params: this.baseParams, actorSpec: this.actorSpec, venueSpec: this.venueSpec, THREE, }); } /** Allow the look to swap the venue without rebuilding the layer. */ setVenue(venueSpec) { this.venueSpec = venueSpec || null; return this; } /** * Bind the composited viz texture onto this layer's screen (venue RTT). * No-op unless the module implements bindVizTexture(instance, texture). * Delegating keeps venue materials owned by the venue module — the * compositor never touches venue meshes directly. */ bindVizTexture(texture) { if (this.module && typeof this.module.bindVizTexture === 'function' && this.instance) { this.module.bindVizTexture(this.instance, texture); } return this; } /** * Base wash for the viz composite. The shot is keyed over this colour * instead of over black: a shot floating on black is the composition * failure the phase 12 rules forbid, miniaturised onto the venue screen — * and it measured ~5x the no-venue section distance (sparse bright things * on nothing maximise layout/region jumps). Defaults to a dark neutral; * venue modules override via washColor(instance) to match their room. */ washColor() { try { if (this.module && typeof this.module.washColor === 'function' && this.instance) { const c = this.module.washColor(this.instance); if (Array.isArray(c) && c.length >= 3) return [c[0], c[1], c[2]]; } } catch { /* fall through to default */ } return [0.03, 0.03, 0.04]; } /** Allow the look to swap the actor without rebuilding the layer. */ setActor(actorSpec) { this.actorSpec = actorSpec || null; return this; } render(renderer, target, ctx) { const { timeline, features } = ctx; const w = target ? target.width : renderer.width; const h = target ? target.height : renderer.height; // Who drives the camera: a module with `directCamera: true` owns its // own rig outright — it drives `camera` inside update() and the shared // rig is ignored for this layer. Everything else borrows the shared // camera when injected (updated centrally from framing/gaze so model // layers share one perspective and one depth buffer), else its own. // Direct control is analytic like everything else: f(timeline, seed, // framing, venueState) — seek === playback still holds. const cam = (this.module.directCamera ? this.camera : (this.sharedCamera || this.camera)); if (cam.aspect !== w / h) { cam.aspect = w / h; cam.updateProjectionMatrix(); } const resolved = this.resolveParams(features); this.module.update({ instance: this.instance, scene: this.scene, camera: cam, timeline, features: features || {}, params: resolved, palette: this.palette, personality: this.personality, framing: this.framing, opacity: this.opacity, actorSpec: this.actorSpec, venueSpec: this.venueSpec, venueState: this.venueState, // Song-level arrangement salt (see ArcDriver.venueJumpAt): which // dressing of the room is live. Null most of the time; the module // re-hangs only when it changes. Plain field, not on Layer base — // only venue modules read it. venueArrangement: this.venueArrangement ?? null, THREE, }); // Materials built mid-update (first update, satellite/passer rebuilds // on discrete-count cuts, venue re-hangs) draw wrong until their // programs link — up to ten frames on the heaviest scene, carried by // feedback long after. Compiling synchronously here converts the // rebuild frame from garbage to correct. Modules set // instance.needsCompile = true wherever they (re)build geometry in // update(); build()-time geometry is covered by prewarm instead. if (this.instance && this.instance.needsCompile) { this.instance.needsCompile = false; try { renderer.compileScene(this.scene, cam); } catch { /* non-fatal: next warm-up frame converges anyway */ } } renderer.renderScene(this.scene, cam, target, true); } dispose() { this.scene.traverse((obj) => { if (obj.geometry) obj.geometry.dispose(); if (obj.material) { const mats = Array.isArray(obj.material) ? obj.material : [obj.material]; mats.forEach((m) => m.dispose()); } }); } } export function createLayer(module, options) { if (module.kind === 'model') return new ModelLayer({ module, ...options }); if (module.kind === 'layer3d') return new SceneLayer({ module, ...options }); return new ShaderLayer({ module, ...options }); }