music-video-gen/flow-state/src/engine/Layer.js

import * as THREE from 'three';
import {
    VERTEX_SHADER, buildFragmentShader, AUDIO_UNIFORMS,
    SIGNATURE_UNIFORMS, IDENTITY_UNIFORMS, IDENTITY_ARRAY_UNIFORMS,
} from './shader-contract.js';
import { signatureUniforms, NEUTRAL_UNIFORMS } from '../look/Personality.js';
import { clampValue, isDiscreteParam } from '../params/schema.js';
import { surfaceOf } from '../scenes/surface.js';
import {
    CANVAS_BEAT_CAP,
    CANVAS_FLUX_CAP,
    CANVAS_BEAT_SOFTEN_SPIKE,
} from '../look/comfort.js';

export const BLEND_MODES = ['normal', 'add', 'screen', 'multiply', 'overlay', 'softlight'];

/**
 * Map a raw feature value through a response curve. `spike` squares the input so
 * beat-driven params punch rather than wobble; `smooth` rounds the shoulders so
 * slow features don't step.
 */
function applyResponse(value, response) {
    switch (response) {
        case 'spike': return value * value;
        case 'smooth': return value * value * (3 - 2 * value);
        case 'inverse': return 1 - value;
        default: return value;
    }
}

/**
 * The standard uniform dictionary every fragment layer is compiled against.
 * Shared by ShaderLayer and the OSD plate so a text layer honours exactly the
 * same contract — palette, audio features and personality — as a scene.
 */
export function buildShaderUniforms(module, baseParams, seed) {
    const uniforms = {
        u_resolution: { value: new THREE.Vector2(1, 1) },
        u_aspect: { value: 1 },
        u_pixelScale: { value: 1 },
        u_time: { value: 0 },
        u_frame: { value: 0 },
        u_progress: { value: 0 },
        u_seed: { value: (seed >>> 0) % 100000 / 1000 },
        u_opacity: { value: 1 },
        u_colors: { value: Array.from({ length: 8 }, () => new THREE.Vector3(1, 1, 1)) },
        u_colorCount: { value: 1 },
        u_prev: { value: null },
        u_hasPrev: { value: 0 },
    };
    for (const name of AUDIO_UNIFORMS) uniforms[name] = { value: 0 };
    for (const [name, type] of Object.entries({ ...SIGNATURE_UNIFORMS, ...IDENTITY_UNIFORMS })) {
        const v = NEUTRAL_UNIFORMS[name];
        uniforms[name] = { value: type === 'vec2' ? new THREE.Vector2(v[0], v[1]) : v };
    }
    // The assembly rides as an array of rows — see IDENTITY_ARRAY_UNIFORMS. The
    // vectors are allocated once and written in place per frame, like u_colors.
    for (const [name, def] of Object.entries(IDENTITY_ARRAY_UNIFORMS)) {
        uniforms[name] = {
            value: Array.from({ length: def.length }, () => new THREE.Vector4()),
        };
    }

    for (const [name, def] of Object.entries(module.params || {})) {
        if (!def.uniform || def.type === 'palette') continue;
        const v = baseParams[name];
        uniforms[def.uniform] = {
            value: def.type === 'vec2'
                ? new THREE.Vector2(v ? v[0] : 0, v ? v[1] : 0)
                : def.type === 'bool' ? (v ? 1 : 0) : (v || 0),
        };
    }
    return uniforms;
}

/**
 * Push the per-frame values into a fragment layer's uniforms: the frame, the
 * audio features, the palette and the personality. Shared by scene layers and
 * the OSD plate so they all see the same inputs — the same render path, so
 * what a scene honours a plate honours too.
 */
export function setFrameUniforms(layer, renderer, target, ctx) {
    const u = layer.uniforms;
    const { timeline, features, prevTexture } = ctx;
    const w = target ? target.width : renderer.width;
    const h = target ? target.height : renderer.height;

    u.u_resolution.value.set(w, h);
    u.u_aspect.value = w / h;
    u.u_pixelScale.value = h / 1080;   // reference height; keeps 720p ≡ 4K
    u.u_time.value = timeline.time;
    u.u_frame.value = timeline.frame;
    u.u_progress.value = timeline.progress;
    u.u_opacity.value = layer.opacity;

    if (features) {
        for (const name of AUDIO_UNIFORMS) {
            const key = name.slice(2); // u_bandLow -> bandLow
            const v = features[key];
            u[name].value = v === undefined ? 0 : v;
        }
    }

    const colors = layer.palette || [];
    u.u_colorCount.value = Math.max(1, Math.min(8, colors.length));
    for (let i = 0; i < 8; i++) {
        const c = colors[i % Math.max(1, colors.length)];
        if (c) u.u_colors.value[i].set(c[0], c[1], c[2]);
    }

    const signature = signatureUniforms(layer.personality, layer.module);
    for (const [name, type] of Object.entries({ ...SIGNATURE_UNIFORMS, ...IDENTITY_UNIFORMS })) {
        const v = signature[name];
        if (type === 'vec2') u[name].value.set(v[0], v[1]);
        else u[name].value = v;
    }
    for (const [name, def] of Object.entries(IDENTITY_ARRAY_UNIFORMS)) {
        const rows = signature[name] || [];
        for (let i = 0; i < def.length; i++) {
            const r = rows[i];
            u[name].value[i].set(r ? r[0] : 0, r ? r[1] : 0, r ? r[2] : 0, r ? r[3] : 0);
        }
    }

    // Framing is per shot and wins over the personality's neutral defaults —
    // it is the operator's hand on a shot that is already set up, not a trait
    // of the track. See look/framing.js.
    if (layer.framing) {
        u.u_sigFrameScale.value = layer.framing.scale;
        u.u_sigFrameShift.value.set(layer.framing.shift[0], layer.framing.shift[1]);
    }

    u.u_prev.value = prevTexture || null;
    u.u_hasPrev.value = prevTexture ? 1 : 0;
}

/** Common surface for shader layers and 3D layers, so Compositor holds one type. */
export class Layer {
    constructor({ module, params = {}, seed = 1, opacity = 1, blend = 'normal' }) {
        this.module = module;
        this.baseParams = { ...params };
        this.params = { ...params };
        this.seed = seed >>> 0;
        this.opacity = opacity;
        this.blend = blend;
        this.palette = [];
        this.personality = null;
        this.venueState = null;
    }

    /**
     * The track's production design. Constant for the whole video — see
     * look/Personality.js — and pushed in the same way the palette is, so a
     * layer never reaches for global state.
     */
    setPersonality(personality) {
        this.personality = personality;
        return this;
    }

    setParams(params) {
        this.baseParams = { ...this.baseParams, ...params };
        return this;
    }

    setPalette(colors) {
        this.palette = colors;
        return this;
    }

    /**
     * This shot's FRAMING — a per-shot scale/recentre pushed by the arc driver.
     *
     * A fragment layer has it applied for it, in the shader epilogue, to the
     * coordinate every scene is handed. A 3D layer receives it in `update` and
     * honours it as a camera move, because only it knows what its camera means.
     * A layer that is never framed renders at the neutral full-frame scale, so
     * nothing predating framing changes.
     */
    setFraming(framing) {
        this.framing = framing || null;
        return this;
    }

    /** Resolve base params + reactive modulation into the values used this frame. */
    resolveParams(features) {
        const defs = this.module.params || {};
        const reactive = this.module.reactive || {};
        const out = this.params;

        for (const name of Object.keys(defs)) out[name] = this.baseParams[name];

        if (!features) return out;

        // A — whole-screen guard: canvas scenes (≥50% coverage) must not swerve
        // or jump with the beat. Every mid-tempo kick (≈2 Hz at 124 BPM) then
        // moves the ENTIRE view in sync — the nausea source on
        // ?song=centre&seed=3837488838. Only smaller regions/elements should
        // react wildly; they light few tiles. This is the immediate, central cap:
        // beat → ≤0.15 and spike→smooth, flux → ≤0.25. Composable scenes are
        // untouched. Tunable in look/comfort.js; stale metadata detected via
        // checks/metadata.js SOURCES. See engine/agitation.js for the tile metric.
        const isCanvas = surfaceOf(this.module) === 'canvas';

        for (const [name, r] of Object.entries(reactive)) {
            const def = defs[name];
            if (!def || def.type === 'palette') continue;
            // Rate params multiply absolute time; modulating them jumps the phase
            // by elapsed * delta. See params/schema.js RATE_FLAG.
            if (def.rate) continue;
            // Continuity rule (see params/schema.js isDiscreteParam): a reactive
            // count pops members on every beat. Discrete params hold per cue.
            if (isDiscreteParam(def)) continue;
            const raw = features[r.feature];
            if (raw === undefined) continue;

            let amount = r.amount;
            let response = r.response;
            if (isCanvas) {
                if (r.feature === 'beat') {
                    amount = Math.min(amount, CANVAS_BEAT_CAP);
                    if (CANVAS_BEAT_SOFTEN_SPIKE && response === 'spike') response = 'smooth';
                } else if (r.feature === 'flux') {
                    amount = Math.min(amount, CANVAS_FLUX_CAP);
                }
            }

            const shaped = applyResponse(Math.max(0, Math.min(1, raw)), response);
            const [lo, hi] = def.range || [0, 1];
            const span = hi - lo;
            const base = out[name] !== undefined ? out[name] : lo;

            if (def.type === 'vec2') {
                out[name] = [
                    clampValue(def, [base[0] + shaped * amount * span, 0])[0],
                    clampValue(def, [0, base[1] + shaped * amount * span])[1],
                ];
            } else if (def.type === 'bool') {
                out[name] = base;
            } else {
                out[name] = clampValue(def, base + shaped * amount * span);
            }
        }
        return out;
    }

    render() { throw new Error('Layer.render not implemented'); }
    dispose() {}
}

/** A fullscreen fragment-shader scene. The common case. */
export class ShaderLayer extends Layer {
    constructor(options) {
        super(options);

        this.uniforms = buildShaderUniforms(this.module, this.baseParams, this.seed);
        this.material = new THREE.ShaderMaterial({
            vertexShader: VERTEX_SHADER,
            fragmentShader: buildFragmentShader(this.module),
            uniforms: this.uniforms,
            depthTest: false,
            depthWrite: false,
        });
    }

    render(renderer, target, ctx) {
        const { features } = ctx;
        setFrameUniforms(this, renderer, target, ctx);

        const resolved = this.resolveParams(features);
        for (const [name, def] of Object.entries(this.module.params || {})) {
            if (!def.uniform || def.type === 'palette') continue;
            const target_u = this.uniforms[def.uniform];
            const v = resolved[name];
            if (v === undefined) continue;
            if (def.type === 'vec2') target_u.value.set(v[0], v[1]);
            else if (def.type === 'bool') target_u.value = v ? 1 : 0;
            else target_u.value = v;
        }

        renderer.blit(this.material, target);
    }

    dispose() {
        this.material.dispose();
    }
}

/**
 * A layer backed by a real three.js scene — particles, geometry, camera motion.
 * The module supplies build/update hooks; everything determinism-related (seeded
 * rng, fixed dt, explicit re-seed at section boundaries) is handled here so 3D
 * modules can't accidentally reintroduce wall-clock or Math.random.
 */
export class SceneLayer extends Layer {
    constructor(options) {
        super(options);
        this.scene = new THREE.Scene();
        this.camera = new THREE.PerspectiveCamera(60, 16 / 9, 0.1, 200);
        this.camera.position.set(0, 0, 5);
        this.instance = this.module.build({
            scene: this.scene,
            camera: this.camera,
            seed: this.seed,
            params: this.baseParams,
            THREE,
        });
    }

    render(renderer, target, ctx) {
        const { timeline, features } = ctx;
        const w = target ? target.width : renderer.width;
        const h = target ? target.height : renderer.height;
        if (this.camera.aspect !== w / h) {
            this.camera.aspect = w / h;
            this.camera.updateProjectionMatrix();
        }
        const resolved = this.resolveParams(features);
        this.module.update({
            instance: this.instance,
            scene: this.scene,
            camera: this.camera,
            timeline,
            features: features || {},
            params: resolved,
            palette: this.palette,
            personality: this.personality,
            // A 3D layer has a literal camera, so framing reaches it as a
            // camera move rather than as a coordinate transform. Fragment
            // scenes get it applied for them in the shader epilogue; this one
            // has to honour it itself, because only it knows what its camera
            // means. See look/framing.js.
            framing: this.framing,
            opacity: this.opacity,
            THREE,
        });
        renderer.renderScene(this.scene, this.camera, target, true);
    }

    dispose() {
        this.scene.traverse((obj) => {
            if (obj.geometry) obj.geometry.dispose();
            if (obj.material) {
                const mats = Array.isArray(obj.material) ? obj.material : [obj.material];
                mats.forEach((m) => m.dispose());
            }
        });
    }
}

/**
 * A 3D model layer — the mesh twin of the shader impostor.
 *
 * Like SceneLayer it owns a THREE.Scene and receives build/update hooks, but
 * its geometry comes from the song's ActorSpec (the mesh assembly), not from
 * a hand-written point cloud. Determinism rule is the same: no integration,
 * only analytic f(time, index, seed). See processors/meshes.js and
 * src/actors/ActorGenerator.js.
 *
 * The camera is borrowed from Compositor.sharedCamera when one exists, so
 * multiple ModelLayers share one perspective and one depth buffer — that is
 * what makes a ground mesh occlude a subject mesh from another layer.
 * Falls back to its own camera when no shared rig is present (tests, solo
 * preview), so existing SceneLayer behaviour is unchanged.
 */
export class ModelLayer extends Layer {
    constructor(options) {
        super(options);
        this.scene = new THREE.Scene();
        // Model layers are sparse: a hero plus a few satellites on a ground.
        // Mark the scene for a transparent clear so Renderer clears to alpha 0
        // and Compositor's lumakey/normal blend shows the composition underneath
        // where there is no mesh — same role as a 'composable' shader layer.
        this.scene.userData.transparentBackground = true;
        // Keep the WebGL clear colour transparent as well; renderScene reads the
        // scene flag but the fallback alpha is set here for safety.
        this.scene.background = null;
        this.camera = new THREE.PerspectiveCamera(60, 16 / 9, 0.1, 200);
        this.camera.position.set(0, 0, 5);
        this.actorSpec = options.actorSpec || null;
        this.venueSpec = options.venueSpec || null;
        // Shared rig injected by Compositor at render time when available.
        this.sharedCamera = null;
        this.instance = this.module.build({
            scene: this.scene,
            camera: this.camera,
            seed: this.seed,
            params: this.baseParams,
            actorSpec: this.actorSpec,
            venueSpec: this.venueSpec,
            THREE,
        });
    }

    /** Allow the look to swap the venue without rebuilding the layer. */
    setVenue(venueSpec) {
        this.venueSpec = venueSpec || null;
        return this;
    }

    /**
     * Bind the composited viz texture onto this layer's screen (venue RTT).
     * No-op unless the module implements bindVizTexture(instance, texture).
     * Delegating keeps venue materials owned by the venue module — the
     * compositor never touches venue meshes directly.
     */
    bindVizTexture(texture) {
        if (this.module && typeof this.module.bindVizTexture === 'function' && this.instance) {
            this.module.bindVizTexture(this.instance, texture);
        }
        return this;
    }

    /**
     * Base wash for the viz composite. The shot is keyed over this colour
     * instead of over black: a shot floating on black is the composition
     * failure the phase 12 rules forbid, miniaturised onto the venue screen —
     * and it measured ~5x the no-venue section distance (sparse bright things
     * on nothing maximise layout/region jumps). Defaults to a dark neutral;
     * venue modules override via washColor(instance) to match their room.
     */
    washColor() {
        try {
            if (this.module && typeof this.module.washColor === 'function' && this.instance) {
                const c = this.module.washColor(this.instance);
                if (Array.isArray(c) && c.length >= 3) return [c[0], c[1], c[2]];
            }
        } catch { /* fall through to default */ }
        return [0.03, 0.03, 0.04];
    }

    /** Allow the look to swap the actor without rebuilding the layer. */
    setActor(actorSpec) {
        this.actorSpec = actorSpec || null;
        return this;
    }

    render(renderer, target, ctx) {
        const { timeline, features } = ctx;
        const w = target ? target.width : renderer.width;
        const h = target ? target.height : renderer.height;
        // Who drives the camera: a module with `directCamera: true` owns its
        // own rig outright — it drives `camera` inside update() and the shared
        // rig is ignored for this layer. Everything else borrows the shared
        // camera when injected (updated centrally from framing/gaze so model
        // layers share one perspective and one depth buffer), else its own.
        // Direct control is analytic like everything else: f(timeline, seed,
        // framing, venueState) — seek === playback still holds.
        const cam = (this.module.directCamera ? this.camera : (this.sharedCamera || this.camera));
        if (cam.aspect !== w / h) {
            cam.aspect = w / h;
            cam.updateProjectionMatrix();
        }
        const resolved = this.resolveParams(features);
        this.module.update({
            instance: this.instance,
            scene: this.scene,
            camera: cam,
            timeline,
            features: features || {},
            params: resolved,
            palette: this.palette,
            personality: this.personality,
            framing: this.framing,
            opacity: this.opacity,
            actorSpec: this.actorSpec,
            venueSpec: this.venueSpec,
            venueState: this.venueState,
            // Song-level arrangement salt (see ArcDriver.venueJumpAt): which
            // dressing of the room is live. Null most of the time; the module
            // re-hangs only when it changes. Plain field, not on Layer base —
            // only venue modules read it.
            venueArrangement: this.venueArrangement ?? null,
            THREE,
        });
        // Materials built mid-update (first update, satellite/passer rebuilds
        // on discrete-count cuts, venue re-hangs) draw wrong until their
        // programs link — up to ten frames on the heaviest scene, carried by
        // feedback long after. Compiling synchronously here converts the
        // rebuild frame from garbage to correct. Modules set
        // instance.needsCompile = true wherever they (re)build geometry in
        // update(); build()-time geometry is covered by prewarm instead.
        if (this.instance && this.instance.needsCompile) {
            this.instance.needsCompile = false;
            try {
                renderer.compileScene(this.scene, cam);
            } catch { /* non-fatal: next warm-up frame converges anyway */ }
        }
        renderer.renderScene(this.scene, cam, target, true);
    }

    dispose() {
        this.scene.traverse((obj) => {
            if (obj.geometry) obj.geometry.dispose();
            if (obj.material) {
                const mats = Array.isArray(obj.material) ? obj.material : [obj.material];
                mats.forEach((m) => m.dispose());
            }
        });
    }
}

export function createLayer(module, options) {
    if (module.kind === 'model') return new ModelLayer({ module, ...options });
    if (module.kind === 'layer3d') return new SceneLayer({ module, ...options });
    return new ShaderLayer({ module, ...options });
}