music-video-gen/flow-state/src/engine/shader-contract.js

// The uniform contract every shader scene is compiled against.
//
// Scenes do not write `main()`. They define:
//
//     vec4 scene(vec2 uv, vec2 p)
//
// where `uv` is 0..1 across the frame and `p` is centred, aspect-corrected,
// roughly -1..1 on the short axis. Everything else — the preamble, the varying,
// main() itself — is injected here. That boilerplate reduction is what makes a
// thirty-scene library affordable to write and to keep consistent.
//
// RESOLUTION INDEPENDENCE: work in `uv`/`p`, never in pixels. If you genuinely
// need a pixel-sized feature, scale it by u_pixelScale so a 720p preview and a
// 4K export agree. The dual-resolution diff in tools/ exists to catch violations.

export const VERTEX_SHADER = `
varying vec2 vUv;
void main() {
    vUv = uv;
    gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
}
`;

/** Audio-reactive uniforms, filled per frame from the FeatureTrack. */
export const AUDIO_UNIFORMS = [
    'u_loudness',   // overall level, track-normalised 0..1
    'u_rms',
    'u_bandSub',    // 20-60 Hz
    'u_bandLow',    // 60-250 Hz
    'u_bandMid',    // 250-2k
    'u_bandHigh',   // 2k-6k
    'u_bandAir',    // 6k-16k
    'u_flux',       // onset strength
    'u_centroid',   // spectral brightness 0..1
    'u_flatness',   // noisy vs tonal 0..1
    'u_width',      // stereo width 0..1
    'u_beat',       // decaying spike on each beat, 1 at the hit
    'u_beatPhase',  // 0..1 within the beat
    'u_barPhase',   // 0..1 within the bar
    'u_phrasePhase',// 0..1 within an 8-bar phrase
    'u_sectionProgress',
    'u_sectionEnergy',
    'u_buildSlope', // >0 while energy is ramping toward the next section
];

/**
 * The track's personality, constant for the whole video. See look/Personality.js.
 *
 * These are what make sixteen unrelated shaders read as one production. A scene
 * declares in `traits` which of them it honours, and the look generator will not
 * cast a scene that cannot express what the track is built on.
 *
 * All of them are neutral by default, so a layer built without a personality
 * renders exactly what it always rendered.
 */
export const SIGNATURE_UNIFORMS = {
    u_sigSides: 'float',    // signature form: 0 = round, else polygon sides
    u_sigRound: 'float',    // corner rounding of that form
    u_sigElong: 'float',    // how far from square the form is
    u_sigTilt: 'float',     // its resting angle

    u_sigDrift: 'vec2',     // camera translation per second
    u_sigSway: 'float',     // camera sway amplitude
    u_sigSwayRate: 'float',
    u_sigSpin: 'float',     // slow camera roll, radians per second
    u_sigBreathe: 'float',  // bar-locked zoom

    u_sigHorizon: 'float',  // where the ground meets the sky, 0..1 up the frame
    u_sigDepth: 'float',    // distance falloff
    u_sigWash: 'vec2',      // background gradient direction and strength

    u_sigLine: 'float',     // line weight
    u_sigSoft: 'float',     // edge softness
    u_sigTexture: 'float',  // surface grain
    u_sigFold: 'float',     // kaleidoscopic folds, 1 = none

    // FRAMING. Not a personality trait — this one is per SHOT, pushed by the
    // arc driver rather than derived from the track. See look/framing.js.
    //
    // Every scene in the library is a locked-off, full-frame wide, and always
    // has been. That is one shot type, held for the length of a song, and it is
    // the reason cutting between two scenes changes the subject but never the
    // FRAMING — which is at least half of how a real edit holds attention.
    //
    // Applied inside sigCamera, in scene coordinates, so a close-up is rendered
    // close rather than being a magnified 720p image. That distinction is the
    // whole reason this is a coordinate transform and not a post pass.
    u_sigFrameScale: 'float',   // >1 pushes in, <1 pulls back
    u_sigFrameShift: 'vec2',    // recentre, in scene units
};

/**
 * The song's CAST and INK — Epic 3's content and style artifacts.
 *
 * These differ from the signature uniforms above in kind, not degree. A
 * signature uniform is a modifier on an image the shader already had, which is
 * why a scene is free to ignore one. A cast uniform IS the image: a stage that
 * ignores it has nothing to draw. See look/Identity.js.
 */
export const IDENTITY_UNIFORMS = {
    u_castSides: 'float',    // protagonist: 0 = round, else polygon sides
    u_castRound: 'float',
    u_castElong: 'float',
    u_castTilt: 'float',
    u_castNotchN: 'float',   // notches cut into the boundary, 0 = none
    u_castNotchD: 'float',
    u_castHollow: 'float',   // >0 makes it an annulus — a form with a hole

    u_chorusSides: 'float',  // the second member: a relative, not a stranger
    u_chorusRound: 'float',
    u_chorusElong: 'float',
    u_chorusTilt: 'float',
    u_chorusNotchN: 'float',
    u_chorusNotchD: 'float',
    u_chorusHollow: 'float',

    u_inkWeight: 'float',    // stroke width
    u_inkEdge: 'float',      // 0 = soft/airbrushed, 1 = hard vector
    u_inkFill: 'float',      // index into Identity.FILLS
    u_inkHatchAngle: 'float',
    u_inkHatchScale: 'float',
    u_inkOutline: 'float',   // 0..1 outline strength on top of the fill
    u_inkPosterize: 'float', // 0 = off, else levels

    u_latKind: 'float',      // index into Identity.LATTICES
    u_latJitter: 'float',    // how far off the lattice things sit
    u_latSpread: 'float',    // how much of the frame it occupies
    u_latScaleSpread: 'float', // 0 = all one size, 1 = a few large, many small
    u_latScaleBias: 'float',   // + puts the large ones in the middle
    u_latScale: 'float',       // the song's element size, ~0.1 tiny .. ~0.9 huge
};

export const FRAME_UNIFORMS = [
    'u_time', 'u_frame', 'u_progress', 'u_seed',
    'u_resolution', 'u_aspect', 'u_pixelScale', 'u_opacity',
];

export const PREAMBLE = `
precision highp float;

uniform vec2  u_resolution;
uniform float u_aspect;
uniform float u_pixelScale;
uniform float u_time;
uniform float u_frame;
uniform float u_progress;
uniform float u_seed;
uniform float u_opacity;

uniform vec3  u_colors[8];
uniform int   u_colorCount;

${AUDIO_UNIFORMS.map((u) => `uniform float ${u};`).join('\n')}

${Object.entries(SIGNATURE_UNIFORMS).map(([u, t]) => `uniform ${t} ${u};`).join('\n')}

${Object.entries(IDENTITY_UNIFORMS).map(([u, t]) => `uniform ${t} ${u};`).join('\n')}

uniform sampler2D u_prev;
uniform int       u_hasPrev;

varying vec2 vUv;

// --- palette helpers -------------------------------------------------------
// Scenes should reach for these instead of hardcoding colours, so the look
// generator can actually recolour them per track.

vec3 pal(int i) {
    int n = max(u_colorCount, 1);
    int k = int(mod(float(i), float(n)));
    for (int j = 0; j < 8; j++) { if (j == k) return u_colors[j]; }
    return u_colors[0];
}

/** Continuous ramp through the palette; t wraps. */
vec3 palRamp(float t) {
    int n = max(u_colorCount, 1);
    float f = fract(t) * float(n);
    int i = int(floor(f));
    return mix(pal(i), pal(i + 1), smoothstep(0.0, 1.0, fract(f)));
}

// --- noise -----------------------------------------------------------------

float hash11(float n) { return fract(sin(n) * 43758.5453123); }
float hash12(vec2 p)  { return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453123); }
vec2  hash22(vec2 p)  {
    vec3 a = fract(vec3(p.xyx) * vec3(123.34, 234.34, 345.65));
    a += dot(a, a + 34.45);
    return fract(vec2(a.x * a.y, a.y * a.z));
}

float vnoise(vec2 p) {
    vec2 i = floor(p), f = fract(p);
    f = f * f * (3.0 - 2.0 * f);
    float a = hash12(i), b = hash12(i + vec2(1.0, 0.0));
    float c = hash12(i + vec2(0.0, 1.0)), d = hash12(i + vec2(1.0, 1.0));
    return mix(mix(a, b, f.x), mix(c, d, f.x), f.y);
}

float fbm(vec2 p, int octaves) {
    float v = 0.0, amp = 0.5;
    for (int i = 0; i < 8; i++) {
        if (i >= octaves) break;
        v += amp * vnoise(p);
        p *= 2.02;
        amp *= 0.5;
    }
    return v;
}

/** Cheap divergence-free-ish flow field. The backbone of the "flow" family. */
vec2 curl(vec2 p, float t) {
    float e = 0.1;
    float n1 = fbm(p + vec2(0.0, e) + t, 4);
    float n2 = fbm(p - vec2(0.0, e) + t, 4);
    float n3 = fbm(p + vec2(e, 0.0) + t, 4);
    float n4 = fbm(p - vec2(e, 0.0) + t, 4);
    return vec2(n1 - n2, n4 - n3) / (2.0 * e);
}

mat2 rot(float a) { float c = cos(a), s = sin(a); return mat2(c, -s, s, c); }

/** N-fold kaleidoscopic fold of a centred coordinate. */
vec2 kaleido(vec2 p, float sides) {
    if (sides < 1.5) return p;
    float a = atan(p.y, p.x);
    float r = length(p);
    float seg = 6.28318530718 / sides;
    a = abs(mod(a + seg * 0.5, seg) - seg * 0.5);
    return vec2(cos(a), sin(a)) * r;
}

// --- personality -----------------------------------------------------------
// The four traits, as functions a scene applies in its own way. A scene that
// calls none of these must not declare the matching trait, or it will be cast
// in a track it cannot express. See look/Personality.js.

/**
 * TRAIT: shape. Signed distance to the track's signature form, radius ~1.
 * Round tracks return a circle, so a scene can call this unconditionally.
 */
float sigShape(vec2 q) {
    q = rot(u_sigTilt) * q;
    q.x /= max(u_sigElong, 0.05);

    if (u_sigSides < 2.5) return length(q) - 1.0;

    // Regular polygon by angular folding, then rounded back toward the circle.
    float seg = 6.28318530718 / u_sigSides;
    float a = atan(q.y, q.x);
    float r = length(q);
    float folded = cos(mod(a + seg * 0.5, seg) - seg * 0.5);
    float poly = r * folded - cos(seg * 0.5);
    return mix(poly, r - 1.0, clamp(u_sigRound, 0.0, 1.0));
}

/** TRAIT: shape, as a filled mask of the given radius, centred on a point. */
float sigForm(vec2 p, vec2 centre, float size) {
    float d = sigShape((p - centre) / max(size, 1e-3)) * max(size, 1e-3);
    return smoothstep(u_sigSoft * 0.25 + 0.004, -u_sigSoft * 0.25, d);
}

/**
 * TRAIT: camera. The same operator filming every scene — a slow drift, a sway,
 * a roll, and a bar-locked breath. Apply to a centred coordinate before using it.
 */
vec2 sigCamera(vec2 p) {
    float t = u_time;
    p = rot(u_sigSpin * t) * p;
    p *= 1.0 - u_sigBreathe * sin(u_barPhase * 6.28318530718);
    p += vec2(sin(t * u_sigSwayRate), cos(t * u_sigSwayRate * 0.83)) * u_sigSway;
    // The pan RETURNS. A constant translation would be a camera on rails: two
    // minutes in, every scene has left the frame entirely. This is a slow track
    // across the subject and back, roughly a two-minute cycle.
    p -= u_sigDrift * 20.0 * sin(t * 0.05);
    return p;
}

/** TRAIT: style. Fold the frame the track's way. 1 fold means no fold. */
vec2 sigFolded(vec2 p) {
    return kaleido(p, u_sigFold);
}

/** TRAIT: style. Turn a signed distance into an edge drawn in the track's hand. */
float sigEdge(float d) {
    float w = 0.004 + u_sigLine * 0.03;
    float soft = w * (0.25 + u_sigSoft * 1.5);
    return smoothstep(w + soft, w - soft, abs(d));
}

/** TRAIT: style. The track's surface grain, for a scene to add at its own weight. */
float sigGrain(vec2 uv) {
    if (u_sigTexture <= 0.001) return 0.0;
    return (hash12(uv * 512.0 + floor(u_frame)) - 0.5) * u_sigTexture;
}

/**
 * TRAIT: space. Height of the shared horizon in the same units as 'p'.
 * Positive is up; a scene with any sense of ground should sit on it.
 */
float sigHorizonY() {
    return (u_sigHorizon - 0.5) * 2.0;
}

/** TRAIT: space. The location's air: distance haze plus the background wash. */
vec3 sigAir(vec3 col, vec2 p, float distance01) {
    vec3 far = pal(0) * (0.25 + 0.35 * u_sigDepth);
    col = mix(col, far, clamp(distance01, 0.0, 1.0) * u_sigDepth);
    col += pal(1) * dot(p, u_sigWash) * 0.35;
    return col;
}

// --- the cast --------------------------------------------------------------
// The song's own forms. A stage that places discrete elements places THESE, and
// that is what makes two stages in one video look like one video — and two
// videos of different songs look like different work.

/** Signed distance to a cast member, radius ~1 at size 1. */
float castSDF(vec2 q, float sides, float rnd, float elong, float tilt,
              float notchN, float notchD, float hollow) {
    q = rot(tilt) * q;
    q.x /= max(elong, 0.05);

    float r = length(q);
    float a = atan(q.y, q.x);

    float d;
    if (sides < 2.5) {
        d = r - 1.0;
    } else {
        float seg = 6.28318530718 / sides;
        float folded = cos(mod(a + seg * 0.5, seg) - seg * 0.5);
        float poly = r * folded - cos(seg * 0.5);
        d = mix(poly, r - 1.0, clamp(rnd, 0.0, 1.0));
    }

    // Notches scallop the boundary. Approximate as a radial perturbation — it
    // is not a true distance any more, but every use here is a thresholded mask
    // and the error is far below a pixel at the sizes these are drawn.
    if (notchN > 0.5) d += notchD * cos(notchN * a);
    // A hole through the middle. Cheap, and the single most recognisable thing
    // a generated form can have.
    if (hollow > 0.001) d = abs(d) - hollow * 0.35;
    return d;
}

/** The protagonist, centred, radius ~1. */
float castMain(vec2 q) {
    return castSDF(q, u_castSides, u_castRound, u_castElong, u_castTilt,
                   u_castNotchN, u_castNotchD, u_castHollow);
}

/** The chorus member — many of these, small. */
float castChorus(vec2 q) {
    return castSDF(q, u_chorusSides, u_chorusRound, u_chorusElong, u_chorusTilt,
                   u_chorusNotchN, u_chorusNotchD, u_chorusHollow);
}

// --- the staging -----------------------------------------------------------
// Where things go. Shared, so two stages in one video agree about composition —
// and so the SIZE HIERARCHY is a decision the song makes once rather than one
// each stage makes for itself. The first four stages all placed similarly-sized
// elements, which left feature scale out of the measurement entirely.

/** Node i of n on the song's lattice: xy position, z scale multiplier. */
vec3 stageNode(float i, float n) {
    vec2 h = hash22(vec2(i * 1.37 + 3.1, i * 0.71 + 7.7));
    float total = max(n, 1.0);
    vec2 pos;

    if (u_latKind < 0.5) {                    // grid
        float cols = max(1.0, floor(sqrt(total) + 0.5));
        float rows = max(1.0, ceil(total / cols));
        pos = vec2((mod(i, cols) / max(cols - 1.0, 1.0) - 0.5) * 2.0,
                   (floor(i / cols) / max(rows - 1.0, 1.0) - 0.5) * 2.0);
    } else if (u_latKind < 1.5) {             // radial rings
        float rings = max(1.0, floor(sqrt(total * 0.5) + 0.5));
        float ring = mod(i, rings) + 1.0;
        float a = (i / total) * 6.28318530718 * 3.0;
        pos = vec2(cos(a), sin(a)) * (ring / rings);
    } else if (u_latKind < 2.5) {             // spiral, golden angle
        float a = i * 2.39996323;
        pos = vec2(cos(a), sin(a)) * sqrt(i / total);
    } else if (u_latKind < 3.5) {             // scatter
        pos = (h - 0.5) * 2.0;
    } else {                                  // strata
        float rows = max(1.0, floor(total / 4.0 + 0.5));
        pos = vec2((h.x - 0.5) * 2.0,
                   (mod(i, rows) / max(rows - 1.0, 1.0) - 0.5) * 2.0);
    }

    pos += (h - 0.5) * u_latJitter;
    pos *= u_latSpread;
    // Sits on the same ground every other scene in the track sits on.
    pos.y += sigHorizonY() * 0.3;

    // A power law when the song wants a hierarchy, near-uniform when it does
    // not. Biased toward the middle or the edges.
    float u = max(hash11(i * 7.13 + 1.7), 0.001);
    float size = mix(1.0, pow(u, 1.0 + u_latScaleSpread * 2.5) * 2.4, u_latScaleSpread);
    size *= 1.0 + u_latScaleBias * (0.5 - length(pos) * 0.5);

    // The song's own element size, relative to the neutral 0.35. A stage
    // multiplies its own size param by this rather than choosing outright, so
    // one song is made of a few huge forms and another of many small ones.
    size *= u_latScale / 0.35;

    return vec3(pos, max(size, 0.05));
}

/** The song's element size as a multiplier a stage applies to its own size. */
float stageScale() { return u_latScale / 0.35; }

// --- the ink ---------------------------------------------------------------
// How the cast is drawn. Changes every pixel of every stage at once, and does
// it structurally rather than chromatically — which is the point, since colour
// was already the only register doing any work.

/** The fill treatment as a 0..1 coverage pattern. 1 everywhere when flat. */
float inkPattern(vec2 uv) {
    int mode = int(u_inkFill + 0.5);
    if (mode == 2) {                                  // hatch
        vec2 h = rot(u_inkHatchAngle) * uv * u_inkHatchScale;
        return smoothstep(0.3, 0.7, 0.5 + 0.5 * sin(h.y));
    }
    if (mode == 3) {                                  // stipple
        return step(0.42, hash12(floor(uv * u_inkHatchScale * 2.0)));
    }
    if (mode == 4) {                                  // halftone
        vec2 g = fract(uv * u_inkHatchScale * 0.25) - 0.5;
        return smoothstep(0.38, 0.28, length(g));
    }
    return 1.0;
}

/**
 * Ink coverage for a signed distance: the fill in the track's treatment, plus
 * its outline. The 'hollow' fill treatment draws the outline only.
 */
float inkMask(float d, vec2 uv) {
    float soft = mix(0.03, 0.0015, clamp(u_inkEdge, 0.0, 1.0));
    int mode = int(u_inkFill + 0.5);

    float fillA = smoothstep(soft, -soft, d) * inkPattern(uv);
    if (mode == 5) fillA = 0.0;

    float w = 0.004 + u_inkWeight * 0.055;
    float strength = (mode == 5) ? 1.0 : u_inkOutline;
    float line = smoothstep(w + soft, w - soft, abs(d)) * strength;

    return clamp(max(fillA, line), 0.0, 1.0);
}

/**
 * Outline only, in the song's hand. The drop-in replacement for sigEdge.
 *
 * sigEdge draws a line at the track's line weight; this draws it at the
 * identity's, which is the same idea one layer up. Kept separate from inkMask
 * because a scene that only ever wanted an edge should not suddenly acquire a
 * fill when it migrates.
 */
float inkStroke(float d) {
    float soft = mix(0.03, 0.0015, clamp(u_inkEdge, 0.0, 1.0));
    float w = 0.004 + u_inkWeight * 0.055;
    return smoothstep(w + soft, w - soft, abs(d));
}

/**
 * The protagonist as a filled, inked mask at a point. Replaces sigForm.
 *
 * Same signature as the thing it supersedes so the substitution is mechanical
 * across the library — see MIGRATION.md. uv is recomputed here rather than
 * passed, so the call site does not have to change shape.
 */
float castForm(vec2 p, vec2 centre, float size) {
    float s = max(size, 1e-3);
    vec2 uv = vec2(p.x / u_aspect, p.y) * 0.5 + 0.5;
    return inkMask(castMain((p - centre) / s) * s, uv);
}

/** The track's value structure. Off unless the identity asked for it. */
vec3 inkValue(vec3 col) {
    if (u_inkPosterize < 1.5) return col;
    float n = u_inkPosterize;
    return floor(col * n + 0.5) / n;
}

vec3 prev(vec2 uv) {
    if (u_hasPrev == 0) return vec3(0.0);
    return texture2D(u_prev, uv).rgb;
}

float sat(float x) { return clamp(x, 0.0, 1.0); }
vec3  sat3(vec3 x) { return clamp(x, 0.0, 1.0); }

/**
 * The framed equivalent of uv, for scenes that build their image in uv space.
 *
 * Screen-space scenes — a scan tear, a vertical transposition — slice the FRAME
 * and are right to do so: a signal artefact happens to the signal, not to the
 * world behind it. But the imagery behind the slicing is still a subject, and a
 * subject can be filmed wide or close. So the slice grid stays on raw uv while
 * the field it displaces is built from this, which carries the shot's framing.
 */
vec2 framedUv(vec2 p) { return vec2(p.x / u_aspect, p.y) * 0.5 + 0.5; }
`;

const EPILOGUE = `
void main() {
    vec2 uv = vUv;
    vec2 p = (uv - 0.5) * 2.0;
    p.x *= u_aspect;

    // FRAMING is applied here, to the coordinate every scene is handed, rather
    // than inside sigCamera where it started out.
    //
    // sigCamera is gated on the camera personality trait: a scene that does
    // not want the track's drift and sway simply never calls it. That is
    // correct for a TRAIT and wrong for framing, which is not one — it is where
    // the camera is standing for this shot, and no scene should be exempt from
    // it because of an unrelated art-direction decision. Measured, that
    // accident left Scan Tear and Pylon Grid completely unframed, and the two
    // are otherwise perfectly good candidates for a close-up.
    //
    // uv is deliberately NOT framed. It is screen space: prev() reads the
    // feedback buffer with it and sigGrain speckles in it, and both of those
    // belong to the output image rather than to the scene being filmed.
    p = p / max(u_sigFrameScale, 0.05) + u_sigFrameShift;

    vec4 col = scene(uv, p);
    gl_FragColor = vec4(col.rgb, col.a * u_opacity);
}
`;

/**
 * Assemble a complete fragment shader from a scene body plus its declared params.
 * Param uniforms are appended to the preamble so a scene never declares them itself —
 * the schema is the single source of truth, which is what the lint checks.
 */
export function buildFragmentShader(sceneModule) {
    const paramUniforms = [];
    for (const [name, def] of Object.entries(sceneModule.params || {})) {
        if (!def.uniform) continue;
        if (def.type === 'palette') continue; // palette rides in u_colors
        const glslType = def.type === 'int' ? 'int' : def.type === 'vec2' ? 'vec2' : 'float';
        paramUniforms.push(`uniform ${glslType} ${def.uniform}; // param: ${name}`);
    }

    return [
        PREAMBLE,
        paramUniforms.join('\n'),
        '\n// ---- scene ----\n',
        sceneModule.shader,
        EPILOGUE,
    ].join('\n');
}