dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add OSD with song name

Browse Source
This commit is contained in:
Dejvino
2026-08-05 21:30:46 +02:00
parent 045ba6a322
commit 2e4e1d7613
8 changed files with 333 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/engine/Layer.js
+89 -70
View File
@@ -19,6 +19,90 @@ function applyResponse(value, response) {
}
}
/**
* The standard uniform dictionary every fragment layer is compiled against.
* Shared by ShaderLayer and the OSD plate so a text layer honours exactly the
* same contract — palette, audio features and personality — as a scene.
*/
export function buildShaderUniforms(module, baseParams, seed) {
const uniforms = {
u_resolution: { value: new THREE.Vector2(1, 1) },
u_aspect: { value: 1 },
u_pixelScale: { value: 1 },
u_time: { value: 0 },
u_frame: { value: 0 },
u_progress: { value: 0 },
u_seed: { value: (seed >>> 0) % 100000 / 1000 },
u_opacity: { value: 1 },
u_colors: { value: Array.from({ length: 8 }, () => new THREE.Vector3(1, 1, 1)) },
u_colorCount: { value: 1 },
u_prev: { value: null },
u_hasPrev: { value: 0 },
};
for (const name of AUDIO_UNIFORMS) uniforms[name] = { value: 0 };
for (const [name, type] of Object.entries(SIGNATURE_UNIFORMS)) {
const v = NEUTRAL_UNIFORMS[name];
uniforms[name] = { value: type === 'vec2' ? new THREE.Vector2(v[0], v[1]) : v };
}
for (const [name, def] of Object.entries(module.params || {})) {
if (!def.uniform || def.type === 'palette') continue;
const v = baseParams[name];
uniforms[def.uniform] = {
value: def.type === 'vec2'
? new THREE.Vector2(v ? v[0] : 0, v ? v[1] : 0)
: def.type === 'bool' ? (v ? 1 : 0) : (v || 0),
};
}
return uniforms;
}
/**
* Push the per-frame values into a fragment layer's uniforms: the frame, the
* audio features, the palette and the personality. Shared by scene layers and
* the OSD plate so they all see the same inputs — the same render path, so
* what a scene honours a plate honours too.
*/
export function setFrameUniforms(layer, renderer, target, ctx) {
const u = layer.uniforms;
const { timeline, features, prevTexture } = ctx;
const w = target ? target.width : renderer.width;
const h = target ? target.height : renderer.height;
u.u_resolution.value.set(w, h);
u.u_aspect.value = w / h;
u.u_pixelScale.value = h / 1080; // reference height; keeps 720p ≡ 4K
u.u_time.value = timeline.time;
u.u_frame.value = timeline.frame;
u.u_progress.value = timeline.progress;
u.u_opacity.value = layer.opacity;
if (features) {
for (const name of AUDIO_UNIFORMS) {
const key = name.slice(2); // u_bandLow -> bandLow
const v = features[key];
u[name].value = v === undefined ? 0 : v;
}
}
const colors = layer.palette || [];
u.u_colorCount.value = Math.max(1, Math.min(8, colors.length));
for (let i = 0; i < 8; i++) {
const c = colors[i % Math.max(1, colors.length)];
if (c) u.u_colors.value[i].set(c[0], c[1], c[2]);
}
const signature = signatureUniforms(layer.personality);
for (const [name, type] of Object.entries(SIGNATURE_UNIFORMS)) {
const v = signature[name];
if (type === 'vec2') u[name].value.set(v[0], v[1]);
else u[name].value = v;
}
u.u_prev.value = prevTexture || null;
u.u_hasPrev.value = prevTexture ? 1 : 0;
}
/** Common surface for shader layers and 3D layers, so Compositor holds one type. */
export class Layer {
constructor({ module, params = {}, seed = 1, opacity = 1, blend = 'normal' }) {
@@ -99,89 +183,24 @@ export class ShaderLayer extends Layer {
constructor(options) {
super(options);
const uniforms = {
u_resolution: { value: new THREE.Vector2(1, 1) },
u_aspect: { value: 1 },
u_pixelScale: { value: 1 },
u_time: { value: 0 },
u_frame: { value: 0 },
u_progress: { value: 0 },
u_seed: { value: (this.seed % 100000) / 1000 },
u_opacity: { value: 1 },
u_colors: { value: Array.from({ length: 8 }, () => new THREE.Vector3(1, 1, 1)) },
u_colorCount: { value: 1 },
u_prev: { value: null },
u_hasPrev: { value: 0 },
};
for (const name of AUDIO_UNIFORMS) uniforms[name] = { value: 0 };
for (const [name, type] of Object.entries(SIGNATURE_UNIFORMS)) {
const v = NEUTRAL_UNIFORMS[name];
uniforms[name] = { value: type === 'vec2' ? new THREE.Vector2(v[0], v[1]) : v };
}
for (const [name, def] of Object.entries(this.module.params || {})) {
if (!def.uniform || def.type === 'palette') continue;
const v = this.baseParams[name];
uniforms[def.uniform] = {
value: def.type === 'vec2'
? new THREE.Vector2(v ? v[0] : 0, v ? v[1] : 0)
: def.type === 'bool' ? (v ? 1 : 0) : (v || 0),
};
}
this.uniforms = uniforms;
this.uniforms = buildShaderUniforms(this.module, this.baseParams, this.seed);
this.material = new THREE.ShaderMaterial({
vertexShader: VERTEX_SHADER,
fragmentShader: buildFragmentShader(this.module),
uniforms,
uniforms: this.uniforms,
depthTest: false,
depthWrite: false,
});
}
render(renderer, target, ctx) {
const { timeline, features, prevTexture } = ctx;
const u = this.uniforms;
const w = target ? target.width : renderer.width;
const h = target ? target.height : renderer.height;
u.u_resolution.value.set(w, h);
u.u_aspect.value = w / h;
u.u_pixelScale.value = h / 1080; // reference height; keeps 720p ≡ 4K
u.u_time.value = timeline.time;
u.u_frame.value = timeline.frame;
u.u_progress.value = timeline.progress;
u.u_opacity.value = this.opacity;
if (features) {
for (const name of AUDIO_UNIFORMS) {
const key = name.slice(2); // u_bandLow -> bandLow
const v = features[key];
u[name].value = v === undefined ? 0 : v;
}
}
const colors = this.palette || [];
u.u_colorCount.value = Math.max(1, Math.min(8, colors.length));
for (let i = 0; i < 8; i++) {
const c = colors[i % Math.max(1, colors.length)];
if (c) u.u_colors.value[i].set(c[0], c[1], c[2]);
}
const signature = signatureUniforms(this.personality);
for (const [name, type] of Object.entries(SIGNATURE_UNIFORMS)) {
const v = signature[name];
if (type === 'vec2') u[name].value.set(v[0], v[1]);
else u[name].value = v;
}
u.u_prev.value = prevTexture || null;
u.u_hasPrev.value = prevTexture ? 1 : 0;
const { features } = ctx;
setFrameUniforms(this, renderer, target, ctx);
const resolved = this.resolveParams(features);
for (const [name, def] of Object.entries(this.module.params || {})) {
if (!def.uniform || def.type === 'palette') continue;
const target_u = u[def.uniform];
const target_u = this.uniforms[def.uniform];
const v = resolved[name];
if (v === undefined) continue;
if (def.type === 'vec2') target_u.value.set(v[0], v[1]);
flow-state/src/engine/OSD.js
+213
View File
@@ -0,0 +1,213 @@
import * as THREE from 'three';
import { VERTEX_SHADER, PREAMBLE } from './shader-contract.js';
import { Layer, buildShaderUniforms, setFrameUniforms } from './Layer.js';
//
// OSD — the on-screen title plate.
//
// A small music-video plate that sits in the corner and names the track. It is
// NOT a DOM overlay: it is rendered into the same layer stack an export runs,
// so the title appears in the exported video exactly where it shows in preview.
//
// Following the personality is not optional fat, it is the point. The plate is
// built out of the same four traits every scene is:
//
// shape — the track's signature form is stamped beside the title as a
// monogram, so the plate carries the same subject the video does.
// camera — the same operator holds the plate: it drifts and sways and takes
// the bar-locked breath every scene takes.
// space — a soft wash of the location's air sits behind it so it stays
// readable over a busy scene.
// style — the title is edged in the track's line weight and softness, folds
// nothing, and carries the same surface grain.
//
// And it is reactive rather than inert: the plate's glow rides the loudness and
// it breathes on the bar, so even a static corner moves with the music.
//
// DETERMINISM: the plate's glyph mask is rasterised once off-frame and held as
// a texture. It uses no clock, no Math.random, no wall time — the same seed
// always produces the same title plate, and preview and export share the one
// texture, so they cannot disagree.
/**
* The corner plate, a fullscreen pass like any other layer so it slots into the
* compositor's blend loop. It masks to a block in the bottom-left corner.
*
* "OSD" follows the personality because it shares the Layer contract: the same
* palette, audio features and signature uniforms as every scene, driven by the
* same per-frame pass.
*/
export class OSDLayer extends Layer {
constructor(renderer, { text = '', personality = null, palette = [] } = {}) {
// A layer with no rendered params; it is its own document, not a scene.
// opacity is 0 until a title exists: the compositor skips zero-opacity
// layers before they draw, so a nameless plate never writes over a frame.
super({ module: { name: 'OSD title' }, blend: 'normal', opacity: 0 });
this.renderer = renderer;
this.personality = personality;
this.palette = palette;
this.text = '';
this._titleTex = null;
this.uniforms = buildShaderUniforms({ params: {} }, {}, 0);
this.uniforms.u_title = { value: null };
this.uniforms.u_titleAspect = { value: 1 };
this.material = new THREE.ShaderMaterial({
vertexShader: VERTEX_SHADER,
fragmentShader: PREAMBLE + `\nuniform sampler2D u_title;\nuniform float u_titleAspect;\n${OSD_FRAGMENT}`,
uniforms: this.uniforms,
depthTest: false,
depthWrite: false,
});
this.setText(text, personality, palette);
}
/** Set the track name (and refresh personality/palette it renders with). */
setText(text, personality = null, palette = null) {
this.text = text || '';
if (personality) this.personality = personality;
if (palette) this.palette = palette;
this._rebuildTitleTex();
return this;
}
setPersonality(personality) { this.personality = personality; this._rebuildTitleTex(); return this; }
setPalette(palette) {
// Palette lives in uniforms (setFrameUniforms), not the texture, so a
// recolour needs no re-raster. Keep the reference for parity anyway.
this.palette = palette;
return this;
}
/** Rasterise the song name once, off-frame. See module notes on determinism. */
_rebuildTitleTex() {
if (this._titleTex) { this._titleTex.dispose(); this._titleTex = null; }
const canvas = renderTitlePlate(this.text, this.personality);
this.uniforms.u_title.value = null;
this.opacity = canvas ? 1 : 0;
if (!canvas) return;
const tex = new THREE.CanvasTexture(canvas);
tex.colorSpace = THREE.SRGBColorSpace;
this.uniforms.u_title.value = tex;
this.uniforms.u_titleAspect.value = canvas.width / canvas.height;
this._titleTex = tex;
}
render(renderer, target, ctx) {
if (!this.uniforms.u_title.value) return;
setFrameUniforms(this, renderer, target, ctx);
renderer.blit(this.material, target);
}
dispose() {
if (this.material) this.material.dispose();
if (this._titleTex) this._titleTex.dispose();
}
}
/**
* Rasterise the title as a white-on-transparent mask the shader recolours.
* Sized to the glyph run, so the texture's aspect is the plate's aspect.
* Returns a canvas, or null if there is nothing to draw.
*/
function renderTitlePlate(text, personality) {
const label = (text || '').trim().toUpperCase();
if (!label) return null;
// Letterspacing reflects the track's line weight; a heavier art direction
// spreads the type further apart.
const spacing = (personality && personality.style ? personality.style.lineWeight : 0.6) * 14;
const font = (px) => `700 ${px}px "Arial Black", "Avenir Next", "Helvetica Neue", "Segoe UI", sans-serif`;
const widthCap = 2048;
let canvas = document.createElement('canvas');
let ctx = canvas.getContext('2d');
const measure = (px) => {
ctx.font = font(px);
try { ctx.letterSpacing = `${spacing}px`; } catch { /* older engines ignore it */ }
return ctx.measureText(label).width;
};
let px = 1500;
let width = measure(px);
while (width > widthCap && px > 260) { px *= 0.8; width = measure(px); }
const pad = Math.round(px * 0.06);
canvas.width = Math.ceil(width + pad * 2);
canvas.height = Math.ceil(px + pad * 2);
ctx = canvas.getContext('2d');
ctx.fillStyle = '#fff';
ctx.font = font(px);
try { ctx.letterSpacing = `${spacing}px`; } catch { /* older engines */}
ctx.textBaseline = 'middle';
ctx.textAlign = 'center';
ctx.fillText(label, canvas.width / 2, canvas.height / 2);
return canvas;
}
// The plate pass. Works in centred aspect-corrected coordinates same as every
// scene, and reads the personality and audio uniforms the contract provides.
const OSD_FRAGMENT = `
void main() {
vec2 uv = vUv;
vec2 p = (uv - 0.5) * 2.0;
p.x *= u_aspect;
// --- the operator films the plate too: sway, and the bar-locked breath ---
vec2 cam = vec2(
sin(u_time * u_sigSwayRate) * u_sigSway,
cos(u_time * u_sigSwayRate * 0.83) * u_sigSway
) * 0.08;
float breath = 1.0 + u_sigBreathe * sin(u_barPhase * 6.28318530718);
vec2 corner = vec2(-u_aspect + 0.2, -1.0 + 0.1) + cam;
float blockH = 0.09 * breath;
float blockW = blockH * u_titleAspect;
vec2 blockBL = corner;
vec2 blockTR = corner + vec2(blockW, blockH);
vec2 blockC = (blockBL + blockTR) * 0.5;
vec3 acc = vec3(0.0);
float alpha = 0.0;
// --- space: the location's backing wash, soft, so the plate reads ---
vec2 rel = (p - blockC) / vec2(blockW + blockH, blockH * 2.2);
float back = exp(-dot(rel, rel) * 4.0) * 0.32;
acc += pal(0) * back * 0.4;
alpha += back;
// --- shape: the signature form stamped as a monogram left of the title ---
float emD = blockH * 0.95;
vec2 emC = vec2(blockBL.x - blockH * 0.35, blockC.y);
float d = sigShape((p - emC) / max(emD * 0.5, 1e-3)) * (emD * 0.5);
float emFill = smoothstep(u_sigSoft * emD * 0.15, -u_sigSoft * emD * 0.15, d);
float emEdge = sigEdge(d);
vec3 emCol = mix(pal(2), pal(1), 0.35);
acc += emCol * emFill * 0.2 + emCol * emEdge;
alpha += emFill * 0.2 + emEdge;
// --- the title itself, recoloured by the track ---
vec2 tuv = (p - blockBL) / vec2(blockW, blockH);
if (tuv.x >= 0.0 && tuv.x <= 1.0 && tuv.y >= 0.0 && tuv.y <= 1.0) {
float mask = texture2D(u_title, tuv).a;
float core = smoothstep(0.5, 0.62, mask);
// A soft glow instead of a hard outline: offset-sampled outlines break
// into a dotted line around the glyphs once the plate is downsampled.
float fringe = smoothstep(0.06, 0.5, mask) * (1.0 - core);
float live = 0.55 + u_loudness * 0.6;
vec3 fill = pal(1);
vec3 glow = pal(0);
acc += fill * core + glow * fringe * 0.55 * live;
alpha += core + fringe * 0.55 * live;
}
// --- the same surface grain every scene carries ---
acc += sigGrain(uv) * 0.3;
gl_FragColor = vec4(acc, min(alpha, 1.0) * u_opacity);
}
`;
flow-state/src/engine/shader-contract.js
+1 -1
View File
@@ -80,7 +80,7 @@ export const FRAME_UNIFORMS = [
'u_resolution', 'u_aspect', 'u_pixelScale', 'u_opacity',
];
const PREAMBLE = `
export const PREAMBLE = `
precision highp float;
uniform vec2 u_resolution;