dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage 3.6 — models visible: transparent model compositing + Assembly density

Browse Source 
Model layers render with transparent clear so Compositor lumakey/normal
can show composition underneath where no mesh covers — Assembly is now
sparse (hero + satellites on ground, not opaque fill), satellites wired
to count/spread/density so the density slider visibly changes the stage
and a seek is still just f(t,seed). Shared camera plumbing already
drove framing->dolly/personality->drift for real parallax.

Gates: lint 108 clean / 70 literals / 69 scenes; vite built.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino 2026-08-20 18:58:32 +02:00
parent 93a5dc8437
commit b98aa94dda
3 changed files with 128 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
flow-state/src/engine/Layer.js
View File

@ -336,6 +336,14 @@ export class ModelLayer extends Layer {
constructor(options) {
super(options);
this.scene = new THREE.Scene();
// Model layers are sparse: a hero plus a few satellites on a ground.
// Mark the scene for a transparent clear so Renderer clears to alpha 0
// and Compositor's lumakey/normal blend shows the composition underneath
// where there is no mesh — same role as a 'composable' shader layer.
this.scene.userData.transparentBackground = true;
// Keep the WebGL clear colour transparent as well; renderScene reads the
// scene flag but the fallback alpha is set here for safety.
this.scene.background = null;
this.camera = new THREE.PerspectiveCamera(60, 16 / 9, 0.1, 200);
this.camera.position.set(0, 0, 5);
this.actorSpec = options.actorSpec || null;

17
flow-state/src/engine/Renderer.js
View File

@ -100,7 +100,22 @@ export class Renderer {
/** Render a real three.js scene (used by 3D layers). */
renderScene(scene, camera, target = null, clear = true) {
this.gl.setRenderTarget(target);
if (clear) this.gl.clear(true, true, true);
if (clear) {
// Model layers need a transparent clear so the compositor's
// lumakey/normal blend can show the layer underneath where the
// mesh does not cover. Shader layers are opaque by construction
// and keep the existing opaque clear via Compositor.
const isTransparent = scene && scene.userData && scene.userData.transparentBackground;
if (isTransparent) {
const prev = this.gl.getClearColor(new THREE.Color());
const prevA = this.gl.getClearAlpha();
this.gl.setClearColor(new THREE.Color(0, 0, 0), 0);
this.gl.clear(true, true, true);
this.gl.setClearColor(prev, prevA);
} else {
this.gl.clear(true, true, true);
}
}
this.gl.render(scene, camera);
this.gl.setRenderTarget(null);
}

160
flow-state/src/scenes/stage/assembly.js
View File

@ -3,10 +3,11 @@
// The flat stages stamp castSDF as impostors (castSolid per instance). This one
// is the mesh twin: the same Identity.form assembly as BufferGeometry, with real
// occlusion, parallax and scale foreshortening. One hero (the protagonist body)
// over a shared ground, lit by the same key the shader's castLit uses. The
// palette and the lattice are the same data the flat stages consume — so two
// Assembly videos of different songs are different bodies in different worlds,
// and two seeds on one song are different readings of one body.
// plus a small orbiting field so count/spread read as density, over a shared
// ground, lit by the same key the shader's castLit uses. The palette and the
// lattice are the same data the flat stages consume — so two Assembly videos of
// different songs are different bodies in different worlds, and two seeds on one
// song are different readings of one body.
//
// Analytic motion only: f(t,seed). No integration, so seek === playback like
// particles.js. Camera is the shared rig (Compositor.sharedCamera) driven from
@ -25,7 +26,7 @@ export const assembly = {
params: {
size: { type: 'float', range: [0.6, 1.9], default: 1.05, uniform: 'u_size', bias: 'energy' },
count: { type: 'int', range: [4, 36], default: 14, uniform: 'u_count', bias: 'density' },
count: { type: 'int', range: [4, 24], default: 10, uniform: 'u_count', bias: 'density' },
spread: { type: 'float', range: [0.55, 1.6],default: 1.05, uniform: 'u_spread' },
spin: { type: 'float', range: [0.05, 0.9],default: 0.32, uniform: 'u_spin', bias: 'motion', rate: true },
lift: { type: 'float', range: [0.0, 1.0], default: 0.42, uniform: 'u_lift' },
@ -48,46 +49,40 @@ export const assembly = {
});
const ground = new THREE.Mesh(groundGeo, groundMat);
ground.rotation.x = -Math.PI / 2;
// Place at horizon so it reads as the same world the flat scenes share.
ground.position.y = -1.15;
ground.receiveShadow = false;
scene.add(ground);
// Lighting — matches shader castLit key/fill/rim direction.
const ambient = new THREE.AmbientLight(0xffffff, 0.55);
const ambient = new THREE.AmbientLight(0xffffff, 0.58);
scene.add(ambient);
const key = new THREE.DirectionalLight(0xffffff, 1.15);
const key = new THREE.DirectionalLight(0xffffff, 1.2);
key.position.set(2.2, 4.5, 2.8);
key.castShadow = false;
scene.add(key);
const fill = new THREE.DirectionalLight(0xffffff, 0.35);
const fill = new THREE.DirectionalLight(0xffffff, 0.34);
fill.position.set(-2.8, 1.6, -2.2);
scene.add(fill);
// Hero — the actor's solid, built from Identity.form via actorToGeometry.
// Seeded from the stage seed + actorSpec.seed so two stages with the same
// actorSpec don't produce identical groups when one is instanced later.
let heroGroup = null;
if (actorSpec && actorSpec.form) {
// actorToGeometry expects (actorSpec, identity, THREE). Identity is not
// available at build time (it arrives in update via personality), so build
// a placeholder group now and rebuild the geometry on first update when
// identity is known — same pattern particles.js uses for its Points.
heroGroup = new THREE.Group();
const heroGroup = new THREE.Group();
heroGroup.name = 'hero';
scene.add(heroGroup);
} else {
heroGroup = new THREE.Group();
heroGroup.name = 'hero';
scene.add(heroGroup);
}
// Chorus field — InstancedMesh will be created on first update when we
// know palette + identity; keep a slot for it.
// Satellites — small chorus instances so count/spread are visibly the
// density control. Built on first update once identity+palette exist.
const satellites = new THREE.Group();
satellites.name = 'satellites';
scene.add(satellites);
scene.background = null;
scene.fog = new THREE.Fog(0x0a0a0f, 9, 26);
return { ground, groundMat, ambient, key, fill, heroGroup, heroBuilt: false, seed, actorSeed: actorSpec ? actorSpec.seed : seed };
return {
ground, groundMat, ambient, key, fill,
heroGroup, heroBuilt: false,
satellites, satBuilt: false, satCount: -1,
seed, actorSeed: actorSpec ? actorSpec.seed : seed,
};
},
update({ instance, scene, camera, timeline, features, params, palette, personality, framing, opacity, actorSpec, THREE }) {
@ -100,20 +95,15 @@ export const assembly = {
// --- hero geometry: built once identity is known, so the cast profile is correct ---
if (!instance.heroBuilt && identity && actorSpec && actorSpec.form) {
// Clear placeholder children that may have been left from a hot rebuild.
for (const child of [...instance.heroGroup.children]) {
instance.heroGroup.remove(child);
if (child.geometry) child.geometry.dispose();
if (child.material) child.material.dispose();
}
// Build the real assembly group and transplant its meshes into heroGroup
// so the heroGroup object identity stays stable (ArcDriver caches layers).
const built = actorToGeometry(actorSpec, identity, THREE);
while (built.children.length) {
const m = built.children[0];
built.remove(m);
// Bake palette per-part via paletteMap so the hero keeps the song's
// colour rhythm across updates — palette may shift via paletteArc.
const partIndex = m.userData.partIndex ?? 0;
const palIndex = actorSpec.paletteMap ? actorSpec.paletteMap[partIndex % actorSpec.paletteMap.length] : partIndex;
const c = pal[palIndex % pal.length];
@ -126,50 +116,95 @@ export const assembly = {
m.receiveShadow = false;
instance.heroGroup.add(m);
}
// Geometry cache for recolour on palette arc moves without rebuilding.
instance.heroPaletteMap = actorSpec.paletteMap || [];
instance.heroBuilt = true;
}
// Palette re-bind each frame so paletteArc / director blend is visible on
// the mesh immediately — paletteMaterial bakes pal(i) at setPalette time for
// shaders, but meshes need it live.
const groundC = pal[0];
instance.groundMat.color.setRGB(groundC[0] * 0.22, groundC[1] * 0.22, groundC[2] * 0.26);
// Horizon sync: ground colour haze toward pal[1] with depth factor from
// personality.space.depth so the mesh world and the shader world agree.
// Keep it subtle — this is a bed, not the subject.
// --- satellites: small instances so count/spread visibly matter ---
const needCount = Math.max(0, Math.min(24, Math.round(params.count)));
const spread = Math.max(0.35, params.spread);
const satDirty = instance.satCount !== needCount || !instance.satBuilt;
if (instance.heroBuilt && satDirty && needCount > 0) {
for (const child of [...instance.satellites.children]) {
instance.satellites.remove(child);
if (child.geometry) child.geometry.dispose();
if (child.material) child.material.dispose();
}
// Deterministic satellite distribution — ring + jitter from seed,
// so a probe and a seek at same frame agree.
let state = (instance.seed ^ 0x9e3779b9) >>> 0;
const rnd = () => {
state = (state + 0x6d2b79f5) >>> 0;
let tt = state; tt = Math.imul(tt ^ (tt >>> 15), tt | 1);
tt ^= tt + Math.imul(tt ^ (tt >>> 7), tt | 61);
return ((tt ^ (tt >>> 14)) >>> 0) / 4294967296;
};
// Satellite template: small scaled clone of the hero's first mesh
// geometry where available, else a cheap icosahedron. Shares the
// song's palette rhythm (paletteMap offset by 1 so satellites and
// hero are not the same colour).
const template = instance.heroGroup.children.find((m) => m.isMesh);
for (let i = 0; i < needCount; i++) {
const ang = (i / Math.max(1, needCount)) * Math.PI * 2 + rnd() * 0.35;
const rad = spread * (0.65 + rnd() * 0.55) + (identity ? identity.lattice.spread * 0.18 : 0);
const yOff = (rnd() - 0.5) * 0.45;
const scale = 0.18 + rnd() * 0.14;
let mesh;
if (template && template.geometry) {
mesh = new THREE.Mesh(template.geometry, new THREE.MeshStandardMaterial({
roughness: 0.5, metalness: 0.06,
}));
} else {
mesh = new THREE.Mesh(new THREE.IcosahedronGeometry(0.22, 1),
new THREE.MeshStandardMaterial({ roughness: 0.5, metalness: 0.06 }));
}
const palIndex = instance.heroPaletteMap[(i + 1) % Math.max(1, instance.heroPaletteMap.length)] ?? (i + 1);
// Palette shift per satellite so the field reads as the song's
// lattice rather than as cloned heroes.
const c = pal[(palIndex + i) % pal.length];
mesh.material.color.setRGB(c[0], c[1], c[2]);
mesh.position.set(Math.cos(ang) * rad, yOff, Math.sin(ang) * rad);
mesh.scale.setScalar(scale);
mesh.userData.baseAng = ang;
mesh.userData.baseRad = rad;
mesh.userData.baseY = yOff;
mesh.userData.spinPhase = rnd() * Math.PI * 2;
instance.satellites.add(mesh);
}
instance.satBuilt = true;
instance.satCount = needCount;
}
// Recolour hero parts when palette moves (paletteArc/plan). Cheap: just
// set material.color since geometry is stable.
// Palette re-bind so paletteArc is live on mesh — ground + hero + sats.
const groundC = pal[0];
instance.groundMat.color.setRGB(groundC[0] * 0.26, groundC[1] * 0.26, groundC[2] * 0.30);
if (instance.heroBuilt) {
for (const m of instance.heroGroup.children) {
if (!m.isMesh) continue;
const partIndex = m.userData.partIndex ?? 0;
const palIndex = instance.heroPaletteMap[partIndex % instance.heroPaletteMap.length] ?? partIndex;
const c = pal[palIndex % pal.length];
// Mix toward white with lift so ink outline would still read if we
// added it — matches shader inkMask's fill+outline balance.
m.material.color.setRGB(c[0], c[1], c[2]);
m.material.opacity = opacity;
m.material.transparent = opacity < 0.999;
}
for (const m of instance.satellites.children) {
if (!m.isMesh) continue;
m.material.opacity = opacity * 0.92;
m.material.transparent = opacity < 0.999;
}
}
// --- hero pose: fully analytic f(t,seed,params,actor.motion) ---
const spin = Math.max(0.01, params.spin);
const size = Math.max(0.2, params.size) * (1 + beat * 0.12);
const lift = params.lift + bandLow * 0.10 + Math.sin(t * (actorSpec ? actorSpec.motion.bobRate : 0.6) + instance.seed * 0.0007) * (actorSpec ? actorSpec.motion.bobAmp : 0.012);
const lift = params.lift + bandLow * 0.10
+ Math.sin(t * (actorSpec ? actorSpec.motion.bobRate : 0.6) + instance.seed * 0.0007)
* (actorSpec ? actorSpec.motion.bobAmp : 0.012);
const orbit = params.orbit;
// Slow yaw + pitched tumble so the assembly's sillhouette CHANGES as it
// turns — that is the promise Identity.js:112 makes ("outline changes as
// it turns, which needs either the object turning or the camera travelling").
const yaw = t * spin * 0.55 + (actorSpec ? actorSpec.motion.spin * 0.18 : 0) + instance.seed * 0.0003;
const pitch = Math.sin(t * 0.31 + instance.seed * 0.0011) * 0.55;
// Analytic orbit around the ground's centre — stage owns motion, identity
// owns placement.
const ox = Math.sin(t * orbit * 0.5 + instance.seed * 0.002) * 0.55;
const oz = Math.cos(t * orbit * 0.45 + instance.seed * 0.0023) * 0.35;
@ -177,19 +212,28 @@ export const assembly = {
instance.heroGroup.rotation.set(pitch, yaw, Math.sin(t * 0.18) * 0.12);
instance.heroGroup.scale.setScalar(size * 0.95);
// Ground framing: scale/shift already drives the shared camera dolly, but
// the ground itself benefits from a subtle analytic drift so a locked-off
// close-up still evolves over a long section.
// Satellites orbit analytically around the hero — stage owns motion.
for (const m of instance.satellites.children) {
const ang = m.userData.baseAng + t * 0.32 + Math.sin(t * 0.12 + m.userData.spinPhase) * 0.15;
m.position.x = Math.cos(ang) * m.userData.baseRad + ox * 0.35;
m.position.z = Math.sin(ang) * m.userData.baseRad + oz * 0.35;
m.position.y = m.userData.baseY + Math.sin(t * 0.7 + m.userData.spinPhase) * 0.07;
m.rotation.y = t * 0.9 + m.userData.spinPhase;
m.rotation.x = Math.sin(t * 0.5 + m.userData.spinPhase) * 0.4;
}
// Subtle ground drift so a locked-off close-up still evolves.
instance.ground.position.x = Math.sin(t * 0.04 + instance.seed * 0.0009) * 0.18;
instance.ground.position.z = Math.cos(t * 0.03 + instance.seed * 0.0007) * 0.12;
// Opacity crossfade — ModelLayer opacity is already the cue eased blend, so
// apply it to the hero's materials (ground stays at full so the bed never
// goes black under a dissolving shot).
for (const m of instance.heroGroup.children) {
if (!m.isMesh) continue;
m.visible = opacity > 0.01;
}
for (const m of instance.satellites.children) {
if (!m.isMesh) continue;
m.visible = opacity > 0.01;
}
},
};