// Pure prompt/schema logic — no gateway, no LookGenerator/Scene imports. // // Split from llmTweak.js so `node --test` can import it without pulling in // `src/scenes/metadata.json` (which requires Vite's JSON handling and fails // under bare Node's import-assertion check). import { z } from 'zod'; import { ARRANGEMENTS } from '../audio/synth.js'; const ARRANGEMENT_VALUES = Object.keys(ARRANGEMENTS); export { ARRANGEMENT_VALUES }; export const TweakEditsSchema = z.object({ bpm: z.number().int().min(60).max(180).nullable(), brightness: z.number().min(0).max(1).nullable(), noise: z.number().min(0).max(1).nullable(), noiseColour: z.number().min(0).max(1).nullable(), dynamics: z.number().min(0).max(1).nullable(), transition: z.number().min(0).max(1).nullable(), arrangement: z.enum(ARRANGEMENT_VALUES).nullable(), }); export const TweakResultSchema = z.object({ rationale: z.string().min(1).max(800), edits: TweakEditsSchema, confidence: z.number().min(0).max(1), }); export const ALLOWED_KEYS = ['bpm', 'brightness', 'noise', 'noiseColour', 'dynamics', 'transition', 'arrangement']; const clamp01 = (x) => Math.max(0, Math.min(1, x)); const round2 = (x) => Math.round(x * 100) / 100; export function clampEdits(edits) { if (!edits || typeof edits !== 'object') return {}; const out = {}; if (edits.bpm != null) { const n = Number(edits.bpm); if (Number.isFinite(n)) out.bpm = Math.max(60, Math.min(180, Math.round(n))); } for (const k of ['brightness', 'noise', 'dynamics', 'transition']) { if (edits[k] != null) { const n = Number(edits[k]); if (Number.isFinite(n)) out[k] = round2(clamp01(n)); } } if (Object.prototype.hasOwnProperty.call(edits, 'noiseColour')) { const v = edits.noiseColour; if (v == null) out.noiseColour = null; else { const n = Number(v); if (Number.isFinite(n)) out.noiseColour = round2(clamp01(n)); } } if (edits.arrangement != null && ARRANGEMENT_VALUES.includes(edits.arrangement)) { out.arrangement = edits.arrangement; } return out; } /** * Build chat messages from already-serializable inputs. * * `lookDigest` is the pre-built digest (describeLook + validateLook) — callers * that have a live Look should build it via llmTweak.lookDigestForPrompt(look) * and pass it here, so this module never imports scene metadata. * * Kept compatible with the old `{spec,track,look,hint}` shape by accepting * either `look` (no-op here) or `lookDigest`. */ export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) { const r2 = (v) => { const n = Number(v); return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0; }; const r3 = r2; const summary = track ? track.summary : null; const digest = lookDigest ?? (look ? look : null); const specForPrompt = spec ? { bpm: spec.bpm != null ? Math.round(Number(spec.bpm)) : null, brightness: spec.brightness != null ? r2(spec.brightness) : null, noise: spec.noise != null ? r2(spec.noise) : null, noiseColour: spec.noiseColour != null ? r2(spec.noiseColour) : null, dynamics: spec.dynamics != null ? r2(spec.dynamics) : null, transition: spec.transition != null ? r2(spec.transition) : null, arrangement: spec.arrangement ?? null, duration: spec.duration != null ? r2(spec.duration) : null, seed: spec.seed ?? null, } : null; const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({ index: Number.isFinite(s.index) ? s.index : i, kind: s.kind || 'unknown', start: r2(s.startSec != null ? s.startSec : (s.start ?? 0)), end: r2(s.endSec != null ? s.endSec : (s.end ?? 0)), duration: r2(s.duration ?? ((s.endSec != null ? s.endSec : (s.end ?? 0)) - (s.startSec != null ? s.startSec : (s.start ?? 0)))), energy: r3(s.energy ?? 0), flux: r3(s.flux ?? 0), centroid: r3(s.centroid ?? 0), flatness: r3(s.flatness ?? 0), width: r3(s.width ?? 0), slope: r3(s.slope ?? 0), endEnergy: r3(s.endEnergy ?? 0), })) : []; // trackSummary — extended let trackSummary = null; if (summary) { trackSummary = { bpm: summary.bpm ?? track?.tempo?.bpm ?? null, meanCentroid: summary.meanCentroid != null ? r3(summary.meanCentroid) : null, meanFlatness: summary.meanFlatness != null ? r3(summary.meanFlatness) : null, meanWidth: summary.meanWidth != null ? r3(summary.meanWidth) : undefined, meanLoudness: summary.meanLoudness != null ? r3(summary.meanLoudness) : null, dynamicRange: summary.dynamicRange != null ? r3(summary.dynamicRange) : null, tempoConfidence: summary.tempoConfidence != null ? r3(summary.tempoConfidence) : (track?.tempo?.confidence != null ? r3(track.tempo.confidence) : null), frameCount: track?.frameCount ?? summary.frameCount ?? null, fps: track?.fps ?? summary.fps ?? null, duration: r2(track?.duration ?? summary.duration ?? 0), sections: sectionsDetailed, }; if (summary.bandBalance) { trackSummary.bandBalance = { sub: r3(summary.bandBalance.sub ?? 0), low: r3(summary.bandBalance.low ?? 0), mid: r3(summary.bandBalance.mid ?? 0), high: r3(summary.bandBalance.high ?? 0), air: r3(summary.bandBalance.air ?? 0), }; } // remove undefined meanWidth if not present to keep payload compact if (trackSummary.meanWidth === undefined) delete trackSummary.meanWidth; } else if (sectionsDetailed.length || track) { // fallback when summary missing but track exists (e.g. synthetic test track) trackSummary = { bpm: track?.tempo?.bpm ?? null, frameCount: track?.frameCount ?? null, fps: track?.fps ?? null, duration: r2(track?.duration ?? 0), sections: sectionsDetailed, }; if (track?.summary?.bandBalance) { trackSummary.bandBalance = { sub: r3(track.summary.bandBalance.sub ?? 0), low: r3(track.summary.bandBalance.low ?? 0), mid: r3(track.summary.bandBalance.mid ?? 0), high: r3(track.summary.bandBalance.high ?? 0), air: r3(track.summary.bandBalance.air ?? 0), }; } } // tempo let tempo = null; if (track?.tempo) { tempo = { bpm: r2(track.tempo.bpm ?? summary?.bpm ?? 0), confidence: r3(track.tempo.confidence ?? 0), beatsPerBar: track.tempo.beatsPerBar ?? 4, downbeats: Array.isArray(track.tempo.downbeats) ? track.tempo.downbeats.length : 0, }; } else if (summary?.bpm != null) { tempo = { bpm: r2(summary.bpm), confidence: r3(summary.tempoConfidence ?? 0), beatsPerBar: 4, downbeats: 0 }; } // normalization let normalization = null; if (track?.normalization && typeof track.normalization === 'object') { normalization = {}; for (const [k, v] of Object.entries(track.normalization)) { if (v && typeof v === 'object' && v.lo != null && v.hi != null) { normalization[k] = { lo: r3(v.lo), hi: r3(v.hi) }; } } if (!Object.keys(normalization).length) normalization = null; } // stats — derived let stats = null; if (sectionsDetailed.length || tempo || track) { const kinds = [...new Set(sectionsDetailed.map((s) => s.kind))]; const avgEnergy = sectionsDetailed.length ? r3(sectionsDetailed.reduce((a, s) => a + s.energy, 0) / sectionsDetailed.length) : 0; let maxSec = null; let maxE = -Infinity; for (const s of sectionsDetailed) if (s.energy > maxE) { maxE = s.energy; maxSec = s; } const duration = track?.duration ?? summary?.duration ?? 0; const bpmVal = tempo?.bpm ?? summary?.bpm ?? 120; const bpb = tempo?.beatsPerBar ?? 4; const bars = bpmVal > 0 && duration > 0 ? Math.round(duration / ((60 / bpmVal) * bpb)) : 0; stats = { totalSections: sectionsDetailed.length, kinds, avgEnergy, maxEnergySection: maxSec ? { index: maxSec.index, kind: maxSec.kind, energy: maxSec.energy } : null, bpm: tempo?.bpm ?? null, beatsPerBar: tempo?.beatsPerBar ?? null, downbeats: tempo?.downbeats ?? 0, bars, duration: r2(duration), fps: track?.fps ?? summary?.fps ?? null, frameCount: track?.frameCount ?? summary?.frameCount ?? null, }; } const hintBlock = hint && String(hint).trim() ? `User hint: "${String(hint).trim().slice(0, 300)}"` : 'No user hint.'; const frameNote = Array.isArray(frames) && frames.length ? `You are also given ${frames.length} low-res thumbnails (32×32) sampled across the song in order — use them to judge color, structure, and motion.` : null; const system = [ 'You are a producer for a generative music-video system called flow-state.', 'You suggest tweaks to the *song synthesis parameters* so the next generation looks better.', 'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.', '', 'Rules:', 'You have FULL control — change any subset of the 7 knobs by any amount within valid ranges, including large jumps, arrangement swaps, or bpm shifts. Prefer impactful changes that clearly improve visuals; keep duration fixed.', '- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.', '- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.', '- All 7 edit fields MUST be present in edits — use null for unchanged (not omitted). Example: { bpm: null, brightness: 0.6, noise: null, noiseColour: null, dynamics: null, transition: null, arrangement: null }', '- Explain your reasoning in 1-2 sentences (rationale).', '- If the look is already good, return all-null edits.', hintBlock, frameNote, ].filter(Boolean).join('\n'); const userPayload = { songSpec: specForPrompt, trackSummary, sections: sectionsDetailed, tempo, normalization, lookDigest: digest, stats, }; const user = JSON.stringify(userPayload, null, 2); return [ { role: 'system', content: system }, { role: 'user', content: `Current generation:\n\`\`\`json\n${user}\n\`\`\`\n\nSuggest edits as { rationale, edits, confidence }.` }, ]; }