LLM song-tweak: AI Gateway via Vercel AI SDK, budgeted tune flow

Add a ✨ tune button that lets an LLM inspect the proposed visualization and tweak the proc-song knobs so the next generation looks better. Edits only the 7 songbank params (bpm, brightness, noise, noiseColour, dynamics, transition, arrangement) — never the Look — then re-synthesizes and re-generates locally for preview. - ai + @ai-sdk/openai + zod (library per plan) against the Vercel AI Gateway (OpenAI-compatible, configurable via VITE_AI_GATEWAY_URL/MODEL/API_KEY, localStorage override, .env.example). - Structured output (TweakResultSchema) + clampEdits enforces ranges; prompt ≤ ~2.5k tokens, max 2-3 knobs ±0.15 / bpm ±8. - Budget caps: 3/load + 10/session (sessionStorage), checked before network, NO_EDITS refunded, offline/auth/rate-limit mapped. - Prompt split into src/llm/spec.js (pure, node-testable) + llmTweak.js (gateway + digest) to keep `node --test` off the Vite JSON path. - UI: ⚙️ gateway/model/key dialog (localStorage flow-state:llm), confirm(prompt) with hint, diff modal with Apply/Dismiss; Apply synthesizes → FeatureTrack → generateLook → Show.useTrack. - Tests: 7 new LLM cases (clamp, schema, prompt, budget), 18/18 pass. Build + lint:scenes green. Co-Authored-By: Claude <noreply@anthropic.com>
2026-09-13 20:03:27 +02:00
parent 7736390f9c
commit b7c3537fcb
12 changed files with 980 additions and 2 deletions
@@ -0,0 +1,129 @@
+// Pure prompt/schema logic — no gateway, no LookGenerator/Scene imports.
+//
+// Split from llmTweak.js so `node --test` can import it without pulling in
+// `src/scenes/metadata.json` (which requires Vite's JSON handling and fails
+// under bare Node's import-assertion check).
+
+import { z } from 'zod';
+import { ARRANGEMENTS } from '../audio/synth.js';
+
+const ARRANGEMENT_VALUES = Object.keys(ARRANGEMENTS);
+
+export { ARRANGEMENT_VALUES };
+
+export const TweakEditsSchema = z.object({
+    bpm: z.number().int().min(60).max(180).optional(),
+    brightness: z.number().min(0).max(1).optional(),
+    noise: z.number().min(0).max(1).optional(),
+    noiseColour: z.number().min(0).max(1).nullable().optional(),
+    dynamics: z.number().min(0).max(1).optional(),
+    transition: z.number().min(0).max(1).optional(),
+    arrangement: z.enum(ARRANGEMENT_VALUES).optional(),
+}).strict();
+
+export const TweakResultSchema = z.object({
+    rationale: z.string().min(1).max(800),
+    edits: TweakEditsSchema,
+    confidence: z.number().min(0).max(1),
+});
+
+export const ALLOWED_KEYS = ['bpm', 'brightness', 'noise', 'noiseColour', 'dynamics', 'transition', 'arrangement'];
+
+const clamp01 = (x) => Math.max(0, Math.min(1, x));
+const round2 = (x) => Math.round(x * 100) / 100;
+
+export function clampEdits(edits) {
+    if (!edits || typeof edits !== 'object') return {};
+    const out = {};
+    if (edits.bpm != null) {
+        const n = Number(edits.bpm);
+        if (Number.isFinite(n)) out.bpm = Math.max(60, Math.min(180, Math.round(n)));
+    }
+    for (const k of ['brightness', 'noise', 'dynamics', 'transition']) {
+        if (edits[k] != null) {
+            const n = Number(edits[k]);
+            if (Number.isFinite(n)) out[k] = round2(clamp01(n));
+        }
+    }
+    if (Object.prototype.hasOwnProperty.call(edits, 'noiseColour')) {
+        const v = edits.noiseColour;
+        if (v == null) out.noiseColour = null;
+        else {
+            const n = Number(v);
+            if (Number.isFinite(n)) out.noiseColour = round2(clamp01(n));
+        }
+    }
+    if (edits.arrangement != null && ARRANGEMENT_VALUES.includes(edits.arrangement)) {
+        out.arrangement = edits.arrangement;
+    }
+    return out;
+}
+
+/**
+ * Build chat messages from already-serializable inputs.
+ *
+ * `lookDigest` is the pre-built digest (describeLook + validateLook) — callers
+ * that have a live Look should build it via llmTweak.lookDigestForPrompt(look)
+ * and pass it here, so this module never imports scene metadata.
+ *
+ * Kept compatible with the old `{spec,track,look,hint}` shape by accepting
+ * either `look` (no-op here) or `lookDigest`.
+ */
+export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
+    const summary = track ? track.summary : null;
+    const sections = track ? (track.sections || []).map((s) => ({
+        kind: s.kind,
+        start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1),
+        end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1),
+        energy: Number((s.energy ?? 0).toFixed(3)),
+    })) : [];
+
+    const digest = lookDigest ?? (look ? look : null);
+
+    const specForPrompt = spec ? {
+        bpm: spec.bpm ?? null,
+        brightness: spec.brightness ?? null,
+        noise: spec.noise ?? null,
+        noiseColour: spec.noiseColour ?? null,
+        dynamics: spec.dynamics ?? null,
+        transition: spec.transition ?? null,
+        arrangement: spec.arrangement ?? null,
+        duration: spec.duration ?? null,
+        seed: spec.seed ?? null,
+    } : null;
+
+    const hintBlock = hint && String(hint).trim()
+        ? `User hint: "${String(hint).trim().slice(0, 300)}"`
+        : 'No user hint.';
+
+    const system = [
+        'You are a producer for a generative music-video system called flow-state.',
+        'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.',
+        'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.',
+        '',
+        'Rules:',
+        '- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.',
+        '- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.',
+        '- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.',
+        '- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.',
+        '- Explain your reasoning in 1-2 sentences (rationale).',
+        '- If the look is already good, return empty edits {}.',
+        hintBlock,
+    ].join('\n');
+
+    const user = JSON.stringify({
+        songSpec: specForPrompt,
+        trackSummary: summary ? {
+            bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness,
+            dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness,
+            tempoConfidence: summary.tempoConfidence,
+        } : null,
+        sections,
+        lookDigest: digest,
+    }, null, 2);
+
+    return [
+        { role: 'system', content: system },
+        { role: 'user', content: `Current generation:\n\`\`\`json\n${user}\n\`\`\`\n\nSuggest edits as { rationale, edits, confidence }.` },
+    ];
+}