diff --git a/flow-state/.env.example b/flow-state/.env.example new file mode 100644 index 0000000..d9ca802 --- /dev/null +++ b/flow-state/.env.example @@ -0,0 +1,5 @@ +# Vercel AI Gateway — copy to .env.local and fill VITE_AI_API_KEY +# Get a token at https://vercel.com/dashboard/ai-gateway +VITE_AI_GATEWAY_URL=https://ai-gateway.vercel.sh/v1 +VITE_AI_MODEL=openai/gpt-4o-mini +VITE_AI_API_KEY= diff --git a/flow-state/.gitignore b/flow-state/.gitignore index 042ff31..1c0d0b5 100644 --- a/flow-state/.gitignore +++ b/flow-state/.gitignore @@ -2,6 +2,7 @@ node_modules dist .vite out +.env.local # Scratch probes: one-off harnesses for debugging decode/encode/mux issues. # They are throwaway by nature and were committed once by accident. diff --git a/flow-state/index.html b/flow-state/index.html index 029b189..7b9a9b6 100644 --- a/flow-state/index.html +++ b/flow-state/index.html @@ -59,6 +59,8 @@ + + diff --git a/flow-state/package-lock.json b/flow-state/package-lock.json index 324f2e2..5c258ef 100644 --- a/flow-state/package-lock.json +++ b/flow-state/package-lock.json @@ -9,14 +9,80 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "@ai-sdk/openai": "^3.0.112", + "ai": "^6.0.282", "mp4-muxer": "^5.2.2", - "three": "^0.181.1" + "three": "^0.181.1", + "zod": "^4.6.4" }, "devDependencies": { "@playwright/test": "^1.62.1", "vite": "^7.2.2" } }, + "node_modules/@ai-sdk/gateway": { + "version": "3.0.193", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.193.tgz", + "integrity": "sha512-T2vq1EKPhbFF2Sk2MUFL/RmCg/2mp1LGbCZZSx5vh0M6Sr1EAj3wRr1EnpcwCV3/Imj2q56pT89XLiUFnB+ZgQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.51", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "3.0.112", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.112.tgz", + "integrity": "sha512-0ho3a9CJ+3KCcrPdBQx8fX7/gyUxFHsfs1M6++SpunYk8+pgGOPMskr0GVOOxY+94jsJbuGxksKfMuSYhYHcxw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.51" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.16.tgz", + "integrity": "sha512-9Av6kg0t/IN/dcYAAmEJ4B9OPhcEJqwSR+GfHEA8olRkindXpUHadc3p3cyvgFUQFTVm1G5thtsmgZ9yVb2w3A==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "4.0.51", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.51.tgz", + "integrity": "sha512-ukLTs9x1Xm6lxSIwbJIxYQxbZHmVeIczNntARZrBcOa6pjdBhqeZnATNnJMHa7M9dZ8Ji5NJbpKpvz7o5XyBtg==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "3.0.16", + "@standard-schema/spec": "^1.1.0", + "eventsource-parser": "^3.0.8", + "undici": "^6.28.0" + }, + "engines": { + "node": ">=18.17" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -476,6 +542,15 @@ "node": "^22.20 || ^24.12 || >=25" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@playwright/test": { "version": "1.62.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", @@ -842,6 +917,12 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, "node_modules/@types/dom-webcodecs": { "version": "0.1.18", "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.18.tgz", @@ -861,6 +942,33 @@ "integrity": "sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==", "license": "MIT" }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/ai": { + "version": "6.0.282", + "resolved": "https://registry.npmjs.org/ai/-/ai-6.0.282.tgz", + "integrity": "sha512-fdzEzSppVTYlgyW/Gp6tMAFIU88yv7k86x77m5vAJo7hXV31hDJCsAkq/aIpE3tP3k4LjrrLcUANPHcBZ0YWsw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "3.0.193", + "@ai-sdk/provider": "3.0.16", + "@ai-sdk/provider-utils": "4.0.51", + "@opentelemetry/api": "^1.9.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -903,6 +1011,15 @@ "@esbuild/win32-x64": "0.28.1" } }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -936,6 +1053,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/mp4-muxer": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/mp4-muxer/-/mp4-muxer-5.2.2.tgz", @@ -1141,6 +1264,15 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/undici": { + "version": "6.28.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz", + "integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/vite": { "version": "7.3.6", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", @@ -1215,6 +1347,15 @@ "optional": true } } + }, + "node_modules/zod": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.4.tgz", + "integrity": "sha512-AXSD6hvGdvRjajG/l1cC+d6IrhH+sjmPKtYeQdJIK8MFJl3LyClzS+o/YsVC+zQZPupAaeH5skwwm8YqYH7BqA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/flow-state/package.json b/flow-state/package.json index 897625e..20ab3c8 100644 --- a/flow-state/package.json +++ b/flow-state/package.json @@ -20,7 +20,10 @@ "vite": "^7.2.2" }, "dependencies": { + "@ai-sdk/openai": "^3.0.112", + "ai": "^6.0.282", "mp4-muxer": "^5.2.2", - "three": "^0.181.1" + "three": "^0.181.1", + "zod": "^4.6.4" } } diff --git a/flow-state/src/llm/budget.js b/flow-state/src/llm/budget.js new file mode 100644 index 0000000..c945a72 --- /dev/null +++ b/flow-state/src/llm/budget.js @@ -0,0 +1,85 @@ +// Invocation budget — caps cost by refusing before the network. +// +// perLoad: how many LLM calls per page load +// perSession: how many per sessionStorage lifetime (survives reload until tab close) + +import { getLlmConfig } from './config.js'; + +export const LLM_BUDGET = Object.freeze({ + perLoad: 3, + perSession: 10, +}); + +const SESSION_KEY = 'flow-state:llm-budget'; + +let loadCount = 0; + +function readSessionCount() { + try { + const raw = sessionStorage.getItem(SESSION_KEY); + if (!raw) return 0; + const n = Number(JSON.parse(raw)); + return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0; + } catch { + return 0; + } +} + +function writeSessionCount(n) { + try { + sessionStorage.setItem(SESSION_KEY, JSON.stringify(Math.max(0, Math.floor(n)))); + } catch { /* ignore */ } +} + +export function getBudgetCounts() { + return { + loadCount, + sessionCount: readSessionCount(), + perLoad: LLM_BUDGET.perLoad, + perSession: LLM_BUDGET.perSession, + }; +} + +export function canInvoke() { + const cfg = getLlmConfig(); + if (!cfg.isConfigured) return { ok: false, reason: 'not-configured', detail: 'Set an AI Gateway API key in ⚙️ settings.' }; + if (typeof navigator !== 'undefined' && navigator.onLine === false) { + return { ok: false, reason: 'offline', detail: 'Offline — LLM unavailable.' }; + } + const counts = getBudgetCounts(); + if (counts.loadCount >= counts.perLoad) { + return { ok: false, reason: 'per-load', detail: `Budget exhausted for this page load (${counts.perLoad}). Reload to reset.` }; + } + if (counts.sessionCount >= counts.perSession) { + return { ok: false, reason: 'per-session', detail: `Session budget exhausted (${counts.perSession}). Close the tab or clear sessionStorage.` }; + } + return { ok: true, reason: 'ok', detail: '', counts }; +} + +export function recordInvocation() { + loadCount += 1; + writeSessionCount(readSessionCount() + 1); + return getBudgetCounts(); +} + +/** Refund one invocation — when the model returned empty/useless edits. */ +export function refundInvocation() { + if (loadCount > 0) loadCount -= 1; + const sc = readSessionCount(); + if (sc > 0) writeSessionCount(sc - 1); + return getBudgetCounts(); +} + +export function remainingText() { + const c = getBudgetCounts(); + const perLoadLeft = Math.max(0, c.perLoad - c.loadCount); + const perSessionLeft = Math.max(0, c.perSession - c.sessionCount); + return `${perLoadLeft}/${c.perLoad} this load · ${perSessionLeft}/${c.perSession} this session`; +} + +export function clearBudget() { + loadCount = 0; + try { sessionStorage.removeItem(SESSION_KEY); } catch { /* ignore */ } +} + +export const LLM_BUDGET_SESSION_KEY = SESSION_KEY; diff --git a/flow-state/src/llm/config.js b/flow-state/src/llm/config.js new file mode 100644 index 0000000..3b06153 --- /dev/null +++ b/flow-state/src/llm/config.js @@ -0,0 +1,87 @@ +// LLM gateway config — localStorage overrides > VITE env > defaults. +// +// The app is a static Vite site with no server; the gateway credential lives +// in the browser. Never commit a key — paste it in the panel (→ localStorage) +// or in an untracked .env.local. + +const STORAGE_KEY = 'flow-state:llm'; + +const DEFAULTS = { + gatewayUrl: 'https://ai-gateway.vercel.sh/v1', + model: 'openai/gpt-4o-mini', + apiKey: '', +}; + +function readEnv() { + try { + const env = import.meta.env || {}; + return { + gatewayUrl: env.VITE_AI_GATEWAY_URL || null, + model: env.VITE_AI_MODEL || null, + apiKey: env.VITE_AI_API_KEY || null, + }; + } catch { + return { gatewayUrl: null, model: null, apiKey: null }; + } +} + +function readStorage() { + try { + const raw = localStorage.getItem(STORAGE_KEY); + if (!raw) return {}; + const parsed = JSON.parse(raw); + return parsed && typeof parsed === 'object' ? parsed : {}; + } catch { + return {}; + } +} + +/** Resolved config — localStorage wins, then env, then defaults. */ +export function getLlmConfig() { + const env = readEnv(); + const stored = readStorage(); + const gatewayUrl = stored.gatewayUrl || env.gatewayUrl || DEFAULTS.gatewayUrl; + const model = stored.model || env.model || DEFAULTS.model; + const apiKey = stored.apiKey != null ? stored.apiKey : (env.apiKey != null ? env.apiKey : DEFAULTS.apiKey); + const key = typeof apiKey === 'string' ? apiKey.trim() : ''; + return { + gatewayUrl: String(gatewayUrl || DEFAULTS.gatewayUrl).trim() || DEFAULTS.gatewayUrl, + model: String(model || DEFAULTS.model).trim() || DEFAULTS.model, + apiKey: key, + isConfigured: key.length > 0, + defaults: { ...DEFAULTS }, + env: { ...env }, + stored: { ...stored }, + }; +} + +/** Merge patch into localStorage. Pass null/'' to clear a field. */ +export function setLlmConfig(patch) { + const current = readStorage(); + const next = { ...current }; + for (const k of ['gatewayUrl', 'model', 'apiKey']) { + if (patch && Object.prototype.hasOwnProperty.call(patch, k)) { + const v = patch[k]; + if (v == null || v === '') { + delete next[k]; + } else { + next[k] = String(v).trim(); + } + } + } + try { + if (!Object.keys(next).length) localStorage.removeItem(STORAGE_KEY); + else localStorage.setItem(STORAGE_KEY, JSON.stringify(next)); + } catch { + // storage full / blocked — ignore, caller sees isConfigured false + } + return getLlmConfig(); +} + +export function clearLlmConfig() { + try { localStorage.removeItem(STORAGE_KEY); } catch { /* ignore */ } + return getLlmConfig(); +} + +export const LLM_STORAGE_KEY = STORAGE_KEY; +export const LLM_DEFAULTS = DEFAULTS; diff --git a/flow-state/src/llm/llmTweak.js b/flow-state/src/llm/llmTweak.js new file mode 100644 index 0000000..ffc8402 --- /dev/null +++ b/flow-state/src/llm/llmTweak.js @@ -0,0 +1,129 @@ +// LLM song-tweak — gateway caller (Vite/browser). +// +// Prompt/schema live in ./spec.js so `node --test` can import them without +// pulling in `src/scenes/metadata.json` (which needs Vite's JSON handling). +// This file adds the look digest (describeLook/validateLook) and the actual +// `ai`/`@ai-sdk/openai` call, which only run in the browser bundle. + +import { generateObject } from 'ai'; +import { createOpenAI } from '@ai-sdk/openai'; +import { getLlmConfig } from './config.js'; +import { canInvoke, recordInvocation, refundInvocation } from './budget.js'; +import { describeLook } from '../look/LookGenerator.js'; +import { validateLook } from '../look/composition-validator.js'; +import { + TweakResultSchema, + TweakEditsSchema, + ALLOWED_KEYS, + ARRANGEMENT_VALUES, + clampEdits, + buildTweakPrompt as buildTweakPromptPure, +} from './spec.js'; + +export { TweakResultSchema, TweakEditsSchema, ALLOWED_KEYS, ARRANGEMENT_VALUES, clampEdits }; + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); + +function lookDigestForPrompt(look) { + let validation = null; + try { validation = validateLook(look, null); } catch { /* ignore */ } + const sections = (look.sections || []).map((s) => { + const primary = s.layers && s.layers.find((l) => !l.role || l.role !== 'ground'); + const name = primary ? primary.module.name : (s.layers && s.layers[0] ? s.layers[0].module.name : '?'); + const layerNames = (s.layers || []).map((l) => `${l.role === 'ground' ? 'ground:' : ''}${l.module.name}(${l.blend || '?'})`).join(' + '); + return { index: s.index, kind: s.kind, name, layers: layerNames }; + }); + return { + describe: (() => { try { return describeLook(look); } catch { return ''; } })(), + director: look.director, + paletteScheme: look.paletteScheme || (look.paletteSchemes && look.paletteSchemes[0]) || null, + sections, + validation: validation ? { + ok: validation.ok, + refused: validation.refused || [], + songRefused: !!validation.songRefused, + perSection: (validation.sections || []).map((v) => ({ + index: v.index, kind: v.kind, ok: v.ok, desired: Number(v.desired.toFixed(3)), predicted: Number(v.predicted.toFixed(3)), reason: v.reason || null, + })), + } : null, + }; +} + +/** Prompt builder — wraps the pure one with a Look→digest step for browser callers. */ +export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) { + const digest = lookDigest != null ? lookDigest : (look ? lookDigestForPrompt(look) : null); + return buildTweakPromptPure({ spec, track, lookDigest: digest, hint }); +} + +export function lookDigestForPromptExport(look) { return lookDigestForPrompt(look); } + +function toErrorMessage(err) { + if (!err) return 'Unknown error'; + if (err instanceof Error) return err.message; + if (typeof err === 'string') return err; + try { return JSON.stringify(err).slice(0, 600); } catch { return String(err); } +} + +/** + * Call the gateway for one tweak. Checks budget, builds prompt, calls + * generateObject, clamps and returns. Counts toward budget on every network + * attempt; refunds if the result is empty/useless. + */ +export async function requestSongTweak({ spec, track, look, hint = '', config = null } = {}) { + const cfg = config || getLlmConfig(); + if (!cfg.isConfigured) { + const err = new Error('LLM not configured — set an AI Gateway API key in ⚙️ settings.'); + err.code = 'NOT_CONFIGURED'; + throw err; + } + + const gate = canInvoke(); + if (!gate.ok) { + const err = new Error(gate.detail || 'Budget exhausted'); + err.code = gate.reason === 'per-load' ? 'BUDGET_PER_LOAD' : gate.reason === 'per-session' ? 'BUDGET_PER_SESSION' : 'BUDGET'; + throw err; + } + + const messages = buildTweakPrompt({ spec, track, look, hint }); + recordInvocation(); + + const provider = createOpenAI({ baseURL: cfg.gatewayUrl, apiKey: cfg.apiKey }); + const modelId = cfg.model; + + try { + const result = await generateObject({ + model: provider(modelId), + schema: TweakResultSchema, + messages, + temperature: 0.4, + maxRetries: 0, + }); + + const raw = result.object || {}; + const edits = clampEdits(raw.edits || {}); + const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.'; + const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5; + + if (!Object.keys(edits).length) { + refundInvocation(); + const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.'); + err.code = 'NO_EDITS'; + err.result = { rationale, edits: {}, confidence, usage: result.usage, raw: result.object }; + throw err; + } + + return { rationale, edits, confidence, usage: result.usage || null, raw: result.object }; + } catch (err) { + const msg = toErrorMessage(err); + const lower = msg.toLowerCase(); + if (err && err.code === 'NO_EDITS') throw err; + if (!err.code) { + if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH'; + else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT'; + else if (lower.includes('timeout') || lower.includes('timed out')) err.code = 'TIMEOUT'; + else err.code = 'LLM_ERROR'; + } + if (!err.message || err.message === '[object Object]') err.message = msg; + throw err; + } +} diff --git a/flow-state/src/llm/spec.js b/flow-state/src/llm/spec.js new file mode 100644 index 0000000..f1e9794 --- /dev/null +++ b/flow-state/src/llm/spec.js @@ -0,0 +1,129 @@ +// Pure prompt/schema logic — no gateway, no LookGenerator/Scene imports. +// +// Split from llmTweak.js so `node --test` can import it without pulling in +// `src/scenes/metadata.json` (which requires Vite's JSON handling and fails +// under bare Node's import-assertion check). + +import { z } from 'zod'; +import { ARRANGEMENTS } from '../audio/synth.js'; + +const ARRANGEMENT_VALUES = Object.keys(ARRANGEMENTS); + +export { ARRANGEMENT_VALUES }; + +export const TweakEditsSchema = z.object({ + bpm: z.number().int().min(60).max(180).optional(), + brightness: z.number().min(0).max(1).optional(), + noise: z.number().min(0).max(1).optional(), + noiseColour: z.number().min(0).max(1).nullable().optional(), + dynamics: z.number().min(0).max(1).optional(), + transition: z.number().min(0).max(1).optional(), + arrangement: z.enum(ARRANGEMENT_VALUES).optional(), +}).strict(); + +export const TweakResultSchema = z.object({ + rationale: z.string().min(1).max(800), + edits: TweakEditsSchema, + confidence: z.number().min(0).max(1), +}); + +export const ALLOWED_KEYS = ['bpm', 'brightness', 'noise', 'noiseColour', 'dynamics', 'transition', 'arrangement']; + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); +const round2 = (x) => Math.round(x * 100) / 100; + +export function clampEdits(edits) { + if (!edits || typeof edits !== 'object') return {}; + const out = {}; + if (edits.bpm != null) { + const n = Number(edits.bpm); + if (Number.isFinite(n)) out.bpm = Math.max(60, Math.min(180, Math.round(n))); + } + for (const k of ['brightness', 'noise', 'dynamics', 'transition']) { + if (edits[k] != null) { + const n = Number(edits[k]); + if (Number.isFinite(n)) out[k] = round2(clamp01(n)); + } + } + if (Object.prototype.hasOwnProperty.call(edits, 'noiseColour')) { + const v = edits.noiseColour; + if (v == null) out.noiseColour = null; + else { + const n = Number(v); + if (Number.isFinite(n)) out.noiseColour = round2(clamp01(n)); + } + } + if (edits.arrangement != null && ARRANGEMENT_VALUES.includes(edits.arrangement)) { + out.arrangement = edits.arrangement; + } + return out; +} + +/** + * Build chat messages from already-serializable inputs. + * + * `lookDigest` is the pre-built digest (describeLook + validateLook) — callers + * that have a live Look should build it via llmTweak.lookDigestForPrompt(look) + * and pass it here, so this module never imports scene metadata. + * + * Kept compatible with the old `{spec,track,look,hint}` shape by accepting + * either `look` (no-op here) or `lookDigest`. + */ +export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) { + const summary = track ? track.summary : null; + const sections = track ? (track.sections || []).map((s) => ({ + kind: s.kind, + start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1), + end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1), + energy: Number((s.energy ?? 0).toFixed(3)), + })) : []; + + const digest = lookDigest ?? (look ? look : null); + + const specForPrompt = spec ? { + bpm: spec.bpm ?? null, + brightness: spec.brightness ?? null, + noise: spec.noise ?? null, + noiseColour: spec.noiseColour ?? null, + dynamics: spec.dynamics ?? null, + transition: spec.transition ?? null, + arrangement: spec.arrangement ?? null, + duration: spec.duration ?? null, + seed: spec.seed ?? null, + } : null; + + const hintBlock = hint && String(hint).trim() + ? `User hint: "${String(hint).trim().slice(0, 300)}"` + : 'No user hint.'; + + const system = [ + 'You are a producer for a generative music-video system called flow-state.', + 'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.', + 'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.', + '', + 'Rules:', + '- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.', + '- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.', + '- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.', + '- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.', + '- Explain your reasoning in 1-2 sentences (rationale).', + '- If the look is already good, return empty edits {}.', + hintBlock, + ].join('\n'); + + const user = JSON.stringify({ + songSpec: specForPrompt, + trackSummary: summary ? { + bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness, + dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness, + tempoConfidence: summary.tempoConfidence, + } : null, + sections, + lookDigest: digest, + }, null, 2); + + return [ + { role: 'system', content: system }, + { role: 'user', content: `Current generation:\n\`\`\`json\n${user}\n\`\`\`\n\nSuggest edits as { rationale, edits, confidence }.` }, + ]; +} diff --git a/flow-state/src/main.js b/flow-state/src/main.js index 87f5f6c..18612f8 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -9,6 +9,13 @@ import { renderClickTrack, audioBufferToWavBlob } from './audio/metronome.js'; import { Exporter, exportSegment, downloadBlob, PRESETS, isSupported, probeExportCodec } from './export/Exporter.js'; import { subjectOf, groundOf, overlaysOf, stackCoverage } from './look/stack.js'; import { coverageOf as sceneCoverage } from './scenes/surface.js'; +import { getLlmConfig, setLlmConfig } from './llm/config.js'; +import { canInvoke, remainingText, getBudgetCounts } from './llm/budget.js'; +import { requestSongTweak } from './llm/llmTweak.js'; +import { synthesizeSong } from './audio/synth.js'; +import { FeatureTrack } from './audio/FeatureTrack.js'; +import { generateLook } from './look/LookGenerator.js'; +import { SONGS } from './audio/songbank.js'; const QUALITY = { draft: 0.5, // half resolution — for scrubbing heavy stacks @@ -836,3 +843,305 @@ if (query.get('song')) { const seed = query.get('seed'); loadBankSong(query.get('song'), seed === null ? null : (Number(seed) >>> 0)); } + +// ---------------------------------------------------------------- LLM song tweak + +let llmTweakInFlight = false; +let currentSongSpec = null; // last synthesizeSong spec (for bank songs); null for dropped files + +function specForTrack(track) { + // For bank songs we have the original spec; for dropped files we derive + // a plausible spec from the measured summary so the LLM has something to edit. + if (currentSongSpec) return { ...currentSongSpec }; + if (!track || !track.summary) return null; + const s = track.summary; + return { + bpm: Math.round(s.bpm || 120), + brightness: Math.max(0, Math.min(1, s.meanCentroid ?? 0.5)), + noise: Math.max(0, Math.min(1, s.meanFlatness ?? 0.15)), + noiseColour: null, + dynamics: Math.max(0, Math.min(1, s.dynamicRange ?? 0.6)), + transition: 0.5, + arrangement: 'classic', + duration: Math.round(track.duration || 120), + seed: track.summary ? 1 : 1, + }; +} + +function setLlmTweakButtonState() { + const btn = document.getElementById('btn-llm-tweak'); + const gear = document.getElementById('btn-llm-config'); + if (!btn) return; + const gate = canInvoke(); + const cfg = getLlmConfig(); + const ready = state.show && state.show.ready; + btn.disabled = llmTweakInFlight || !ready || !gate.ok; + if (!ready) btn.title = 'Load a track first'; + else if (!cfg.isConfigured) btn.title = 'Set an AI Gateway API key — click ⚙️'; + else if (!gate.ok) btn.title = gate.detail; + else btn.title = `Let an LLM tune the generated song · ${remainingText()} · click to review`; + if (gear) { + const dot = cfg.isConfigured ? '●' : '○'; + gear.textContent = `${dot} ⚙️`; + gear.title = cfg.isConfigured + ? `LLM: ${cfg.model} @ ${cfg.gatewayUrl} · ${remainingText()} — click to configure` + : 'LLM not configured — click to set gateway URL / model / API key'; + gear.classList.toggle('llm-not-configured', !cfg.isConfigured); + } +} + +function openLlmConfigDialog() { + const cfg = getLlmConfig(); + const counts = getBudgetCounts(); + const overlay = document.createElement('div'); + overlay.className = 'llm-overlay'; + overlay.innerHTML = ` +
+ `; + document.body.appendChild(overlay); + const close = () => overlay.remove(); + overlay.querySelector('.llm-backdrop').addEventListener('click', close); + overlay.querySelector('#llm-cancel').addEventListener('click', close); + overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); }); + overlay.querySelector('#llm-save').addEventListener('click', () => { + const gw = overlay.querySelector('#llm-gw').value.trim(); + const model = overlay.querySelector('#llm-model').value.trim(); + const key = overlay.querySelector('#llm-key').value; + setLlmConfig({ + gatewayUrl: gw || null, + model: model || null, + apiKey: key, + }); + close(); + setLlmTweakButtonState(); + showToast('LLM settings saved', { ok: true, timeout: 2000 }); + }); + overlay.querySelector('#llm-gw').focus(); +} + +function escapeHtml(s) { + return String(s).replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); +} + +function openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter) { + const diffRows = Object.keys(specAfter || {}).filter((k) => { + if (k === 'seed' || k === 'duration') return false; + return specBefore[k] !== specAfter[k]; + }).map((k) => { + const a = specBefore[k]; + const b = specAfter[k]; + const fmt = (v) => v == null ? '∅' : (typeof v === 'number' ? (Number.isInteger(v) ? String(v) : v.toFixed(2)) : String(v)); + return `
${k}${fmt(a)} → ${fmt(b)}
`; + }).join('') || '
No song knobs changed (model returned empty edits — this dialog should not have opened).
'; + + const overlay = document.createElement('div'); + overlay.className = 'llm-overlay'; + overlay.innerHTML = ` +
+ `; + document.body.appendChild(overlay); + const close = () => overlay.remove(); + overlay.querySelector('.llm-backdrop').addEventListener('click', close); + overlay.querySelector('#llm-dismiss').addEventListener('click', close); + overlay.querySelector('#llm-apply').addEventListener('click', () => { + try { + const newDuration = specAfter.duration || trackBefore.duration; + // Synthesize a fresh buffer from the edited spec, derive a new FeatureTrack, + // then install it via Show.useTrack so prewarm/arc are correct. + const buf = synthesizeSong({ + bpm: specAfter.bpm, + duration: newDuration, + arrangement: specAfter.arrangement, + brightness: specAfter.brightness, + noise: specAfter.noise, + noiseColour: specAfter.noiseColour, + dynamics: specAfter.dynamics, + transition: specAfter.transition, + seed: specAfter.seed ?? (specBefore.seed ?? 1), + }); + const newTrack = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps }); + // Derive a seed from the new spec so generateLook is deterministic on the edit. + const newSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1); + const newLook = generateLook(newTrack, { seed: newSeed }); + // Swap in — useTrack handles timeline/provider/arc/prewarm + currentSongSpec = { ...specAfter, seed: newSeed }; + state.show.useTrack(newTrack, newLook); + // Keep the displayed name but mark as LLM-tuned + state.show.fileName = `${state.show.fileName || 'track'} · llm-tuned`; + state.show.osd.setText(state.show.fileName, newLook.personality, newLook.palette); + strip.setShow(state.show); + seekTo(0); + renderPanel(); + showToast(`Applied LLM tweak: ${result.rationale.slice(0, 120)}`, { ok: true, timeout: 4000 }); + } catch (err) { + showToast(`Failed to apply LLM tweak: ${err.message}`); + console.error(err); + } finally { + close(); + setLlmTweakButtonState(); + } + }); + overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); }); + // focus Apply for quick keyboard confirm + setTimeout(() => overlay.querySelector('#llm-apply')?.focus(), 50); +} + +async function handleLlmTweak() { + if (llmTweakInFlight) return; + if (!state.show || !state.show.ready) { + showToast('Load a track first'); + return; + } + const cfg = getLlmConfig(); + if (!cfg.isConfigured) { + openLlmConfigDialog(); + return; + } + const gate = canInvoke(); + if (!gate.ok) { + showToast(gate.detail || 'LLM budget exhausted'); + return; + } + // Confirm — preview cost/budget + const remaining = remainingText(); + const hint = window.prompt( + `Let the LLM tune the generated song?` + + `\nBudget: ${remaining}` + + `\nModel: ${cfg.model}` + + `\n` + + `\nOptional hint for the model (e.g. "make the drop calmer") — leave empty to skip:`, + '' + ); + if (hint === null) return; // cancelled + + const spec = specForTrack(state.show.track); + if (!spec) { + showToast('No song spec available for this track'); + return; + } + // Capture before for diff + const specBefore = { ...spec }; + const trackBefore = state.show.track; + const lookBefore = state.show.look; + + llmTweakInFlight = true; + setLlmTweakButtonState(); + const btn = document.getElementById('btn-llm-tweak'); + const prevLabel = btn ? btn.textContent : ''; + if (btn) btn.textContent = '… tuning'; + + try { + const result = await requestSongTweak({ + spec: specBefore, + track: trackBefore, + look: lookBefore, + hint: hint || '', + config: cfg, + }); + + const specAfter = { ...specBefore, ...result.edits }; + // Validate locally by re-synthesizing and measuring — show diff before applying. + // Do a lightweight preview synthesis so we can report section/agitation delta + // without mutating the live Show. + let trackAfter = null, lookAfter = null; + try { + const buf = synthesizeSong({ + bpm: specAfter.bpm, + duration: specAfter.duration, + arrangement: specAfter.arrangement, + brightness: specAfter.brightness, + noise: specAfter.noise, + noiseColour: specAfter.noiseColour, + dynamics: specAfter.dynamics, + transition: specAfter.transition, + seed: specAfter.seed ?? specBefore.seed ?? 1, + }); + trackAfter = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps }); + const previewSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1); + lookAfter = generateLook(trackAfter, { seed: previewSeed }); + } catch (e) { + console.warn('[llm] preview synthesis failed', e); + } + + openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter); + } catch (err) { + const code = err && err.code; + if (code === 'NO_EDITS') { + showToast(err.message, { ok: true, timeout: 5000 }); + } else if (code === 'BUDGET_PER_LOAD' || code === 'BUDGET_PER_SESSION' || code === 'BUDGET') { + showToast(err.message); + } else if (code === 'AUTH') { + showToast(`LLM auth failed (401) — check your gateway API key in ⚙️\n${err.message}`); + } else if (code === 'RATE_LIMIT') { + showToast(`LLM rate limited (429) — try again shortly\n${err.message}`); + } else if (code === 'NOT_CONFIGURED') { + openLlmConfigDialog(); + } else { + showToast(`LLM tweak failed: ${err.message}`); + console.error('[llm] request failed', err); + } + } finally { + llmTweakInFlight = false; + if (btn) btn.textContent = prevLabel || '✨ tune'; + setLlmTweakButtonState(); + } +} + +document.getElementById('btn-llm-tweak')?.addEventListener('click', handleLlmTweak); +document.getElementById('btn-llm-config')?.addEventListener('click', openLlmConfigDialog); + +// Keep the tune button in sync with ready/config/budget — renderPanel is +// called after every load/reroll/param change. +const __prevRenderPanel = renderPanel; +renderPanel = function llmAwareRenderPanel(...args) { + const r = __prevRenderPanel.apply(this, args); + setLlmTweakButtonState(); + return r; +}; + +// Initial state — disabled until a track is loaded or a previous session had one +setLlmTweakButtonState(); + +// Keep loadBankSong in sync with LLM spec tracking +const __origLoadBankSong = loadBankSong; +loadBankSong = async function trackedLoadBankSong(name, seed) { + const spec = SONGS.find((s) => s.name === name); + if (spec) { + currentSongSpec = { + bpm: spec.bpm, + brightness: spec.brightness, + noise: spec.noise, + noiseColour: spec.noiseColour ?? null, + dynamics: spec.dynamics, + transition: spec.transition ?? 0.5, + arrangement: spec.arrangement, + duration: 120, + seed: seed != null ? (Number(seed) >>> 0) : (1 + SONGS.indexOf(spec)), + }; + } + return __origLoadBankSong(name, seed); +}; diff --git a/flow-state/src/ui/style.css b/flow-state/src/ui/style.css index 758b5de..e5af93d 100644 --- a/flow-state/src/ui/style.css +++ b/flow-state/src/ui/style.css @@ -284,6 +284,21 @@ input[type=range] { accent-color: var(--accent); background: transparent; } } } +/* LLM gateway dialogs (main app). Minimal, inherits panel/modal language. */ +.llm-overlay { position: fixed; inset: 0; display: flex; align-items: center; justify-content: center; z-index: 50; } +.llm-backdrop { position: absolute; inset: 0; background: rgba(0,0,0,0.62); backdrop-filter: blur(2px); } +.llm-box { position: relative; width: min(560px, calc(100vw - 32px)); max-height: min(78vh, 720px); overflow: auto; background: var(--panel); border: 1px solid var(--line); border-left: 3px solid var(--accent); padding: 16px 18px 14px; box-shadow: 0 20px 40px rgba(0,0,0,0.55); } +.llm-box.llm-diff { width: min(640px, calc(100vw - 32px)); } +.llm-title { font-weight: 700; font-size: 12px; letter-spacing: .08em; text-transform: uppercase; margin-bottom: 10px; } +.llm-field { display: grid; grid-template-columns: 110px 1fr; gap: 8px; align-items: center; margin: 8px 0; font-size: 12px; } +.llm-field span { color: var(--dim); } +.llm-field input { width: 100%; background: #171a22; border: 1px solid var(--line); color: var(--text); padding: 6px 8px; font: inherit; font-size: 12px; } +.llm-actions { display: flex; gap: 8px; justify-content: flex-end; margin-top: 14px; } +.llm-rationale { font-size: 12px; line-height: 1.6; white-space: pre-wrap; color: var(--text); margin-bottom: 6px; } +.llm-confidence { margin-bottom: 4px; } +.llm-not-configured { opacity: 0.85; } +#btn-llm-tweak:disabled { opacity: 0.45; cursor: not-allowed; } + /* Single-scene gate output (checks.html?scene=Name). Monospaced so the PASS/FAIL column lines up and the whole verdict reads in one glance. */ .scene-gate { diff --git a/flow-state/test/llm.test.js b/flow-state/test/llm.test.js new file mode 100644 index 0000000..19ca39f --- /dev/null +++ b/flow-state/test/llm.test.js @@ -0,0 +1,72 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { clampEdits, buildTweakPrompt, TweakResultSchema, ALLOWED_KEYS } from '../src/llm/spec.js'; +import { LLM_BUDGET, getBudgetCounts, canInvoke, remainingText } from '../src/llm/budget.js'; +import { LLM_DEFAULTS, LLM_STORAGE_KEY } from '../src/llm/config.js'; + +test('clampEdits clamps to valid ranges and drops invalid arrangement', () => { + const out = clampEdits({ bpm: 300, brightness: 1.5, noise: -1, arrangement: 'nope', dynamics: 0.1234 }); + assert.equal(out.bpm, 180); + assert.equal(out.brightness, 1); + assert.equal(out.noise, 0); + assert.equal(out.arrangement, undefined); + assert.equal(out.dynamics, 0.12); +}); + +test('clampEdits preserves noiseColour null and rounds', () => { + assert.deepEqual(clampEdits({ noiseColour: null }), { noiseColour: null }); + assert.deepEqual(clampEdits({ noiseColour: 0.3333 }), { noiseColour: 0.33 }); + assert.deepEqual(clampEdits({ arrangement: 'classic' }), { arrangement: 'classic' }); +}); + +test('ALLOWED_KEYS covers the 7 song knobs', () => { + assert.deepEqual([...ALLOWED_KEYS].sort(), ['arrangement', 'bpm', 'brightness', 'dynamics', 'noise', 'noiseColour', 'transition'].sort()); +}); + +test('buildTweakPrompt returns system+user messages and respects hint', () => { + const fakeDigest = { + describe: 'seed abc · geometer · triad', + director: 'geometer', + paletteScheme: 'triad', + sections: [{ index: 0, kind: 'intro', name: 'Shot', layers: 'Shot' }], + validation: null, + }; + const msgs = buildTweakPrompt({ + spec: { bpm: 124, brightness: 0.5, noise: 0.2, noiseColour: null, dynamics: 0.55, transition: 0.6, arrangement: 'classic', duration: 120, seed: 42 }, + track: { summary: { bpm: 124, meanCentroid: 0.5, meanFlatness: 0.2, dynamicRange: 0.55, meanLoudness: 0.4, tempoConfidence: 0.9 }, sections: [{ kind: 'intro', start: 0, end: 15, energy: 0.2 }], duration: 120 }, + lookDigest: fakeDigest, + hint: 'make the drop calmer', + }); + assert.equal(msgs.length, 2); + assert.equal(msgs[0].role, 'system'); + assert.equal(msgs[1].role, 'user'); + assert.ok(msgs[0].content.includes('make the drop calmer') || msgs[0].content.includes('User hint')); + assert.ok(msgs[1].content.includes('songSpec')); +}); + +test('buildTweakPrompt truncates long hint in system', () => { + const long = 'x'.repeat(1000); + const msgs = buildTweakPrompt({ spec: null, track: null, lookDigest: null, hint: long }); + assert.ok(msgs[0].content.length < 5000); +}); + +test('TweakResultSchema validates a minimal valid result', () => { + const parsed = TweakResultSchema.parse({ rationale: 'Lower noise to calm the drop.', edits: { noise: 0.3 }, confidence: 0.8 }); + assert.equal(parsed.edits.noise, 0.3); + assert.ok(parsed.rationale.length > 0); +}); + +test('budget defaults are sane and canInvoke reports not-configured without a key', () => { + assert.equal(LLM_BUDGET.perLoad, 3); + assert.equal(LLM_BUDGET.perSession, 10); + assert.equal(LLM_STORAGE_KEY, 'flow-state:llm'); + assert.equal(LLM_DEFAULTS.gatewayUrl, 'https://ai-gateway.vercel.sh/v1'); + const counts = getBudgetCounts(); + assert.ok(counts.perLoad === 3 && counts.perSession === 10); + assert.ok(typeof counts.loadCount === 'number' && typeof counts.sessionCount === 'number'); + const gate = canInvoke(); + assert.equal(gate.ok, false); + assert.equal(gate.reason, 'not-configured'); + assert.ok(typeof remainingText() === 'string'); +});