LLM song-tweak: AI Gateway via Vercel AI SDK, budgeted tune flow
Add a ✨ tune button that lets an LLM inspect the proposed visualization and tweak the proc-song knobs so the next generation looks better. Edits only the 7 songbank params (bpm, brightness, noise, noiseColour, dynamics, transition, arrangement) — never the Look — then re-synthesizes and re-generates locally for preview. - ai + @ai-sdk/openai + zod (library per plan) against the Vercel AI Gateway (OpenAI-compatible, configurable via VITE_AI_GATEWAY_URL/MODEL/API_KEY, localStorage override, .env.example). - Structured output (TweakResultSchema) + clampEdits enforces ranges; prompt ≤ ~2.5k tokens, max 2-3 knobs ±0.15 / bpm ±8. - Budget caps: 3/load + 10/session (sessionStorage), checked before network, NO_EDITS refunded, offline/auth/rate-limit mapped. - Prompt split into src/llm/spec.js (pure, node-testable) + llmTweak.js (gateway + digest) to keep `node --test` off the Vite JSON path. - UI: ⚙️ gateway/model/key dialog (localStorage flow-state:llm), confirm(prompt) with hint, diff modal with Apply/Dismiss; Apply synthesizes → FeatureTrack → generateLook → Show.useTrack. - Tests: 7 new LLM cases (clamp, schema, prompt, budget), 18/18 pass. Build + lint:scenes green. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
@@ -0,0 +1,5 @@
|
||||
# Vercel AI Gateway — copy to .env.local and fill VITE_AI_API_KEY
|
||||
# Get a token at https://vercel.com/dashboard/ai-gateway
|
||||
VITE_AI_GATEWAY_URL=https://ai-gateway.vercel.sh/v1
|
||||
VITE_AI_MODEL=openai/gpt-4o-mini
|
||||
VITE_AI_API_KEY=
|
||||
@@ -2,6 +2,7 @@ node_modules
|
||||
dist
|
||||
.vite
|
||||
out
|
||||
.env.local
|
||||
|
||||
# Scratch probes: one-off harnesses for debugging decode/encode/mux issues.
|
||||
# They are throwaway by nature and were committed once by accident.
|
||||
|
||||
@@ -59,6 +59,8 @@
|
||||
<button id="btn-reroll" title="New seed for the whole track">reroll</button>
|
||||
<button id="btn-reroll-section" title="New scene for this section only">reroll section</button>
|
||||
<button id="btn-lock" title="Lock this section against rerolls">lock</button>
|
||||
<button id="btn-llm-tweak" title="Let an LLM tune the generated song">✨ tune</button>
|
||||
<button id="btn-llm-config" title="LLM gateway settings">⚙️</button>
|
||||
<button id="btn-hud" title="Toggle debug HUD (D)">hud</button>
|
||||
<button id="btn-osd" title="Toggle the title plate in the corner (O)">osd</button>
|
||||
<button id="btn-segment" title="Render 20s around the playhead at export quality">test render</button>
|
||||
|
||||
Generated
+142
-1
@@ -9,14 +9,80 @@
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^3.0.112",
|
||||
"ai": "^6.0.282",
|
||||
"mp4-muxer": "^5.2.2",
|
||||
"three": "^0.181.1"
|
||||
"three": "^0.181.1",
|
||||
"zod": "^4.6.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.62.1",
|
||||
"vite": "^7.2.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/gateway": {
|
||||
"version": "3.0.193",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-3.0.193.tgz",
|
||||
"integrity": "sha512-T2vq1EKPhbFF2Sk2MUFL/RmCg/2mp1LGbCZZSx5vh0M6Sr1EAj3wRr1EnpcwCV3/Imj2q56pT89XLiUFnB+ZgQ==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "3.0.16",
|
||||
"@ai-sdk/provider-utils": "4.0.51",
|
||||
"@vercel/oidc": "3.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/openai": {
|
||||
"version": "3.0.112",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-3.0.112.tgz",
|
||||
"integrity": "sha512-0ho3a9CJ+3KCcrPdBQx8fX7/gyUxFHsfs1M6++SpunYk8+pgGOPMskr0GVOOxY+94jsJbuGxksKfMuSYhYHcxw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "3.0.16",
|
||||
"@ai-sdk/provider-utils": "4.0.51"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider": {
|
||||
"version": "3.0.16",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-3.0.16.tgz",
|
||||
"integrity": "sha512-9Av6kg0t/IN/dcYAAmEJ4B9OPhcEJqwSR+GfHEA8olRkindXpUHadc3p3cyvgFUQFTVm1G5thtsmgZ9yVb2w3A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"json-schema": "^0.4.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@ai-sdk/provider-utils": {
|
||||
"version": "4.0.51",
|
||||
"resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-4.0.51.tgz",
|
||||
"integrity": "sha512-ukLTs9x1Xm6lxSIwbJIxYQxbZHmVeIczNntARZrBcOa6pjdBhqeZnATNnJMHa7M9dZ8Ji5NJbpKpvz7o5XyBtg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/provider": "3.0.16",
|
||||
"@standard-schema/spec": "^1.1.0",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"undici": "^6.28.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.17"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz",
|
||||
@@ -476,6 +542,15 @@
|
||||
"node": "^22.20 || ^24.12 || >=25"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/api": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz",
|
||||
"integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
|
||||
@@ -842,6 +917,12 @@
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@standard-schema/spec": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz",
|
||||
"integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/dom-webcodecs": {
|
||||
"version": "0.1.18",
|
||||
"resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.18.tgz",
|
||||
@@ -861,6 +942,33 @@
|
||||
"integrity": "sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@vercel/oidc": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz",
|
||||
"integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">= 20"
|
||||
}
|
||||
},
|
||||
"node_modules/ai": {
|
||||
"version": "6.0.282",
|
||||
"resolved": "https://registry.npmjs.org/ai/-/ai-6.0.282.tgz",
|
||||
"integrity": "sha512-fdzEzSppVTYlgyW/Gp6tMAFIU88yv7k86x77m5vAJo7hXV31hDJCsAkq/aIpE3tP3k4LjrrLcUANPHcBZ0YWsw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@ai-sdk/gateway": "3.0.193",
|
||||
"@ai-sdk/provider": "3.0.16",
|
||||
"@ai-sdk/provider-utils": "4.0.51",
|
||||
"@opentelemetry/api": "^1.9.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"zod": "^3.25.76 || ^4.1.8"
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.28.1",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz",
|
||||
@@ -903,6 +1011,15 @@
|
||||
"@esbuild/win32-x64": "0.28.1"
|
||||
}
|
||||
},
|
||||
"node_modules/eventsource-parser": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz",
|
||||
"integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fdir": {
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
|
||||
@@ -936,6 +1053,12 @@
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/json-schema": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz",
|
||||
"integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==",
|
||||
"license": "(AFL-2.1 OR BSD-3-Clause)"
|
||||
},
|
||||
"node_modules/mp4-muxer": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/mp4-muxer/-/mp4-muxer-5.2.2.tgz",
|
||||
@@ -1141,6 +1264,15 @@
|
||||
"url": "https://github.com/sponsors/SuperchupuDev"
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "6.28.1",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-6.28.1.tgz",
|
||||
"integrity": "sha512-zWpdTVD54H48CIybL0rWQ3ukpb9d23wM7eH5RtfdmeP70cWHNjtfo7P4vZX+5CoDcO53J4Pu5uXp7lNfjc6DRA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18.17"
|
||||
}
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "7.3.6",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz",
|
||||
@@ -1215,6 +1347,15 @@
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/zod": {
|
||||
"version": "4.6.4",
|
||||
"resolved": "https://registry.npmjs.org/zod/-/zod-4.6.4.tgz",
|
||||
"integrity": "sha512-AXSD6hvGdvRjajG/l1cC+d6IrhH+sjmPKtYeQdJIK8MFJl3LyClzS+o/YsVC+zQZPupAaeH5skwwm8YqYH7BqA==",
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/colinhacks"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,10 @@
|
||||
"vite": "^7.2.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^3.0.112",
|
||||
"ai": "^6.0.282",
|
||||
"mp4-muxer": "^5.2.2",
|
||||
"three": "^0.181.1"
|
||||
"three": "^0.181.1",
|
||||
"zod": "^4.6.4"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Invocation budget — caps cost by refusing before the network.
|
||||
//
|
||||
// perLoad: how many LLM calls per page load
|
||||
// perSession: how many per sessionStorage lifetime (survives reload until tab close)
|
||||
|
||||
import { getLlmConfig } from './config.js';
|
||||
|
||||
export const LLM_BUDGET = Object.freeze({
|
||||
perLoad: 3,
|
||||
perSession: 10,
|
||||
});
|
||||
|
||||
const SESSION_KEY = 'flow-state:llm-budget';
|
||||
|
||||
let loadCount = 0;
|
||||
|
||||
function readSessionCount() {
|
||||
try {
|
||||
const raw = sessionStorage.getItem(SESSION_KEY);
|
||||
if (!raw) return 0;
|
||||
const n = Number(JSON.parse(raw));
|
||||
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function writeSessionCount(n) {
|
||||
try {
|
||||
sessionStorage.setItem(SESSION_KEY, JSON.stringify(Math.max(0, Math.floor(n))));
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export function getBudgetCounts() {
|
||||
return {
|
||||
loadCount,
|
||||
sessionCount: readSessionCount(),
|
||||
perLoad: LLM_BUDGET.perLoad,
|
||||
perSession: LLM_BUDGET.perSession,
|
||||
};
|
||||
}
|
||||
|
||||
export function canInvoke() {
|
||||
const cfg = getLlmConfig();
|
||||
if (!cfg.isConfigured) return { ok: false, reason: 'not-configured', detail: 'Set an AI Gateway API key in ⚙️ settings.' };
|
||||
if (typeof navigator !== 'undefined' && navigator.onLine === false) {
|
||||
return { ok: false, reason: 'offline', detail: 'Offline — LLM unavailable.' };
|
||||
}
|
||||
const counts = getBudgetCounts();
|
||||
if (counts.loadCount >= counts.perLoad) {
|
||||
return { ok: false, reason: 'per-load', detail: `Budget exhausted for this page load (${counts.perLoad}). Reload to reset.` };
|
||||
}
|
||||
if (counts.sessionCount >= counts.perSession) {
|
||||
return { ok: false, reason: 'per-session', detail: `Session budget exhausted (${counts.perSession}). Close the tab or clear sessionStorage.` };
|
||||
}
|
||||
return { ok: true, reason: 'ok', detail: '', counts };
|
||||
}
|
||||
|
||||
export function recordInvocation() {
|
||||
loadCount += 1;
|
||||
writeSessionCount(readSessionCount() + 1);
|
||||
return getBudgetCounts();
|
||||
}
|
||||
|
||||
/** Refund one invocation — when the model returned empty/useless edits. */
|
||||
export function refundInvocation() {
|
||||
if (loadCount > 0) loadCount -= 1;
|
||||
const sc = readSessionCount();
|
||||
if (sc > 0) writeSessionCount(sc - 1);
|
||||
return getBudgetCounts();
|
||||
}
|
||||
|
||||
export function remainingText() {
|
||||
const c = getBudgetCounts();
|
||||
const perLoadLeft = Math.max(0, c.perLoad - c.loadCount);
|
||||
const perSessionLeft = Math.max(0, c.perSession - c.sessionCount);
|
||||
return `${perLoadLeft}/${c.perLoad} this load · ${perSessionLeft}/${c.perSession} this session`;
|
||||
}
|
||||
|
||||
export function clearBudget() {
|
||||
loadCount = 0;
|
||||
try { sessionStorage.removeItem(SESSION_KEY); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export const LLM_BUDGET_SESSION_KEY = SESSION_KEY;
|
||||
@@ -0,0 +1,87 @@
|
||||
// LLM gateway config — localStorage overrides > VITE env > defaults.
|
||||
//
|
||||
// The app is a static Vite site with no server; the gateway credential lives
|
||||
// in the browser. Never commit a key — paste it in the panel (→ localStorage)
|
||||
// or in an untracked .env.local.
|
||||
|
||||
const STORAGE_KEY = 'flow-state:llm';
|
||||
|
||||
const DEFAULTS = {
|
||||
gatewayUrl: 'https://ai-gateway.vercel.sh/v1',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
apiKey: '',
|
||||
};
|
||||
|
||||
function readEnv() {
|
||||
try {
|
||||
const env = import.meta.env || {};
|
||||
return {
|
||||
gatewayUrl: env.VITE_AI_GATEWAY_URL || null,
|
||||
model: env.VITE_AI_MODEL || null,
|
||||
apiKey: env.VITE_AI_API_KEY || null,
|
||||
};
|
||||
} catch {
|
||||
return { gatewayUrl: null, model: null, apiKey: null };
|
||||
}
|
||||
}
|
||||
|
||||
function readStorage() {
|
||||
try {
|
||||
const raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (!raw) return {};
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed && typeof parsed === 'object' ? parsed : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolved config — localStorage wins, then env, then defaults. */
|
||||
export function getLlmConfig() {
|
||||
const env = readEnv();
|
||||
const stored = readStorage();
|
||||
const gatewayUrl = stored.gatewayUrl || env.gatewayUrl || DEFAULTS.gatewayUrl;
|
||||
const model = stored.model || env.model || DEFAULTS.model;
|
||||
const apiKey = stored.apiKey != null ? stored.apiKey : (env.apiKey != null ? env.apiKey : DEFAULTS.apiKey);
|
||||
const key = typeof apiKey === 'string' ? apiKey.trim() : '';
|
||||
return {
|
||||
gatewayUrl: String(gatewayUrl || DEFAULTS.gatewayUrl).trim() || DEFAULTS.gatewayUrl,
|
||||
model: String(model || DEFAULTS.model).trim() || DEFAULTS.model,
|
||||
apiKey: key,
|
||||
isConfigured: key.length > 0,
|
||||
defaults: { ...DEFAULTS },
|
||||
env: { ...env },
|
||||
stored: { ...stored },
|
||||
};
|
||||
}
|
||||
|
||||
/** Merge patch into localStorage. Pass null/'' to clear a field. */
|
||||
export function setLlmConfig(patch) {
|
||||
const current = readStorage();
|
||||
const next = { ...current };
|
||||
for (const k of ['gatewayUrl', 'model', 'apiKey']) {
|
||||
if (patch && Object.prototype.hasOwnProperty.call(patch, k)) {
|
||||
const v = patch[k];
|
||||
if (v == null || v === '') {
|
||||
delete next[k];
|
||||
} else {
|
||||
next[k] = String(v).trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (!Object.keys(next).length) localStorage.removeItem(STORAGE_KEY);
|
||||
else localStorage.setItem(STORAGE_KEY, JSON.stringify(next));
|
||||
} catch {
|
||||
// storage full / blocked — ignore, caller sees isConfigured false
|
||||
}
|
||||
return getLlmConfig();
|
||||
}
|
||||
|
||||
export function clearLlmConfig() {
|
||||
try { localStorage.removeItem(STORAGE_KEY); } catch { /* ignore */ }
|
||||
return getLlmConfig();
|
||||
}
|
||||
|
||||
export const LLM_STORAGE_KEY = STORAGE_KEY;
|
||||
export const LLM_DEFAULTS = DEFAULTS;
|
||||
@@ -0,0 +1,129 @@
|
||||
// LLM song-tweak — gateway caller (Vite/browser).
|
||||
//
|
||||
// Prompt/schema live in ./spec.js so `node --test` can import them without
|
||||
// pulling in `src/scenes/metadata.json` (which needs Vite's JSON handling).
|
||||
// This file adds the look digest (describeLook/validateLook) and the actual
|
||||
// `ai`/`@ai-sdk/openai` call, which only run in the browser bundle.
|
||||
|
||||
import { generateObject } from 'ai';
|
||||
import { createOpenAI } from '@ai-sdk/openai';
|
||||
import { getLlmConfig } from './config.js';
|
||||
import { canInvoke, recordInvocation, refundInvocation } from './budget.js';
|
||||
import { describeLook } from '../look/LookGenerator.js';
|
||||
import { validateLook } from '../look/composition-validator.js';
|
||||
import {
|
||||
TweakResultSchema,
|
||||
TweakEditsSchema,
|
||||
ALLOWED_KEYS,
|
||||
ARRANGEMENT_VALUES,
|
||||
clampEdits,
|
||||
buildTweakPrompt as buildTweakPromptPure,
|
||||
} from './spec.js';
|
||||
|
||||
export { TweakResultSchema, TweakEditsSchema, ALLOWED_KEYS, ARRANGEMENT_VALUES, clampEdits };
|
||||
|
||||
const clamp01 = (x) => Math.max(0, Math.min(1, x));
|
||||
|
||||
function lookDigestForPrompt(look) {
|
||||
let validation = null;
|
||||
try { validation = validateLook(look, null); } catch { /* ignore */ }
|
||||
const sections = (look.sections || []).map((s) => {
|
||||
const primary = s.layers && s.layers.find((l) => !l.role || l.role !== 'ground');
|
||||
const name = primary ? primary.module.name : (s.layers && s.layers[0] ? s.layers[0].module.name : '?');
|
||||
const layerNames = (s.layers || []).map((l) => `${l.role === 'ground' ? 'ground:' : ''}${l.module.name}(${l.blend || '?'})`).join(' + ');
|
||||
return { index: s.index, kind: s.kind, name, layers: layerNames };
|
||||
});
|
||||
return {
|
||||
describe: (() => { try { return describeLook(look); } catch { return ''; } })(),
|
||||
director: look.director,
|
||||
paletteScheme: look.paletteScheme || (look.paletteSchemes && look.paletteSchemes[0]) || null,
|
||||
sections,
|
||||
validation: validation ? {
|
||||
ok: validation.ok,
|
||||
refused: validation.refused || [],
|
||||
songRefused: !!validation.songRefused,
|
||||
perSection: (validation.sections || []).map((v) => ({
|
||||
index: v.index, kind: v.kind, ok: v.ok, desired: Number(v.desired.toFixed(3)), predicted: Number(v.predicted.toFixed(3)), reason: v.reason || null,
|
||||
})),
|
||||
} : null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Prompt builder — wraps the pure one with a Look→digest step for browser callers. */
|
||||
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
|
||||
const digest = lookDigest != null ? lookDigest : (look ? lookDigestForPrompt(look) : null);
|
||||
return buildTweakPromptPure({ spec, track, lookDigest: digest, hint });
|
||||
}
|
||||
|
||||
export function lookDigestForPromptExport(look) { return lookDigestForPrompt(look); }
|
||||
|
||||
function toErrorMessage(err) {
|
||||
if (!err) return 'Unknown error';
|
||||
if (err instanceof Error) return err.message;
|
||||
if (typeof err === 'string') return err;
|
||||
try { return JSON.stringify(err).slice(0, 600); } catch { return String(err); }
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the gateway for one tweak. Checks budget, builds prompt, calls
|
||||
* generateObject, clamps and returns. Counts toward budget on every network
|
||||
* attempt; refunds if the result is empty/useless.
|
||||
*/
|
||||
export async function requestSongTweak({ spec, track, look, hint = '', config = null } = {}) {
|
||||
const cfg = config || getLlmConfig();
|
||||
if (!cfg.isConfigured) {
|
||||
const err = new Error('LLM not configured — set an AI Gateway API key in ⚙️ settings.');
|
||||
err.code = 'NOT_CONFIGURED';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const gate = canInvoke();
|
||||
if (!gate.ok) {
|
||||
const err = new Error(gate.detail || 'Budget exhausted');
|
||||
err.code = gate.reason === 'per-load' ? 'BUDGET_PER_LOAD' : gate.reason === 'per-session' ? 'BUDGET_PER_SESSION' : 'BUDGET';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const messages = buildTweakPrompt({ spec, track, look, hint });
|
||||
recordInvocation();
|
||||
|
||||
const provider = createOpenAI({ baseURL: cfg.gatewayUrl, apiKey: cfg.apiKey });
|
||||
const modelId = cfg.model;
|
||||
|
||||
try {
|
||||
const result = await generateObject({
|
||||
model: provider(modelId),
|
||||
schema: TweakResultSchema,
|
||||
messages,
|
||||
temperature: 0.4,
|
||||
maxRetries: 0,
|
||||
});
|
||||
|
||||
const raw = result.object || {};
|
||||
const edits = clampEdits(raw.edits || {});
|
||||
const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.';
|
||||
const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5;
|
||||
|
||||
if (!Object.keys(edits).length) {
|
||||
refundInvocation();
|
||||
const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.');
|
||||
err.code = 'NO_EDITS';
|
||||
err.result = { rationale, edits: {}, confidence, usage: result.usage, raw: result.object };
|
||||
throw err;
|
||||
}
|
||||
|
||||
return { rationale, edits, confidence, usage: result.usage || null, raw: result.object };
|
||||
} catch (err) {
|
||||
const msg = toErrorMessage(err);
|
||||
const lower = msg.toLowerCase();
|
||||
if (err && err.code === 'NO_EDITS') throw err;
|
||||
if (!err.code) {
|
||||
if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH';
|
||||
else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT';
|
||||
else if (lower.includes('timeout') || lower.includes('timed out')) err.code = 'TIMEOUT';
|
||||
else err.code = 'LLM_ERROR';
|
||||
}
|
||||
if (!err.message || err.message === '[object Object]') err.message = msg;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// Pure prompt/schema logic — no gateway, no LookGenerator/Scene imports.
|
||||
//
|
||||
// Split from llmTweak.js so `node --test` can import it without pulling in
|
||||
// `src/scenes/metadata.json` (which requires Vite's JSON handling and fails
|
||||
// under bare Node's import-assertion check).
|
||||
|
||||
import { z } from 'zod';
|
||||
import { ARRANGEMENTS } from '../audio/synth.js';
|
||||
|
||||
const ARRANGEMENT_VALUES = Object.keys(ARRANGEMENTS);
|
||||
|
||||
export { ARRANGEMENT_VALUES };
|
||||
|
||||
export const TweakEditsSchema = z.object({
|
||||
bpm: z.number().int().min(60).max(180).optional(),
|
||||
brightness: z.number().min(0).max(1).optional(),
|
||||
noise: z.number().min(0).max(1).optional(),
|
||||
noiseColour: z.number().min(0).max(1).nullable().optional(),
|
||||
dynamics: z.number().min(0).max(1).optional(),
|
||||
transition: z.number().min(0).max(1).optional(),
|
||||
arrangement: z.enum(ARRANGEMENT_VALUES).optional(),
|
||||
}).strict();
|
||||
|
||||
export const TweakResultSchema = z.object({
|
||||
rationale: z.string().min(1).max(800),
|
||||
edits: TweakEditsSchema,
|
||||
confidence: z.number().min(0).max(1),
|
||||
});
|
||||
|
||||
export const ALLOWED_KEYS = ['bpm', 'brightness', 'noise', 'noiseColour', 'dynamics', 'transition', 'arrangement'];
|
||||
|
||||
const clamp01 = (x) => Math.max(0, Math.min(1, x));
|
||||
const round2 = (x) => Math.round(x * 100) / 100;
|
||||
|
||||
export function clampEdits(edits) {
|
||||
if (!edits || typeof edits !== 'object') return {};
|
||||
const out = {};
|
||||
if (edits.bpm != null) {
|
||||
const n = Number(edits.bpm);
|
||||
if (Number.isFinite(n)) out.bpm = Math.max(60, Math.min(180, Math.round(n)));
|
||||
}
|
||||
for (const k of ['brightness', 'noise', 'dynamics', 'transition']) {
|
||||
if (edits[k] != null) {
|
||||
const n = Number(edits[k]);
|
||||
if (Number.isFinite(n)) out[k] = round2(clamp01(n));
|
||||
}
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(edits, 'noiseColour')) {
|
||||
const v = edits.noiseColour;
|
||||
if (v == null) out.noiseColour = null;
|
||||
else {
|
||||
const n = Number(v);
|
||||
if (Number.isFinite(n)) out.noiseColour = round2(clamp01(n));
|
||||
}
|
||||
}
|
||||
if (edits.arrangement != null && ARRANGEMENT_VALUES.includes(edits.arrangement)) {
|
||||
out.arrangement = edits.arrangement;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build chat messages from already-serializable inputs.
|
||||
*
|
||||
* `lookDigest` is the pre-built digest (describeLook + validateLook) — callers
|
||||
* that have a live Look should build it via llmTweak.lookDigestForPrompt(look)
|
||||
* and pass it here, so this module never imports scene metadata.
|
||||
*
|
||||
* Kept compatible with the old `{spec,track,look,hint}` shape by accepting
|
||||
* either `look` (no-op here) or `lookDigest`.
|
||||
*/
|
||||
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
|
||||
const summary = track ? track.summary : null;
|
||||
const sections = track ? (track.sections || []).map((s) => ({
|
||||
kind: s.kind,
|
||||
start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1),
|
||||
end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1),
|
||||
energy: Number((s.energy ?? 0).toFixed(3)),
|
||||
})) : [];
|
||||
|
||||
const digest = lookDigest ?? (look ? look : null);
|
||||
|
||||
const specForPrompt = spec ? {
|
||||
bpm: spec.bpm ?? null,
|
||||
brightness: spec.brightness ?? null,
|
||||
noise: spec.noise ?? null,
|
||||
noiseColour: spec.noiseColour ?? null,
|
||||
dynamics: spec.dynamics ?? null,
|
||||
transition: spec.transition ?? null,
|
||||
arrangement: spec.arrangement ?? null,
|
||||
duration: spec.duration ?? null,
|
||||
seed: spec.seed ?? null,
|
||||
} : null;
|
||||
|
||||
const hintBlock = hint && String(hint).trim()
|
||||
? `User hint: "${String(hint).trim().slice(0, 300)}"`
|
||||
: 'No user hint.';
|
||||
|
||||
const system = [
|
||||
'You are a producer for a generative music-video system called flow-state.',
|
||||
'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.',
|
||||
'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.',
|
||||
'',
|
||||
'Rules:',
|
||||
'- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.',
|
||||
'- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.',
|
||||
'- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.',
|
||||
'- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.',
|
||||
'- Explain your reasoning in 1-2 sentences (rationale).',
|
||||
'- If the look is already good, return empty edits {}.',
|
||||
hintBlock,
|
||||
].join('\n');
|
||||
|
||||
const user = JSON.stringify({
|
||||
songSpec: specForPrompt,
|
||||
trackSummary: summary ? {
|
||||
bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness,
|
||||
dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness,
|
||||
tempoConfidence: summary.tempoConfidence,
|
||||
} : null,
|
||||
sections,
|
||||
lookDigest: digest,
|
||||
}, null, 2);
|
||||
|
||||
return [
|
||||
{ role: 'system', content: system },
|
||||
{ role: 'user', content: `Current generation:\n\`\`\`json\n${user}\n\`\`\`\n\nSuggest edits as { rationale, edits, confidence }.` },
|
||||
];
|
||||
}
|
||||
@@ -9,6 +9,13 @@ import { renderClickTrack, audioBufferToWavBlob } from './audio/metronome.js';
|
||||
import { Exporter, exportSegment, downloadBlob, PRESETS, isSupported, probeExportCodec } from './export/Exporter.js';
|
||||
import { subjectOf, groundOf, overlaysOf, stackCoverage } from './look/stack.js';
|
||||
import { coverageOf as sceneCoverage } from './scenes/surface.js';
|
||||
import { getLlmConfig, setLlmConfig } from './llm/config.js';
|
||||
import { canInvoke, remainingText, getBudgetCounts } from './llm/budget.js';
|
||||
import { requestSongTweak } from './llm/llmTweak.js';
|
||||
import { synthesizeSong } from './audio/synth.js';
|
||||
import { FeatureTrack } from './audio/FeatureTrack.js';
|
||||
import { generateLook } from './look/LookGenerator.js';
|
||||
import { SONGS } from './audio/songbank.js';
|
||||
|
||||
const QUALITY = {
|
||||
draft: 0.5, // half resolution — for scrubbing heavy stacks
|
||||
@@ -836,3 +843,305 @@ if (query.get('song')) {
|
||||
const seed = query.get('seed');
|
||||
loadBankSong(query.get('song'), seed === null ? null : (Number(seed) >>> 0));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- LLM song tweak
|
||||
|
||||
let llmTweakInFlight = false;
|
||||
let currentSongSpec = null; // last synthesizeSong spec (for bank songs); null for dropped files
|
||||
|
||||
function specForTrack(track) {
|
||||
// For bank songs we have the original spec; for dropped files we derive
|
||||
// a plausible spec from the measured summary so the LLM has something to edit.
|
||||
if (currentSongSpec) return { ...currentSongSpec };
|
||||
if (!track || !track.summary) return null;
|
||||
const s = track.summary;
|
||||
return {
|
||||
bpm: Math.round(s.bpm || 120),
|
||||
brightness: Math.max(0, Math.min(1, s.meanCentroid ?? 0.5)),
|
||||
noise: Math.max(0, Math.min(1, s.meanFlatness ?? 0.15)),
|
||||
noiseColour: null,
|
||||
dynamics: Math.max(0, Math.min(1, s.dynamicRange ?? 0.6)),
|
||||
transition: 0.5,
|
||||
arrangement: 'classic',
|
||||
duration: Math.round(track.duration || 120),
|
||||
seed: track.summary ? 1 : 1,
|
||||
};
|
||||
}
|
||||
|
||||
function setLlmTweakButtonState() {
|
||||
const btn = document.getElementById('btn-llm-tweak');
|
||||
const gear = document.getElementById('btn-llm-config');
|
||||
if (!btn) return;
|
||||
const gate = canInvoke();
|
||||
const cfg = getLlmConfig();
|
||||
const ready = state.show && state.show.ready;
|
||||
btn.disabled = llmTweakInFlight || !ready || !gate.ok;
|
||||
if (!ready) btn.title = 'Load a track first';
|
||||
else if (!cfg.isConfigured) btn.title = 'Set an AI Gateway API key — click ⚙️';
|
||||
else if (!gate.ok) btn.title = gate.detail;
|
||||
else btn.title = `Let an LLM tune the generated song · ${remainingText()} · click to review`;
|
||||
if (gear) {
|
||||
const dot = cfg.isConfigured ? '●' : '○';
|
||||
gear.textContent = `${dot} ⚙️`;
|
||||
gear.title = cfg.isConfigured
|
||||
? `LLM: ${cfg.model} @ ${cfg.gatewayUrl} · ${remainingText()} — click to configure`
|
||||
: 'LLM not configured — click to set gateway URL / model / API key';
|
||||
gear.classList.toggle('llm-not-configured', !cfg.isConfigured);
|
||||
}
|
||||
}
|
||||
|
||||
function openLlmConfigDialog() {
|
||||
const cfg = getLlmConfig();
|
||||
const counts = getBudgetCounts();
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'llm-overlay';
|
||||
overlay.innerHTML = `
|
||||
<div class="llm-backdrop"></div>
|
||||
<div class="llm-box" role="dialog" aria-modal="true" aria-labelledby="llm-cfg-title">
|
||||
<div class="llm-title" id="llm-cfg-title">LLM gateway settings</div>
|
||||
<div class="hint" style="margin-bottom:10px;">Stored in <code>localStorage</code> (<code>flow-state:llm</code>). Never committed. Leave API key empty to disable.</div>
|
||||
<label class="llm-field"><span>Gateway URL</span><input id="llm-gw" type="text" placeholder="${cfg.defaults.gatewayUrl}" value="${escapeHtml(cfg.stored.gatewayUrl || '')}"></label>
|
||||
<label class="llm-field"><span>Model</span><input id="llm-model" type="text" placeholder="${cfg.defaults.model}" value="${escapeHtml(cfg.stored.model || '')}"></label>
|
||||
<label class="llm-field"><span>API key</span><input id="llm-key" type="password" autocomplete="off" placeholder="(env or empty)" value="${escapeHtml(cfg.stored.apiKey || '')}"></label>
|
||||
<div class="hint">Env fallback: <code>VITE_AI_GATEWAY_URL</code> / <code>VITE_AI_MODEL</code> / <code>VITE_AI_API_KEY</code> in <code>.env.local</code>. See <code>.env.example</code>.</div>
|
||||
<div class="hint">Budget: ${counts.loadCount}/${counts.perLoad} this load · ${counts.sessionCount}/${counts.perSession} this session. Counters reset on reload / tab close.</div>
|
||||
<div class="llm-actions">
|
||||
<button id="llm-cancel">Cancel</button>
|
||||
<button id="llm-save" class="primary">Save</button>
|
||||
</div>
|
||||
</div>`;
|
||||
document.body.appendChild(overlay);
|
||||
const close = () => overlay.remove();
|
||||
overlay.querySelector('.llm-backdrop').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-cancel').addEventListener('click', close);
|
||||
overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); });
|
||||
overlay.querySelector('#llm-save').addEventListener('click', () => {
|
||||
const gw = overlay.querySelector('#llm-gw').value.trim();
|
||||
const model = overlay.querySelector('#llm-model').value.trim();
|
||||
const key = overlay.querySelector('#llm-key').value;
|
||||
setLlmConfig({
|
||||
gatewayUrl: gw || null,
|
||||
model: model || null,
|
||||
apiKey: key,
|
||||
});
|
||||
close();
|
||||
setLlmTweakButtonState();
|
||||
showToast('LLM settings saved', { ok: true, timeout: 2000 });
|
||||
});
|
||||
overlay.querySelector('#llm-gw').focus();
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return String(s).replace(/&/g, '&').replace(/"/g, '"').replace(/</g, '<').replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter) {
|
||||
const diffRows = Object.keys(specAfter || {}).filter((k) => {
|
||||
if (k === 'seed' || k === 'duration') return false;
|
||||
return specBefore[k] !== specAfter[k];
|
||||
}).map((k) => {
|
||||
const a = specBefore[k];
|
||||
const b = specAfter[k];
|
||||
const fmt = (v) => v == null ? '∅' : (typeof v === 'number' ? (Number.isInteger(v) ? String(v) : v.toFixed(2)) : String(v));
|
||||
return `<div class="kv"><span>${k}</span><b>${fmt(a)} → ${fmt(b)}</b></div>`;
|
||||
}).join('') || '<div class="hint">No song knobs changed (model returned empty edits — this dialog should not have opened).</div>';
|
||||
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'llm-overlay';
|
||||
overlay.innerHTML = `
|
||||
<div class="llm-backdrop"></div>
|
||||
<div class="llm-box llm-diff" role="dialog" aria-modal="true" aria-labelledby="llm-diff-title">
|
||||
<div class="llm-title" id="llm-diff-title">LLM suggestion — review before applying</div>
|
||||
<div class="llm-rationale">${escapeHtml(result.rationale || '')}</div>
|
||||
<div class="llm-confidence hint">confidence ${(result.confidence ?? 0).toFixed(2)}${result.usage ? ` · tokens ${result.usage.totalTokens ?? result.usage.total_tokens ?? '—'}` : ''}</div>
|
||||
<div class="pp-sub" style="margin-top:10px;">song spec diff</div>
|
||||
${diffRows}
|
||||
<div class="pp-sub">effect (approx)</div>
|
||||
<div class="hint">Re-synthesized and re-analysed locally — what you see when you Apply is what you get.</div>
|
||||
<div class="llm-actions">
|
||||
<button id="llm-dismiss">Dismiss</button>
|
||||
<button id="llm-apply" class="primary">Apply</button>
|
||||
</div>
|
||||
</div>`;
|
||||
document.body.appendChild(overlay);
|
||||
const close = () => overlay.remove();
|
||||
overlay.querySelector('.llm-backdrop').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-dismiss').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-apply').addEventListener('click', () => {
|
||||
try {
|
||||
const newDuration = specAfter.duration || trackBefore.duration;
|
||||
// Synthesize a fresh buffer from the edited spec, derive a new FeatureTrack,
|
||||
// then install it via Show.useTrack so prewarm/arc are correct.
|
||||
const buf = synthesizeSong({
|
||||
bpm: specAfter.bpm,
|
||||
duration: newDuration,
|
||||
arrangement: specAfter.arrangement,
|
||||
brightness: specAfter.brightness,
|
||||
noise: specAfter.noise,
|
||||
noiseColour: specAfter.noiseColour,
|
||||
dynamics: specAfter.dynamics,
|
||||
transition: specAfter.transition,
|
||||
seed: specAfter.seed ?? (specBefore.seed ?? 1),
|
||||
});
|
||||
const newTrack = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps });
|
||||
// Derive a seed from the new spec so generateLook is deterministic on the edit.
|
||||
const newSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1);
|
||||
const newLook = generateLook(newTrack, { seed: newSeed });
|
||||
// Swap in — useTrack handles timeline/provider/arc/prewarm
|
||||
currentSongSpec = { ...specAfter, seed: newSeed };
|
||||
state.show.useTrack(newTrack, newLook);
|
||||
// Keep the displayed name but mark as LLM-tuned
|
||||
state.show.fileName = `${state.show.fileName || 'track'} · llm-tuned`;
|
||||
state.show.osd.setText(state.show.fileName, newLook.personality, newLook.palette);
|
||||
strip.setShow(state.show);
|
||||
seekTo(0);
|
||||
renderPanel();
|
||||
showToast(`Applied LLM tweak: ${result.rationale.slice(0, 120)}`, { ok: true, timeout: 4000 });
|
||||
} catch (err) {
|
||||
showToast(`Failed to apply LLM tweak: ${err.message}`);
|
||||
console.error(err);
|
||||
} finally {
|
||||
close();
|
||||
setLlmTweakButtonState();
|
||||
}
|
||||
});
|
||||
overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); });
|
||||
// focus Apply for quick keyboard confirm
|
||||
setTimeout(() => overlay.querySelector('#llm-apply')?.focus(), 50);
|
||||
}
|
||||
|
||||
async function handleLlmTweak() {
|
||||
if (llmTweakInFlight) return;
|
||||
if (!state.show || !state.show.ready) {
|
||||
showToast('Load a track first');
|
||||
return;
|
||||
}
|
||||
const cfg = getLlmConfig();
|
||||
if (!cfg.isConfigured) {
|
||||
openLlmConfigDialog();
|
||||
return;
|
||||
}
|
||||
const gate = canInvoke();
|
||||
if (!gate.ok) {
|
||||
showToast(gate.detail || 'LLM budget exhausted');
|
||||
return;
|
||||
}
|
||||
// Confirm — preview cost/budget
|
||||
const remaining = remainingText();
|
||||
const hint = window.prompt(
|
||||
`Let the LLM tune the generated song?` +
|
||||
`\nBudget: ${remaining}` +
|
||||
`\nModel: ${cfg.model}` +
|
||||
`\n` +
|
||||
`\nOptional hint for the model (e.g. "make the drop calmer") — leave empty to skip:`,
|
||||
''
|
||||
);
|
||||
if (hint === null) return; // cancelled
|
||||
|
||||
const spec = specForTrack(state.show.track);
|
||||
if (!spec) {
|
||||
showToast('No song spec available for this track');
|
||||
return;
|
||||
}
|
||||
// Capture before for diff
|
||||
const specBefore = { ...spec };
|
||||
const trackBefore = state.show.track;
|
||||
const lookBefore = state.show.look;
|
||||
|
||||
llmTweakInFlight = true;
|
||||
setLlmTweakButtonState();
|
||||
const btn = document.getElementById('btn-llm-tweak');
|
||||
const prevLabel = btn ? btn.textContent : '';
|
||||
if (btn) btn.textContent = '… tuning';
|
||||
|
||||
try {
|
||||
const result = await requestSongTweak({
|
||||
spec: specBefore,
|
||||
track: trackBefore,
|
||||
look: lookBefore,
|
||||
hint: hint || '',
|
||||
config: cfg,
|
||||
});
|
||||
|
||||
const specAfter = { ...specBefore, ...result.edits };
|
||||
// Validate locally by re-synthesizing and measuring — show diff before applying.
|
||||
// Do a lightweight preview synthesis so we can report section/agitation delta
|
||||
// without mutating the live Show.
|
||||
let trackAfter = null, lookAfter = null;
|
||||
try {
|
||||
const buf = synthesizeSong({
|
||||
bpm: specAfter.bpm,
|
||||
duration: specAfter.duration,
|
||||
arrangement: specAfter.arrangement,
|
||||
brightness: specAfter.brightness,
|
||||
noise: specAfter.noise,
|
||||
noiseColour: specAfter.noiseColour,
|
||||
dynamics: specAfter.dynamics,
|
||||
transition: specAfter.transition,
|
||||
seed: specAfter.seed ?? specBefore.seed ?? 1,
|
||||
});
|
||||
trackAfter = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps });
|
||||
const previewSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1);
|
||||
lookAfter = generateLook(trackAfter, { seed: previewSeed });
|
||||
} catch (e) {
|
||||
console.warn('[llm] preview synthesis failed', e);
|
||||
}
|
||||
|
||||
openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter);
|
||||
} catch (err) {
|
||||
const code = err && err.code;
|
||||
if (code === 'NO_EDITS') {
|
||||
showToast(err.message, { ok: true, timeout: 5000 });
|
||||
} else if (code === 'BUDGET_PER_LOAD' || code === 'BUDGET_PER_SESSION' || code === 'BUDGET') {
|
||||
showToast(err.message);
|
||||
} else if (code === 'AUTH') {
|
||||
showToast(`LLM auth failed (401) — check your gateway API key in ⚙️\n${err.message}`);
|
||||
} else if (code === 'RATE_LIMIT') {
|
||||
showToast(`LLM rate limited (429) — try again shortly\n${err.message}`);
|
||||
} else if (code === 'NOT_CONFIGURED') {
|
||||
openLlmConfigDialog();
|
||||
} else {
|
||||
showToast(`LLM tweak failed: ${err.message}`);
|
||||
console.error('[llm] request failed', err);
|
||||
}
|
||||
} finally {
|
||||
llmTweakInFlight = false;
|
||||
if (btn) btn.textContent = prevLabel || '✨ tune';
|
||||
setLlmTweakButtonState();
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('btn-llm-tweak')?.addEventListener('click', handleLlmTweak);
|
||||
document.getElementById('btn-llm-config')?.addEventListener('click', openLlmConfigDialog);
|
||||
|
||||
// Keep the tune button in sync with ready/config/budget — renderPanel is
|
||||
// called after every load/reroll/param change.
|
||||
const __prevRenderPanel = renderPanel;
|
||||
renderPanel = function llmAwareRenderPanel(...args) {
|
||||
const r = __prevRenderPanel.apply(this, args);
|
||||
setLlmTweakButtonState();
|
||||
return r;
|
||||
};
|
||||
|
||||
// Initial state — disabled until a track is loaded or a previous session had one
|
||||
setLlmTweakButtonState();
|
||||
|
||||
// Keep loadBankSong in sync with LLM spec tracking
|
||||
const __origLoadBankSong = loadBankSong;
|
||||
loadBankSong = async function trackedLoadBankSong(name, seed) {
|
||||
const spec = SONGS.find((s) => s.name === name);
|
||||
if (spec) {
|
||||
currentSongSpec = {
|
||||
bpm: spec.bpm,
|
||||
brightness: spec.brightness,
|
||||
noise: spec.noise,
|
||||
noiseColour: spec.noiseColour ?? null,
|
||||
dynamics: spec.dynamics,
|
||||
transition: spec.transition ?? 0.5,
|
||||
arrangement: spec.arrangement,
|
||||
duration: 120,
|
||||
seed: seed != null ? (Number(seed) >>> 0) : (1 + SONGS.indexOf(spec)),
|
||||
};
|
||||
}
|
||||
return __origLoadBankSong(name, seed);
|
||||
};
|
||||
|
||||
@@ -284,6 +284,21 @@ input[type=range] { accent-color: var(--accent); background: transparent; }
|
||||
}
|
||||
}
|
||||
|
||||
/* LLM gateway dialogs (main app). Minimal, inherits panel/modal language. */
|
||||
.llm-overlay { position: fixed; inset: 0; display: flex; align-items: center; justify-content: center; z-index: 50; }
|
||||
.llm-backdrop { position: absolute; inset: 0; background: rgba(0,0,0,0.62); backdrop-filter: blur(2px); }
|
||||
.llm-box { position: relative; width: min(560px, calc(100vw - 32px)); max-height: min(78vh, 720px); overflow: auto; background: var(--panel); border: 1px solid var(--line); border-left: 3px solid var(--accent); padding: 16px 18px 14px; box-shadow: 0 20px 40px rgba(0,0,0,0.55); }
|
||||
.llm-box.llm-diff { width: min(640px, calc(100vw - 32px)); }
|
||||
.llm-title { font-weight: 700; font-size: 12px; letter-spacing: .08em; text-transform: uppercase; margin-bottom: 10px; }
|
||||
.llm-field { display: grid; grid-template-columns: 110px 1fr; gap: 8px; align-items: center; margin: 8px 0; font-size: 12px; }
|
||||
.llm-field span { color: var(--dim); }
|
||||
.llm-field input { width: 100%; background: #171a22; border: 1px solid var(--line); color: var(--text); padding: 6px 8px; font: inherit; font-size: 12px; }
|
||||
.llm-actions { display: flex; gap: 8px; justify-content: flex-end; margin-top: 14px; }
|
||||
.llm-rationale { font-size: 12px; line-height: 1.6; white-space: pre-wrap; color: var(--text); margin-bottom: 6px; }
|
||||
.llm-confidence { margin-bottom: 4px; }
|
||||
.llm-not-configured { opacity: 0.85; }
|
||||
#btn-llm-tweak:disabled { opacity: 0.45; cursor: not-allowed; }
|
||||
|
||||
/* Single-scene gate output (checks.html?scene=Name). Monospaced so the
|
||||
PASS/FAIL column lines up and the whole verdict reads in one glance. */
|
||||
.scene-gate {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { clampEdits, buildTweakPrompt, TweakResultSchema, ALLOWED_KEYS } from '../src/llm/spec.js';
|
||||
import { LLM_BUDGET, getBudgetCounts, canInvoke, remainingText } from '../src/llm/budget.js';
|
||||
import { LLM_DEFAULTS, LLM_STORAGE_KEY } from '../src/llm/config.js';
|
||||
|
||||
test('clampEdits clamps to valid ranges and drops invalid arrangement', () => {
|
||||
const out = clampEdits({ bpm: 300, brightness: 1.5, noise: -1, arrangement: 'nope', dynamics: 0.1234 });
|
||||
assert.equal(out.bpm, 180);
|
||||
assert.equal(out.brightness, 1);
|
||||
assert.equal(out.noise, 0);
|
||||
assert.equal(out.arrangement, undefined);
|
||||
assert.equal(out.dynamics, 0.12);
|
||||
});
|
||||
|
||||
test('clampEdits preserves noiseColour null and rounds', () => {
|
||||
assert.deepEqual(clampEdits({ noiseColour: null }), { noiseColour: null });
|
||||
assert.deepEqual(clampEdits({ noiseColour: 0.3333 }), { noiseColour: 0.33 });
|
||||
assert.deepEqual(clampEdits({ arrangement: 'classic' }), { arrangement: 'classic' });
|
||||
});
|
||||
|
||||
test('ALLOWED_KEYS covers the 7 song knobs', () => {
|
||||
assert.deepEqual([...ALLOWED_KEYS].sort(), ['arrangement', 'bpm', 'brightness', 'dynamics', 'noise', 'noiseColour', 'transition'].sort());
|
||||
});
|
||||
|
||||
test('buildTweakPrompt returns system+user messages and respects hint', () => {
|
||||
const fakeDigest = {
|
||||
describe: 'seed abc · geometer · triad',
|
||||
director: 'geometer',
|
||||
paletteScheme: 'triad',
|
||||
sections: [{ index: 0, kind: 'intro', name: 'Shot', layers: 'Shot' }],
|
||||
validation: null,
|
||||
};
|
||||
const msgs = buildTweakPrompt({
|
||||
spec: { bpm: 124, brightness: 0.5, noise: 0.2, noiseColour: null, dynamics: 0.55, transition: 0.6, arrangement: 'classic', duration: 120, seed: 42 },
|
||||
track: { summary: { bpm: 124, meanCentroid: 0.5, meanFlatness: 0.2, dynamicRange: 0.55, meanLoudness: 0.4, tempoConfidence: 0.9 }, sections: [{ kind: 'intro', start: 0, end: 15, energy: 0.2 }], duration: 120 },
|
||||
lookDigest: fakeDigest,
|
||||
hint: 'make the drop calmer',
|
||||
});
|
||||
assert.equal(msgs.length, 2);
|
||||
assert.equal(msgs[0].role, 'system');
|
||||
assert.equal(msgs[1].role, 'user');
|
||||
assert.ok(msgs[0].content.includes('make the drop calmer') || msgs[0].content.includes('User hint'));
|
||||
assert.ok(msgs[1].content.includes('songSpec'));
|
||||
});
|
||||
|
||||
test('buildTweakPrompt truncates long hint in system', () => {
|
||||
const long = 'x'.repeat(1000);
|
||||
const msgs = buildTweakPrompt({ spec: null, track: null, lookDigest: null, hint: long });
|
||||
assert.ok(msgs[0].content.length < 5000);
|
||||
});
|
||||
|
||||
test('TweakResultSchema validates a minimal valid result', () => {
|
||||
const parsed = TweakResultSchema.parse({ rationale: 'Lower noise to calm the drop.', edits: { noise: 0.3 }, confidence: 0.8 });
|
||||
assert.equal(parsed.edits.noise, 0.3);
|
||||
assert.ok(parsed.rationale.length > 0);
|
||||
});
|
||||
|
||||
test('budget defaults are sane and canInvoke reports not-configured without a key', () => {
|
||||
assert.equal(LLM_BUDGET.perLoad, 3);
|
||||
assert.equal(LLM_BUDGET.perSession, 10);
|
||||
assert.equal(LLM_STORAGE_KEY, 'flow-state:llm');
|
||||
assert.equal(LLM_DEFAULTS.gatewayUrl, 'https://ai-gateway.vercel.sh/v1');
|
||||
const counts = getBudgetCounts();
|
||||
assert.ok(counts.perLoad === 3 && counts.perSession === 10);
|
||||
assert.ok(typeof counts.loadCount === 'number' && typeof counts.sessionCount === 'number');
|
||||
const gate = canInvoke();
|
||||
assert.equal(gate.ok, false);
|
||||
assert.equal(gate.reason, 'not-configured');
|
||||
assert.ok(typeof remainingText() === 'string');
|
||||
});
|
||||
Reference in New Issue
Block a user