LLM song-tweak: AI Gateway via Vercel AI SDK, budgeted tune flow
Add a ✨ tune button that lets an LLM inspect the proposed visualization and tweak the proc-song knobs so the next generation looks better. Edits only the 7 songbank params (bpm, brightness, noise, noiseColour, dynamics, transition, arrangement) — never the Look — then re-synthesizes and re-generates locally for preview. - ai + @ai-sdk/openai + zod (library per plan) against the Vercel AI Gateway (OpenAI-compatible, configurable via VITE_AI_GATEWAY_URL/MODEL/API_KEY, localStorage override, .env.example). - Structured output (TweakResultSchema) + clampEdits enforces ranges; prompt ≤ ~2.5k tokens, max 2-3 knobs ±0.15 / bpm ±8. - Budget caps: 3/load + 10/session (sessionStorage), checked before network, NO_EDITS refunded, offline/auth/rate-limit mapped. - Prompt split into src/llm/spec.js (pure, node-testable) + llmTweak.js (gateway + digest) to keep `node --test` off the Vite JSON path. - UI: ⚙️ gateway/model/key dialog (localStorage flow-state:llm), confirm(prompt) with hint, diff modal with Apply/Dismiss; Apply synthesizes → FeatureTrack → generateLook → Show.useTrack. - Tests: 7 new LLM cases (clamp, schema, prompt, budget), 18/18 pass. Build + lint:scenes green. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
@@ -9,6 +9,13 @@ import { renderClickTrack, audioBufferToWavBlob } from './audio/metronome.js';
|
||||
import { Exporter, exportSegment, downloadBlob, PRESETS, isSupported, probeExportCodec } from './export/Exporter.js';
|
||||
import { subjectOf, groundOf, overlaysOf, stackCoverage } from './look/stack.js';
|
||||
import { coverageOf as sceneCoverage } from './scenes/surface.js';
|
||||
import { getLlmConfig, setLlmConfig } from './llm/config.js';
|
||||
import { canInvoke, remainingText, getBudgetCounts } from './llm/budget.js';
|
||||
import { requestSongTweak } from './llm/llmTweak.js';
|
||||
import { synthesizeSong } from './audio/synth.js';
|
||||
import { FeatureTrack } from './audio/FeatureTrack.js';
|
||||
import { generateLook } from './look/LookGenerator.js';
|
||||
import { SONGS } from './audio/songbank.js';
|
||||
|
||||
const QUALITY = {
|
||||
draft: 0.5, // half resolution — for scrubbing heavy stacks
|
||||
@@ -836,3 +843,305 @@ if (query.get('song')) {
|
||||
const seed = query.get('seed');
|
||||
loadBankSong(query.get('song'), seed === null ? null : (Number(seed) >>> 0));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- LLM song tweak
|
||||
|
||||
let llmTweakInFlight = false;
|
||||
let currentSongSpec = null; // last synthesizeSong spec (for bank songs); null for dropped files
|
||||
|
||||
function specForTrack(track) {
|
||||
// For bank songs we have the original spec; for dropped files we derive
|
||||
// a plausible spec from the measured summary so the LLM has something to edit.
|
||||
if (currentSongSpec) return { ...currentSongSpec };
|
||||
if (!track || !track.summary) return null;
|
||||
const s = track.summary;
|
||||
return {
|
||||
bpm: Math.round(s.bpm || 120),
|
||||
brightness: Math.max(0, Math.min(1, s.meanCentroid ?? 0.5)),
|
||||
noise: Math.max(0, Math.min(1, s.meanFlatness ?? 0.15)),
|
||||
noiseColour: null,
|
||||
dynamics: Math.max(0, Math.min(1, s.dynamicRange ?? 0.6)),
|
||||
transition: 0.5,
|
||||
arrangement: 'classic',
|
||||
duration: Math.round(track.duration || 120),
|
||||
seed: track.summary ? 1 : 1,
|
||||
};
|
||||
}
|
||||
|
||||
function setLlmTweakButtonState() {
|
||||
const btn = document.getElementById('btn-llm-tweak');
|
||||
const gear = document.getElementById('btn-llm-config');
|
||||
if (!btn) return;
|
||||
const gate = canInvoke();
|
||||
const cfg = getLlmConfig();
|
||||
const ready = state.show && state.show.ready;
|
||||
btn.disabled = llmTweakInFlight || !ready || !gate.ok;
|
||||
if (!ready) btn.title = 'Load a track first';
|
||||
else if (!cfg.isConfigured) btn.title = 'Set an AI Gateway API key — click ⚙️';
|
||||
else if (!gate.ok) btn.title = gate.detail;
|
||||
else btn.title = `Let an LLM tune the generated song · ${remainingText()} · click to review`;
|
||||
if (gear) {
|
||||
const dot = cfg.isConfigured ? '●' : '○';
|
||||
gear.textContent = `${dot} ⚙️`;
|
||||
gear.title = cfg.isConfigured
|
||||
? `LLM: ${cfg.model} @ ${cfg.gatewayUrl} · ${remainingText()} — click to configure`
|
||||
: 'LLM not configured — click to set gateway URL / model / API key';
|
||||
gear.classList.toggle('llm-not-configured', !cfg.isConfigured);
|
||||
}
|
||||
}
|
||||
|
||||
function openLlmConfigDialog() {
|
||||
const cfg = getLlmConfig();
|
||||
const counts = getBudgetCounts();
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'llm-overlay';
|
||||
overlay.innerHTML = `
|
||||
<div class="llm-backdrop"></div>
|
||||
<div class="llm-box" role="dialog" aria-modal="true" aria-labelledby="llm-cfg-title">
|
||||
<div class="llm-title" id="llm-cfg-title">LLM gateway settings</div>
|
||||
<div class="hint" style="margin-bottom:10px;">Stored in <code>localStorage</code> (<code>flow-state:llm</code>). Never committed. Leave API key empty to disable.</div>
|
||||
<label class="llm-field"><span>Gateway URL</span><input id="llm-gw" type="text" placeholder="${cfg.defaults.gatewayUrl}" value="${escapeHtml(cfg.stored.gatewayUrl || '')}"></label>
|
||||
<label class="llm-field"><span>Model</span><input id="llm-model" type="text" placeholder="${cfg.defaults.model}" value="${escapeHtml(cfg.stored.model || '')}"></label>
|
||||
<label class="llm-field"><span>API key</span><input id="llm-key" type="password" autocomplete="off" placeholder="(env or empty)" value="${escapeHtml(cfg.stored.apiKey || '')}"></label>
|
||||
<div class="hint">Env fallback: <code>VITE_AI_GATEWAY_URL</code> / <code>VITE_AI_MODEL</code> / <code>VITE_AI_API_KEY</code> in <code>.env.local</code>. See <code>.env.example</code>.</div>
|
||||
<div class="hint">Budget: ${counts.loadCount}/${counts.perLoad} this load · ${counts.sessionCount}/${counts.perSession} this session. Counters reset on reload / tab close.</div>
|
||||
<div class="llm-actions">
|
||||
<button id="llm-cancel">Cancel</button>
|
||||
<button id="llm-save" class="primary">Save</button>
|
||||
</div>
|
||||
</div>`;
|
||||
document.body.appendChild(overlay);
|
||||
const close = () => overlay.remove();
|
||||
overlay.querySelector('.llm-backdrop').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-cancel').addEventListener('click', close);
|
||||
overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); });
|
||||
overlay.querySelector('#llm-save').addEventListener('click', () => {
|
||||
const gw = overlay.querySelector('#llm-gw').value.trim();
|
||||
const model = overlay.querySelector('#llm-model').value.trim();
|
||||
const key = overlay.querySelector('#llm-key').value;
|
||||
setLlmConfig({
|
||||
gatewayUrl: gw || null,
|
||||
model: model || null,
|
||||
apiKey: key,
|
||||
});
|
||||
close();
|
||||
setLlmTweakButtonState();
|
||||
showToast('LLM settings saved', { ok: true, timeout: 2000 });
|
||||
});
|
||||
overlay.querySelector('#llm-gw').focus();
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return String(s).replace(/&/g, '&').replace(/"/g, '"').replace(/</g, '<').replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter) {
|
||||
const diffRows = Object.keys(specAfter || {}).filter((k) => {
|
||||
if (k === 'seed' || k === 'duration') return false;
|
||||
return specBefore[k] !== specAfter[k];
|
||||
}).map((k) => {
|
||||
const a = specBefore[k];
|
||||
const b = specAfter[k];
|
||||
const fmt = (v) => v == null ? '∅' : (typeof v === 'number' ? (Number.isInteger(v) ? String(v) : v.toFixed(2)) : String(v));
|
||||
return `<div class="kv"><span>${k}</span><b>${fmt(a)} → ${fmt(b)}</b></div>`;
|
||||
}).join('') || '<div class="hint">No song knobs changed (model returned empty edits — this dialog should not have opened).</div>';
|
||||
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'llm-overlay';
|
||||
overlay.innerHTML = `
|
||||
<div class="llm-backdrop"></div>
|
||||
<div class="llm-box llm-diff" role="dialog" aria-modal="true" aria-labelledby="llm-diff-title">
|
||||
<div class="llm-title" id="llm-diff-title">LLM suggestion — review before applying</div>
|
||||
<div class="llm-rationale">${escapeHtml(result.rationale || '')}</div>
|
||||
<div class="llm-confidence hint">confidence ${(result.confidence ?? 0).toFixed(2)}${result.usage ? ` · tokens ${result.usage.totalTokens ?? result.usage.total_tokens ?? '—'}` : ''}</div>
|
||||
<div class="pp-sub" style="margin-top:10px;">song spec diff</div>
|
||||
${diffRows}
|
||||
<div class="pp-sub">effect (approx)</div>
|
||||
<div class="hint">Re-synthesized and re-analysed locally — what you see when you Apply is what you get.</div>
|
||||
<div class="llm-actions">
|
||||
<button id="llm-dismiss">Dismiss</button>
|
||||
<button id="llm-apply" class="primary">Apply</button>
|
||||
</div>
|
||||
</div>`;
|
||||
document.body.appendChild(overlay);
|
||||
const close = () => overlay.remove();
|
||||
overlay.querySelector('.llm-backdrop').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-dismiss').addEventListener('click', close);
|
||||
overlay.querySelector('#llm-apply').addEventListener('click', () => {
|
||||
try {
|
||||
const newDuration = specAfter.duration || trackBefore.duration;
|
||||
// Synthesize a fresh buffer from the edited spec, derive a new FeatureTrack,
|
||||
// then install it via Show.useTrack so prewarm/arc are correct.
|
||||
const buf = synthesizeSong({
|
||||
bpm: specAfter.bpm,
|
||||
duration: newDuration,
|
||||
arrangement: specAfter.arrangement,
|
||||
brightness: specAfter.brightness,
|
||||
noise: specAfter.noise,
|
||||
noiseColour: specAfter.noiseColour,
|
||||
dynamics: specAfter.dynamics,
|
||||
transition: specAfter.transition,
|
||||
seed: specAfter.seed ?? (specBefore.seed ?? 1),
|
||||
});
|
||||
const newTrack = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps });
|
||||
// Derive a seed from the new spec so generateLook is deterministic on the edit.
|
||||
const newSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1);
|
||||
const newLook = generateLook(newTrack, { seed: newSeed });
|
||||
// Swap in — useTrack handles timeline/provider/arc/prewarm
|
||||
currentSongSpec = { ...specAfter, seed: newSeed };
|
||||
state.show.useTrack(newTrack, newLook);
|
||||
// Keep the displayed name but mark as LLM-tuned
|
||||
state.show.fileName = `${state.show.fileName || 'track'} · llm-tuned`;
|
||||
state.show.osd.setText(state.show.fileName, newLook.personality, newLook.palette);
|
||||
strip.setShow(state.show);
|
||||
seekTo(0);
|
||||
renderPanel();
|
||||
showToast(`Applied LLM tweak: ${result.rationale.slice(0, 120)}`, { ok: true, timeout: 4000 });
|
||||
} catch (err) {
|
||||
showToast(`Failed to apply LLM tweak: ${err.message}`);
|
||||
console.error(err);
|
||||
} finally {
|
||||
close();
|
||||
setLlmTweakButtonState();
|
||||
}
|
||||
});
|
||||
overlay.addEventListener('keydown', (e) => { if (e.key === 'Escape') close(); });
|
||||
// focus Apply for quick keyboard confirm
|
||||
setTimeout(() => overlay.querySelector('#llm-apply')?.focus(), 50);
|
||||
}
|
||||
|
||||
async function handleLlmTweak() {
|
||||
if (llmTweakInFlight) return;
|
||||
if (!state.show || !state.show.ready) {
|
||||
showToast('Load a track first');
|
||||
return;
|
||||
}
|
||||
const cfg = getLlmConfig();
|
||||
if (!cfg.isConfigured) {
|
||||
openLlmConfigDialog();
|
||||
return;
|
||||
}
|
||||
const gate = canInvoke();
|
||||
if (!gate.ok) {
|
||||
showToast(gate.detail || 'LLM budget exhausted');
|
||||
return;
|
||||
}
|
||||
// Confirm — preview cost/budget
|
||||
const remaining = remainingText();
|
||||
const hint = window.prompt(
|
||||
`Let the LLM tune the generated song?` +
|
||||
`\nBudget: ${remaining}` +
|
||||
`\nModel: ${cfg.model}` +
|
||||
`\n` +
|
||||
`\nOptional hint for the model (e.g. "make the drop calmer") — leave empty to skip:`,
|
||||
''
|
||||
);
|
||||
if (hint === null) return; // cancelled
|
||||
|
||||
const spec = specForTrack(state.show.track);
|
||||
if (!spec) {
|
||||
showToast('No song spec available for this track');
|
||||
return;
|
||||
}
|
||||
// Capture before for diff
|
||||
const specBefore = { ...spec };
|
||||
const trackBefore = state.show.track;
|
||||
const lookBefore = state.show.look;
|
||||
|
||||
llmTweakInFlight = true;
|
||||
setLlmTweakButtonState();
|
||||
const btn = document.getElementById('btn-llm-tweak');
|
||||
const prevLabel = btn ? btn.textContent : '';
|
||||
if (btn) btn.textContent = '… tuning';
|
||||
|
||||
try {
|
||||
const result = await requestSongTweak({
|
||||
spec: specBefore,
|
||||
track: trackBefore,
|
||||
look: lookBefore,
|
||||
hint: hint || '',
|
||||
config: cfg,
|
||||
});
|
||||
|
||||
const specAfter = { ...specBefore, ...result.edits };
|
||||
// Validate locally by re-synthesizing and measuring — show diff before applying.
|
||||
// Do a lightweight preview synthesis so we can report section/agitation delta
|
||||
// without mutating the live Show.
|
||||
let trackAfter = null, lookAfter = null;
|
||||
try {
|
||||
const buf = synthesizeSong({
|
||||
bpm: specAfter.bpm,
|
||||
duration: specAfter.duration,
|
||||
arrangement: specAfter.arrangement,
|
||||
brightness: specAfter.brightness,
|
||||
noise: specAfter.noise,
|
||||
noiseColour: specAfter.noiseColour,
|
||||
dynamics: specAfter.dynamics,
|
||||
transition: specAfter.transition,
|
||||
seed: specAfter.seed ?? specBefore.seed ?? 1,
|
||||
});
|
||||
trackAfter = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps });
|
||||
const previewSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1);
|
||||
lookAfter = generateLook(trackAfter, { seed: previewSeed });
|
||||
} catch (e) {
|
||||
console.warn('[llm] preview synthesis failed', e);
|
||||
}
|
||||
|
||||
openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter);
|
||||
} catch (err) {
|
||||
const code = err && err.code;
|
||||
if (code === 'NO_EDITS') {
|
||||
showToast(err.message, { ok: true, timeout: 5000 });
|
||||
} else if (code === 'BUDGET_PER_LOAD' || code === 'BUDGET_PER_SESSION' || code === 'BUDGET') {
|
||||
showToast(err.message);
|
||||
} else if (code === 'AUTH') {
|
||||
showToast(`LLM auth failed (401) — check your gateway API key in ⚙️\n${err.message}`);
|
||||
} else if (code === 'RATE_LIMIT') {
|
||||
showToast(`LLM rate limited (429) — try again shortly\n${err.message}`);
|
||||
} else if (code === 'NOT_CONFIGURED') {
|
||||
openLlmConfigDialog();
|
||||
} else {
|
||||
showToast(`LLM tweak failed: ${err.message}`);
|
||||
console.error('[llm] request failed', err);
|
||||
}
|
||||
} finally {
|
||||
llmTweakInFlight = false;
|
||||
if (btn) btn.textContent = prevLabel || '✨ tune';
|
||||
setLlmTweakButtonState();
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('btn-llm-tweak')?.addEventListener('click', handleLlmTweak);
|
||||
document.getElementById('btn-llm-config')?.addEventListener('click', openLlmConfigDialog);
|
||||
|
||||
// Keep the tune button in sync with ready/config/budget — renderPanel is
|
||||
// called after every load/reroll/param change.
|
||||
const __prevRenderPanel = renderPanel;
|
||||
renderPanel = function llmAwareRenderPanel(...args) {
|
||||
const r = __prevRenderPanel.apply(this, args);
|
||||
setLlmTweakButtonState();
|
||||
return r;
|
||||
};
|
||||
|
||||
// Initial state — disabled until a track is loaded or a previous session had one
|
||||
setLlmTweakButtonState();
|
||||
|
||||
// Keep loadBankSong in sync with LLM spec tracking
|
||||
const __origLoadBankSong = loadBankSong;
|
||||
loadBankSong = async function trackedLoadBankSong(name, seed) {
|
||||
const spec = SONGS.find((s) => s.name === name);
|
||||
if (spec) {
|
||||
currentSongSpec = {
|
||||
bpm: spec.bpm,
|
||||
brightness: spec.brightness,
|
||||
noise: spec.noise,
|
||||
noiseColour: spec.noiseColour ?? null,
|
||||
dynamics: spec.dynamics,
|
||||
transition: spec.transition ?? 0.5,
|
||||
arrangement: spec.arrangement,
|
||||
duration: 120,
|
||||
seed: seed != null ? (Number(seed) >>> 0) : (1 + SONGS.indexOf(spec)),
|
||||
};
|
||||
}
|
||||
return __origLoadBankSong(name, seed);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user