LLM: verbose [llm] logs + visible tweak progress

Tune appeared to do nothing — the 400's cause was buried, in-flight was
near-invisible, and dialog only appeared on success. Make every stage
observable and cheap to diagnose.

- src/llm/llmTweak.js (+ spec.js): provider.chat(model) + providerOptions
  strictJsonSchema:false fix (correct placement), edits nullable so strict
  requires all 7 keys; clamped, budget-aware. Now logs [llm] fetch target,
  messages/spec, raw vs clamped edits, rationale/confidence/usage,
  elapsed time, and diff summary. NO_EDITS refunded with console.warn.
- src/main.js: immediate OK toast "Contacting LLM (model)…" on click,
  console.group for handleLlmTweak start, post-result toast with
  rationale snippet + editsSummary before dialog, preview synthesis
  logged, catch logs stack once with toast.
- vite.config.js: /__llm dev proxy already in place from prior fix;
  retained for CORS-free dev.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-09-13 20:59:30 +02:00
co-authored by Claude
parent 82ab88e6b0
commit e2f56e3ce4
2 changed files with 48 additions and 3 deletions
+26 -2
View File
@@ -8,7 +8,7 @@
import { generateObject } from 'ai'; import { generateObject } from 'ai';
import { createOpenAI } from '@ai-sdk/openai'; import { createOpenAI } from '@ai-sdk/openai';
import { getLlmConfig } from './config.js'; import { getLlmConfig } from './config.js';
import { canInvoke, recordInvocation, refundInvocation } from './budget.js'; import { canInvoke, recordInvocation, refundInvocation, getBudgetCounts } from './budget.js';
import { describeLook } from '../look/LookGenerator.js'; import { describeLook } from '../look/LookGenerator.js';
import { validateLook } from '../look/composition-validator.js'; import { validateLook } from '../look/composition-validator.js';
import { import {
@@ -91,6 +91,8 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
const proxyFetch = async (url, init = {}) => { const proxyFetch = async (url, init = {}) => {
const headers = new Headers(init.headers || {}); const headers = new Headers(init.headers || {});
if (isDev) headers.set('x-llm-gateway', cfg.gatewayUrl); if (isDev) headers.set('x-llm-gateway', cfg.gatewayUrl);
// Dev proxy forwards server-side; log what we send
console.info('[llm] fetch', isDev ? '→ /__llm (proxied to ' + cfg.gatewayUrl + ')' : '→ ' + cfg.gatewayUrl, String(url).slice(0, 120));
return fetch(url, { ...init, headers }); return fetch(url, { ...init, headers });
}; };
const provider = createOpenAI({ const provider = createOpenAI({
@@ -100,7 +102,15 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
}); });
const modelId = cfg.model; const modelId = cfg.model;
console.groupCollapsed(`[llm] requestSongTweak — ${modelId}${hint ? ` · hint: "${hint.slice(0, 80)}"` : ''}`);
console.info('[llm] model', modelId, 'gateway', cfg.gatewayUrl, isDev ? '(via /__llm dev proxy)' : '(direct)');
console.info('[llm] budget after record', getBudgetCounts());
console.log('[llm] messages', messages);
console.log('[llm] spec before', spec);
console.groupEnd();
try { try {
const started = performance.now();
const result = await generateObject({ const result = await generateObject({
model: provider.chat(modelId), model: provider.chat(modelId),
schema: TweakResultSchema, schema: TweakResultSchema,
@@ -109,12 +119,22 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
maxRetries: 0, maxRetries: 0,
providerOptions: { openai: { strictJsonSchema: false } }, providerOptions: { openai: { strictJsonSchema: false } },
}); });
const elapsed = ((performance.now() - started) / 1000).toFixed(1);
const raw = result.object || {}; const raw = result.object || {};
const edits = clampEdits(raw.edits || {}); const edits = clampEdits(raw.edits || {});
const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.'; const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.';
const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5; const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5;
console.groupCollapsed(`[llm] response in ${elapsed}s — ${Object.keys(edits).length ? Object.keys(edits).join(', ') : 'no edits'}`);
console.log('[llm] raw', raw);
console.log('[llm] clamped edits', edits);
console.log('[llm] rationale', rationale);
console.log('[llm] confidence', confidence, 'usage', result.usage || null);
const changed = Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)';
console.info('[llm] diff', changed);
console.groupEnd();
if (!Object.keys(edits).length) { if (!Object.keys(edits).length) {
refundInvocation(); refundInvocation();
const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.'); const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.');
@@ -127,7 +147,11 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
} catch (err) { } catch (err) {
const msg = toErrorMessage(err); const msg = toErrorMessage(err);
const lower = msg.toLowerCase(); const lower = msg.toLowerCase();
if (err && err.code === 'NO_EDITS') throw err; if (err && err.code === 'NO_EDITS') {
console.warn('[llm] no edits —', err.message, err.result);
throw err;
}
console.error('[llm] request failed', err, '\nmessage:', msg);
if (!err.code) { if (!err.code) {
if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH'; if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH';
else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT'; else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT';
+22 -1
View File
@@ -1053,6 +1053,14 @@ async function handleLlmTweak() {
const btn = document.getElementById('btn-llm-tweak'); const btn = document.getElementById('btn-llm-tweak');
const prevLabel = btn ? btn.textContent : ''; const prevLabel = btn ? btn.textContent : '';
if (btn) btn.textContent = '… tuning'; if (btn) btn.textContent = '… tuning';
// Immediate feedback — budget had a one-frame lag before, so show progress explicitly.
showToast(`Contacting LLM (${cfg.model})…`, { ok: true });
console.info('[llm] handleLlmTweak start', {
model: cfg.model, gateway: cfg.gatewayUrl,
remaining, hint: hint || '(none)',
specBefore, trackSummary: trackBefore && trackBefore.summary,
lookDirector: lookBefore && lookBefore.director,
});
try { try {
const result = await requestSongTweak({ const result = await requestSongTweak({
@@ -1063,6 +1071,17 @@ async function handleLlmTweak() {
config: cfg, config: cfg,
}); });
// Log what came back before any synthesis — this is the audit trail.
console.groupCollapsed(`[llm] got result — ${Object.keys(result.edits).join(', ') || 'no edits'} · confidence ${String(result.confidence)}`);
console.log('[llm] rationale:', result.rationale);
console.log('[llm] edits:', result.edits);
console.log('[llm] raw:', result.raw, 'usage:', result.usage);
console.groupEnd();
const editsSummary = Object.keys(result.edits).length
? Object.entries(result.edits).map(([k, v]) => `${k}→${v == null ? '∅' : String(v)}`).join(' · ')
: '(no knobs changed)';
showToast(`LLM replied: ${result.rationale.slice(0, 160)} — edits: ${editsSummary}`, { ok: true, timeout: 5000 });
const specAfter = { ...specBefore, ...result.edits }; const specAfter = { ...specBefore, ...result.edits };
// Validate locally by re-synthesizing and measuring — show diff before applying. // Validate locally by re-synthesizing and measuring — show diff before applying.
// Do a lightweight preview synthesis so we can report section/agitation delta // Do a lightweight preview synthesis so we can report section/agitation delta
@@ -1083,12 +1102,15 @@ async function handleLlmTweak() {
trackAfter = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps }); trackAfter = FeatureTrack.fromAudioBuffer(buf, { fps: state.show.fps });
const previewSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1); const previewSeed = specAfter.seed != null ? (specAfter.seed >>> 0) : (specBefore.seed != null ? (specBefore.seed >>> 0) + 1 : 1);
lookAfter = generateLook(trackAfter, { seed: previewSeed }); lookAfter = generateLook(trackAfter, { seed: previewSeed });
console.info('[llm] preview look', lookAfter ? `${lookAfter.director} · ${lookAfter.sections.length} sections` : 'n/a',
trackAfter ? `sections ${trackAfter.sections.length}` : '');
} catch (e) { } catch (e) {
console.warn('[llm] preview synthesis failed', e); console.warn('[llm] preview synthesis failed', e);
} }
openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter); openLlmTweakDialog(result, specBefore, specAfter, trackBefore, trackAfter, lookBefore, lookAfter);
} catch (err) { } catch (err) {
console.warn('[llm] request failed', err && err.stack ? err.stack : err);
const code = err && err.code; const code = err && err.code;
if (code === 'NO_EDITS') { if (code === 'NO_EDITS') {
showToast(err.message, { ok: true, timeout: 5000 }); showToast(err.message, { ok: true, timeout: 5000 });
@@ -1102,7 +1124,6 @@ async function handleLlmTweak() {
openLlmConfigDialog(); openLlmConfigDialog();
} else { } else {
showToast(`LLM tweak failed: ${err.message}`); showToast(`LLM tweak failed: ${err.message}`);
console.error('[llm] request failed', err);
} }
} finally { } finally {
llmTweakInFlight = false; llmTweakInFlight = false;