LLM: streaming + rich context + 32×32 frames + proxy log

- Streaming: requestSongTweak now uses streamObject when a progress
  callback is provided (via handleLlmTweak), emitting partial {rationale,
  edits} and updating toast/console as text arrives; falls back to
  generateObject if gateway doesn't support streaming. Models always
  called via provider.chat(modelId) / /v1/chat/completions, the correct
  path for meta/muse-spark-1.3-contributor via gateway.

- Rich context: spec.js gains FULL control wording (any amount/arrangement/
  bpm allowed), and detailed payload — trackSummary with bandBalance,
  tempo, normalization (per-feature lo/hi), sectionsDetailed (energy/
  flux/centroid/flatness/width/slope/endEnergy/duration), stats
  (kinds, avg/max energy, bars). LLM told You have FULL control.

- Low-res vision: new src/llm/thumbs.js — captureThumbnails(show, {8,32})
  samples section midpoints, warms up state, renders via Show, box-averages
  down to 32×32 JPEG data URLs, restores timeline. main.js now captures 8
  frames with two animation frames of pause, then passes them as {type:
  image, image: dataUrl} parts in the user message (llmTweak.js
  attachFramesToMessages).

- Proxy logging (no bearer): vite.config.js llmGatewayProxy now logs
  sanitized requests/responses to logs/llm-proxy.log (JSON lines, 1MB
  rotation, redacts apiKey/authorization from path/body/headers) so the
  coding agent can inspect prompts without re-calling. .gitignore adds
  logs/.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-09-13 21:54:07 +02:00
co-authored by Claude
parent e2f56e3ce4
commit e0e619af85
6 changed files with 548 additions and 49 deletions
+29 -2
View File
@@ -16,6 +16,7 @@ import { synthesizeSong } from './audio/synth.js';
import { FeatureTrack } from './audio/FeatureTrack.js';
import { generateLook } from './look/LookGenerator.js';
import { SONGS } from './audio/songbank.js';
import { captureThumbnails } from './llm/thumbs.js';
const QUALITY = {
draft: 0.5, // half resolution — for scrubbing heavy stacks
@@ -1053,8 +1054,9 @@ async function handleLlmTweak() {
const btn = document.getElementById('btn-llm-tweak');
const prevLabel = btn ? btn.textContent : '';
if (btn) btn.textContent = '… tuning';
// Immediate feedback — budget had a one-frame lag before, so show progress explicitly.
showToast(`Contacting LLM (${cfg.model})…`, { ok: true });
// Live progress: start capturing thumbs, then stream. Toast updates as stream arrives.
let llmProgressToast = `Contacting LLM (${cfg.model})…`;
showToast(llmProgressToast, { ok: true });
console.info('[llm] handleLlmTweak start', {
model: cfg.model, gateway: cfg.gatewayUrl,
remaining, hint: hint || '(none)',
@@ -1062,13 +1064,38 @@ async function handleLlmTweak() {
lookDirector: lookBefore && lookBefore.director,
});
// Capture 32×32 thumbnails — low-res strip for the vision model, non-blocking.
showToast('Capturing thumbnails…', { ok: true });
let llmFrames = [];
try {
// Keep the preview responsive — allow one paint before capture.
await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
llmFrames = await captureThumbnails(state.show, { count: 8, size: 32 });
console.info('[llm] captured frames', llmFrames.length, llmFrames.map((f) => `${f.kind}@${f.time.toFixed(1)}s`));
} catch (e) {
console.warn('[llm] captureThumbnails failed', e);
}
showToast(`Contacting LLM (${cfg.model}) · ${llmFrames.length ? llmFrames.length + ' frames' : 'no frames'}…`, { ok: true });
try {
const result = await requestSongTweak({
spec: specBefore,
track: trackBefore,
look: lookBefore,
hint: hint || '',
frames: llmFrames,
config: cfg,
onProgress: (evt) => {
if (!evt) return;
const text = evt.text ? evt.text.slice(0, 80) : '';
if (evt.stage === 'partial') {
const partialKeys = evt.partial && evt.partial.edits ? Object.keys(evt.partial.edits).filter((k) => evt.partial.edits[k] != null) : [];
showToast(`LLM streaming… ${text}${partialKeys.length ? ` · ${partialKeys.join(', ')}` : ''}`, { ok: true });
console.info('[llm] stream', evt);
} else if (evt.stage === 'streaming') {
showToast(`LLM streaming… ${text}`, { ok: true });
}
},
});
// Log what came back before any synthesis — this is the audit trail.