LLM: streaming + rich context + 32×32 frames + proxy log
- Streaming: requestSongTweak now uses streamObject when a progress
callback is provided (via handleLlmTweak), emitting partial {rationale,
edits} and updating toast/console as text arrives; falls back to
generateObject if gateway doesn't support streaming. Models always
called via provider.chat(modelId) / /v1/chat/completions, the correct
path for meta/muse-spark-1.3-contributor via gateway.
- Rich context: spec.js gains FULL control wording (any amount/arrangement/
bpm allowed), and detailed payload — trackSummary with bandBalance,
tempo, normalization (per-feature lo/hi), sectionsDetailed (energy/
flux/centroid/flatness/width/slope/endEnergy/duration), stats
(kinds, avg/max energy, bars). LLM told You have FULL control.
- Low-res vision: new src/llm/thumbs.js — captureThumbnails(show, {8,32})
samples section midpoints, warms up state, renders via Show, box-averages
down to 32×32 JPEG data URLs, restores timeline. main.js now captures 8
frames with two animation frames of pause, then passes them as {type:
image, image: dataUrl} parts in the user message (llmTweak.js
attachFramesToMessages).
- Proxy logging (no bearer): vite.config.js llmGatewayProxy now logs
sanitized requests/responses to logs/llm-proxy.log (JSON lines, 1MB
rotation, redacts apiKey/authorization from path/body/headers) so the
coding agent can inspect prompts without re-calling. .gitignore adds
logs/.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
+29
-2
@@ -16,6 +16,7 @@ import { synthesizeSong } from './audio/synth.js';
|
||||
import { FeatureTrack } from './audio/FeatureTrack.js';
|
||||
import { generateLook } from './look/LookGenerator.js';
|
||||
import { SONGS } from './audio/songbank.js';
|
||||
import { captureThumbnails } from './llm/thumbs.js';
|
||||
|
||||
const QUALITY = {
|
||||
draft: 0.5, // half resolution — for scrubbing heavy stacks
|
||||
@@ -1053,8 +1054,9 @@ async function handleLlmTweak() {
|
||||
const btn = document.getElementById('btn-llm-tweak');
|
||||
const prevLabel = btn ? btn.textContent : '';
|
||||
if (btn) btn.textContent = '… tuning';
|
||||
// Immediate feedback — budget had a one-frame lag before, so show progress explicitly.
|
||||
showToast(`Contacting LLM (${cfg.model})…`, { ok: true });
|
||||
// Live progress: start capturing thumbs, then stream. Toast updates as stream arrives.
|
||||
let llmProgressToast = `Contacting LLM (${cfg.model})…`;
|
||||
showToast(llmProgressToast, { ok: true });
|
||||
console.info('[llm] handleLlmTweak start', {
|
||||
model: cfg.model, gateway: cfg.gatewayUrl,
|
||||
remaining, hint: hint || '(none)',
|
||||
@@ -1062,13 +1064,38 @@ async function handleLlmTweak() {
|
||||
lookDirector: lookBefore && lookBefore.director,
|
||||
});
|
||||
|
||||
// Capture 32×32 thumbnails — low-res strip for the vision model, non-blocking.
|
||||
showToast('Capturing thumbnails…', { ok: true });
|
||||
let llmFrames = [];
|
||||
try {
|
||||
// Keep the preview responsive — allow one paint before capture.
|
||||
await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
|
||||
llmFrames = await captureThumbnails(state.show, { count: 8, size: 32 });
|
||||
console.info('[llm] captured frames', llmFrames.length, llmFrames.map((f) => `${f.kind}@${f.time.toFixed(1)}s`));
|
||||
} catch (e) {
|
||||
console.warn('[llm] captureThumbnails failed', e);
|
||||
}
|
||||
showToast(`Contacting LLM (${cfg.model}) · ${llmFrames.length ? llmFrames.length + ' frames' : 'no frames'}…`, { ok: true });
|
||||
|
||||
try {
|
||||
const result = await requestSongTweak({
|
||||
spec: specBefore,
|
||||
track: trackBefore,
|
||||
look: lookBefore,
|
||||
hint: hint || '',
|
||||
frames: llmFrames,
|
||||
config: cfg,
|
||||
onProgress: (evt) => {
|
||||
if (!evt) return;
|
||||
const text = evt.text ? evt.text.slice(0, 80) : '';
|
||||
if (evt.stage === 'partial') {
|
||||
const partialKeys = evt.partial && evt.partial.edits ? Object.keys(evt.partial.edits).filter((k) => evt.partial.edits[k] != null) : [];
|
||||
showToast(`LLM streaming… ${text}${partialKeys.length ? ` · ${partialKeys.join(', ')}` : ''}`, { ok: true });
|
||||
console.info('[llm] stream', evt);
|
||||
} else if (evt.stage === 'streaming') {
|
||||
showToast(`LLM streaming… ${text}`, { ok: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Log what came back before any synthesis — this is the audit trail.
|
||||
|
||||
Reference in New Issue
Block a user