LLM: streaming + rich context + 32×32 frames + proxy log

- Streaming: requestSongTweak now uses streamObject when a progress
  callback is provided (via handleLlmTweak), emitting partial {rationale,
  edits} and updating toast/console as text arrives; falls back to
  generateObject if gateway doesn't support streaming. Models always
  called via provider.chat(modelId) / /v1/chat/completions, the correct
  path for meta/muse-spark-1.3-contributor via gateway.

- Rich context: spec.js gains FULL control wording (any amount/arrangement/
  bpm allowed), and detailed payload — trackSummary with bandBalance,
  tempo, normalization (per-feature lo/hi), sectionsDetailed (energy/
  flux/centroid/flatness/width/slope/endEnergy/duration), stats
  (kinds, avg/max energy, bars). LLM told You have FULL control.

- Low-res vision: new src/llm/thumbs.js — captureThumbnails(show, {8,32})
  samples section midpoints, warms up state, renders via Show, box-averages
  down to 32×32 JPEG data URLs, restores timeline. main.js now captures 8
  frames with two animation frames of pause, then passes them as {type:
  image, image: dataUrl} parts in the user message (llmTweak.js
  attachFramesToMessages).

- Proxy logging (no bearer): vite.config.js llmGatewayProxy now logs
  sanitized requests/responses to logs/llm-proxy.log (JSON lines, 1MB
  rotation, redacts apiKey/authorization from path/body/headers) so the
  coding agent can inspect prompts without re-calling. .gitignore adds
  logs/.

Co-Authored-By: Claude <noreply@anthropic.com>

flow-state/.gitignore

@@ -3,6 +3,7 @@ dist
 .vite
 out
 .env.local
+logs/
 
 # Scratch probes: one-off harnesses for debugging decode/encode/mux issues.
 # They are throwaway by nature and were committed once by accident.

flow-state/src/llm/llmTweak.js

@@ -5,7 +5,7 @@
 // This file adds the look digest (describeLook/validateLook) and the actual
 // `ai`/`@ai-sdk/openai` call, which only run in the browser bundle.
 
-import { generateObject } from 'ai';
+import { generateObject, streamObject } from 'ai';
 import { createOpenAI } from '@ai-sdk/openai';
 import { getLlmConfig } from './config.js';
 import { canInvoke, recordInvocation, refundInvocation, getBudgetCounts } from './budget.js';
@@ -50,9 +50,11 @@ function lookDigestForPrompt(look) {
 }
 
 /** Prompt builder — wraps the pure one with a Look→digest step for browser callers. */
-export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
+export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) {
     const digest = lookDigest != null ? lookDigest : (look ? lookDigestForPrompt(look) : null);
-    return buildTweakPromptPure({ spec, track, lookDigest: digest, hint });
+    // Frames are not embedded as text — they become image parts in the message.
+    // Pass them through so spec can note count in system prompt.
+    return buildTweakPromptPure({ spec, track, lookDigest: digest, hint, frames });
 }
 
 export function lookDigestForPromptExport(look) { return lookDigestForPrompt(look); }
@@ -64,12 +66,33 @@ function toErrorMessage(err) {
     try { return JSON.stringify(err).slice(0, 600); } catch { return String(err); }
 }
 
+function attachFramesToMessages(messages, frames) {
+    if (!frames || !frames.length) return messages;
+    // Find the user message and turn its string content into parts with images.
+    return messages.map((m) => {
+        if (m.role !== 'user') return m;
+        const text = typeof m.content === 'string' ? m.content : String(m.content || '');
+        const parts = [{ type: 'text', text }];
+        for (const f of frames) {
+            if (!f || !f.dataUrl || typeof f.dataUrl !== 'string') continue;
+            // Only keep reasonable length data URLs; skip broken placeholders.
+            if (f.dataUrl.length < 100 || !f.dataUrl.startsWith('data:')) continue;
+            parts.push({ type: 'image', image: f.dataUrl });
+        }
+        // If only text, keep simple string form for models that prefer it.
+        if (parts.length === 1) return m;
+        return { role: 'user', content: parts };
+    });
+}
+
 /**
- * Call the gateway for one tweak. Checks budget, builds prompt, calls
+ * Call the gateway for one tweak. Checks budget, builds prompt (with optional
- * generateObject, clamps and returns. Counts toward budget on every network
+ * 32×32 thumbnails), streams or generates the object, clamps and returns.
- * attempt; refunds if the result is empty/useless.
+ *
+ * @param {{spec: object, track: object, look: object, hint?: string, frames?: Array, config?: object, onProgress?: (evt:{stage:string, partial?:object, text?:string})=>void}} opts
+ * @returns {Promise<{rationale:string, edits:object, confidence:number, usage?:object, raw:any}>}
  */
-export async function requestSongTweak({ spec, track, look, hint = '', config = null } = {}) {
+export async function requestSongTweak({ spec, track, look, hint = '', frames = null, config = null, onProgress = null } = {}) {
     const cfg = config || getLlmConfig();
     if (!cfg.isConfigured) {
         const err = new Error('LLM not configured — set an AI Gateway API key in ⚙️ settings.');
@@ -84,14 +107,14 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
         throw err;
     }
 
-    const messages = buildTweakPrompt({ spec, track, look, hint });
+    const safeFrames = Array.isArray(frames) ? frames.filter((f) => f && f.dataUrl && f.dataUrl.length > 100) : [];
+    const messages = attachFramesToMessages(buildTweakPrompt({ spec, track, look, hint, frames: safeFrames }), safeFrames);
     recordInvocation();
 
     const isDev = import.meta.env && import.meta.env.DEV;
     const proxyFetch = async (url, init = {}) => {
         const headers = new Headers(init.headers || {});
         if (isDev) headers.set('x-llm-gateway', cfg.gatewayUrl);
-        // Dev proxy forwards server-side; log what we send
         console.info('[llm] fetch', isDev ? '→ /__llm (proxied to ' + cfg.gatewayUrl + ')' : '→ ' + cfg.gatewayUrl, String(url).slice(0, 120));
         return fetch(url, { ...init, headers });
     };
@@ -102,20 +125,104 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
     });
     const modelId = cfg.model;
 
-    console.groupCollapsed(`[llm] requestSongTweak — ${modelId}${hint ? ` · hint: "${hint.slice(0, 80)}"` : ''}`);
+    console.groupCollapsed(`[llm] requestSongTweak — ${modelId}${hint ? ` · hint: "${hint.slice(0, 80)}"` : ''}${safeFrames.length ? ` · ${safeFrames.length} frames` : ''}`);
     console.info('[llm] model', modelId, 'gateway', cfg.gatewayUrl, isDev ? '(via /__llm dev proxy)' : '(direct)');
     console.info('[llm] budget after record', getBudgetCounts());
     console.log('[llm] messages', messages);
+    if (safeFrames.length) console.log('[llm] frames', safeFrames.map((f) => ({ kind: f.kind, time: f.time, frame: f.frame, bytes: f.dataUrl.length })));
     console.log('[llm] spec before', spec);
     console.groupEnd();
 
+    const emit = (evt) => { try { if (typeof onProgress === 'function') onProgress(evt); } catch { /* ignore */ } };
+
+    // Prefer streaming so the UI can show status as we wait. Fall back to
+    // generateObject if the gateway/model doesn't support streaming.
+    const useStream = typeof onProgress === 'function';
+    if (useStream) {
+        try {
+            const started = performance.now();
+            emit({ stage: 'streaming', text: 'Contacting LLM…' });
+            const stream = await streamObject({
+                model: provider.chat(modelId),
+                schema: TweakResultSchema,
+                messages,
+                temperature: 0.7,
+                providerOptions: { openai: { strictJsonSchema: false } },
+            });
+
+            let lastPartial = null;
+            let lastText = '';
+            for await (const part of stream.partialObjectStream) {
+                lastPartial = part;
+                // Derive a short status from whatever is already materialized.
+                const r = part && part.rationale ? String(part.rationale).slice(0, 120) : '';
+                const editsKeys = part && part.edits ? Object.keys(part.edits).filter((k) => part.edits[k] != null) : [];
+                const text = r ? `… ${r}` : (editsKeys.length ? `edits: ${editsKeys.join(', ')}` : '… thinking');
+                if (text !== lastText) {
+                    lastText = text;
+                    emit({ stage: 'partial', partial: part, text });
+                    console.info('[llm] stream partial', part);
+                }
+            }
+
+            const final = await stream.object;
+            const usage = await stream.usage.catch(() => null);
+            const elapsed = ((performance.now() - started) / 1000).toFixed(1);
+            const raw = final || lastPartial || {};
+            const edits = clampEdits(raw.edits || {});
+            const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.';
+            const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5;
+
+            console.groupCollapsed(`[llm] stream done in ${elapsed}s — ${Object.keys(edits).length ? Object.keys(edits).join(', ') : 'no edits'}`);
+            console.log('[llm] raw', raw);
+            console.log('[llm] clamped edits', edits);
+            console.log('[llm] rationale', rationale);
+            console.log('[llm] confidence', confidence, 'usage', usage || null);
+            console.info('[llm] diff', Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)');
+            console.groupEnd();
+
+            emit({ stage: 'done', partial: raw, text: rationale.slice(0, 120) });
+
+            if (!Object.keys(edits).length) {
+                refundInvocation();
+                const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.');
+                err.code = 'NO_EDITS';
+                err.result = { rationale, edits: {}, confidence, usage: usage || null, raw };
+                throw err;
+            }
+            return { rationale, edits, confidence, usage: usage || null, raw };
+        } catch (err) {
+            // If streaming failed before producing a result (e.g. gateway doesn't
+            // support it), fall through to non-streaming generateObject below —
+            // unless it's our own NO_EDITS signal.
+            if (err && err.code === 'NO_EDITS') throw err;
+            const msg = toErrorMessage(err).toLowerCase();
+            const isStreamUnsupported = msg.includes('stream') || msg.includes('not supported') || msg.includes('unsupported');
+            if (!isStreamUnsupported) {
+                // Real LLM error — surface as usual (don't silently fall back and
+                // hide it behind a second call that also fails).
+                const lower = msg.toLowerCase();
+                if (!err.code) {
+                    if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH';
+                    else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT';
+                    else if (lower.includes('timeout') || lower.includes('timed out')) err.code = 'TIMEOUT';
+                    else err.code = 'LLM_ERROR';
+                }
+                if (!err.message || err.message === '[object Object]') err.message = toErrorMessage(err);
+                console.error('[llm] stream failed', err);
+                throw err;
+            }
+            console.warn('[llm] stream not supported, falling back to generateObject —', err.message);
+        }
+    }
+
     try {
         const started = performance.now();
         const result = await generateObject({
             model: provider.chat(modelId),
             schema: TweakResultSchema,
             messages,
-            temperature: 0.4,
+            temperature: 0.7,
             maxRetries: 0,
             providerOptions: { openai: { strictJsonSchema: false } },
         });
@@ -131,8 +238,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', config =
         console.log('[llm] clamped edits', edits);
         console.log('[llm] rationale', rationale);
         console.log('[llm] confidence', confidence, 'usage', result.usage || null);
-        const changed = Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)';
+        console.info('[llm] diff', Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)');
-        console.info('[llm] diff', changed);
         console.groupEnd();
 
         if (!Object.keys(edits).length) {

flow-state/src/llm/spec.js

@@ -69,15 +69,8 @@ export function clampEdits(edits) {
  * Kept compatible with the old `{spec,track,look,hint}` shape by accepting
  * either `look` (no-op here) or `lookDigest`.
  */
-export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
+export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) {
     const summary = track ? track.summary : null;
-    const sections = track ? (track.sections || []).map((s) => ({
-        kind: s.kind,
-        start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1),
-        end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1),
-        energy: Number((s.energy ?? 0).toFixed(3)),
-    })) : [];
-
     const digest = lookDigest ?? (look ? look : null);
 
     const specForPrompt = spec ? {
@@ -92,36 +85,167 @@ export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint =
         seed: spec.seed ?? null,
     } : null;
 
+    // --- helpers for richer context ---
+    const r2 = (v) => {
+        const n = Number(v);
+        return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0;
+    };
+    const r3 = (v) => {
+        const n = Number(v);
+        return Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0;
+    };
+
+    const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({
+        index: Number.isFinite(s.index) ? s.index : i,
+        kind: s.kind || 'unknown',
+        start: r2(s.startSec != null ? s.startSec : (s.start ?? 0)),
+        end: r2(s.endSec != null ? s.endSec : (s.end ?? 0)),
+        duration: r2(s.duration ?? ((s.endSec != null ? s.endSec : (s.end ?? 0)) - (s.startSec != null ? s.startSec : (s.start ?? 0)))),
+        energy: r3(s.energy ?? 0),
+        flux: r3(s.flux ?? 0),
+        centroid: r3(s.centroid ?? 0),
+        flatness: r3(s.flatness ?? 0),
+        width: r3(s.width ?? 0),
+        slope: r3(s.slope ?? 0),
+        endEnergy: r3(s.endEnergy ?? 0),
+    })) : [];
+
+    // trackSummary — extended
+    let trackSummary = null;
+    if (summary) {
+        trackSummary = {
+            bpm: summary.bpm ?? track?.tempo?.bpm ?? null,
+            meanCentroid: summary.meanCentroid != null ? r3(summary.meanCentroid) : null,
+            meanFlatness: summary.meanFlatness != null ? r3(summary.meanFlatness) : null,
+            meanWidth: summary.meanWidth != null ? r3(summary.meanWidth) : undefined,
+            meanLoudness: summary.meanLoudness != null ? r3(summary.meanLoudness) : null,
+            dynamicRange: summary.dynamicRange != null ? r3(summary.dynamicRange) : null,
+            tempoConfidence: summary.tempoConfidence != null ? r3(summary.tempoConfidence) : (track?.tempo?.confidence != null ? r3(track.tempo.confidence) : null),
+            frameCount: track?.frameCount ?? summary.frameCount ?? null,
+            fps: track?.fps ?? summary.fps ?? null,
+            duration: r2(track?.duration ?? summary.duration ?? 0),
+            sections: sectionsDetailed,
+        };
+        if (summary.bandBalance) {
+            trackSummary.bandBalance = {
+                sub: r3(summary.bandBalance.sub ?? 0),
+                low: r3(summary.bandBalance.low ?? 0),
+                mid: r3(summary.bandBalance.mid ?? 0),
+                high: r3(summary.bandBalance.high ?? 0),
+                air: r3(summary.bandBalance.air ?? 0),
+            };
+        }
+        // remove undefined meanWidth if not present to keep payload compact
+        if (trackSummary.meanWidth === undefined) delete trackSummary.meanWidth;
+    } else if (sectionsDetailed.length || track) {
+        // fallback when summary missing but track exists (e.g. synthetic test track)
+        trackSummary = {
+            bpm: track?.tempo?.bpm ?? null,
+            frameCount: track?.frameCount ?? null,
+            fps: track?.fps ?? null,
+            duration: r2(track?.duration ?? 0),
+            sections: sectionsDetailed,
+        };
+        if (track?.summary?.bandBalance) {
+            trackSummary.bandBalance = {
+                sub: r3(track.summary.bandBalance.sub ?? 0),
+                low: r3(track.summary.bandBalance.low ?? 0),
+                mid: r3(track.summary.bandBalance.mid ?? 0),
+                high: r3(track.summary.bandBalance.high ?? 0),
+                air: r3(track.summary.bandBalance.air ?? 0),
+            };
+        }
+    }
+
+    // tempo
+    let tempo = null;
+    if (track?.tempo) {
+        tempo = {
+            bpm: r2(track.tempo.bpm ?? summary?.bpm ?? 0),
+            confidence: r3(track.tempo.confidence ?? 0),
+            beatsPerBar: track.tempo.beatsPerBar ?? 4,
+            downbeats: Array.isArray(track.tempo.downbeats) ? track.tempo.downbeats.length : 0,
+        };
+    } else if (summary?.bpm != null) {
+        tempo = { bpm: r2(summary.bpm), confidence: r3(summary.tempoConfidence ?? 0), beatsPerBar: 4, downbeats: 0 };
+    }
+
+    // normalization
+    let normalization = null;
+    if (track?.normalization && typeof track.normalization === 'object') {
+        normalization = {};
+        for (const [k, v] of Object.entries(track.normalization)) {
+            if (v && typeof v === 'object' && v.lo != null && v.hi != null) {
+                normalization[k] = { lo: r3(v.lo), hi: r3(v.hi) };
+            }
+        }
+        if (!Object.keys(normalization).length) normalization = null;
+    }
+
+    // stats — derived
+    let stats = null;
+    if (sectionsDetailed.length || tempo || track) {
+        const kinds = [...new Set(sectionsDetailed.map((s) => s.kind))];
+        const avgEnergy = sectionsDetailed.length
+            ? r3(sectionsDetailed.reduce((a, s) => a + s.energy, 0) / sectionsDetailed.length)
+            : 0;
+        let maxSec = null;
+        let maxE = -Infinity;
+        for (const s of sectionsDetailed) if (s.energy > maxE) { maxE = s.energy; maxSec = s; }
+        const duration = track?.duration ?? summary?.duration ?? 0;
+        const bpmVal = tempo?.bpm ?? summary?.bpm ?? 120;
+        const bpb = tempo?.beatsPerBar ?? 4;
+        const bars = bpmVal > 0 && duration > 0 ? Math.round(duration / ((60 / bpmVal) * bpb)) : 0;
+        stats = {
+            totalSections: sectionsDetailed.length,
+            kinds,
+            avgEnergy,
+            maxEnergySection: maxSec ? { index: maxSec.index, kind: maxSec.kind, energy: maxSec.energy } : null,
+            bpm: tempo?.bpm ?? null,
+            beatsPerBar: tempo?.beatsPerBar ?? null,
+            downbeats: tempo?.downbeats ?? 0,
+            bars,
+            duration: r2(duration),
+            fps: track?.fps ?? summary?.fps ?? null,
+            frameCount: track?.frameCount ?? summary?.frameCount ?? null,
+        };
+    }
+
     const hintBlock = hint && String(hint).trim()
         ? `User hint: "${String(hint).trim().slice(0, 300)}"`
         : 'No user hint.';
 
+    const frameNote = Array.isArray(frames) && frames.length
+        ? `You are also given ${frames.length} low-res thumbnails (32×32) sampled across the song in order — use them to judge color, structure, and motion.`
+        : null;
+
     const system = [
         'You are a producer for a generative music-video system called flow-state.',
-        'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.',
+        'You suggest tweaks to the *song synthesis parameters* so the next generation looks better.',
         'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.',
         '',
         'Rules:',
-        '- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.',
+        'You have FULL control — change any subset of the 7 knobs by any amount within valid ranges, including large jumps, arrangement swaps, or bpm shifts. Prefer impactful changes that clearly improve visuals; keep duration fixed.',
-        '- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.',
         '- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.',
         '- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.',
         '- All 7 edit fields MUST be present in edits — use null for unchanged (not omitted). Example: { bpm: null, brightness: 0.6, noise: null, noiseColour: null, dynamics: null, transition: null, arrangement: null }',
         '- Explain your reasoning in 1-2 sentences (rationale).',
         '- If the look is already good, return all-null edits.',
         hintBlock,
-    ].join('\n');
+        frameNote,
+    ].filter(Boolean).join('\n');
 
-    const user = JSON.stringify({
+    const userPayload = {
         songSpec: specForPrompt,
-        trackSummary: summary ? {
+        trackSummary,
-            bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness,
+        sections: sectionsDetailed,
-            dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness,
+        tempo,
-            tempoConfidence: summary.tempoConfidence,
+        normalization,
-        } : null,
-        sections,
         lookDigest: digest,
-    }, null, 2);
+        stats,
+    };
+
+    const user = JSON.stringify(userPayload, null, 2);
 
     return [
         { role: 'system', content: system },

flow-state/src/llm/thumbs.js

@@ -0,0 +1,128 @@
+export function downscaleRgbaToDataUrl(pixels, srcW, srcH, dstSize) {
+    const dst = new Uint8ClampedArray(dstSize * dstSize * 4);
+    for (let y = 0; y < dstSize; y++) {
+        const y0 = Math.floor((y * srcH) / dstSize);
+        const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * srcH) / dstSize));
+        for (let x = 0; x < dstSize; x++) {
+            const x0 = Math.floor((x * srcW) / dstSize);
+            const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * srcW) / dstSize));
+            let r = 0, g = 0, b = 0, c = 0;
+            for (let sy = y0; sy < y1; sy++)
+                for (let sx = x0; sx < x1; sx++) {
+                    const i = (sy * srcW + sx) * 4;
+                    r += pixels[i]; g += pixels[i + 1]; b += pixels[i + 2]; c++;
+                }
+            const o = (y * dstSize + x) * 4;
+            dst[o] = c ? Math.round(r / c) : 0;
+            dst[o + 1] = c ? Math.round(g / c) : 0;
+            dst[o + 2] = c ? Math.round(b / c) : 0;
+            dst[o + 3] = 255;
+        }
+    }
+    if (typeof document !== 'undefined' && document.createElement) {
+        try {
+            const canvas = document.createElement('canvas');
+            canvas.width = dstSize; canvas.height = dstSize;
+            const ctx = canvas.getContext('2d');
+            if (ctx) { ctx.putImageData(new ImageData(dst, dstSize, dstSize), 0, 0); return canvas.toDataURL('image/jpeg', 0.6); }
+        } catch { /* fallback */ }
+    }
+    return 'data:image/jpeg;base64,';
+}
+
+function uniformFrames(count, frameCount) {
+    const seen = new Set(); const out = [];
+    for (let i = 0; i < count; i++) {
+        const f = Math.max(0, Math.min(frameCount - 1, Math.round(((i + 0.5) / count) * frameCount)));
+        if (!seen.has(f)) { seen.add(f); out.push(f); }
+    }
+    return out;
+}
+
+function pickFrames(show, count, frameCount, fps) {
+    const sections = show.track && show.track.sections;
+    if (sections && sections.length) {
+        const mids = []; const seen = new Set();
+        for (const s of sections) {
+            let mid;
+            if (Number.isFinite(s.startFrame) && Number.isFinite(s.endFrame)) mid = Math.round((s.startFrame + s.endFrame) / 2);
+            else if (Number.isFinite(s.start) && Number.isFinite(s.end)) mid = Math.round(((s.start + s.end) / 2) * fps);
+            else continue;
+            mid = Math.max(0, Math.min(frameCount - 1, mid));
+            if (!seen.has(mid)) { seen.add(mid); mids.push(mid); }
+        }
+        if (!mids.length) return uniformFrames(count, frameCount);
+        if (mids.length >= count) {
+            if (mids.length === count) return mids;
+            mids.sort((a, b) => a - b);
+            const out = [];
+            for (let i = 0; i < count; i++) {
+                const idx = Math.min(mids.length - 1, Math.floor(((i + 0.5) / count) * mids.length));
+                const v = mids[idx]; if (!out.includes(v)) out.push(v);
+            }
+            return out;
+        }
+        const need = count - mids.length;
+        for (const f of uniformFrames(need, frameCount)) if (!seen.has(f)) { seen.add(f); mids.push(f); }
+        if (mids.length < count) for (const f of uniformFrames(count, frameCount)) if (!seen.has(f) && mids.length < count) { seen.add(f); mids.push(f); }
+        const sorted = [...seen].sort((a, b) => a - b);
+        if (sorted.length <= count) return sorted;
+        const out = [];
+        for (let i = 0; i < count; i++) {
+            const idx = Math.min(sorted.length - 1, Math.floor(((i + 0.5) / count) * sorted.length));
+            const v = sorted[idx]; if (!out.includes(v)) out.push(v);
+        }
+        return out;
+    }
+    return uniformFrames(count, frameCount);
+}
+
+export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
+    if (!show || !show.ready || !show.track || !show.engine) return [];
+    const fps = show.fps || 60;
+    const frameCount = show.frameCount || (show.track && show.track.frameCount) || Math.round((show.track.duration || 0) * fps) || 0;
+    if (!frameCount) return [];
+    const originalFrame = show.timeline ? show.timeline.frame : 0;
+    const origW = show.engine.width, origH = show.engine.height;
+    const frames = pickFrames(show, count, frameCount, fps).sort((a, b) => a - b);
+    const out = [];
+    try {
+        for (const frame of frames) {
+            try {
+                if (typeof show.warmUp === 'function') {
+                    const wf = typeof show.warmupFrames === 'function' ? show.warmupFrames() : 120;
+                    show.warmUp(frame, Math.min(120, wf));
+                } else if (show.engine.compositor) {
+                    if (typeof show.engine.warmUp === 'function') show.engine.warmUp(frame, 120);
+                    else show.engine.compositor.reset();
+                }
+            } catch { /* ignore */ }
+            let target; try { target = show.renderFrame(frame); } catch { continue; }
+            let pixels; try { pixels = show.readPixels(target); } catch { continue; }
+            if (!pixels || !pixels.length) continue;
+            let dataUrl = ''; try { dataUrl = downscaleRgbaToDataUrl(pixels, origW, origH, size); } catch { dataUrl = 'data:image/jpeg;base64,'; }
+            let kind = '';
+            try {
+                const sec = show.track.sectionAt ? show.track.sectionAt(frame) : null;
+                if (sec && sec.kind) kind = sec.kind;
+                else if (show.track.sections) for (const s of show.track.sections) if (Number.isFinite(s.startFrame) && frame >= s.startFrame && frame < s.endFrame) { kind = s.kind || ''; break; }
+            } catch { /* ignore */ }
+            out.push({ kind: kind || '', time: frame / fps, frame, dataUrl });
+        }
+    } finally {
+        try {
+            if (typeof show.seek === 'function') show.seek(originalFrame);
+            else if (show.timeline && typeof show.timeline.seek === 'function') {
+                show.timeline.seek(originalFrame);
+                try { const t = show.renderFrame(originalFrame); if (typeof show.present === 'function') show.present(t); } catch { /* ignore */ }
+            }
+        } catch { /* ignore */ }
+        try {
+            if (show.engine && (show.engine.width !== origW || show.engine.height !== origH)) {
+                if (typeof show.engine.setSize === 'function') show.engine.setSize(origW, origH);
+                else if (typeof show.setSize === 'function') show.setSize(origW, origH);
+            }
+        } catch { /* ignore */ }
+    }
+    return out;
+}

flow-state/src/main.js

@@ -16,6 +16,7 @@ import { synthesizeSong } from './audio/synth.js';
 import { FeatureTrack } from './audio/FeatureTrack.js';
 import { generateLook } from './look/LookGenerator.js';
 import { SONGS } from './audio/songbank.js';
+import { captureThumbnails } from './llm/thumbs.js';
 
 const QUALITY = {
     draft: 0.5,     // half resolution — for scrubbing heavy stacks
@@ -1053,8 +1054,9 @@ async function handleLlmTweak() {
     const btn = document.getElementById('btn-llm-tweak');
     const prevLabel = btn ? btn.textContent : '';
     if (btn) btn.textContent = '… tuning';
-    // Immediate feedback — budget had a one-frame lag before, so show progress explicitly.
+    // Live progress: start capturing thumbs, then stream. Toast updates as stream arrives.
-    showToast(`Contacting LLM (${cfg.model})…`, { ok: true });
+    let llmProgressToast = `Contacting LLM (${cfg.model})…`;
+    showToast(llmProgressToast, { ok: true });
     console.info('[llm] handleLlmTweak start', {
         model: cfg.model, gateway: cfg.gatewayUrl,
         remaining, hint: hint || '(none)',
@@ -1062,13 +1064,38 @@ async function handleLlmTweak() {
         lookDirector: lookBefore && lookBefore.director,
     });
 
+    // Capture 32×32 thumbnails — low-res strip for the vision model, non-blocking.
+    showToast('Capturing thumbnails…', { ok: true });
+    let llmFrames = [];
+    try {
+        // Keep the preview responsive — allow one paint before capture.
+        await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
+        llmFrames = await captureThumbnails(state.show, { count: 8, size: 32 });
+        console.info('[llm] captured frames', llmFrames.length, llmFrames.map((f) => `${f.kind}@${f.time.toFixed(1)}s`));
+    } catch (e) {
+        console.warn('[llm] captureThumbnails failed', e);
+    }
+    showToast(`Contacting LLM (${cfg.model}) · ${llmFrames.length ? llmFrames.length + ' frames' : 'no frames'}…`, { ok: true });
+
     try {
         const result = await requestSongTweak({
             spec: specBefore,
             track: trackBefore,
             look: lookBefore,
             hint: hint || '',
+            frames: llmFrames,
             config: cfg,
+            onProgress: (evt) => {
+                if (!evt) return;
+                const text = evt.text ? evt.text.slice(0, 80) : '';
+                if (evt.stage === 'partial') {
+                    const partialKeys = evt.partial && evt.partial.edits ? Object.keys(evt.partial.edits).filter((k) => evt.partial.edits[k] != null) : [];
+                    showToast(`LLM streaming… ${text}${partialKeys.length ? ` · ${partialKeys.join(', ')}` : ''}`, { ok: true });
+                    console.info('[llm] stream', evt);
+                } else if (evt.stage === 'streaming') {
+                    showToast(`LLM streaming… ${text}`, { ok: true });
+                }
+            },
         });
 
         // Log what came back before any synthesis — this is the audit trail.

flow-state/vite.config.js

@@ -1,6 +1,6 @@
 import { defineConfig } from 'vite';
 import { resolve } from 'path';
-import { writeFileSync } from 'fs';
+import { writeFileSync, existsSync, mkdirSync, appendFileSync, statSync } from 'fs';
 
 /**
  * Let the measuring pages write their results back into the source tree.
@@ -63,8 +63,77 @@ function measurementWriter() {
  * Production builds are static; there this route does not exist and the
  * client falls through to a direct fetch. Deployments that need CORS-free
  * production should add an equivalent `/api/llm` serverless proxy.
+ *
+ * Also logs sanitized traffic (no Authorization) to `logs/llm-proxy.log` so
+ * the coding agent can inspect prompts/responses without re-calling the LLM.
  */
 function llmGatewayProxy() {
+    const LOG_DIR = resolve(process.cwd(), 'logs');
+    const LOG_FILE = resolve(LOG_DIR, 'llm-proxy.log');
+
+    function safeAppend(entry) {
+        try {
+            if (!existsSync(LOG_DIR)) mkdirSync(LOG_DIR, { recursive: true });
+            // Rotate: truncate if >1MB to avoid unbounded growth
+            try {
+                if (existsSync(LOG_FILE) && statSync(LOG_FILE).size > 1024 * 1024) {
+                    writeFileSync(LOG_FILE, '');
+                }
+            } catch { /* ignore stat/truncate errors */ }
+            appendFileSync(LOG_FILE, JSON.stringify(entry) + '\n');
+        } catch { /* ignore */ }
+    }
+
+    function sanitizePathForLog(path) {
+        try {
+            const u = new URL(path, 'http://localhost');
+            for (const key of [...u.searchParams.keys()]) {
+                const lower = key.toLowerCase();
+                if (lower === 'apikey' || lower === 'api_key' || lower === 'authorization') {
+                    u.searchParams.delete(key);
+                }
+            }
+            return u.pathname + u.search;
+        } catch {
+            // Fallback: strip sensitive query params via regex
+            return path.replace(/([?&])(apiKey|api_key|authorization)=[^&]*/gi, '').replace(/[?&]$/, '').replace(/\?&/, '?') || '/';
+        }
+    }
+
+    function redactJsonForLog(obj) {
+        if (Array.isArray(obj)) return obj.map(redactJsonForLog);
+        if (obj && typeof obj === 'object') {
+            const out = {};
+            for (const [k, v] of Object.entries(obj)) {
+                const lower = k.toLowerCase();
+                if (lower === 'apikey' || lower === 'api_key' || lower === 'authorization') {
+                    out[k] = '[REDACTED]';
+                } else {
+                    out[k] = redactJsonForLog(v);
+                }
+            }
+            return out;
+        }
+        return obj;
+    }
+
+    function sanitizeBodyForLog(ct, raw) {
+        // Don't log if content-type is missing or body is binary
+        if (!ct || !ct.includes('json')) {
+            if (raw && raw.length > 4096) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`;
+            try { return raw ? raw.toString('utf8').slice(0, 4096) : ''; } catch { return '[unprintable]'; }
+        }
+        try {
+            const text = raw ? raw.toString('utf8') : '';
+            const parsed = JSON.parse(text);
+            // Redact sensitive keys if they somehow appear in JSON body
+            const redacted = redactJsonForLog(parsed);
+            return JSON.stringify(redacted).slice(0, 16384);
+        } catch {
+            return raw ? raw.toString('utf8').slice(0, 4096) : '';
+        }
+    }
+
     return {
         name: 'flow-state-llm-gateway-proxy',
         apply: 'serve',
@@ -85,16 +154,7 @@ function llmGatewayProxy() {
                 }
 
                 const gatewayBase = (req.headers['x-llm-gateway'] && String(req.headers['x-llm-gateway'])) || 'https://ai-gateway.vercel.sh';
-                // Strip the proxy prefix; the client sends `/__llm/v1/...`,
-                // so after stripping we have `/v1/...` which joins correctly.
                 const pathAfter = req.url.replace(/^\/__llm/, '') || '/';
-                // Preserve query string
-                const targetUrl = new URL(pathAfter, gatewayBase.endsWith('/') ? gatewayBase : gatewayBase + '/');
-                // But `new URL('/v1/x', 'https://host/v1')` drops the base path —
-                // `/v1/x` is absolute. Since `pathAfter` already starts with `/v1`,
-                // joining against the origin is correct; gatewayBase's path is just
-                // the fallback origin. So rebuild against origin only.
-                // Simpler: construct from origin + pathAfter
                 let origin;
                 try {
                     origin = new URL(gatewayBase).origin;
@@ -106,14 +166,28 @@ function llmGatewayProxy() {
                 let bodyChunks = [];
                 req.on('data', (c) => bodyChunks.push(c));
                 req.on('end', async () => {
+                    const started = Date.now();
+                    const reqCt = req.headers['content-type'] || '';
+                    const rawReq = bodyChunks.length ? Buffer.concat(bodyChunks) : null;
+                    const reqLog = sanitizeBodyForLog(reqCt, rawReq);
+                    // Always log the request (sanitized — no Authorization/apiKey)
+                    safeAppend({
+                        ts: new Date().toISOString(),
+                        dir: '→',
+                        method: req.method,
+                        path: sanitizePathForLog(pathAfter),
+                        gateway: origin,
+                        contentType: reqCt,
+                        // Authorization header and apiKey query param never logged
+                        body: reqLog,
+                    });
+
                     try {
-                        const body = bodyChunks.length ? Buffer.concat(bodyChunks) : undefined;
+                        const body = rawReq && rawReq.length ? rawReq : undefined;
                         const headers = {};
-                        // Forward relevant headers
                         if (req.headers['content-type']) headers['content-type'] = req.headers['content-type'];
                         if (req.headers['authorization']) headers['authorization'] = req.headers['authorization'];
                         else if (req.headers['x-api-key']) headers['authorization'] = `Bearer ${req.headers['x-api-key']}`;
-                        // Also forward any x- header the gateway might care about
                         for (const [k, v] of Object.entries(req.headers)) {
                             if (k.startsWith('x-') && k !== 'x-llm-gateway') headers[k] = v;
                         }
@@ -124,7 +198,6 @@ function llmGatewayProxy() {
                             body: req.method === 'GET' || req.method === 'HEAD' ? undefined : body,
                         });
 
-                        // Mirror status and content headers
                         const contentType = upstream.headers.get('content-type');
                         res.writeHead(upstream.status, {
                             ...(contentType ? { 'content-type': contentType } : {}),
@@ -133,7 +206,47 @@ function llmGatewayProxy() {
 
                         const buf = Buffer.from(await upstream.arrayBuffer());
                         res.end(buf);
+
+                        // Log response (sanitized — still no token in body)
+                        const elapsed = Date.now() - started;
+                        let respLog = '';
+                        try {
+                            const ct = contentType || '';
+                            if (ct.includes('json') || ct.includes('event-stream') || buf.length < 32768) {
+                                const text = buf.toString('utf8');
+                                // For SSE streams, keep first 16k; for JSON, keep parsed form
+                                if (ct.includes('json')) {
+                                    try { respLog = JSON.stringify(JSON.parse(text)).slice(0, 16384); } catch { respLog = text.slice(0, 8192); }
+                                } else {
+                                    respLog = text.slice(0, 8192);
+                                }
+                            } else {
+                                respLog = `[${contentType} ${buf.length} bytes]`;
+                            }
+                        } catch { respLog = `[${buf.length} bytes]`; }
+
+                        safeAppend({
+                            ts: new Date().toISOString(),
+                            dir: '←',
+                            status: upstream.status,
+                            elapsedMs: elapsed,
+                            contentType: contentType || '',
+                            body: respLog,
+                        });
+
+                        // Also emit to vite's console for live tailing
+                        const tag = upstream.status >= 400 ? '← ERR' : '← OK';
+                        console.info(`[llm-proxy] ${tag} ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${buf.length} bytes)`);
                     } catch (err) {
+                        const elapsed = Date.now() - started;
+                        safeAppend({
+                            ts: new Date().toISOString(),
+                            dir: '←',
+                            status: 502,
+                            elapsedMs: elapsed,
+                            error: String(err && err.message ? err.message : err).slice(0, 2000),
+                        });
+                        console.error(`[llm-proxy] ← 502 ${req.method} ${pathAfter}`, err);
                         res.writeHead(502, { 'access-control-allow-origin': '*' });
                         res.end(String(err && err.message ? err.message : err));
                     }