diff --git a/flow-state/.gitignore b/flow-state/.gitignore index 1c0d0b5..658a84c 100644 --- a/flow-state/.gitignore +++ b/flow-state/.gitignore @@ -3,6 +3,7 @@ dist .vite out .env.local +logs/ # Scratch probes: one-off harnesses for debugging decode/encode/mux issues. # They are throwaway by nature and were committed once by accident. diff --git a/flow-state/src/llm/llmTweak.js b/flow-state/src/llm/llmTweak.js index 8a88a3b..e65e9e8 100644 --- a/flow-state/src/llm/llmTweak.js +++ b/flow-state/src/llm/llmTweak.js @@ -5,7 +5,7 @@ // This file adds the look digest (describeLook/validateLook) and the actual // `ai`/`@ai-sdk/openai` call, which only run in the browser bundle. -import { generateObject } from 'ai'; +import { generateObject, streamObject } from 'ai'; import { createOpenAI } from '@ai-sdk/openai'; import { getLlmConfig } from './config.js'; import { canInvoke, recordInvocation, refundInvocation, getBudgetCounts } from './budget.js'; @@ -50,9 +50,11 @@ function lookDigestForPrompt(look) { } /** Prompt builder — wraps the pure one with a Look→digest step for browser callers. */ -export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) { +export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) { const digest = lookDigest != null ? lookDigest : (look ? lookDigestForPrompt(look) : null); - return buildTweakPromptPure({ spec, track, lookDigest: digest, hint }); + // Frames are not embedded as text — they become image parts in the message. + // Pass them through so spec can note count in system prompt. + return buildTweakPromptPure({ spec, track, lookDigest: digest, hint, frames }); } export function lookDigestForPromptExport(look) { return lookDigestForPrompt(look); } @@ -64,12 +66,33 @@ function toErrorMessage(err) { try { return JSON.stringify(err).slice(0, 600); } catch { return String(err); } } +function attachFramesToMessages(messages, frames) { + if (!frames || !frames.length) return messages; + // Find the user message and turn its string content into parts with images. + return messages.map((m) => { + if (m.role !== 'user') return m; + const text = typeof m.content === 'string' ? m.content : String(m.content || ''); + const parts = [{ type: 'text', text }]; + for (const f of frames) { + if (!f || !f.dataUrl || typeof f.dataUrl !== 'string') continue; + // Only keep reasonable length data URLs; skip broken placeholders. + if (f.dataUrl.length < 100 || !f.dataUrl.startsWith('data:')) continue; + parts.push({ type: 'image', image: f.dataUrl }); + } + // If only text, keep simple string form for models that prefer it. + if (parts.length === 1) return m; + return { role: 'user', content: parts }; + }); +} + /** - * Call the gateway for one tweak. Checks budget, builds prompt, calls - * generateObject, clamps and returns. Counts toward budget on every network - * attempt; refunds if the result is empty/useless. + * Call the gateway for one tweak. Checks budget, builds prompt (with optional + * 32×32 thumbnails), streams or generates the object, clamps and returns. + * + * @param {{spec: object, track: object, look: object, hint?: string, frames?: Array, config?: object, onProgress?: (evt:{stage:string, partial?:object, text?:string})=>void}} opts + * @returns {Promise<{rationale:string, edits:object, confidence:number, usage?:object, raw:any}>} */ -export async function requestSongTweak({ spec, track, look, hint = '', config = null } = {}) { +export async function requestSongTweak({ spec, track, look, hint = '', frames = null, config = null, onProgress = null } = {}) { const cfg = config || getLlmConfig(); if (!cfg.isConfigured) { const err = new Error('LLM not configured — set an AI Gateway API key in ⚙️ settings.'); @@ -84,14 +107,14 @@ export async function requestSongTweak({ spec, track, look, hint = '', config = throw err; } - const messages = buildTweakPrompt({ spec, track, look, hint }); + const safeFrames = Array.isArray(frames) ? frames.filter((f) => f && f.dataUrl && f.dataUrl.length > 100) : []; + const messages = attachFramesToMessages(buildTweakPrompt({ spec, track, look, hint, frames: safeFrames }), safeFrames); recordInvocation(); const isDev = import.meta.env && import.meta.env.DEV; const proxyFetch = async (url, init = {}) => { const headers = new Headers(init.headers || {}); if (isDev) headers.set('x-llm-gateway', cfg.gatewayUrl); - // Dev proxy forwards server-side; log what we send console.info('[llm] fetch', isDev ? '→ /__llm (proxied to ' + cfg.gatewayUrl + ')' : '→ ' + cfg.gatewayUrl, String(url).slice(0, 120)); return fetch(url, { ...init, headers }); }; @@ -102,20 +125,104 @@ export async function requestSongTweak({ spec, track, look, hint = '', config = }); const modelId = cfg.model; - console.groupCollapsed(`[llm] requestSongTweak — ${modelId}${hint ? ` · hint: "${hint.slice(0, 80)}"` : ''}`); + console.groupCollapsed(`[llm] requestSongTweak — ${modelId}${hint ? ` · hint: "${hint.slice(0, 80)}"` : ''}${safeFrames.length ? ` · ${safeFrames.length} frames` : ''}`); console.info('[llm] model', modelId, 'gateway', cfg.gatewayUrl, isDev ? '(via /__llm dev proxy)' : '(direct)'); console.info('[llm] budget after record', getBudgetCounts()); console.log('[llm] messages', messages); + if (safeFrames.length) console.log('[llm] frames', safeFrames.map((f) => ({ kind: f.kind, time: f.time, frame: f.frame, bytes: f.dataUrl.length }))); console.log('[llm] spec before', spec); console.groupEnd(); + const emit = (evt) => { try { if (typeof onProgress === 'function') onProgress(evt); } catch { /* ignore */ } }; + + // Prefer streaming so the UI can show status as we wait. Fall back to + // generateObject if the gateway/model doesn't support streaming. + const useStream = typeof onProgress === 'function'; + if (useStream) { + try { + const started = performance.now(); + emit({ stage: 'streaming', text: 'Contacting LLM…' }); + const stream = await streamObject({ + model: provider.chat(modelId), + schema: TweakResultSchema, + messages, + temperature: 0.7, + providerOptions: { openai: { strictJsonSchema: false } }, + }); + + let lastPartial = null; + let lastText = ''; + for await (const part of stream.partialObjectStream) { + lastPartial = part; + // Derive a short status from whatever is already materialized. + const r = part && part.rationale ? String(part.rationale).slice(0, 120) : ''; + const editsKeys = part && part.edits ? Object.keys(part.edits).filter((k) => part.edits[k] != null) : []; + const text = r ? `… ${r}` : (editsKeys.length ? `edits: ${editsKeys.join(', ')}` : '… thinking'); + if (text !== lastText) { + lastText = text; + emit({ stage: 'partial', partial: part, text }); + console.info('[llm] stream partial', part); + } + } + + const final = await stream.object; + const usage = await stream.usage.catch(() => null); + const elapsed = ((performance.now() - started) / 1000).toFixed(1); + const raw = final || lastPartial || {}; + const edits = clampEdits(raw.edits || {}); + const rationale = String(raw.rationale || '').trim().slice(0, 800) || 'No rationale provided.'; + const confidence = Number.isFinite(raw.confidence) ? clamp01(raw.confidence) : 0.5; + + console.groupCollapsed(`[llm] stream done in ${elapsed}s — ${Object.keys(edits).length ? Object.keys(edits).join(', ') : 'no edits'}`); + console.log('[llm] raw', raw); + console.log('[llm] clamped edits', edits); + console.log('[llm] rationale', rationale); + console.log('[llm] confidence', confidence, 'usage', usage || null); + console.info('[llm] diff', Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)'); + console.groupEnd(); + + emit({ stage: 'done', partial: raw, text: rationale.slice(0, 120) }); + + if (!Object.keys(edits).length) { + refundInvocation(); + const err = new Error('LLM suggested no changes — the current generation already looks good. Try a hint or reroll.'); + err.code = 'NO_EDITS'; + err.result = { rationale, edits: {}, confidence, usage: usage || null, raw }; + throw err; + } + return { rationale, edits, confidence, usage: usage || null, raw }; + } catch (err) { + // If streaming failed before producing a result (e.g. gateway doesn't + // support it), fall through to non-streaming generateObject below — + // unless it's our own NO_EDITS signal. + if (err && err.code === 'NO_EDITS') throw err; + const msg = toErrorMessage(err).toLowerCase(); + const isStreamUnsupported = msg.includes('stream') || msg.includes('not supported') || msg.includes('unsupported'); + if (!isStreamUnsupported) { + // Real LLM error — surface as usual (don't silently fall back and + // hide it behind a second call that also fails). + const lower = msg.toLowerCase(); + if (!err.code) { + if (lower.includes('401') || lower.includes('unauthorized') || lower.includes('invalid api key')) err.code = 'AUTH'; + else if (lower.includes('429') || lower.includes('rate')) err.code = 'RATE_LIMIT'; + else if (lower.includes('timeout') || lower.includes('timed out')) err.code = 'TIMEOUT'; + else err.code = 'LLM_ERROR'; + } + if (!err.message || err.message === '[object Object]') err.message = toErrorMessage(err); + console.error('[llm] stream failed', err); + throw err; + } + console.warn('[llm] stream not supported, falling back to generateObject —', err.message); + } + } + try { const started = performance.now(); const result = await generateObject({ model: provider.chat(modelId), schema: TweakResultSchema, messages, - temperature: 0.4, + temperature: 0.7, maxRetries: 0, providerOptions: { openai: { strictJsonSchema: false } }, }); @@ -131,8 +238,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', config = console.log('[llm] clamped edits', edits); console.log('[llm] rationale', rationale); console.log('[llm] confidence', confidence, 'usage', result.usage || null); - const changed = Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)'; - console.info('[llm] diff', changed); + console.info('[llm] diff', Object.keys(edits).map((k) => `${k}: ${spec[k]} → ${edits[k]}`).join(' · ') || '(none — all null)'); console.groupEnd(); if (!Object.keys(edits).length) { diff --git a/flow-state/src/llm/spec.js b/flow-state/src/llm/spec.js index 1db83b0..5c5a787 100644 --- a/flow-state/src/llm/spec.js +++ b/flow-state/src/llm/spec.js @@ -69,15 +69,8 @@ export function clampEdits(edits) { * Kept compatible with the old `{spec,track,look,hint}` shape by accepting * either `look` (no-op here) or `lookDigest`. */ -export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) { +export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) { const summary = track ? track.summary : null; - const sections = track ? (track.sections || []).map((s) => ({ - kind: s.kind, - start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1), - end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1), - energy: Number((s.energy ?? 0).toFixed(3)), - })) : []; - const digest = lookDigest ?? (look ? look : null); const specForPrompt = spec ? { @@ -92,36 +85,167 @@ export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = seed: spec.seed ?? null, } : null; + // --- helpers for richer context --- + const r2 = (v) => { + const n = Number(v); + return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0; + }; + const r3 = (v) => { + const n = Number(v); + return Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0; + }; + + const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({ + index: Number.isFinite(s.index) ? s.index : i, + kind: s.kind || 'unknown', + start: r2(s.startSec != null ? s.startSec : (s.start ?? 0)), + end: r2(s.endSec != null ? s.endSec : (s.end ?? 0)), + duration: r2(s.duration ?? ((s.endSec != null ? s.endSec : (s.end ?? 0)) - (s.startSec != null ? s.startSec : (s.start ?? 0)))), + energy: r3(s.energy ?? 0), + flux: r3(s.flux ?? 0), + centroid: r3(s.centroid ?? 0), + flatness: r3(s.flatness ?? 0), + width: r3(s.width ?? 0), + slope: r3(s.slope ?? 0), + endEnergy: r3(s.endEnergy ?? 0), + })) : []; + + // trackSummary — extended + let trackSummary = null; + if (summary) { + trackSummary = { + bpm: summary.bpm ?? track?.tempo?.bpm ?? null, + meanCentroid: summary.meanCentroid != null ? r3(summary.meanCentroid) : null, + meanFlatness: summary.meanFlatness != null ? r3(summary.meanFlatness) : null, + meanWidth: summary.meanWidth != null ? r3(summary.meanWidth) : undefined, + meanLoudness: summary.meanLoudness != null ? r3(summary.meanLoudness) : null, + dynamicRange: summary.dynamicRange != null ? r3(summary.dynamicRange) : null, + tempoConfidence: summary.tempoConfidence != null ? r3(summary.tempoConfidence) : (track?.tempo?.confidence != null ? r3(track.tempo.confidence) : null), + frameCount: track?.frameCount ?? summary.frameCount ?? null, + fps: track?.fps ?? summary.fps ?? null, + duration: r2(track?.duration ?? summary.duration ?? 0), + sections: sectionsDetailed, + }; + if (summary.bandBalance) { + trackSummary.bandBalance = { + sub: r3(summary.bandBalance.sub ?? 0), + low: r3(summary.bandBalance.low ?? 0), + mid: r3(summary.bandBalance.mid ?? 0), + high: r3(summary.bandBalance.high ?? 0), + air: r3(summary.bandBalance.air ?? 0), + }; + } + // remove undefined meanWidth if not present to keep payload compact + if (trackSummary.meanWidth === undefined) delete trackSummary.meanWidth; + } else if (sectionsDetailed.length || track) { + // fallback when summary missing but track exists (e.g. synthetic test track) + trackSummary = { + bpm: track?.tempo?.bpm ?? null, + frameCount: track?.frameCount ?? null, + fps: track?.fps ?? null, + duration: r2(track?.duration ?? 0), + sections: sectionsDetailed, + }; + if (track?.summary?.bandBalance) { + trackSummary.bandBalance = { + sub: r3(track.summary.bandBalance.sub ?? 0), + low: r3(track.summary.bandBalance.low ?? 0), + mid: r3(track.summary.bandBalance.mid ?? 0), + high: r3(track.summary.bandBalance.high ?? 0), + air: r3(track.summary.bandBalance.air ?? 0), + }; + } + } + + // tempo + let tempo = null; + if (track?.tempo) { + tempo = { + bpm: r2(track.tempo.bpm ?? summary?.bpm ?? 0), + confidence: r3(track.tempo.confidence ?? 0), + beatsPerBar: track.tempo.beatsPerBar ?? 4, + downbeats: Array.isArray(track.tempo.downbeats) ? track.tempo.downbeats.length : 0, + }; + } else if (summary?.bpm != null) { + tempo = { bpm: r2(summary.bpm), confidence: r3(summary.tempoConfidence ?? 0), beatsPerBar: 4, downbeats: 0 }; + } + + // normalization + let normalization = null; + if (track?.normalization && typeof track.normalization === 'object') { + normalization = {}; + for (const [k, v] of Object.entries(track.normalization)) { + if (v && typeof v === 'object' && v.lo != null && v.hi != null) { + normalization[k] = { lo: r3(v.lo), hi: r3(v.hi) }; + } + } + if (!Object.keys(normalization).length) normalization = null; + } + + // stats — derived + let stats = null; + if (sectionsDetailed.length || tempo || track) { + const kinds = [...new Set(sectionsDetailed.map((s) => s.kind))]; + const avgEnergy = sectionsDetailed.length + ? r3(sectionsDetailed.reduce((a, s) => a + s.energy, 0) / sectionsDetailed.length) + : 0; + let maxSec = null; + let maxE = -Infinity; + for (const s of sectionsDetailed) if (s.energy > maxE) { maxE = s.energy; maxSec = s; } + const duration = track?.duration ?? summary?.duration ?? 0; + const bpmVal = tempo?.bpm ?? summary?.bpm ?? 120; + const bpb = tempo?.beatsPerBar ?? 4; + const bars = bpmVal > 0 && duration > 0 ? Math.round(duration / ((60 / bpmVal) * bpb)) : 0; + stats = { + totalSections: sectionsDetailed.length, + kinds, + avgEnergy, + maxEnergySection: maxSec ? { index: maxSec.index, kind: maxSec.kind, energy: maxSec.energy } : null, + bpm: tempo?.bpm ?? null, + beatsPerBar: tempo?.beatsPerBar ?? null, + downbeats: tempo?.downbeats ?? 0, + bars, + duration: r2(duration), + fps: track?.fps ?? summary?.fps ?? null, + frameCount: track?.frameCount ?? summary?.frameCount ?? null, + }; + } + const hintBlock = hint && String(hint).trim() ? `User hint: "${String(hint).trim().slice(0, 300)}"` : 'No user hint.'; + const frameNote = Array.isArray(frames) && frames.length + ? `You are also given ${frames.length} low-res thumbnails (32×32) sampled across the song in order — use them to judge color, structure, and motion.` + : null; + const system = [ 'You are a producer for a generative music-video system called flow-state.', - 'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.', + 'You suggest tweaks to the *song synthesis parameters* so the next generation looks better.', 'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.', '', 'Rules:', - '- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.', - '- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.', + 'You have FULL control — change any subset of the 7 knobs by any amount within valid ranges, including large jumps, arrangement swaps, or bpm shifts. Prefer impactful changes that clearly improve visuals; keep duration fixed.', '- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.', '- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.', '- All 7 edit fields MUST be present in edits — use null for unchanged (not omitted). Example: { bpm: null, brightness: 0.6, noise: null, noiseColour: null, dynamics: null, transition: null, arrangement: null }', '- Explain your reasoning in 1-2 sentences (rationale).', '- If the look is already good, return all-null edits.', hintBlock, - ].join('\n'); + frameNote, + ].filter(Boolean).join('\n'); - const user = JSON.stringify({ + const userPayload = { songSpec: specForPrompt, - trackSummary: summary ? { - bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness, - dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness, - tempoConfidence: summary.tempoConfidence, - } : null, - sections, + trackSummary, + sections: sectionsDetailed, + tempo, + normalization, lookDigest: digest, - }, null, 2); + stats, + }; + + const user = JSON.stringify(userPayload, null, 2); return [ { role: 'system', content: system }, diff --git a/flow-state/src/llm/thumbs.js b/flow-state/src/llm/thumbs.js new file mode 100644 index 0000000..161d100 --- /dev/null +++ b/flow-state/src/llm/thumbs.js @@ -0,0 +1,128 @@ +export function downscaleRgbaToDataUrl(pixels, srcW, srcH, dstSize) { + const dst = new Uint8ClampedArray(dstSize * dstSize * 4); + for (let y = 0; y < dstSize; y++) { + const y0 = Math.floor((y * srcH) / dstSize); + const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * srcH) / dstSize)); + for (let x = 0; x < dstSize; x++) { + const x0 = Math.floor((x * srcW) / dstSize); + const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * srcW) / dstSize)); + let r = 0, g = 0, b = 0, c = 0; + for (let sy = y0; sy < y1; sy++) + for (let sx = x0; sx < x1; sx++) { + const i = (sy * srcW + sx) * 4; + r += pixels[i]; g += pixels[i + 1]; b += pixels[i + 2]; c++; + } + const o = (y * dstSize + x) * 4; + dst[o] = c ? Math.round(r / c) : 0; + dst[o + 1] = c ? Math.round(g / c) : 0; + dst[o + 2] = c ? Math.round(b / c) : 0; + dst[o + 3] = 255; + } + } + if (typeof document !== 'undefined' && document.createElement) { + try { + const canvas = document.createElement('canvas'); + canvas.width = dstSize; canvas.height = dstSize; + const ctx = canvas.getContext('2d'); + if (ctx) { ctx.putImageData(new ImageData(dst, dstSize, dstSize), 0, 0); return canvas.toDataURL('image/jpeg', 0.6); } + } catch { /* fallback */ } + } + return 'data:image/jpeg;base64,'; +} + +function uniformFrames(count, frameCount) { + const seen = new Set(); const out = []; + for (let i = 0; i < count; i++) { + const f = Math.max(0, Math.min(frameCount - 1, Math.round(((i + 0.5) / count) * frameCount))); + if (!seen.has(f)) { seen.add(f); out.push(f); } + } + return out; +} + +function pickFrames(show, count, frameCount, fps) { + const sections = show.track && show.track.sections; + if (sections && sections.length) { + const mids = []; const seen = new Set(); + for (const s of sections) { + let mid; + if (Number.isFinite(s.startFrame) && Number.isFinite(s.endFrame)) mid = Math.round((s.startFrame + s.endFrame) / 2); + else if (Number.isFinite(s.start) && Number.isFinite(s.end)) mid = Math.round(((s.start + s.end) / 2) * fps); + else continue; + mid = Math.max(0, Math.min(frameCount - 1, mid)); + if (!seen.has(mid)) { seen.add(mid); mids.push(mid); } + } + if (!mids.length) return uniformFrames(count, frameCount); + if (mids.length >= count) { + if (mids.length === count) return mids; + mids.sort((a, b) => a - b); + const out = []; + for (let i = 0; i < count; i++) { + const idx = Math.min(mids.length - 1, Math.floor(((i + 0.5) / count) * mids.length)); + const v = mids[idx]; if (!out.includes(v)) out.push(v); + } + return out; + } + const need = count - mids.length; + for (const f of uniformFrames(need, frameCount)) if (!seen.has(f)) { seen.add(f); mids.push(f); } + if (mids.length < count) for (const f of uniformFrames(count, frameCount)) if (!seen.has(f) && mids.length < count) { seen.add(f); mids.push(f); } + const sorted = [...seen].sort((a, b) => a - b); + if (sorted.length <= count) return sorted; + const out = []; + for (let i = 0; i < count; i++) { + const idx = Math.min(sorted.length - 1, Math.floor(((i + 0.5) / count) * sorted.length)); + const v = sorted[idx]; if (!out.includes(v)) out.push(v); + } + return out; + } + return uniformFrames(count, frameCount); +} + +export async function captureThumbnails(show, { count = 8, size = 32 } = {}) { + if (!show || !show.ready || !show.track || !show.engine) return []; + const fps = show.fps || 60; + const frameCount = show.frameCount || (show.track && show.track.frameCount) || Math.round((show.track.duration || 0) * fps) || 0; + if (!frameCount) return []; + const originalFrame = show.timeline ? show.timeline.frame : 0; + const origW = show.engine.width, origH = show.engine.height; + const frames = pickFrames(show, count, frameCount, fps).sort((a, b) => a - b); + const out = []; + try { + for (const frame of frames) { + try { + if (typeof show.warmUp === 'function') { + const wf = typeof show.warmupFrames === 'function' ? show.warmupFrames() : 120; + show.warmUp(frame, Math.min(120, wf)); + } else if (show.engine.compositor) { + if (typeof show.engine.warmUp === 'function') show.engine.warmUp(frame, 120); + else show.engine.compositor.reset(); + } + } catch { /* ignore */ } + let target; try { target = show.renderFrame(frame); } catch { continue; } + let pixels; try { pixels = show.readPixels(target); } catch { continue; } + if (!pixels || !pixels.length) continue; + let dataUrl = ''; try { dataUrl = downscaleRgbaToDataUrl(pixels, origW, origH, size); } catch { dataUrl = 'data:image/jpeg;base64,'; } + let kind = ''; + try { + const sec = show.track.sectionAt ? show.track.sectionAt(frame) : null; + if (sec && sec.kind) kind = sec.kind; + else if (show.track.sections) for (const s of show.track.sections) if (Number.isFinite(s.startFrame) && frame >= s.startFrame && frame < s.endFrame) { kind = s.kind || ''; break; } + } catch { /* ignore */ } + out.push({ kind: kind || '', time: frame / fps, frame, dataUrl }); + } + } finally { + try { + if (typeof show.seek === 'function') show.seek(originalFrame); + else if (show.timeline && typeof show.timeline.seek === 'function') { + show.timeline.seek(originalFrame); + try { const t = show.renderFrame(originalFrame); if (typeof show.present === 'function') show.present(t); } catch { /* ignore */ } + } + } catch { /* ignore */ } + try { + if (show.engine && (show.engine.width !== origW || show.engine.height !== origH)) { + if (typeof show.engine.setSize === 'function') show.engine.setSize(origW, origH); + else if (typeof show.setSize === 'function') show.setSize(origW, origH); + } + } catch { /* ignore */ } + } + return out; +} diff --git a/flow-state/src/main.js b/flow-state/src/main.js index fab3cd1..4dff5b1 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -16,6 +16,7 @@ import { synthesizeSong } from './audio/synth.js'; import { FeatureTrack } from './audio/FeatureTrack.js'; import { generateLook } from './look/LookGenerator.js'; import { SONGS } from './audio/songbank.js'; +import { captureThumbnails } from './llm/thumbs.js'; const QUALITY = { draft: 0.5, // half resolution — for scrubbing heavy stacks @@ -1053,8 +1054,9 @@ async function handleLlmTweak() { const btn = document.getElementById('btn-llm-tweak'); const prevLabel = btn ? btn.textContent : ''; if (btn) btn.textContent = '… tuning'; - // Immediate feedback — budget had a one-frame lag before, so show progress explicitly. - showToast(`Contacting LLM (${cfg.model})…`, { ok: true }); + // Live progress: start capturing thumbs, then stream. Toast updates as stream arrives. + let llmProgressToast = `Contacting LLM (${cfg.model})…`; + showToast(llmProgressToast, { ok: true }); console.info('[llm] handleLlmTweak start', { model: cfg.model, gateway: cfg.gatewayUrl, remaining, hint: hint || '(none)', @@ -1062,13 +1064,38 @@ async function handleLlmTweak() { lookDirector: lookBefore && lookBefore.director, }); + // Capture 32×32 thumbnails — low-res strip for the vision model, non-blocking. + showToast('Capturing thumbnails…', { ok: true }); + let llmFrames = []; + try { + // Keep the preview responsive — allow one paint before capture. + await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))); + llmFrames = await captureThumbnails(state.show, { count: 8, size: 32 }); + console.info('[llm] captured frames', llmFrames.length, llmFrames.map((f) => `${f.kind}@${f.time.toFixed(1)}s`)); + } catch (e) { + console.warn('[llm] captureThumbnails failed', e); + } + showToast(`Contacting LLM (${cfg.model}) · ${llmFrames.length ? llmFrames.length + ' frames' : 'no frames'}…`, { ok: true }); + try { const result = await requestSongTweak({ spec: specBefore, track: trackBefore, look: lookBefore, hint: hint || '', + frames: llmFrames, config: cfg, + onProgress: (evt) => { + if (!evt) return; + const text = evt.text ? evt.text.slice(0, 80) : ''; + if (evt.stage === 'partial') { + const partialKeys = evt.partial && evt.partial.edits ? Object.keys(evt.partial.edits).filter((k) => evt.partial.edits[k] != null) : []; + showToast(`LLM streaming… ${text}${partialKeys.length ? ` · ${partialKeys.join(', ')}` : ''}`, { ok: true }); + console.info('[llm] stream', evt); + } else if (evt.stage === 'streaming') { + showToast(`LLM streaming… ${text}`, { ok: true }); + } + }, }); // Log what came back before any synthesis — this is the audit trail. diff --git a/flow-state/vite.config.js b/flow-state/vite.config.js index 2d8a66d..9319bc7 100644 --- a/flow-state/vite.config.js +++ b/flow-state/vite.config.js @@ -1,6 +1,6 @@ import { defineConfig } from 'vite'; import { resolve } from 'path'; -import { writeFileSync } from 'fs'; +import { writeFileSync, existsSync, mkdirSync, appendFileSync, statSync } from 'fs'; /** * Let the measuring pages write their results back into the source tree. @@ -63,8 +63,77 @@ function measurementWriter() { * Production builds are static; there this route does not exist and the * client falls through to a direct fetch. Deployments that need CORS-free * production should add an equivalent `/api/llm` serverless proxy. + * + * Also logs sanitized traffic (no Authorization) to `logs/llm-proxy.log` so + * the coding agent can inspect prompts/responses without re-calling the LLM. */ function llmGatewayProxy() { + const LOG_DIR = resolve(process.cwd(), 'logs'); + const LOG_FILE = resolve(LOG_DIR, 'llm-proxy.log'); + + function safeAppend(entry) { + try { + if (!existsSync(LOG_DIR)) mkdirSync(LOG_DIR, { recursive: true }); + // Rotate: truncate if >1MB to avoid unbounded growth + try { + if (existsSync(LOG_FILE) && statSync(LOG_FILE).size > 1024 * 1024) { + writeFileSync(LOG_FILE, ''); + } + } catch { /* ignore stat/truncate errors */ } + appendFileSync(LOG_FILE, JSON.stringify(entry) + '\n'); + } catch { /* ignore */ } + } + + function sanitizePathForLog(path) { + try { + const u = new URL(path, 'http://localhost'); + for (const key of [...u.searchParams.keys()]) { + const lower = key.toLowerCase(); + if (lower === 'apikey' || lower === 'api_key' || lower === 'authorization') { + u.searchParams.delete(key); + } + } + return u.pathname + u.search; + } catch { + // Fallback: strip sensitive query params via regex + return path.replace(/([?&])(apiKey|api_key|authorization)=[^&]*/gi, '').replace(/[?&]$/, '').replace(/\?&/, '?') || '/'; + } + } + + function redactJsonForLog(obj) { + if (Array.isArray(obj)) return obj.map(redactJsonForLog); + if (obj && typeof obj === 'object') { + const out = {}; + for (const [k, v] of Object.entries(obj)) { + const lower = k.toLowerCase(); + if (lower === 'apikey' || lower === 'api_key' || lower === 'authorization') { + out[k] = '[REDACTED]'; + } else { + out[k] = redactJsonForLog(v); + } + } + return out; + } + return obj; + } + + function sanitizeBodyForLog(ct, raw) { + // Don't log if content-type is missing or body is binary + if (!ct || !ct.includes('json')) { + if (raw && raw.length > 4096) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`; + try { return raw ? raw.toString('utf8').slice(0, 4096) : ''; } catch { return '[unprintable]'; } + } + try { + const text = raw ? raw.toString('utf8') : ''; + const parsed = JSON.parse(text); + // Redact sensitive keys if they somehow appear in JSON body + const redacted = redactJsonForLog(parsed); + return JSON.stringify(redacted).slice(0, 16384); + } catch { + return raw ? raw.toString('utf8').slice(0, 4096) : ''; + } + } + return { name: 'flow-state-llm-gateway-proxy', apply: 'serve', @@ -85,16 +154,7 @@ function llmGatewayProxy() { } const gatewayBase = (req.headers['x-llm-gateway'] && String(req.headers['x-llm-gateway'])) || 'https://ai-gateway.vercel.sh'; - // Strip the proxy prefix; the client sends `/__llm/v1/...`, - // so after stripping we have `/v1/...` which joins correctly. const pathAfter = req.url.replace(/^\/__llm/, '') || '/'; - // Preserve query string - const targetUrl = new URL(pathAfter, gatewayBase.endsWith('/') ? gatewayBase : gatewayBase + '/'); - // But `new URL('/v1/x', 'https://host/v1')` drops the base path — - // `/v1/x` is absolute. Since `pathAfter` already starts with `/v1`, - // joining against the origin is correct; gatewayBase's path is just - // the fallback origin. So rebuild against origin only. - // Simpler: construct from origin + pathAfter let origin; try { origin = new URL(gatewayBase).origin; @@ -106,14 +166,28 @@ function llmGatewayProxy() { let bodyChunks = []; req.on('data', (c) => bodyChunks.push(c)); req.on('end', async () => { + const started = Date.now(); + const reqCt = req.headers['content-type'] || ''; + const rawReq = bodyChunks.length ? Buffer.concat(bodyChunks) : null; + const reqLog = sanitizeBodyForLog(reqCt, rawReq); + // Always log the request (sanitized — no Authorization/apiKey) + safeAppend({ + ts: new Date().toISOString(), + dir: '→', + method: req.method, + path: sanitizePathForLog(pathAfter), + gateway: origin, + contentType: reqCt, + // Authorization header and apiKey query param never logged + body: reqLog, + }); + try { - const body = bodyChunks.length ? Buffer.concat(bodyChunks) : undefined; + const body = rawReq && rawReq.length ? rawReq : undefined; const headers = {}; - // Forward relevant headers if (req.headers['content-type']) headers['content-type'] = req.headers['content-type']; if (req.headers['authorization']) headers['authorization'] = req.headers['authorization']; else if (req.headers['x-api-key']) headers['authorization'] = `Bearer ${req.headers['x-api-key']}`; - // Also forward any x- header the gateway might care about for (const [k, v] of Object.entries(req.headers)) { if (k.startsWith('x-') && k !== 'x-llm-gateway') headers[k] = v; } @@ -124,7 +198,6 @@ function llmGatewayProxy() { body: req.method === 'GET' || req.method === 'HEAD' ? undefined : body, }); - // Mirror status and content headers const contentType = upstream.headers.get('content-type'); res.writeHead(upstream.status, { ...(contentType ? { 'content-type': contentType } : {}), @@ -133,7 +206,47 @@ function llmGatewayProxy() { const buf = Buffer.from(await upstream.arrayBuffer()); res.end(buf); + + // Log response (sanitized — still no token in body) + const elapsed = Date.now() - started; + let respLog = ''; + try { + const ct = contentType || ''; + if (ct.includes('json') || ct.includes('event-stream') || buf.length < 32768) { + const text = buf.toString('utf8'); + // For SSE streams, keep first 16k; for JSON, keep parsed form + if (ct.includes('json')) { + try { respLog = JSON.stringify(JSON.parse(text)).slice(0, 16384); } catch { respLog = text.slice(0, 8192); } + } else { + respLog = text.slice(0, 8192); + } + } else { + respLog = `[${contentType} ${buf.length} bytes]`; + } + } catch { respLog = `[${buf.length} bytes]`; } + + safeAppend({ + ts: new Date().toISOString(), + dir: '←', + status: upstream.status, + elapsedMs: elapsed, + contentType: contentType || '', + body: respLog, + }); + + // Also emit to vite's console for live tailing + const tag = upstream.status >= 400 ? '← ERR' : '← OK'; + console.info(`[llm-proxy] ${tag} ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${buf.length} bytes)`); } catch (err) { + const elapsed = Date.now() - started; + safeAppend({ + ts: new Date().toISOString(), + dir: '←', + status: 502, + elapsedMs: elapsed, + error: String(err && err.message ? err.message : err).slice(0, 2000), + }); + console.error(`[llm-proxy] ← 502 ${req.method} ${pathAfter}`, err); res.writeHead(502, { 'access-control-allow-origin': '*' }); res.end(String(err && err.message ? err.message : err)); }