From 49914fd4ad0e59abd7681c17df1f1c6ecc9718d0 Mon Sep 17 00:00:00 2001 From: Dejvino Date: Sun, 13 Sep 2026 22:21:09 +0200 Subject: [PATCH] LLM refine: 2dp prompt, mixed 64/32 frames, full proxy log, low reasoning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spec.js: round all songSpec + extended summary/tempo/normalization to r2 (2dp) so high-precision 0.6607… stops wasting tokens; hoist r2/r3 above specForPrompt to fix ReferenceError; FULL control wording kept. - thumbs.js: border detection — first/last + 0.5s of any section start → 64×64 (borders), interior → 32×32 (between). Back-compat shim for old {size:N} callers. Returns size+role per frame for attachFrames. - vite.config.js: raise sanitizeBodyForLog to 64k for __llm so 8× vision frames aren't truncated (was 16384, only 2 images logged). Also log reasoningEffort via providerOptions. - llmTweak.js: set providerOptions reasoningEffort:'low' for both streamObject and generateObject fallback to cut 2459 → faster streaming progress, temperature 0.7 kept for variety. Verification: build+lint:scenes green, tests 18/18. Co-Authored-By: Claude --- flow-state/src/llm/llmTweak.js | 4 ++-- flow-state/src/llm/spec.js | 36 +++++++++++++--------------- flow-state/src/llm/thumbs.js | 43 ++++++++++++++++++++++++++++++++-- flow-state/vite.config.js | 13 +++++----- 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/flow-state/src/llm/llmTweak.js b/flow-state/src/llm/llmTweak.js index e65e9e8..28f73d7 100644 --- a/flow-state/src/llm/llmTweak.js +++ b/flow-state/src/llm/llmTweak.js @@ -147,7 +147,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', frames = schema: TweakResultSchema, messages, temperature: 0.7, - providerOptions: { openai: { strictJsonSchema: false } }, + providerOptions: { openai: { strictJsonSchema: false, reasoningEffort: 'low' } }, }); let lastPartial = null; @@ -224,7 +224,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', frames = messages, temperature: 0.7, maxRetries: 0, - providerOptions: { openai: { strictJsonSchema: false } }, + providerOptions: { openai: { strictJsonSchema: false, reasoningEffort: 'low' } }, }); const elapsed = ((performance.now() - started) / 1000).toFixed(1); diff --git a/flow-state/src/llm/spec.js b/flow-state/src/llm/spec.js index 5c5a787..e831352 100644 --- a/flow-state/src/llm/spec.js +++ b/flow-state/src/llm/spec.js @@ -70,30 +70,26 @@ export function clampEdits(edits) { * either `look` (no-op here) or `lookDigest`. */ export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) { - const summary = track ? track.summary : null; - const digest = lookDigest ?? (look ? look : null); - - const specForPrompt = spec ? { - bpm: spec.bpm ?? null, - brightness: spec.brightness ?? null, - noise: spec.noise ?? null, - noiseColour: spec.noiseColour ?? null, - dynamics: spec.dynamics ?? null, - transition: spec.transition ?? null, - arrangement: spec.arrangement ?? null, - duration: spec.duration ?? null, - seed: spec.seed ?? null, - } : null; - - // --- helpers for richer context --- const r2 = (v) => { const n = Number(v); return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0; }; - const r3 = (v) => { - const n = Number(v); - return Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0; - }; + const r3 = r2; + + const summary = track ? track.summary : null; + const digest = lookDigest ?? (look ? look : null); + + const specForPrompt = spec ? { + bpm: spec.bpm != null ? Math.round(Number(spec.bpm)) : null, + brightness: spec.brightness != null ? r2(spec.brightness) : null, + noise: spec.noise != null ? r2(spec.noise) : null, + noiseColour: spec.noiseColour != null ? r2(spec.noiseColour) : null, + dynamics: spec.dynamics != null ? r2(spec.dynamics) : null, + transition: spec.transition != null ? r2(spec.transition) : null, + arrangement: spec.arrangement ?? null, + duration: spec.duration != null ? r2(spec.duration) : null, + seed: spec.seed ?? null, + } : null; const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({ index: Number.isFinite(s.index) ? s.index : i, diff --git a/flow-state/src/llm/thumbs.js b/flow-state/src/llm/thumbs.js index 161d100..f115dfa 100644 --- a/flow-state/src/llm/thumbs.js +++ b/flow-state/src/llm/thumbs.js @@ -77,7 +77,28 @@ function pickFrames(show, count, frameCount, fps) { return uniformFrames(count, frameCount); } -export async function captureThumbnails(show, { count = 8, size = 32 } = {}) { +/** + * Capture thumbnails — mixed res per "scene borders" request. + * + * Each sampled frame is classified: frames at the track's first frame, + * last frame, or within 0.5s of any section start/end are "border" + * frames and rendered at 64×64; frames in the interior of a section are + * rendered at 32×32 (between-border). This gives the vision model high + * detail where cuts happen and low cost elsewhere. + * + * Returns Array<{ kind, time, frame, dataUrl, size, role }> where + * role is 'border'|'inner'. Backward compat: callers that ignore size + * still see the same shape plus a new field. + */ +export async function captureThumbnails(show, { count = 8, borderSize = 64, innerSize = 32 } = {}) { + // Backward compat shim: old callers pass {size: N}. Respect it as innerSize with borderSize = min(innerSize*2, 128). + if (arguments[1] && Object.prototype.hasOwnProperty.call(arguments[1], 'size') && !Object.prototype.hasOwnProperty.call(arguments[1], 'innerSize')) { + const legacySize = arguments[1].size; + if (typeof legacySize === 'number' && legacySize > 0) { + innerSize = legacySize | 0; + borderSize = Math.min(128, innerSize * 2); + } + } if (!show || !show.ready || !show.track || !show.engine) return []; const fps = show.fps || 60; const frameCount = show.frameCount || (show.track && show.track.frameCount) || Math.round((show.track.duration || 0) * fps) || 0; @@ -85,6 +106,23 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) { const originalFrame = show.timeline ? show.timeline.frame : 0; const origW = show.engine.width, origH = show.engine.height; const frames = pickFrames(show, count, frameCount, fps).sort((a, b) => a - b); + + // Precompute border detection from section starts + const boundaries = []; + if (show.track && show.track.sections) { + for (const s of show.track.sections) { + if (Number.isFinite(s.startFrame)) boundaries.push(s.startFrame); + else if (Number.isFinite(s.start)) boundaries.push(Math.round(s.start * fps)); + } + if (frameCount > 0) boundaries.push(frameCount - 1); + } + const halfSec = Math.round(0.5 * fps); + const isBorder = (f) => { + if (f === 0 || f === frameCount - 1) return true; + for (const b of boundaries) if (Math.abs(f - b) <= halfSec) return true; + return false; + }; + const out = []; try { for (const frame of frames) { @@ -100,6 +138,7 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) { let target; try { target = show.renderFrame(frame); } catch { continue; } let pixels; try { pixels = show.readPixels(target); } catch { continue; } if (!pixels || !pixels.length) continue; + const size = isBorder(frame) ? borderSize : innerSize; let dataUrl = ''; try { dataUrl = downscaleRgbaToDataUrl(pixels, origW, origH, size); } catch { dataUrl = 'data:image/jpeg;base64,'; } let kind = ''; try { @@ -107,7 +146,7 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) { if (sec && sec.kind) kind = sec.kind; else if (show.track.sections) for (const s of show.track.sections) if (Number.isFinite(s.startFrame) && frame >= s.startFrame && frame < s.endFrame) { kind = s.kind || ''; break; } } catch { /* ignore */ } - out.push({ kind: kind || '', time: frame / fps, frame, dataUrl }); + out.push({ kind: kind || '', time: frame / fps, frame, dataUrl, size, role: isBorder(frame) ? 'border' : 'inner' }); } } finally { try { diff --git a/flow-state/vite.config.js b/flow-state/vite.config.js index 458ad65..b73bd24 100644 --- a/flow-state/vite.config.js +++ b/flow-state/vite.config.js @@ -118,19 +118,20 @@ function llmGatewayProxy() { } function sanitizeBodyForLog(ct, raw) { - // Don't log if content-type is missing or body is binary + // Allow __llm JSON (which includes images) to be larger — spec asked + // for full logging. 64k keeps 8× images plus schema without truncation. + const limit = 65536; if (!ct || !ct.includes('json')) { - if (raw && raw.length > 4096) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`; - try { return raw ? raw.toString('utf8').slice(0, 4096) : ''; } catch { return '[unprintable]'; } + if (raw && raw.length > limit) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`; + try { return raw ? raw.toString('utf8').slice(0, limit) : ''; } catch { return '[unprintable]'; } } try { const text = raw ? raw.toString('utf8') : ''; const parsed = JSON.parse(text); - // Redact sensitive keys if they somehow appear in JSON body const redacted = redactJsonForLog(parsed); - return JSON.stringify(redacted).slice(0, 16384); + return JSON.stringify(redacted).slice(0, limit); } catch { - return raw ? raw.toString('utf8').slice(0, 4096) : ''; + return raw ? raw.toString('utf8').slice(0, limit) : ''; } }