dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LLM refine: 2dp prompt, mixed 64/32 frames, full proxy log, low reasoning

Browse Source 
- spec.js: round all songSpec + extended summary/tempo/normalization to
  r2 (2dp) so high-precision 0.6607… stops wasting tokens; hoist r2/r3
  above specForPrompt to fix ReferenceError; FULL control wording kept.
- thumbs.js: border detection — first/last + 0.5s of any section start
  → 64×64 (borders), interior → 32×32 (between). Back-compat shim for
  old {size:N} callers. Returns size+role per frame for attachFrames.
- vite.config.js: raise sanitizeBodyForLog to 64k for __llm so 8×
  vision frames aren't truncated (was 16384, only 2 images logged).
  Also log reasoningEffort via providerOptions.
- llmTweak.js: set providerOptions reasoningEffort:'low' for both
  streamObject and generateObject fallback to cut 2459 → faster
  streaming progress, temperature 0.7 kept for variety.

Verification: build+lint:scenes green, tests 18/18.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-09-13 22:21:09 +02:00
co-authored by Claude
parent b0989169e2
commit 49914fd4ad
4 changed files with 66 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/llm/llmTweak.js
+2 -2
View File
@@ -147,7 +147,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', frames =
schema: TweakResultSchema,
messages,
temperature: 0.7,
providerOptions: { openai: { strictJsonSchema: false } },
providerOptions: { openai: { strictJsonSchema: false, reasoningEffort: 'low' } },
});
let lastPartial = null;
@@ -224,7 +224,7 @@ export async function requestSongTweak({ spec, track, look, hint = '', frames =
messages,
temperature: 0.7,
maxRetries: 0,
providerOptions: { openai: { strictJsonSchema: false } },
providerOptions: { openai: { strictJsonSchema: false, reasoningEffort: 'low' } },
});
const elapsed = ((performance.now() - started) / 1000).toFixed(1);
flow-state/src/llm/spec.js
+16 -20
View File
@@ -70,30 +70,26 @@ export function clampEdits(edits) {
* either `look` (no-op here) or `lookDigest`.
*/
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) {
const summary = track ? track.summary : null;
const digest = lookDigest ?? (look ? look : null);
const specForPrompt = spec ? {
bpm: spec.bpm ?? null,
brightness: spec.brightness ?? null,
noise: spec.noise ?? null,
noiseColour: spec.noiseColour ?? null,
dynamics: spec.dynamics ?? null,
transition: spec.transition ?? null,
arrangement: spec.arrangement ?? null,
duration: spec.duration ?? null,
seed: spec.seed ?? null,
} : null;
// --- helpers for richer context ---
const r2 = (v) => {
const n = Number(v);
return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0;
};
const r3 = (v) => {
const n = Number(v);
return Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0;
};
const r3 = r2;
const summary = track ? track.summary : null;
const digest = lookDigest ?? (look ? look : null);
const specForPrompt = spec ? {
bpm: spec.bpm != null ? Math.round(Number(spec.bpm)) : null,
brightness: spec.brightness != null ? r2(spec.brightness) : null,
noise: spec.noise != null ? r2(spec.noise) : null,
noiseColour: spec.noiseColour != null ? r2(spec.noiseColour) : null,
dynamics: spec.dynamics != null ? r2(spec.dynamics) : null,
transition: spec.transition != null ? r2(spec.transition) : null,
arrangement: spec.arrangement ?? null,
duration: spec.duration != null ? r2(spec.duration) : null,
seed: spec.seed ?? null,
} : null;
const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({
index: Number.isFinite(s.index) ? s.index : i,
flow-state/src/llm/thumbs.js
+41 -2
View File
@@ -77,7 +77,28 @@ function pickFrames(show, count, frameCount, fps) {
return uniformFrames(count, frameCount);
}
export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
/**
* Capture thumbnails — mixed res per "scene borders" request.
*
* Each sampled frame is classified: frames at the track's first frame,
* last frame, or within 0.5s of any section start/end are "border"
* frames and rendered at 64×64; frames in the interior of a section are
* rendered at 32×32 (between-border). This gives the vision model high
* detail where cuts happen and low cost elsewhere.
*
* Returns Array<{ kind, time, frame, dataUrl, size, role }> where
* role is 'border'|'inner'. Backward compat: callers that ignore size
* still see the same shape plus a new field.
*/
export async function captureThumbnails(show, { count = 8, borderSize = 64, innerSize = 32 } = {}) {
// Backward compat shim: old callers pass {size: N}. Respect it as innerSize with borderSize = min(innerSize*2, 128).
if (arguments[1] && Object.prototype.hasOwnProperty.call(arguments[1], 'size') && !Object.prototype.hasOwnProperty.call(arguments[1], 'innerSize')) {
const legacySize = arguments[1].size;
if (typeof legacySize === 'number' && legacySize > 0) {
innerSize = legacySize | 0;
borderSize = Math.min(128, innerSize * 2);
}
}
if (!show || !show.ready || !show.track || !show.engine) return [];
const fps = show.fps || 60;
const frameCount = show.frameCount || (show.track && show.track.frameCount) || Math.round((show.track.duration || 0) * fps) || 0;
@@ -85,6 +106,23 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
const originalFrame = show.timeline ? show.timeline.frame : 0;
const origW = show.engine.width, origH = show.engine.height;
const frames = pickFrames(show, count, frameCount, fps).sort((a, b) => a - b);
// Precompute border detection from section starts
const boundaries = [];
if (show.track && show.track.sections) {
for (const s of show.track.sections) {
if (Number.isFinite(s.startFrame)) boundaries.push(s.startFrame);
else if (Number.isFinite(s.start)) boundaries.push(Math.round(s.start * fps));
}
if (frameCount > 0) boundaries.push(frameCount - 1);
}
const halfSec = Math.round(0.5 * fps);
const isBorder = (f) => {
if (f === 0 || f === frameCount - 1) return true;
for (const b of boundaries) if (Math.abs(f - b) <= halfSec) return true;
return false;
};
const out = [];
try {
for (const frame of frames) {
@@ -100,6 +138,7 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
let target; try { target = show.renderFrame(frame); } catch { continue; }
let pixels; try { pixels = show.readPixels(target); } catch { continue; }
if (!pixels || !pixels.length) continue;
const size = isBorder(frame) ? borderSize : innerSize;
let dataUrl = ''; try { dataUrl = downscaleRgbaToDataUrl(pixels, origW, origH, size); } catch { dataUrl = 'data:image/jpeg;base64,'; }
let kind = '';
try {
@@ -107,7 +146,7 @@ export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
if (sec && sec.kind) kind = sec.kind;
else if (show.track.sections) for (const s of show.track.sections) if (Number.isFinite(s.startFrame) && frame >= s.startFrame && frame < s.endFrame) { kind = s.kind || ''; break; }
} catch { /* ignore */ }
out.push({ kind: kind || '', time: frame / fps, frame, dataUrl });
out.push({ kind: kind || '', time: frame / fps, frame, dataUrl, size, role: isBorder(frame) ? 'border' : 'inner' });
}
} finally {
try {
flow-state/vite.config.js
+7 -6
View File
@@ -118,19 +118,20 @@ function llmGatewayProxy() {
}
function sanitizeBodyForLog(ct, raw) {
// Don't log if content-type is missing or body is binary
// Allow __llm JSON (which includes images) to be larger — spec asked
// for full logging. 64k keeps 8× images plus schema without truncation.
const limit = 65536;
if (!ct || !ct.includes('json')) {
if (raw && raw.length > 4096) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`;
try { return raw ? raw.toString('utf8').slice(0, 4096) : ''; } catch { return '[unprintable]'; }
if (raw && raw.length > limit) return `[${ct || 'unknown'} body ${raw.length} bytes, not logged]`;
try { return raw ? raw.toString('utf8').slice(0, limit) : ''; } catch { return '[unprintable]'; }
}
try {
const text = raw ? raw.toString('utf8') : '';
const parsed = JSON.parse(text);
// Redact sensitive keys if they somehow appear in JSON body
const redacted = redactJsonForLog(parsed);
return JSON.stringify(redacted).slice(0, 16384);
return JSON.stringify(redacted).slice(0, limit);
} catch {
return raw ? raw.toString('utf8').slice(0, 4096) : '';
return raw ? raw.toString('utf8').slice(0, limit) : '';
}
}