dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LLM: streaming + rich context + 32×32 frames + proxy log

Browse Source 
- Streaming: requestSongTweak now uses streamObject when a progress
  callback is provided (via handleLlmTweak), emitting partial {rationale,
  edits} and updating toast/console as text arrives; falls back to
  generateObject if gateway doesn't support streaming. Models always
  called via provider.chat(modelId) / /v1/chat/completions, the correct
  path for meta/muse-spark-1.3-contributor via gateway.

- Rich context: spec.js gains FULL control wording (any amount/arrangement/
  bpm allowed), and detailed payload — trackSummary with bandBalance,
  tempo, normalization (per-feature lo/hi), sectionsDetailed (energy/
  flux/centroid/flatness/width/slope/endEnergy/duration), stats
  (kinds, avg/max energy, bars). LLM told You have FULL control.

- Low-res vision: new src/llm/thumbs.js — captureThumbnails(show, {8,32})
  samples section midpoints, warms up state, renders via Show, box-averages
  down to 32×32 JPEG data URLs, restores timeline. main.js now captures 8
  frames with two animation frames of pause, then passes them as {type:
  image, image: dataUrl} parts in the user message (llmTweak.js
  attachFramesToMessages).

- Proxy logging (no bearer): vite.config.js llmGatewayProxy now logs
  sanitized requests/responses to logs/llm-proxy.log (JSON lines, 1MB
  rotation, redacts apiKey/authorization from path/body/headers) so the
  coding agent can inspect prompts without re-calling. .gitignore adds
  logs/.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-09-13 21:54:07 +02:00
co-authored by Claude
parent e2f56e3ce4
commit e0e619af85
6 changed files with 548 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/src/llm/spec.js
+144 -20
View File
@@ -69,15 +69,8 @@ export function clampEdits(edits) {
* Kept compatible with the old `{spec,track,look,hint}` shape by accepting
* either `look` (no-op here) or `lookDigest`.
*/
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '' } = {}) {
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) {
const summary = track ? track.summary : null;
const sections = track ? (track.sections || []).map((s) => ({
kind: s.kind,
start: Number(s.startSec != null ? s.startSec : s.start).toFixed(1),
end: Number(s.endSec != null ? s.endSec : s.end).toFixed(1),
energy: Number((s.energy ?? 0).toFixed(3)),
})) : [];
const digest = lookDigest ?? (look ? look : null);
const specForPrompt = spec ? {
@@ -92,36 +85,167 @@ export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint =
seed: spec.seed ?? null,
} : null;
// --- helpers for richer context ---
const r2 = (v) => {
const n = Number(v);
return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0;
};
const r3 = (v) => {
const n = Number(v);
return Number.isFinite(n) ? Math.round(n * 1000) / 1000 : 0;
};
const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({
index: Number.isFinite(s.index) ? s.index : i,
kind: s.kind || 'unknown',
start: r2(s.startSec != null ? s.startSec : (s.start ?? 0)),
end: r2(s.endSec != null ? s.endSec : (s.end ?? 0)),
duration: r2(s.duration ?? ((s.endSec != null ? s.endSec : (s.end ?? 0)) - (s.startSec != null ? s.startSec : (s.start ?? 0)))),
energy: r3(s.energy ?? 0),
flux: r3(s.flux ?? 0),
centroid: r3(s.centroid ?? 0),
flatness: r3(s.flatness ?? 0),
width: r3(s.width ?? 0),
slope: r3(s.slope ?? 0),
endEnergy: r3(s.endEnergy ?? 0),
})) : [];
// trackSummary — extended
let trackSummary = null;
if (summary) {
trackSummary = {
bpm: summary.bpm ?? track?.tempo?.bpm ?? null,
meanCentroid: summary.meanCentroid != null ? r3(summary.meanCentroid) : null,
meanFlatness: summary.meanFlatness != null ? r3(summary.meanFlatness) : null,
meanWidth: summary.meanWidth != null ? r3(summary.meanWidth) : undefined,
meanLoudness: summary.meanLoudness != null ? r3(summary.meanLoudness) : null,
dynamicRange: summary.dynamicRange != null ? r3(summary.dynamicRange) : null,
tempoConfidence: summary.tempoConfidence != null ? r3(summary.tempoConfidence) : (track?.tempo?.confidence != null ? r3(track.tempo.confidence) : null),
frameCount: track?.frameCount ?? summary.frameCount ?? null,
fps: track?.fps ?? summary.fps ?? null,
duration: r2(track?.duration ?? summary.duration ?? 0),
sections: sectionsDetailed,
};
if (summary.bandBalance) {
trackSummary.bandBalance = {
sub: r3(summary.bandBalance.sub ?? 0),
low: r3(summary.bandBalance.low ?? 0),
mid: r3(summary.bandBalance.mid ?? 0),
high: r3(summary.bandBalance.high ?? 0),
air: r3(summary.bandBalance.air ?? 0),
};
}
// remove undefined meanWidth if not present to keep payload compact
if (trackSummary.meanWidth === undefined) delete trackSummary.meanWidth;
} else if (sectionsDetailed.length || track) {
// fallback when summary missing but track exists (e.g. synthetic test track)
trackSummary = {
bpm: track?.tempo?.bpm ?? null,
frameCount: track?.frameCount ?? null,
fps: track?.fps ?? null,
duration: r2(track?.duration ?? 0),
sections: sectionsDetailed,
};
if (track?.summary?.bandBalance) {
trackSummary.bandBalance = {
sub: r3(track.summary.bandBalance.sub ?? 0),
low: r3(track.summary.bandBalance.low ?? 0),
mid: r3(track.summary.bandBalance.mid ?? 0),
high: r3(track.summary.bandBalance.high ?? 0),
air: r3(track.summary.bandBalance.air ?? 0),
};
}
}
// tempo
let tempo = null;
if (track?.tempo) {
tempo = {
bpm: r2(track.tempo.bpm ?? summary?.bpm ?? 0),
confidence: r3(track.tempo.confidence ?? 0),
beatsPerBar: track.tempo.beatsPerBar ?? 4,
downbeats: Array.isArray(track.tempo.downbeats) ? track.tempo.downbeats.length : 0,
};
} else if (summary?.bpm != null) {
tempo = { bpm: r2(summary.bpm), confidence: r3(summary.tempoConfidence ?? 0), beatsPerBar: 4, downbeats: 0 };
}
// normalization
let normalization = null;
if (track?.normalization && typeof track.normalization === 'object') {
normalization = {};
for (const [k, v] of Object.entries(track.normalization)) {
if (v && typeof v === 'object' && v.lo != null && v.hi != null) {
normalization[k] = { lo: r3(v.lo), hi: r3(v.hi) };
}
}
if (!Object.keys(normalization).length) normalization = null;
}
// stats — derived
let stats = null;
if (sectionsDetailed.length || tempo || track) {
const kinds = [...new Set(sectionsDetailed.map((s) => s.kind))];
const avgEnergy = sectionsDetailed.length
? r3(sectionsDetailed.reduce((a, s) => a + s.energy, 0) / sectionsDetailed.length)
: 0;
let maxSec = null;
let maxE = -Infinity;
for (const s of sectionsDetailed) if (s.energy > maxE) { maxE = s.energy; maxSec = s; }
const duration = track?.duration ?? summary?.duration ?? 0;
const bpmVal = tempo?.bpm ?? summary?.bpm ?? 120;
const bpb = tempo?.beatsPerBar ?? 4;
const bars = bpmVal > 0 && duration > 0 ? Math.round(duration / ((60 / bpmVal) * bpb)) : 0;
stats = {
totalSections: sectionsDetailed.length,
kinds,
avgEnergy,
maxEnergySection: maxSec ? { index: maxSec.index, kind: maxSec.kind, energy: maxSec.energy } : null,
bpm: tempo?.bpm ?? null,
beatsPerBar: tempo?.beatsPerBar ?? null,
downbeats: tempo?.downbeats ?? 0,
bars,
duration: r2(duration),
fps: track?.fps ?? summary?.fps ?? null,
frameCount: track?.frameCount ?? summary?.frameCount ?? null,
};
}
const hintBlock = hint && String(hint).trim()
? `User hint: "${String(hint).trim().slice(0, 300)}"`
: 'No user hint.';
const frameNote = Array.isArray(frames) && frames.length
? `You are also given ${frames.length} low-res thumbnails (32×32) sampled across the song in order — use them to judge color, structure, and motion.`
: null;
const system = [
'You are a producer for a generative music-video system called flow-state.',
'You suggest SMALL tweaks to the *song synthesis parameters* so the next generation looks better.',
'You suggest tweaks to the *song synthesis parameters* so the next generation looks better.',
'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.',
'',
'Rules:',
'- Change at most 2-3 knobs, each by ≤0.15 (brightness/noise/dynamics/transition/noiseColour) or bpm ±8, or one arrangement step.',
'- Prefer lowering agitation for breakdowns/intros; keep section count ≥3 if possible.',
'You have FULL control — change any subset of the 7 knobs by any amount within valid ranges, including large jumps, arrangement swaps, or bpm shifts. Prefer impactful changes that clearly improve visuals; keep duration fixed.',
'- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.',
'- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.',
'- All 7 edit fields MUST be present in edits — use null for unchanged (not omitted). Example: { bpm: null, brightness: 0.6, noise: null, noiseColour: null, dynamics: null, transition: null, arrangement: null }',
'- Explain your reasoning in 1-2 sentences (rationale).',
'- If the look is already good, return all-null edits.',
hintBlock,
].join('\n');
frameNote,
].filter(Boolean).join('\n');
const user = JSON.stringify({
const userPayload = {
songSpec: specForPrompt,
trackSummary: summary ? {
bpm: summary.bpm, meanCentroid: summary.meanCentroid, meanFlatness: summary.meanFlatness,
dynamicRange: summary.dynamicRange, meanLoudness: summary.meanLoudness,
tempoConfidence: summary.tempoConfidence,
} : null,
sections,
trackSummary,
sections: sectionsDetailed,
tempo,
normalization,
lookDigest: digest,
}, null, 2);
stats,
};
const user = JSON.stringify(userPayload, null, 2);
return [
{ role: 'system', content: system },