DejvinoandClaude
49914fd4ad
- spec.js: round all songSpec + extended summary/tempo/normalization to
r2 (2dp) so high-precision 0.6607… stops wasting tokens; hoist r2/r3
above specForPrompt to fix ReferenceError; FULL control wording kept.
- thumbs.js: border detection — first/last + 0.5s of any section start
→ 64×64 (borders), interior → 32×32 (between). Back-compat shim for
old {size:N} callers. Returns size+role per frame for attachFrames.
- vite.config.js: raise sanitizeBodyForLog to 64k for __llm so 8×
vision frames aren't truncated (was 16384, only 2 images logged).
Also log reasoningEffort via providerOptions.
- llmTweak.js: set providerOptions reasoningEffort:'low' for both
streamObject and generateObject fallback to cut 2459 → faster
streaming progress, temperature 0.7 kept for variety.
Verification: build+lint:scenes green, tests 18/18.
Co-Authored-By: Claude <noreply@anthropic.com>
251 lines
11 KiB
JavaScript
251 lines
11 KiB
JavaScript
// Pure prompt/schema logic — no gateway, no LookGenerator/Scene imports.
|
||
//
|
||
// Split from llmTweak.js so `node --test` can import it without pulling in
|
||
// `src/scenes/metadata.json` (which requires Vite's JSON handling and fails
|
||
// under bare Node's import-assertion check).
|
||
|
||
import { z } from 'zod';
|
||
import { ARRANGEMENTS } from '../audio/synth.js';
|
||
|
||
const ARRANGEMENT_VALUES = Object.keys(ARRANGEMENTS);
|
||
|
||
export { ARRANGEMENT_VALUES };
|
||
|
||
export const TweakEditsSchema = z.object({
|
||
bpm: z.number().int().min(60).max(180).nullable(),
|
||
brightness: z.number().min(0).max(1).nullable(),
|
||
noise: z.number().min(0).max(1).nullable(),
|
||
noiseColour: z.number().min(0).max(1).nullable(),
|
||
dynamics: z.number().min(0).max(1).nullable(),
|
||
transition: z.number().min(0).max(1).nullable(),
|
||
arrangement: z.enum(ARRANGEMENT_VALUES).nullable(),
|
||
});
|
||
|
||
export const TweakResultSchema = z.object({
|
||
rationale: z.string().min(1).max(800),
|
||
edits: TweakEditsSchema,
|
||
confidence: z.number().min(0).max(1),
|
||
});
|
||
|
||
export const ALLOWED_KEYS = ['bpm', 'brightness', 'noise', 'noiseColour', 'dynamics', 'transition', 'arrangement'];
|
||
|
||
const clamp01 = (x) => Math.max(0, Math.min(1, x));
|
||
const round2 = (x) => Math.round(x * 100) / 100;
|
||
|
||
export function clampEdits(edits) {
|
||
if (!edits || typeof edits !== 'object') return {};
|
||
const out = {};
|
||
if (edits.bpm != null) {
|
||
const n = Number(edits.bpm);
|
||
if (Number.isFinite(n)) out.bpm = Math.max(60, Math.min(180, Math.round(n)));
|
||
}
|
||
for (const k of ['brightness', 'noise', 'dynamics', 'transition']) {
|
||
if (edits[k] != null) {
|
||
const n = Number(edits[k]);
|
||
if (Number.isFinite(n)) out[k] = round2(clamp01(n));
|
||
}
|
||
}
|
||
if (Object.prototype.hasOwnProperty.call(edits, 'noiseColour')) {
|
||
const v = edits.noiseColour;
|
||
if (v == null) out.noiseColour = null;
|
||
else {
|
||
const n = Number(v);
|
||
if (Number.isFinite(n)) out.noiseColour = round2(clamp01(n));
|
||
}
|
||
}
|
||
if (edits.arrangement != null && ARRANGEMENT_VALUES.includes(edits.arrangement)) {
|
||
out.arrangement = edits.arrangement;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Build chat messages from already-serializable inputs.
|
||
*
|
||
* `lookDigest` is the pre-built digest (describeLook + validateLook) — callers
|
||
* that have a live Look should build it via llmTweak.lookDigestForPrompt(look)
|
||
* and pass it here, so this module never imports scene metadata.
|
||
*
|
||
* Kept compatible with the old `{spec,track,look,hint}` shape by accepting
|
||
* either `look` (no-op here) or `lookDigest`.
|
||
*/
|
||
export function buildTweakPrompt({ spec, track, look, lookDigest = null, hint = '', frames = [] } = {}) {
|
||
const r2 = (v) => {
|
||
const n = Number(v);
|
||
return Number.isFinite(n) ? Math.round(n * 100) / 100 : 0;
|
||
};
|
||
const r3 = r2;
|
||
|
||
const summary = track ? track.summary : null;
|
||
const digest = lookDigest ?? (look ? look : null);
|
||
|
||
const specForPrompt = spec ? {
|
||
bpm: spec.bpm != null ? Math.round(Number(spec.bpm)) : null,
|
||
brightness: spec.brightness != null ? r2(spec.brightness) : null,
|
||
noise: spec.noise != null ? r2(spec.noise) : null,
|
||
noiseColour: spec.noiseColour != null ? r2(spec.noiseColour) : null,
|
||
dynamics: spec.dynamics != null ? r2(spec.dynamics) : null,
|
||
transition: spec.transition != null ? r2(spec.transition) : null,
|
||
arrangement: spec.arrangement ?? null,
|
||
duration: spec.duration != null ? r2(spec.duration) : null,
|
||
seed: spec.seed ?? null,
|
||
} : null;
|
||
|
||
const sectionsDetailed = track ? (track.sections || []).map((s, i) => ({
|
||
index: Number.isFinite(s.index) ? s.index : i,
|
||
kind: s.kind || 'unknown',
|
||
start: r2(s.startSec != null ? s.startSec : (s.start ?? 0)),
|
||
end: r2(s.endSec != null ? s.endSec : (s.end ?? 0)),
|
||
duration: r2(s.duration ?? ((s.endSec != null ? s.endSec : (s.end ?? 0)) - (s.startSec != null ? s.startSec : (s.start ?? 0)))),
|
||
energy: r3(s.energy ?? 0),
|
||
flux: r3(s.flux ?? 0),
|
||
centroid: r3(s.centroid ?? 0),
|
||
flatness: r3(s.flatness ?? 0),
|
||
width: r3(s.width ?? 0),
|
||
slope: r3(s.slope ?? 0),
|
||
endEnergy: r3(s.endEnergy ?? 0),
|
||
})) : [];
|
||
|
||
// trackSummary — extended
|
||
let trackSummary = null;
|
||
if (summary) {
|
||
trackSummary = {
|
||
bpm: summary.bpm ?? track?.tempo?.bpm ?? null,
|
||
meanCentroid: summary.meanCentroid != null ? r3(summary.meanCentroid) : null,
|
||
meanFlatness: summary.meanFlatness != null ? r3(summary.meanFlatness) : null,
|
||
meanWidth: summary.meanWidth != null ? r3(summary.meanWidth) : undefined,
|
||
meanLoudness: summary.meanLoudness != null ? r3(summary.meanLoudness) : null,
|
||
dynamicRange: summary.dynamicRange != null ? r3(summary.dynamicRange) : null,
|
||
tempoConfidence: summary.tempoConfidence != null ? r3(summary.tempoConfidence) : (track?.tempo?.confidence != null ? r3(track.tempo.confidence) : null),
|
||
frameCount: track?.frameCount ?? summary.frameCount ?? null,
|
||
fps: track?.fps ?? summary.fps ?? null,
|
||
duration: r2(track?.duration ?? summary.duration ?? 0),
|
||
sections: sectionsDetailed,
|
||
};
|
||
if (summary.bandBalance) {
|
||
trackSummary.bandBalance = {
|
||
sub: r3(summary.bandBalance.sub ?? 0),
|
||
low: r3(summary.bandBalance.low ?? 0),
|
||
mid: r3(summary.bandBalance.mid ?? 0),
|
||
high: r3(summary.bandBalance.high ?? 0),
|
||
air: r3(summary.bandBalance.air ?? 0),
|
||
};
|
||
}
|
||
// remove undefined meanWidth if not present to keep payload compact
|
||
if (trackSummary.meanWidth === undefined) delete trackSummary.meanWidth;
|
||
} else if (sectionsDetailed.length || track) {
|
||
// fallback when summary missing but track exists (e.g. synthetic test track)
|
||
trackSummary = {
|
||
bpm: track?.tempo?.bpm ?? null,
|
||
frameCount: track?.frameCount ?? null,
|
||
fps: track?.fps ?? null,
|
||
duration: r2(track?.duration ?? 0),
|
||
sections: sectionsDetailed,
|
||
};
|
||
if (track?.summary?.bandBalance) {
|
||
trackSummary.bandBalance = {
|
||
sub: r3(track.summary.bandBalance.sub ?? 0),
|
||
low: r3(track.summary.bandBalance.low ?? 0),
|
||
mid: r3(track.summary.bandBalance.mid ?? 0),
|
||
high: r3(track.summary.bandBalance.high ?? 0),
|
||
air: r3(track.summary.bandBalance.air ?? 0),
|
||
};
|
||
}
|
||
}
|
||
|
||
// tempo
|
||
let tempo = null;
|
||
if (track?.tempo) {
|
||
tempo = {
|
||
bpm: r2(track.tempo.bpm ?? summary?.bpm ?? 0),
|
||
confidence: r3(track.tempo.confidence ?? 0),
|
||
beatsPerBar: track.tempo.beatsPerBar ?? 4,
|
||
downbeats: Array.isArray(track.tempo.downbeats) ? track.tempo.downbeats.length : 0,
|
||
};
|
||
} else if (summary?.bpm != null) {
|
||
tempo = { bpm: r2(summary.bpm), confidence: r3(summary.tempoConfidence ?? 0), beatsPerBar: 4, downbeats: 0 };
|
||
}
|
||
|
||
// normalization
|
||
let normalization = null;
|
||
if (track?.normalization && typeof track.normalization === 'object') {
|
||
normalization = {};
|
||
for (const [k, v] of Object.entries(track.normalization)) {
|
||
if (v && typeof v === 'object' && v.lo != null && v.hi != null) {
|
||
normalization[k] = { lo: r3(v.lo), hi: r3(v.hi) };
|
||
}
|
||
}
|
||
if (!Object.keys(normalization).length) normalization = null;
|
||
}
|
||
|
||
// stats — derived
|
||
let stats = null;
|
||
if (sectionsDetailed.length || tempo || track) {
|
||
const kinds = [...new Set(sectionsDetailed.map((s) => s.kind))];
|
||
const avgEnergy = sectionsDetailed.length
|
||
? r3(sectionsDetailed.reduce((a, s) => a + s.energy, 0) / sectionsDetailed.length)
|
||
: 0;
|
||
let maxSec = null;
|
||
let maxE = -Infinity;
|
||
for (const s of sectionsDetailed) if (s.energy > maxE) { maxE = s.energy; maxSec = s; }
|
||
const duration = track?.duration ?? summary?.duration ?? 0;
|
||
const bpmVal = tempo?.bpm ?? summary?.bpm ?? 120;
|
||
const bpb = tempo?.beatsPerBar ?? 4;
|
||
const bars = bpmVal > 0 && duration > 0 ? Math.round(duration / ((60 / bpmVal) * bpb)) : 0;
|
||
stats = {
|
||
totalSections: sectionsDetailed.length,
|
||
kinds,
|
||
avgEnergy,
|
||
maxEnergySection: maxSec ? { index: maxSec.index, kind: maxSec.kind, energy: maxSec.energy } : null,
|
||
bpm: tempo?.bpm ?? null,
|
||
beatsPerBar: tempo?.beatsPerBar ?? null,
|
||
downbeats: tempo?.downbeats ?? 0,
|
||
bars,
|
||
duration: r2(duration),
|
||
fps: track?.fps ?? summary?.fps ?? null,
|
||
frameCount: track?.frameCount ?? summary?.frameCount ?? null,
|
||
};
|
||
}
|
||
|
||
const hintBlock = hint && String(hint).trim()
|
||
? `User hint: "${String(hint).trim().slice(0, 300)}"`
|
||
: 'No user hint.';
|
||
|
||
const frameNote = Array.isArray(frames) && frames.length
|
||
? `You are also given ${frames.length} low-res thumbnails (32×32) sampled across the song in order — use them to judge color, structure, and motion.`
|
||
: null;
|
||
|
||
const system = [
|
||
'You are a producer for a generative music-video system called flow-state.',
|
||
'You suggest tweaks to the *song synthesis parameters* so the next generation looks better.',
|
||
'You do NOT edit the visualization directly — only the 7 song knobs below. The system re-synthesizes audio and regenerates the look.',
|
||
'',
|
||
'Rules:',
|
||
'You have FULL control — change any subset of the 7 knobs by any amount within valid ranges, including large jumps, arrangement swaps, or bpm shifts. Prefer impactful changes that clearly improve visuals; keep duration fixed.',
|
||
'- Valid bpm 60-180 int, brightness/noise/dynamics/transition/noiseColour 0-1, arrangement one of: ' + ARRANGEMENT_VALUES.join(', ') + '.',
|
||
'- noiseColour may be null (follows brightness) or 0-1; leave it null unless you have a reason.',
|
||
'- All 7 edit fields MUST be present in edits — use null for unchanged (not omitted). Example: { bpm: null, brightness: 0.6, noise: null, noiseColour: null, dynamics: null, transition: null, arrangement: null }',
|
||
'- Explain your reasoning in 1-2 sentences (rationale).',
|
||
'- If the look is already good, return all-null edits.',
|
||
hintBlock,
|
||
frameNote,
|
||
].filter(Boolean).join('\n');
|
||
|
||
const userPayload = {
|
||
songSpec: specForPrompt,
|
||
trackSummary,
|
||
sections: sectionsDetailed,
|
||
tempo,
|
||
normalization,
|
||
lookDigest: digest,
|
||
stats,
|
||
};
|
||
|
||
const user = JSON.stringify(userPayload, null, 2);
|
||
|
||
return [
|
||
{ role: 'system', content: system },
|
||
{ role: 'user', content: `Current generation:\n\`\`\`json\n${user}\n\`\`\`\n\nSuggest edits as { rationale, edits, confidence }.` },
|
||
];
|
||
}
|