LLM: streaming + rich context + 32×32 frames + proxy log
- Streaming: requestSongTweak now uses streamObject when a progress
callback is provided (via handleLlmTweak), emitting partial {rationale,
edits} and updating toast/console as text arrives; falls back to
generateObject if gateway doesn't support streaming. Models always
called via provider.chat(modelId) / /v1/chat/completions, the correct
path for meta/muse-spark-1.3-contributor via gateway.
- Rich context: spec.js gains FULL control wording (any amount/arrangement/
bpm allowed), and detailed payload — trackSummary with bandBalance,
tempo, normalization (per-feature lo/hi), sectionsDetailed (energy/
flux/centroid/flatness/width/slope/endEnergy/duration), stats
(kinds, avg/max energy, bars). LLM told You have FULL control.
- Low-res vision: new src/llm/thumbs.js — captureThumbnails(show, {8,32})
samples section midpoints, warms up state, renders via Show, box-averages
down to 32×32 JPEG data URLs, restores timeline. main.js now captures 8
frames with two animation frames of pause, then passes them as {type:
image, image: dataUrl} parts in the user message (llmTweak.js
attachFramesToMessages).
- Proxy logging (no bearer): vite.config.js llmGatewayProxy now logs
sanitized requests/responses to logs/llm-proxy.log (JSON lines, 1MB
rotation, redacts apiKey/authorization from path/body/headers) so the
coding agent can inspect prompts without re-calling. .gitignore adds
logs/.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
@@ -0,0 +1,128 @@
|
||||
export function downscaleRgbaToDataUrl(pixels, srcW, srcH, dstSize) {
|
||||
const dst = new Uint8ClampedArray(dstSize * dstSize * 4);
|
||||
for (let y = 0; y < dstSize; y++) {
|
||||
const y0 = Math.floor((y * srcH) / dstSize);
|
||||
const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * srcH) / dstSize));
|
||||
for (let x = 0; x < dstSize; x++) {
|
||||
const x0 = Math.floor((x * srcW) / dstSize);
|
||||
const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * srcW) / dstSize));
|
||||
let r = 0, g = 0, b = 0, c = 0;
|
||||
for (let sy = y0; sy < y1; sy++)
|
||||
for (let sx = x0; sx < x1; sx++) {
|
||||
const i = (sy * srcW + sx) * 4;
|
||||
r += pixels[i]; g += pixels[i + 1]; b += pixels[i + 2]; c++;
|
||||
}
|
||||
const o = (y * dstSize + x) * 4;
|
||||
dst[o] = c ? Math.round(r / c) : 0;
|
||||
dst[o + 1] = c ? Math.round(g / c) : 0;
|
||||
dst[o + 2] = c ? Math.round(b / c) : 0;
|
||||
dst[o + 3] = 255;
|
||||
}
|
||||
}
|
||||
if (typeof document !== 'undefined' && document.createElement) {
|
||||
try {
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = dstSize; canvas.height = dstSize;
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (ctx) { ctx.putImageData(new ImageData(dst, dstSize, dstSize), 0, 0); return canvas.toDataURL('image/jpeg', 0.6); }
|
||||
} catch { /* fallback */ }
|
||||
}
|
||||
return 'data:image/jpeg;base64,';
|
||||
}
|
||||
|
||||
function uniformFrames(count, frameCount) {
|
||||
const seen = new Set(); const out = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const f = Math.max(0, Math.min(frameCount - 1, Math.round(((i + 0.5) / count) * frameCount)));
|
||||
if (!seen.has(f)) { seen.add(f); out.push(f); }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function pickFrames(show, count, frameCount, fps) {
|
||||
const sections = show.track && show.track.sections;
|
||||
if (sections && sections.length) {
|
||||
const mids = []; const seen = new Set();
|
||||
for (const s of sections) {
|
||||
let mid;
|
||||
if (Number.isFinite(s.startFrame) && Number.isFinite(s.endFrame)) mid = Math.round((s.startFrame + s.endFrame) / 2);
|
||||
else if (Number.isFinite(s.start) && Number.isFinite(s.end)) mid = Math.round(((s.start + s.end) / 2) * fps);
|
||||
else continue;
|
||||
mid = Math.max(0, Math.min(frameCount - 1, mid));
|
||||
if (!seen.has(mid)) { seen.add(mid); mids.push(mid); }
|
||||
}
|
||||
if (!mids.length) return uniformFrames(count, frameCount);
|
||||
if (mids.length >= count) {
|
||||
if (mids.length === count) return mids;
|
||||
mids.sort((a, b) => a - b);
|
||||
const out = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const idx = Math.min(mids.length - 1, Math.floor(((i + 0.5) / count) * mids.length));
|
||||
const v = mids[idx]; if (!out.includes(v)) out.push(v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const need = count - mids.length;
|
||||
for (const f of uniformFrames(need, frameCount)) if (!seen.has(f)) { seen.add(f); mids.push(f); }
|
||||
if (mids.length < count) for (const f of uniformFrames(count, frameCount)) if (!seen.has(f) && mids.length < count) { seen.add(f); mids.push(f); }
|
||||
const sorted = [...seen].sort((a, b) => a - b);
|
||||
if (sorted.length <= count) return sorted;
|
||||
const out = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const idx = Math.min(sorted.length - 1, Math.floor(((i + 0.5) / count) * sorted.length));
|
||||
const v = sorted[idx]; if (!out.includes(v)) out.push(v);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return uniformFrames(count, frameCount);
|
||||
}
|
||||
|
||||
export async function captureThumbnails(show, { count = 8, size = 32 } = {}) {
|
||||
if (!show || !show.ready || !show.track || !show.engine) return [];
|
||||
const fps = show.fps || 60;
|
||||
const frameCount = show.frameCount || (show.track && show.track.frameCount) || Math.round((show.track.duration || 0) * fps) || 0;
|
||||
if (!frameCount) return [];
|
||||
const originalFrame = show.timeline ? show.timeline.frame : 0;
|
||||
const origW = show.engine.width, origH = show.engine.height;
|
||||
const frames = pickFrames(show, count, frameCount, fps).sort((a, b) => a - b);
|
||||
const out = [];
|
||||
try {
|
||||
for (const frame of frames) {
|
||||
try {
|
||||
if (typeof show.warmUp === 'function') {
|
||||
const wf = typeof show.warmupFrames === 'function' ? show.warmupFrames() : 120;
|
||||
show.warmUp(frame, Math.min(120, wf));
|
||||
} else if (show.engine.compositor) {
|
||||
if (typeof show.engine.warmUp === 'function') show.engine.warmUp(frame, 120);
|
||||
else show.engine.compositor.reset();
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
let target; try { target = show.renderFrame(frame); } catch { continue; }
|
||||
let pixels; try { pixels = show.readPixels(target); } catch { continue; }
|
||||
if (!pixels || !pixels.length) continue;
|
||||
let dataUrl = ''; try { dataUrl = downscaleRgbaToDataUrl(pixels, origW, origH, size); } catch { dataUrl = 'data:image/jpeg;base64,'; }
|
||||
let kind = '';
|
||||
try {
|
||||
const sec = show.track.sectionAt ? show.track.sectionAt(frame) : null;
|
||||
if (sec && sec.kind) kind = sec.kind;
|
||||
else if (show.track.sections) for (const s of show.track.sections) if (Number.isFinite(s.startFrame) && frame >= s.startFrame && frame < s.endFrame) { kind = s.kind || ''; break; }
|
||||
} catch { /* ignore */ }
|
||||
out.push({ kind: kind || '', time: frame / fps, frame, dataUrl });
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
if (typeof show.seek === 'function') show.seek(originalFrame);
|
||||
else if (show.timeline && typeof show.timeline.seek === 'function') {
|
||||
show.timeline.seek(originalFrame);
|
||||
try { const t = show.renderFrame(originalFrame); if (typeof show.present === 'function') show.present(t); } catch { /* ignore */ }
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
if (show.engine && (show.engine.width !== origW || show.engine.height !== origH)) {
|
||||
if (typeof show.engine.setSize === 'function') show.engine.setSize(origW, origH);
|
||||
else if (typeof show.setSize === 'function') show.setSize(origW, origH);
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
Reference in New Issue
Block a user