dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LLM proxy: stream SSE through without buffering

Browse Source 
The dev proxy buffered the entire gateway response via
arrayBuffer() before forwarding, which defeats streamObject —
no partials arrive until the full completion. Now event-stream
responses are streamed chunk-by-chunk via the upstream ReadableStream,
so the UI's "LLM streaming…" toast and console partials update live.
Non-streaming JSON is still buffered as before.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Dejvino
2026-09-13 21:55:51 +02:00
co-authored by Claude
parent e0e619af85
commit b0989169e2
1 changed files with 41 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flow-state/vite.config.js
+41 -9
View File
@@ -198,24 +198,57 @@ function llmGatewayProxy() {
body: req.method === 'GET' || req.method === 'HEAD' ? undefined : body,
});
const contentType = upstream.headers.get('content-type');
const contentType = upstream.headers.get('content-type') || '';
const isStream = contentType.includes('event-stream') || contentType.includes('text/event-stream');
// Stream SSE straight through so streamObject sees incremental chunks.
if (isStream && upstream.body) {
res.writeHead(upstream.status, {
'content-type': contentType,
'access-control-allow-origin': '*',
'cache-control': 'no-cache',
});
const reader = upstream.body.getReader();
let totalBytes = 0;
const chunks = [];
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
totalBytes += value.byteLength;
if (chunks.length < 64) chunks.push(Buffer.from(value).toString('utf8'));
res.write(value);
}
} finally {
try { await reader.cancel(); } catch { /* ignore */ }
}
res.end();
const elapsed = Date.now() - started;
safeAppend({
ts: new Date().toISOString(),
dir: '←',
status: upstream.status,
elapsedMs: elapsed,
contentType,
body: chunks.join('').slice(0, 16384) || `[stream ${totalBytes} bytes]`,
});
console.info(`[llm-proxy] ← OK (stream) ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${totalBytes} bytes)`);
return;
}
// Non-streaming: buffer then forward
const buf = Buffer.from(await upstream.arrayBuffer());
res.writeHead(upstream.status, {
...(contentType ? { 'content-type': contentType } : {}),
'access-control-allow-origin': '*',
});
const buf = Buffer.from(await upstream.arrayBuffer());
res.end(buf);
// Log response (sanitized — still no token in body)
const elapsed = Date.now() - started;
let respLog = '';
try {
const ct = contentType || '';
if (ct.includes('json') || ct.includes('event-stream') || buf.length < 32768) {
if (contentType.includes('json') || buf.length < 32768) {
const text = buf.toString('utf8');
// For SSE streams, keep first 16k; for JSON, keep parsed form
if (ct.includes('json')) {
if (contentType.includes('json')) {
try { respLog = JSON.stringify(JSON.parse(text)).slice(0, 16384); } catch { respLog = text.slice(0, 8192); }
} else {
respLog = text.slice(0, 8192);
@@ -234,7 +267,6 @@ function llmGatewayProxy() {
body: respLog,
});
// Also emit to vite's console for live tailing
const tag = upstream.status >= 400 ? '← ERR' : '← OK';
console.info(`[llm-proxy] ${tag} ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${buf.length} bytes)`);
} catch (err) {