LLM proxy: stream SSE through without buffering
The dev proxy buffered the entire gateway response via arrayBuffer() before forwarding, which defeats streamObject — no partials arrive until the full completion. Now event-stream responses are streamed chunk-by-chunk via the upstream ReadableStream, so the UI's "LLM streaming…" toast and console partials update live. Non-streaming JSON is still buffered as before. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -198,24 +198,57 @@ function llmGatewayProxy() {
|
|||||||
body: req.method === 'GET' || req.method === 'HEAD' ? undefined : body,
|
body: req.method === 'GET' || req.method === 'HEAD' ? undefined : body,
|
||||||
});
|
});
|
||||||
|
|
||||||
const contentType = upstream.headers.get('content-type');
|
const contentType = upstream.headers.get('content-type') || '';
|
||||||
|
const isStream = contentType.includes('event-stream') || contentType.includes('text/event-stream');
|
||||||
|
// Stream SSE straight through so streamObject sees incremental chunks.
|
||||||
|
if (isStream && upstream.body) {
|
||||||
|
res.writeHead(upstream.status, {
|
||||||
|
'content-type': contentType,
|
||||||
|
'access-control-allow-origin': '*',
|
||||||
|
'cache-control': 'no-cache',
|
||||||
|
});
|
||||||
|
const reader = upstream.body.getReader();
|
||||||
|
let totalBytes = 0;
|
||||||
|
const chunks = [];
|
||||||
|
try {
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
totalBytes += value.byteLength;
|
||||||
|
if (chunks.length < 64) chunks.push(Buffer.from(value).toString('utf8'));
|
||||||
|
res.write(value);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
try { await reader.cancel(); } catch { /* ignore */ }
|
||||||
|
}
|
||||||
|
res.end();
|
||||||
|
const elapsed = Date.now() - started;
|
||||||
|
safeAppend({
|
||||||
|
ts: new Date().toISOString(),
|
||||||
|
dir: '←',
|
||||||
|
status: upstream.status,
|
||||||
|
elapsedMs: elapsed,
|
||||||
|
contentType,
|
||||||
|
body: chunks.join('').slice(0, 16384) || `[stream ${totalBytes} bytes]`,
|
||||||
|
});
|
||||||
|
console.info(`[llm-proxy] ← OK (stream) ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${totalBytes} bytes)`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-streaming: buffer then forward
|
||||||
|
const buf = Buffer.from(await upstream.arrayBuffer());
|
||||||
res.writeHead(upstream.status, {
|
res.writeHead(upstream.status, {
|
||||||
...(contentType ? { 'content-type': contentType } : {}),
|
...(contentType ? { 'content-type': contentType } : {}),
|
||||||
'access-control-allow-origin': '*',
|
'access-control-allow-origin': '*',
|
||||||
});
|
});
|
||||||
|
|
||||||
const buf = Buffer.from(await upstream.arrayBuffer());
|
|
||||||
res.end(buf);
|
res.end(buf);
|
||||||
|
|
||||||
// Log response (sanitized — still no token in body)
|
|
||||||
const elapsed = Date.now() - started;
|
const elapsed = Date.now() - started;
|
||||||
let respLog = '';
|
let respLog = '';
|
||||||
try {
|
try {
|
||||||
const ct = contentType || '';
|
if (contentType.includes('json') || buf.length < 32768) {
|
||||||
if (ct.includes('json') || ct.includes('event-stream') || buf.length < 32768) {
|
|
||||||
const text = buf.toString('utf8');
|
const text = buf.toString('utf8');
|
||||||
// For SSE streams, keep first 16k; for JSON, keep parsed form
|
if (contentType.includes('json')) {
|
||||||
if (ct.includes('json')) {
|
|
||||||
try { respLog = JSON.stringify(JSON.parse(text)).slice(0, 16384); } catch { respLog = text.slice(0, 8192); }
|
try { respLog = JSON.stringify(JSON.parse(text)).slice(0, 16384); } catch { respLog = text.slice(0, 8192); }
|
||||||
} else {
|
} else {
|
||||||
respLog = text.slice(0, 8192);
|
respLog = text.slice(0, 8192);
|
||||||
@@ -234,7 +267,6 @@ function llmGatewayProxy() {
|
|||||||
body: respLog,
|
body: respLog,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Also emit to vite's console for live tailing
|
|
||||||
const tag = upstream.status >= 400 ? '← ERR' : '← OK';
|
const tag = upstream.status >= 400 ? '← ERR' : '← OK';
|
||||||
console.info(`[llm-proxy] ${tag} ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${buf.length} bytes)`);
|
console.info(`[llm-proxy] ${tag} ${req.method} ${pathAfter} → ${upstream.status} (${elapsed}ms, ${buf.length} bytes)`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
Reference in New Issue
Block a user