From b7c6fd1c5a53b8884d507c8f59ab988d725a3d0f Mon Sep 17 00:00:00 2001 From: Dejvino Date: Thu, 6 Aug 2026 13:10:24 +0200 Subject: [PATCH] Fix export frame loss: reject codec profiles that reorder frames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exporter was losing roughly three of every four frames. A 20s render muxed 319 samples instead of 1200 and played at 15.9 fps, with no error reported and a file that looked superficially fine. The cause was B-frames. WebCodecs delivers chunks in decode order, but EncodedVideoChunk carries only a presentation timestamp — there is no decode timestamp to recover the real order from. Handed presentation timestamps as if they were decode timestamps, mp4-muxer saw DTS run backwards and rejected every reordered chunk. That throw happened inside the encoder's output callback, where it could not reach the export loop, so it surfaced as an uncaught error and the render carried on. Only the I/P frames survived, one per 4-frame GOP, which is exactly the stts pattern the files showed. Writing the correct timeline instead is not available to us: it needs negative composition offsets, and mp4-muxer emits ctts as a version-0 box, which is unsigned. isConfigSupported says nothing about reordering, and measurement showed latencyMode: 'realtime' does not prevent it either. So pickVideoConfig now test-encodes 12 frames per candidate and checks the order they come back in, taking the first profile that does not reorder. Candidates stay in quality order, high profile down to baseline, so browsers that never reorder keep the better profiles; baseline forbids B-slices by spec and is the guaranteed floor. If every supported profile reorders the export fails up front rather than after minutes of rendering. Two guards so this class of loss cannot be silent again: - The output callback catches, routing muxer rejections to the error list the loop actually checks. - Frames in and chunks accepted are counted and compared after flush, with the reordering count and a gap histogram alongside. The count deliberately tracks chunks the muxer took, not chunks that arrived — counting arrivals reports success for frames rejected a line later. Failures now raise a toast over the stage that stays until dismissed. An export that dies after a long render should not sit unread in a panel. Also corrects the record from d31d0fc, which claimed VideoEncoder.encode() silently drops frames once its queue saturates. It does not: the queue grows without bound and the only cost is memory. That commit's dequeue-gated backpressure addressed a mechanism that does not exist and is reverted here; the queue poll it replaced is restored, described honestly as a memory bound. The opus resampling from that commit was a real fix and is untouched. tools/probe-mp4.js reports per-track timescale, sample count and the stts table, which is what identified the fault and what verifies a good export: one row of [N x 1]. Co-Authored-By: Claude Opus 5 --- flow-state/index.html | 1 + flow-state/src/export/Exporter.js | 189 ++++++++++++++++++++++++------ flow-state/src/main.js | 23 ++++ flow-state/src/ui/style.css | 23 ++++ flow-state/tools/probe-mp4.js | 88 ++++++++++++++ 5 files changed, 290 insertions(+), 34 deletions(-) create mode 100644 flow-state/tools/probe-mp4.js diff --git a/flow-state/index.html b/flow-state/index.html index 51ee63c..78d7966 100644 --- a/flow-state/index.html +++ b/flow-state/index.html @@ -20,6 +20,7 @@ +
diff --git a/flow-state/src/export/Exporter.js b/flow-state/src/export/Exporter.js index 9cd78a8..9cc7e66 100644 --- a/flow-state/src/export/Exporter.js +++ b/flow-state/src/export/Exporter.js @@ -25,11 +25,79 @@ export function isSupported() { return typeof VideoEncoder !== 'undefined' && typeof VideoFrame !== 'undefined'; } -/** Probe for a codec configuration the browser will actually accept. */ +/** + * Frames to run through a candidate encoder when testing it for reordering. + * + * Hierarchical B-pyramids repeat every 4 frames, so two GOPs is enough to see + * the pattern while costing almost nothing even at 4K. + */ +const ORDER_PROBE_FRAMES = 12; + +/** + * Does this configuration emit chunks in presentation order? + * + * It matters because of what WebCodecs does NOT expose. An encoder that emits + * B-frames delivers chunks in *decode* order, but `EncodedVideoChunk` carries + * only a presentation timestamp and no decode timestamp — so there is no way to + * recover the decode timeline after the fact. mp4-muxer, handed presentation + * timestamps, sees DTS run backwards and rejects the chunk; and because that + * throw happens inside the encoder's output callback it cannot reach the export + * loop at all. The result is a file missing three quarters of its frames with + * nothing reported. Writing the correct timeline instead would need negative + * composition offsets, which mp4-muxer emits as a version-0 `ctts` box — + * unsigned, so it cannot represent them. + * + * `isConfigSupported` says nothing about reordering, and neither does + * `latencyMode: 'realtime'` — measured, Chrome still emits B-frames under it. + * So actually encode a few frames and watch what order they come back in. + */ +async function emitsInPresentationOrder(config) { + const stamps = []; + let failed = false; + let encoder = null; + try { + encoder = new VideoEncoder({ + output: (chunk) => stamps.push(chunk.timestamp), + error: () => { failed = true; }, + }); + encoder.configure(config); + + const canvas = new OffscreenCanvas(config.width, config.height); + const ctx = canvas.getContext('2d'); + const period = Math.round(1e6 / config.framerate); + for (let i = 0; i < ORDER_PROBE_FRAMES; i++) { + // Vary the content: an encoder fed identical frames may collapse + // them and never exercise its reordering path. + ctx.fillStyle = `rgb(${(i * 37) % 256} ${(i * 91) % 256} ${(i * 17) % 256})`; + ctx.fillRect(0, 0, config.width, config.height); + const frame = new VideoFrame(canvas, { timestamp: i * period, duration: period }); + encoder.encode(frame, { keyFrame: i === 0 }); + frame.close(); + } + await encoder.flush(); + } catch { + failed = true; + } finally { + try { if (encoder) encoder.close(); } catch { /* already closed */ } + } + if (failed || stamps.length !== ORDER_PROBE_FRAMES) return false; + return stamps.every((t, i) => i === 0 || t > stamps[i - 1]); +} + +/** + * Probe for a codec configuration the browser will actually accept — and will + * encode in presentation order. + * + * The candidates run high profile first for compression efficiency, down to + * baseline last. Baseline forbids B-slices outright, so it is the profile that + * cannot reorder; the earlier entries are tried first because when a browser + * does not reorder there is no reason to give up their quality. + */ async function pickVideoConfig(width, height, bitrate, fps) { const candidates = [ 'avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e', ]; + const supported = []; for (const codec of candidates) { const config = { codec, width, height, bitrate, framerate: fps, @@ -37,8 +105,19 @@ async function pickVideoConfig(width, height, bitrate, fps) { }; try { const support = await VideoEncoder.isConfigSupported(config); - if (support.supported) return config; - } catch { /* try the next one */ } + if (!support.supported) continue; + } catch { continue; } + supported.push(config); + if (await emitsInPresentationOrder(config)) return config; + } + // Every supported profile reorders. Refuse rather than write a file that + // silently loses most of its frames — see emitsInPresentationOrder. + if (supported.length) { + throw new Error( + 'every supported H.264 profile emits frames out of order on this browser ' + + `(tried ${supported.map((c) => c.codec).join(', ')}), which this exporter ` + + 'cannot mux correctly', + ); } return null; } @@ -98,30 +177,34 @@ function resampleInterleaved(input, fromRate, toRate, channels) { } /** - * Backpressure: VideoEncoder.encode() silently drops frames once its internal - * queue is full (encodeQueueSize is capped at an implementation-defined - * limit). Polling that number is unreliable — the only signal that a frame - * actually left the queue is the 'dequeue' event, so wait on it before every - * encode. Keeping the queue small (rather than draining it fully) leaves the - * hardware encoder a pipeline to chew on while guaranteeing nothing is dropped. + * Describe what actually came out of the encoder. + * + * A frame that goes in and never comes out is otherwise invisible — no error + * fires, the muxer simply receives fewer samples and spreads their timestamps + * across the full duration, so the file plays at a fraction of the intended + * rate. The gap histogram says whether losses were a steady decimation (one + * dominant gap size) or bursts (a long tail), which are different bugs. */ -const MAX_ENCODER_QUEUE = 8; - -function waitForQueue(encoder, errors) { - if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve(); - return new Promise((resolve, reject) => { - const cleanup = () => { - encoder.removeEventListener('dequeue', onDequeue); - encoder.removeEventListener('error', onError); - }; - const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); }; - const onDequeue = () => { - if (errors.length) { cleanup(); reject(errors[0]); return; } - if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); } - }; - encoder.addEventListener('dequeue', onDequeue); - encoder.addEventListener('error', onError); - }); +function frameStats(emittedAt, encoded, emitted, fps) { + const period = 1e6 / fps; + // Chunks arrive in decode order. A timestamp that goes backwards means the + // encoder reordered — the condition that silently ate three quarters of + // every export until latencyMode pinned it down. + const reordered = emittedAt.filter((t, i) => i > 0 && t < emittedAt[i - 1]).length; + const stamps = emittedAt.slice().sort((a, b) => a - b); + const gaps = stamps.slice(1).map((t, i) => Math.round((t - stamps[i]) / period)); + const gapHistogram = {}; + for (const g of gaps) gapHistogram[g] = (gapHistogram[g] || 0) + 1; + const firstGap = gaps.findIndex((g) => g !== 1); + return { + encoded, + emitted, + reordered, + effectiveFps: encoded > 0 ? (emitted / encoded) * fps : 0, + maxGap: gaps.length ? Math.max(...gaps) : 0, + gapHistogram, + firstGapAt: firstGap < 0 ? null : Math.round(stamps[firstGap] / period), + }; } export class Exporter { @@ -187,8 +270,30 @@ export class Exporter { }); const errors = []; + // A frame that goes into the encoder and never comes out is invisible: + // no error fires, the muxer just receives fewer samples and spreads + // their timestamps over the full duration, so the file plays at a + // fraction of the intended rate. Count both ends and refuse to hand + // back a video that lost frames. + let framesEncoded = 0; + let chunksEmitted = 0; + const emittedAt = []; const videoEncoder = new VideoEncoder({ - output: (chunk, meta) => muxer.addVideoChunk(chunk, meta), + // This callback runs from the encoder, not from the export loop, so + // a throw here escapes as an uncaught error and the loop never + // learns the chunk was lost. Catch it and route it to `errors`, + // which the loop does check. Count only chunks the muxer accepted — + // counting them on arrival would report success for frames that + // were rejected a line later. + output: (chunk, meta) => { + try { + muxer.addVideoChunk(chunk, meta); + chunksEmitted++; + emittedAt.push(chunk.timestamp); + } catch (e) { + errors.push(e); + } + }, error: (e) => errors.push(e), }); videoEncoder.configure(videoConfig); @@ -221,12 +326,6 @@ export class Exporter { const target = show.renderFrame(frameIndex); show.present(target); // encode from the canvas, which now holds this frame - // True backpressure before every encode — VideoEncoder drops - // frames silently once its queue saturates, which would turn a - // smooth render into the stutter that first shipped here. - await waitForQueue(videoEncoder, errors); - if (errors.length) throw errors[0]; - const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps); const videoFrame = new VideoFrame(show.engine.renderer.canvas, { timestamp, @@ -235,9 +334,17 @@ export class Exporter { // Keyframe every two seconds: seekable output without bloating size. videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 }); videoFrame.close(); + framesEncoded++; - // Yield periodically so the progress UI paints. + // Yield periodically so the progress UI paints, and cap how far + // the encoder may fall behind. This is a memory bound, not a + // correctness one: encode() queues without limit and does not + // drop, so the only cost of an unbounded queue is holding every + // pending frame's pixels at once — which at 4K is gigabytes. if (i % 10 === 0) { + while (videoEncoder.encodeQueueSize > 30) { + await new Promise((r) => setTimeout(r, 4)); + } onProgress && onProgress({ frame: i, total, fraction: i / total, stage: 'rendering', }); @@ -249,6 +356,20 @@ export class Exporter { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'finishing video' }); await videoEncoder.flush(); + this.frameStats = frameStats(emittedAt, framesEncoded, chunksEmitted, fps); + // Always report, not just on mismatch: a silent success that lost + // frames is exactly the failure this is here to catch. + console.info('[export] frameStats', this.frameStats); + if (chunksEmitted !== framesEncoded) { + const s = this.frameStats; + throw new Error( + `encoder lost ${framesEncoded - chunksEmitted} of ${framesEncoded} frames ` + + `(${chunksEmitted} encoded chunks) — the export would play at ` + + `${s.effectiveFps.toFixed(1)} fps instead of ${fps}. ` + + `First gap at frame ${s.firstGapAt}; gap sizes ${JSON.stringify(s.gapHistogram)}`, + ); + } + if (hasAudio) { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' }); try { diff --git a/flow-state/src/main.js b/flow-state/src/main.js index 66f3e24..0569bfb 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -26,6 +26,7 @@ const dom = { panelBody: document.getElementById('panel-body'), panelTabs: document.getElementById('panel-tabs'), hud: document.getElementById('hud'), + toast: document.getElementById('toast'), osd: document.getElementById('btn-osd'), play: document.getElementById('btn-play'), thLabel: document.getElementById('th-label'), @@ -490,6 +491,24 @@ function currentPreset() { return el ? el.value : '1080p'; } +/** + * Surface a message over the stage. + * + * Errors stay until dismissed: an export that fails after several minutes of + * rendering must not scroll past unnoticed, which is exactly what the status + * line inside the export panel allowed. + */ +function showToast(message, { ok = false, timeout = 0 } = {}) { + if (!dom.toast) return; + clearTimeout(showToast._timer); + dom.toast.textContent = message; + dom.toast.classList.toggle('ok', ok); + dom.toast.hidden = false; + if (timeout) showToast._timer = setTimeout(() => { dom.toast.hidden = true; }, timeout); +} + +if (dom.toast) dom.toast.addEventListener('click', () => { dom.toast.hidden = true; }); + function exportProgress(p) { const status = document.getElementById('export-status'); const fill = document.getElementById('export-fill'); @@ -505,6 +524,7 @@ async function runExport(segment) { [...dom.panelTabs.children].forEach((b) => b.classList.toggle('active', b.dataset.tab === 'export')); renderPanel(); + if (dom.toast) dom.toast.hidden = true; try { const preset = currentPreset(); const exporter = new Exporter(state.show); @@ -523,9 +543,12 @@ async function runExport(segment) { status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` + (warnings ? ` (${warnings})` : ''); } + showToast(`export done — ${(blob.size / 1e6).toFixed(1)} MB` + + (warnings ? `\n${warnings}` : ''), { ok: true, timeout: 6000 }); } catch (err) { const status = document.getElementById('export-status'); if (status) status.textContent = `failed: ${err.message}`; + showToast(`export failed\n${err.message}`); console.error(err); } finally { state.busy = false; diff --git a/flow-state/src/ui/style.css b/flow-state/src/ui/style.css index ea577c6..0aa8208 100644 --- a/flow-state/src/ui/style.css +++ b/flow-state/src/ui/style.css @@ -85,6 +85,29 @@ body { pointer-events: none; } +/* Export failures used to land only in a status line inside a panel the user + may not be looking at — a lost render deserves to interrupt. */ +#toast { + position: absolute; bottom: 16px; left: 50%; + transform: translateX(-50%); + max-width: min(680px, calc(100% - 32px)); + background: rgba(7,8,12,0.94); + border: 1px solid #7f1d1d; + border-left: 3px solid #ef4444; + padding: 10px 34px 10px 12px; + font-size: 12px; line-height: 1.6; + color: var(--text); + white-space: pre-wrap; + overflow-wrap: anywhere; + cursor: pointer; +} +#toast.ok { border-color: #14532d; border-left-color: var(--accent); } +#toast::after { + content: '×'; + position: absolute; top: 6px; right: 10px; + color: var(--dim); font-size: 14px; +} + #transport { grid-area: transport; background: var(--panel); } #timeline { height: 46px; } #timeline-canvas { width: 100%; height: 46px; display: block; cursor: pointer; } diff --git a/flow-state/tools/probe-mp4.js b/flow-state/tools/probe-mp4.js new file mode 100644 index 0000000..f247b84 --- /dev/null +++ b/flow-state/tools/probe-mp4.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node +/** + * Report what an exported mp4 actually contains, per track: timescale, sample + * count, and the sample-duration table (stts). + * + * The point is to separate "the exporter dropped frames" from "the player is + * stuttering on a file that is fine". A 60 fps, N-second video must show + * timescale 60 and one stts row of [N*60, 1]. Several rows — or any row with a + * duration of 0 — means the muxed timeline is wrong, and the frames that are + * there will not be shown. + * + * node tools/probe-mp4.js path/to/export.mp4 + */ +import fs from 'fs'; + +const path = process.argv[2]; +if (!path) { + console.error('usage: node tools/probe-mp4.js '); + process.exit(1); +} +const buf = fs.readFileSync(path); + +const CONTAINERS = ['moov', 'trak', 'mdia', 'minf', 'stbl']; +const tracks = []; +let cur = null; + +function walk(start, end) { + let off = start; + while (off + 8 <= end) { + let size = buf.readUInt32BE(off); + const type = buf.toString('latin1', off + 4, off + 8); + let hdr = 8; + if (size === 1) { size = Number(buf.readBigUInt64BE(off + 8)); hdr = 16; } + if (size === 0) size = end - off; + if (CONTAINERS.includes(type)) { + if (type === 'trak') { cur = {}; tracks.push(cur); } + walk(off + hdr, off + size); + } else if (cur) { + box(type, off + hdr); + } + off += size; + } +} + +function box(type, s) { + if (type === 'mdhd') { + cur.timescale = buf.readUInt32BE(s + 12); + cur.duration = buf.readUInt32BE(s + 16); + } else if (type === 'hdlr') { + cur.kind = buf.toString('latin1', s + 8, s + 12); + } else if (type === 'stsd') { + cur.format = buf.toString('latin1', s + 12, s + 16); + } else if (type === 'stsz') { + cur.sampleCount = buf.readUInt32BE(s + 8); + } else if (type === 'stts') { + const n = buf.readUInt32BE(s + 4); + const rows = []; + let samples = 0; + let total = 0; + for (let i = 0; i < n; i++) { + const count = buf.readUInt32BE(s + 8 + i * 8); + const delta = buf.readUInt32BE(s + 12 + i * 8); + rows.push({ count, delta }); + samples += count; + total += count * delta; + } + cur.stts = { rows, samples, total }; + } +} + +walk(0, buf.length); + +for (const t of tracks) { + const stts = t.stts || { rows: [], samples: 0, total: 0 }; + const seconds = t.timescale ? stts.total / t.timescale : 0; + console.log(`\n[${t.kind}] ${t.format} · timescale ${t.timescale}`); + console.log(` samples: ${t.sampleCount} · media duration: ${seconds.toFixed(3)}s`); + if (t.kind === 'vide' && seconds > 0) { + console.log(` effective frame rate: ${(stts.samples / seconds).toFixed(3)} fps`); + } + console.log(` stts rows (${stts.rows.length}):`); + for (const r of stts.rows.slice(0, 20)) { + console.log(` ${r.count} × ${r.delta}${r.delta === 0 ? ' <-- zero duration: these frames never display' : ''}`); + } + if (stts.rows.length > 20) console.log(` … ${stts.rows.length - 20} more rows`); + const zeros = stts.rows.filter((r) => r.delta === 0).reduce((a, r) => a + r.count, 0); + if (zeros) console.log(` ${zeros} samples have zero duration`); +}