diff --git a/flow-state/index.html b/flow-state/index.html index 51ee63c..78d7966 100644 --- a/flow-state/index.html +++ b/flow-state/index.html @@ -20,6 +20,7 @@ +
diff --git a/flow-state/src/export/Exporter.js b/flow-state/src/export/Exporter.js index 9cd78a8..9cc7e66 100644 --- a/flow-state/src/export/Exporter.js +++ b/flow-state/src/export/Exporter.js @@ -25,11 +25,79 @@ export function isSupported() { return typeof VideoEncoder !== 'undefined' && typeof VideoFrame !== 'undefined'; } -/** Probe for a codec configuration the browser will actually accept. */ +/** + * Frames to run through a candidate encoder when testing it for reordering. + * + * Hierarchical B-pyramids repeat every 4 frames, so two GOPs is enough to see + * the pattern while costing almost nothing even at 4K. + */ +const ORDER_PROBE_FRAMES = 12; + +/** + * Does this configuration emit chunks in presentation order? + * + * It matters because of what WebCodecs does NOT expose. An encoder that emits + * B-frames delivers chunks in *decode* order, but `EncodedVideoChunk` carries + * only a presentation timestamp and no decode timestamp — so there is no way to + * recover the decode timeline after the fact. mp4-muxer, handed presentation + * timestamps, sees DTS run backwards and rejects the chunk; and because that + * throw happens inside the encoder's output callback it cannot reach the export + * loop at all. The result is a file missing three quarters of its frames with + * nothing reported. Writing the correct timeline instead would need negative + * composition offsets, which mp4-muxer emits as a version-0 `ctts` box — + * unsigned, so it cannot represent them. + * + * `isConfigSupported` says nothing about reordering, and neither does + * `latencyMode: 'realtime'` — measured, Chrome still emits B-frames under it. + * So actually encode a few frames and watch what order they come back in. + */ +async function emitsInPresentationOrder(config) { + const stamps = []; + let failed = false; + let encoder = null; + try { + encoder = new VideoEncoder({ + output: (chunk) => stamps.push(chunk.timestamp), + error: () => { failed = true; }, + }); + encoder.configure(config); + + const canvas = new OffscreenCanvas(config.width, config.height); + const ctx = canvas.getContext('2d'); + const period = Math.round(1e6 / config.framerate); + for (let i = 0; i < ORDER_PROBE_FRAMES; i++) { + // Vary the content: an encoder fed identical frames may collapse + // them and never exercise its reordering path. + ctx.fillStyle = `rgb(${(i * 37) % 256} ${(i * 91) % 256} ${(i * 17) % 256})`; + ctx.fillRect(0, 0, config.width, config.height); + const frame = new VideoFrame(canvas, { timestamp: i * period, duration: period }); + encoder.encode(frame, { keyFrame: i === 0 }); + frame.close(); + } + await encoder.flush(); + } catch { + failed = true; + } finally { + try { if (encoder) encoder.close(); } catch { /* already closed */ } + } + if (failed || stamps.length !== ORDER_PROBE_FRAMES) return false; + return stamps.every((t, i) => i === 0 || t > stamps[i - 1]); +} + +/** + * Probe for a codec configuration the browser will actually accept — and will + * encode in presentation order. + * + * The candidates run high profile first for compression efficiency, down to + * baseline last. Baseline forbids B-slices outright, so it is the profile that + * cannot reorder; the earlier entries are tried first because when a browser + * does not reorder there is no reason to give up their quality. + */ async function pickVideoConfig(width, height, bitrate, fps) { const candidates = [ 'avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e', ]; + const supported = []; for (const codec of candidates) { const config = { codec, width, height, bitrate, framerate: fps, @@ -37,8 +105,19 @@ async function pickVideoConfig(width, height, bitrate, fps) { }; try { const support = await VideoEncoder.isConfigSupported(config); - if (support.supported) return config; - } catch { /* try the next one */ } + if (!support.supported) continue; + } catch { continue; } + supported.push(config); + if (await emitsInPresentationOrder(config)) return config; + } + // Every supported profile reorders. Refuse rather than write a file that + // silently loses most of its frames — see emitsInPresentationOrder. + if (supported.length) { + throw new Error( + 'every supported H.264 profile emits frames out of order on this browser ' + + `(tried ${supported.map((c) => c.codec).join(', ')}), which this exporter ` + + 'cannot mux correctly', + ); } return null; } @@ -98,30 +177,34 @@ function resampleInterleaved(input, fromRate, toRate, channels) { } /** - * Backpressure: VideoEncoder.encode() silently drops frames once its internal - * queue is full (encodeQueueSize is capped at an implementation-defined - * limit). Polling that number is unreliable — the only signal that a frame - * actually left the queue is the 'dequeue' event, so wait on it before every - * encode. Keeping the queue small (rather than draining it fully) leaves the - * hardware encoder a pipeline to chew on while guaranteeing nothing is dropped. + * Describe what actually came out of the encoder. + * + * A frame that goes in and never comes out is otherwise invisible — no error + * fires, the muxer simply receives fewer samples and spreads their timestamps + * across the full duration, so the file plays at a fraction of the intended + * rate. The gap histogram says whether losses were a steady decimation (one + * dominant gap size) or bursts (a long tail), which are different bugs. */ -const MAX_ENCODER_QUEUE = 8; - -function waitForQueue(encoder, errors) { - if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve(); - return new Promise((resolve, reject) => { - const cleanup = () => { - encoder.removeEventListener('dequeue', onDequeue); - encoder.removeEventListener('error', onError); - }; - const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); }; - const onDequeue = () => { - if (errors.length) { cleanup(); reject(errors[0]); return; } - if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); } - }; - encoder.addEventListener('dequeue', onDequeue); - encoder.addEventListener('error', onError); - }); +function frameStats(emittedAt, encoded, emitted, fps) { + const period = 1e6 / fps; + // Chunks arrive in decode order. A timestamp that goes backwards means the + // encoder reordered — the condition that silently ate three quarters of + // every export until latencyMode pinned it down. + const reordered = emittedAt.filter((t, i) => i > 0 && t < emittedAt[i - 1]).length; + const stamps = emittedAt.slice().sort((a, b) => a - b); + const gaps = stamps.slice(1).map((t, i) => Math.round((t - stamps[i]) / period)); + const gapHistogram = {}; + for (const g of gaps) gapHistogram[g] = (gapHistogram[g] || 0) + 1; + const firstGap = gaps.findIndex((g) => g !== 1); + return { + encoded, + emitted, + reordered, + effectiveFps: encoded > 0 ? (emitted / encoded) * fps : 0, + maxGap: gaps.length ? Math.max(...gaps) : 0, + gapHistogram, + firstGapAt: firstGap < 0 ? null : Math.round(stamps[firstGap] / period), + }; } export class Exporter { @@ -187,8 +270,30 @@ export class Exporter { }); const errors = []; + // A frame that goes into the encoder and never comes out is invisible: + // no error fires, the muxer just receives fewer samples and spreads + // their timestamps over the full duration, so the file plays at a + // fraction of the intended rate. Count both ends and refuse to hand + // back a video that lost frames. + let framesEncoded = 0; + let chunksEmitted = 0; + const emittedAt = []; const videoEncoder = new VideoEncoder({ - output: (chunk, meta) => muxer.addVideoChunk(chunk, meta), + // This callback runs from the encoder, not from the export loop, so + // a throw here escapes as an uncaught error and the loop never + // learns the chunk was lost. Catch it and route it to `errors`, + // which the loop does check. Count only chunks the muxer accepted — + // counting them on arrival would report success for frames that + // were rejected a line later. + output: (chunk, meta) => { + try { + muxer.addVideoChunk(chunk, meta); + chunksEmitted++; + emittedAt.push(chunk.timestamp); + } catch (e) { + errors.push(e); + } + }, error: (e) => errors.push(e), }); videoEncoder.configure(videoConfig); @@ -221,12 +326,6 @@ export class Exporter { const target = show.renderFrame(frameIndex); show.present(target); // encode from the canvas, which now holds this frame - // True backpressure before every encode — VideoEncoder drops - // frames silently once its queue saturates, which would turn a - // smooth render into the stutter that first shipped here. - await waitForQueue(videoEncoder, errors); - if (errors.length) throw errors[0]; - const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps); const videoFrame = new VideoFrame(show.engine.renderer.canvas, { timestamp, @@ -235,9 +334,17 @@ export class Exporter { // Keyframe every two seconds: seekable output without bloating size. videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 }); videoFrame.close(); + framesEncoded++; - // Yield periodically so the progress UI paints. + // Yield periodically so the progress UI paints, and cap how far + // the encoder may fall behind. This is a memory bound, not a + // correctness one: encode() queues without limit and does not + // drop, so the only cost of an unbounded queue is holding every + // pending frame's pixels at once — which at 4K is gigabytes. if (i % 10 === 0) { + while (videoEncoder.encodeQueueSize > 30) { + await new Promise((r) => setTimeout(r, 4)); + } onProgress && onProgress({ frame: i, total, fraction: i / total, stage: 'rendering', }); @@ -249,6 +356,20 @@ export class Exporter { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'finishing video' }); await videoEncoder.flush(); + this.frameStats = frameStats(emittedAt, framesEncoded, chunksEmitted, fps); + // Always report, not just on mismatch: a silent success that lost + // frames is exactly the failure this is here to catch. + console.info('[export] frameStats', this.frameStats); + if (chunksEmitted !== framesEncoded) { + const s = this.frameStats; + throw new Error( + `encoder lost ${framesEncoded - chunksEmitted} of ${framesEncoded} frames ` + + `(${chunksEmitted} encoded chunks) — the export would play at ` + + `${s.effectiveFps.toFixed(1)} fps instead of ${fps}. ` + + `First gap at frame ${s.firstGapAt}; gap sizes ${JSON.stringify(s.gapHistogram)}`, + ); + } + if (hasAudio) { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' }); try { diff --git a/flow-state/src/main.js b/flow-state/src/main.js index 66f3e24..0569bfb 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -26,6 +26,7 @@ const dom = { panelBody: document.getElementById('panel-body'), panelTabs: document.getElementById('panel-tabs'), hud: document.getElementById('hud'), + toast: document.getElementById('toast'), osd: document.getElementById('btn-osd'), play: document.getElementById('btn-play'), thLabel: document.getElementById('th-label'), @@ -490,6 +491,24 @@ function currentPreset() { return el ? el.value : '1080p'; } +/** + * Surface a message over the stage. + * + * Errors stay until dismissed: an export that fails after several minutes of + * rendering must not scroll past unnoticed, which is exactly what the status + * line inside the export panel allowed. + */ +function showToast(message, { ok = false, timeout = 0 } = {}) { + if (!dom.toast) return; + clearTimeout(showToast._timer); + dom.toast.textContent = message; + dom.toast.classList.toggle('ok', ok); + dom.toast.hidden = false; + if (timeout) showToast._timer = setTimeout(() => { dom.toast.hidden = true; }, timeout); +} + +if (dom.toast) dom.toast.addEventListener('click', () => { dom.toast.hidden = true; }); + function exportProgress(p) { const status = document.getElementById('export-status'); const fill = document.getElementById('export-fill'); @@ -505,6 +524,7 @@ async function runExport(segment) { [...dom.panelTabs.children].forEach((b) => b.classList.toggle('active', b.dataset.tab === 'export')); renderPanel(); + if (dom.toast) dom.toast.hidden = true; try { const preset = currentPreset(); const exporter = new Exporter(state.show); @@ -523,9 +543,12 @@ async function runExport(segment) { status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` + (warnings ? ` (${warnings})` : ''); } + showToast(`export done — ${(blob.size / 1e6).toFixed(1)} MB` + + (warnings ? `\n${warnings}` : ''), { ok: true, timeout: 6000 }); } catch (err) { const status = document.getElementById('export-status'); if (status) status.textContent = `failed: ${err.message}`; + showToast(`export failed\n${err.message}`); console.error(err); } finally { state.busy = false; diff --git a/flow-state/src/ui/style.css b/flow-state/src/ui/style.css index ea577c6..0aa8208 100644 --- a/flow-state/src/ui/style.css +++ b/flow-state/src/ui/style.css @@ -85,6 +85,29 @@ body { pointer-events: none; } +/* Export failures used to land only in a status line inside a panel the user + may not be looking at — a lost render deserves to interrupt. */ +#toast { + position: absolute; bottom: 16px; left: 50%; + transform: translateX(-50%); + max-width: min(680px, calc(100% - 32px)); + background: rgba(7,8,12,0.94); + border: 1px solid #7f1d1d; + border-left: 3px solid #ef4444; + padding: 10px 34px 10px 12px; + font-size: 12px; line-height: 1.6; + color: var(--text); + white-space: pre-wrap; + overflow-wrap: anywhere; + cursor: pointer; +} +#toast.ok { border-color: #14532d; border-left-color: var(--accent); } +#toast::after { + content: '×'; + position: absolute; top: 6px; right: 10px; + color: var(--dim); font-size: 14px; +} + #transport { grid-area: transport; background: var(--panel); } #timeline { height: 46px; } #timeline-canvas { width: 100%; height: 46px; display: block; cursor: pointer; } diff --git a/flow-state/tools/probe-mp4.js b/flow-state/tools/probe-mp4.js new file mode 100644 index 0000000..f247b84 --- /dev/null +++ b/flow-state/tools/probe-mp4.js @@ -0,0 +1,88 @@ +#!/usr/bin/env node +/** + * Report what an exported mp4 actually contains, per track: timescale, sample + * count, and the sample-duration table (stts). + * + * The point is to separate "the exporter dropped frames" from "the player is + * stuttering on a file that is fine". A 60 fps, N-second video must show + * timescale 60 and one stts row of [N*60, 1]. Several rows — or any row with a + * duration of 0 — means the muxed timeline is wrong, and the frames that are + * there will not be shown. + * + * node tools/probe-mp4.js path/to/export.mp4 + */ +import fs from 'fs'; + +const path = process.argv[2]; +if (!path) { + console.error('usage: node tools/probe-mp4.js '); + process.exit(1); +} +const buf = fs.readFileSync(path); + +const CONTAINERS = ['moov', 'trak', 'mdia', 'minf', 'stbl']; +const tracks = []; +let cur = null; + +function walk(start, end) { + let off = start; + while (off + 8 <= end) { + let size = buf.readUInt32BE(off); + const type = buf.toString('latin1', off + 4, off + 8); + let hdr = 8; + if (size === 1) { size = Number(buf.readBigUInt64BE(off + 8)); hdr = 16; } + if (size === 0) size = end - off; + if (CONTAINERS.includes(type)) { + if (type === 'trak') { cur = {}; tracks.push(cur); } + walk(off + hdr, off + size); + } else if (cur) { + box(type, off + hdr); + } + off += size; + } +} + +function box(type, s) { + if (type === 'mdhd') { + cur.timescale = buf.readUInt32BE(s + 12); + cur.duration = buf.readUInt32BE(s + 16); + } else if (type === 'hdlr') { + cur.kind = buf.toString('latin1', s + 8, s + 12); + } else if (type === 'stsd') { + cur.format = buf.toString('latin1', s + 12, s + 16); + } else if (type === 'stsz') { + cur.sampleCount = buf.readUInt32BE(s + 8); + } else if (type === 'stts') { + const n = buf.readUInt32BE(s + 4); + const rows = []; + let samples = 0; + let total = 0; + for (let i = 0; i < n; i++) { + const count = buf.readUInt32BE(s + 8 + i * 8); + const delta = buf.readUInt32BE(s + 12 + i * 8); + rows.push({ count, delta }); + samples += count; + total += count * delta; + } + cur.stts = { rows, samples, total }; + } +} + +walk(0, buf.length); + +for (const t of tracks) { + const stts = t.stts || { rows: [], samples: 0, total: 0 }; + const seconds = t.timescale ? stts.total / t.timescale : 0; + console.log(`\n[${t.kind}] ${t.format} · timescale ${t.timescale}`); + console.log(` samples: ${t.sampleCount} · media duration: ${seconds.toFixed(3)}s`); + if (t.kind === 'vide' && seconds > 0) { + console.log(` effective frame rate: ${(stts.samples / seconds).toFixed(3)} fps`); + } + console.log(` stts rows (${stts.rows.length}):`); + for (const r of stts.rows.slice(0, 20)) { + console.log(` ${r.count} × ${r.delta}${r.delta === 0 ? ' <-- zero duration: these frames never display' : ''}`); + } + if (stts.rows.length > 20) console.log(` … ${stts.rows.length - 20} more rows`); + const zeros = stts.rows.filter((r) => r.delta === 0).reduce((a, r) => a + r.count, 0); + if (zeros) console.log(` ${zeros} samples have zero duration`); +}