Fix export frame loss: reject codec profiles that reorder frames
The exporter was losing roughly three of every four frames. A 20s render
muxed 319 samples instead of 1200 and played at 15.9 fps, with no error
reported and a file that looked superficially fine.
The cause was B-frames. WebCodecs delivers chunks in decode order, but
EncodedVideoChunk carries only a presentation timestamp — there is no decode
timestamp to recover the real order from. Handed presentation timestamps as
if they were decode timestamps, mp4-muxer saw DTS run backwards and rejected
every reordered chunk. That throw happened inside the encoder's output
callback, where it could not reach the export loop, so it surfaced as an
uncaught error and the render carried on. Only the I/P frames survived, one
per 4-frame GOP, which is exactly the stts pattern the files showed.
Writing the correct timeline instead is not available to us: it needs
negative composition offsets, and mp4-muxer emits ctts as a version-0 box,
which is unsigned. isConfigSupported says nothing about reordering, and
measurement showed latencyMode: 'realtime' does not prevent it either.
So pickVideoConfig now test-encodes 12 frames per candidate and checks the
order they come back in, taking the first profile that does not reorder.
Candidates stay in quality order, high profile down to baseline, so browsers
that never reorder keep the better profiles; baseline forbids B-slices by
spec and is the guaranteed floor. If every supported profile reorders the
export fails up front rather than after minutes of rendering.
Two guards so this class of loss cannot be silent again:
- The output callback catches, routing muxer rejections to the error list
the loop actually checks.
- Frames in and chunks accepted are counted and compared after flush, with
the reordering count and a gap histogram alongside. The count deliberately
tracks chunks the muxer took, not chunks that arrived — counting arrivals
reports success for frames rejected a line later.
Failures now raise a toast over the stage that stays until dismissed. An
export that dies after a long render should not sit unread in a panel.
Also corrects the record from d31d0fc, which claimed VideoEncoder.encode()
silently drops frames once its queue saturates. It does not: the queue grows
without bound and the only cost is memory. That commit's dequeue-gated
backpressure addressed a mechanism that does not exist and is reverted here;
the queue poll it replaced is restored, described honestly as a memory bound.
The opus resampling from that commit was a real fix and is untouched.
tools/probe-mp4.js reports per-track timescale, sample count and the stts
table, which is what identified the fault and what verifies a good export:
one row of [N x 1].
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Dejvino
parent
d31d0fc3a1
commit
b7c6fd1c5a
@ -20,6 +20,7 @@
|
||||
</div>
|
||||
</div>
|
||||
<div id="hud" hidden></div>
|
||||
<div id="toast" hidden></div>
|
||||
</div>
|
||||
|
||||
<div id="transport">
|
||||
|
||||
@ -25,11 +25,79 @@ export function isSupported() {
|
||||
return typeof VideoEncoder !== 'undefined' && typeof VideoFrame !== 'undefined';
|
||||
}
|
||||
|
||||
/** Probe for a codec configuration the browser will actually accept. */
|
||||
/**
|
||||
* Frames to run through a candidate encoder when testing it for reordering.
|
||||
*
|
||||
* Hierarchical B-pyramids repeat every 4 frames, so two GOPs is enough to see
|
||||
* the pattern while costing almost nothing even at 4K.
|
||||
*/
|
||||
const ORDER_PROBE_FRAMES = 12;
|
||||
|
||||
/**
|
||||
* Does this configuration emit chunks in presentation order?
|
||||
*
|
||||
* It matters because of what WebCodecs does NOT expose. An encoder that emits
|
||||
* B-frames delivers chunks in *decode* order, but `EncodedVideoChunk` carries
|
||||
* only a presentation timestamp and no decode timestamp — so there is no way to
|
||||
* recover the decode timeline after the fact. mp4-muxer, handed presentation
|
||||
* timestamps, sees DTS run backwards and rejects the chunk; and because that
|
||||
* throw happens inside the encoder's output callback it cannot reach the export
|
||||
* loop at all. The result is a file missing three quarters of its frames with
|
||||
* nothing reported. Writing the correct timeline instead would need negative
|
||||
* composition offsets, which mp4-muxer emits as a version-0 `ctts` box —
|
||||
* unsigned, so it cannot represent them.
|
||||
*
|
||||
* `isConfigSupported` says nothing about reordering, and neither does
|
||||
* `latencyMode: 'realtime'` — measured, Chrome still emits B-frames under it.
|
||||
* So actually encode a few frames and watch what order they come back in.
|
||||
*/
|
||||
async function emitsInPresentationOrder(config) {
|
||||
const stamps = [];
|
||||
let failed = false;
|
||||
let encoder = null;
|
||||
try {
|
||||
encoder = new VideoEncoder({
|
||||
output: (chunk) => stamps.push(chunk.timestamp),
|
||||
error: () => { failed = true; },
|
||||
});
|
||||
encoder.configure(config);
|
||||
|
||||
const canvas = new OffscreenCanvas(config.width, config.height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
const period = Math.round(1e6 / config.framerate);
|
||||
for (let i = 0; i < ORDER_PROBE_FRAMES; i++) {
|
||||
// Vary the content: an encoder fed identical frames may collapse
|
||||
// them and never exercise its reordering path.
|
||||
ctx.fillStyle = `rgb(${(i * 37) % 256} ${(i * 91) % 256} ${(i * 17) % 256})`;
|
||||
ctx.fillRect(0, 0, config.width, config.height);
|
||||
const frame = new VideoFrame(canvas, { timestamp: i * period, duration: period });
|
||||
encoder.encode(frame, { keyFrame: i === 0 });
|
||||
frame.close();
|
||||
}
|
||||
await encoder.flush();
|
||||
} catch {
|
||||
failed = true;
|
||||
} finally {
|
||||
try { if (encoder) encoder.close(); } catch { /* already closed */ }
|
||||
}
|
||||
if (failed || stamps.length !== ORDER_PROBE_FRAMES) return false;
|
||||
return stamps.every((t, i) => i === 0 || t > stamps[i - 1]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe for a codec configuration the browser will actually accept — and will
|
||||
* encode in presentation order.
|
||||
*
|
||||
* The candidates run high profile first for compression efficiency, down to
|
||||
* baseline last. Baseline forbids B-slices outright, so it is the profile that
|
||||
* cannot reorder; the earlier entries are tried first because when a browser
|
||||
* does not reorder there is no reason to give up their quality.
|
||||
*/
|
||||
async function pickVideoConfig(width, height, bitrate, fps) {
|
||||
const candidates = [
|
||||
'avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e',
|
||||
];
|
||||
const supported = [];
|
||||
for (const codec of candidates) {
|
||||
const config = {
|
||||
codec, width, height, bitrate, framerate: fps,
|
||||
@ -37,8 +105,19 @@ async function pickVideoConfig(width, height, bitrate, fps) {
|
||||
};
|
||||
try {
|
||||
const support = await VideoEncoder.isConfigSupported(config);
|
||||
if (support.supported) return config;
|
||||
} catch { /* try the next one */ }
|
||||
if (!support.supported) continue;
|
||||
} catch { continue; }
|
||||
supported.push(config);
|
||||
if (await emitsInPresentationOrder(config)) return config;
|
||||
}
|
||||
// Every supported profile reorders. Refuse rather than write a file that
|
||||
// silently loses most of its frames — see emitsInPresentationOrder.
|
||||
if (supported.length) {
|
||||
throw new Error(
|
||||
'every supported H.264 profile emits frames out of order on this browser ' +
|
||||
`(tried ${supported.map((c) => c.codec).join(', ')}), which this exporter ` +
|
||||
'cannot mux correctly',
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@ -98,30 +177,34 @@ function resampleInterleaved(input, fromRate, toRate, channels) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Backpressure: VideoEncoder.encode() silently drops frames once its internal
|
||||
* queue is full (encodeQueueSize is capped at an implementation-defined
|
||||
* limit). Polling that number is unreliable — the only signal that a frame
|
||||
* actually left the queue is the 'dequeue' event, so wait on it before every
|
||||
* encode. Keeping the queue small (rather than draining it fully) leaves the
|
||||
* hardware encoder a pipeline to chew on while guaranteeing nothing is dropped.
|
||||
* Describe what actually came out of the encoder.
|
||||
*
|
||||
* A frame that goes in and never comes out is otherwise invisible — no error
|
||||
* fires, the muxer simply receives fewer samples and spreads their timestamps
|
||||
* across the full duration, so the file plays at a fraction of the intended
|
||||
* rate. The gap histogram says whether losses were a steady decimation (one
|
||||
* dominant gap size) or bursts (a long tail), which are different bugs.
|
||||
*/
|
||||
const MAX_ENCODER_QUEUE = 8;
|
||||
|
||||
function waitForQueue(encoder, errors) {
|
||||
if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
encoder.removeEventListener('dequeue', onDequeue);
|
||||
encoder.removeEventListener('error', onError);
|
||||
function frameStats(emittedAt, encoded, emitted, fps) {
|
||||
const period = 1e6 / fps;
|
||||
// Chunks arrive in decode order. A timestamp that goes backwards means the
|
||||
// encoder reordered — the condition that silently ate three quarters of
|
||||
// every export until latencyMode pinned it down.
|
||||
const reordered = emittedAt.filter((t, i) => i > 0 && t < emittedAt[i - 1]).length;
|
||||
const stamps = emittedAt.slice().sort((a, b) => a - b);
|
||||
const gaps = stamps.slice(1).map((t, i) => Math.round((t - stamps[i]) / period));
|
||||
const gapHistogram = {};
|
||||
for (const g of gaps) gapHistogram[g] = (gapHistogram[g] || 0) + 1;
|
||||
const firstGap = gaps.findIndex((g) => g !== 1);
|
||||
return {
|
||||
encoded,
|
||||
emitted,
|
||||
reordered,
|
||||
effectiveFps: encoded > 0 ? (emitted / encoded) * fps : 0,
|
||||
maxGap: gaps.length ? Math.max(...gaps) : 0,
|
||||
gapHistogram,
|
||||
firstGapAt: firstGap < 0 ? null : Math.round(stamps[firstGap] / period),
|
||||
};
|
||||
const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); };
|
||||
const onDequeue = () => {
|
||||
if (errors.length) { cleanup(); reject(errors[0]); return; }
|
||||
if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); }
|
||||
};
|
||||
encoder.addEventListener('dequeue', onDequeue);
|
||||
encoder.addEventListener('error', onError);
|
||||
});
|
||||
}
|
||||
|
||||
export class Exporter {
|
||||
@ -187,8 +270,30 @@ export class Exporter {
|
||||
});
|
||||
|
||||
const errors = [];
|
||||
// A frame that goes into the encoder and never comes out is invisible:
|
||||
// no error fires, the muxer just receives fewer samples and spreads
|
||||
// their timestamps over the full duration, so the file plays at a
|
||||
// fraction of the intended rate. Count both ends and refuse to hand
|
||||
// back a video that lost frames.
|
||||
let framesEncoded = 0;
|
||||
let chunksEmitted = 0;
|
||||
const emittedAt = [];
|
||||
const videoEncoder = new VideoEncoder({
|
||||
output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
|
||||
// This callback runs from the encoder, not from the export loop, so
|
||||
// a throw here escapes as an uncaught error and the loop never
|
||||
// learns the chunk was lost. Catch it and route it to `errors`,
|
||||
// which the loop does check. Count only chunks the muxer accepted —
|
||||
// counting them on arrival would report success for frames that
|
||||
// were rejected a line later.
|
||||
output: (chunk, meta) => {
|
||||
try {
|
||||
muxer.addVideoChunk(chunk, meta);
|
||||
chunksEmitted++;
|
||||
emittedAt.push(chunk.timestamp);
|
||||
} catch (e) {
|
||||
errors.push(e);
|
||||
}
|
||||
},
|
||||
error: (e) => errors.push(e),
|
||||
});
|
||||
videoEncoder.configure(videoConfig);
|
||||
@ -221,12 +326,6 @@ export class Exporter {
|
||||
const target = show.renderFrame(frameIndex);
|
||||
show.present(target); // encode from the canvas, which now holds this frame
|
||||
|
||||
// True backpressure before every encode — VideoEncoder drops
|
||||
// frames silently once its queue saturates, which would turn a
|
||||
// smooth render into the stutter that first shipped here.
|
||||
await waitForQueue(videoEncoder, errors);
|
||||
if (errors.length) throw errors[0];
|
||||
|
||||
const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps);
|
||||
const videoFrame = new VideoFrame(show.engine.renderer.canvas, {
|
||||
timestamp,
|
||||
@ -235,9 +334,17 @@ export class Exporter {
|
||||
// Keyframe every two seconds: seekable output without bloating size.
|
||||
videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 });
|
||||
videoFrame.close();
|
||||
framesEncoded++;
|
||||
|
||||
// Yield periodically so the progress UI paints.
|
||||
// Yield periodically so the progress UI paints, and cap how far
|
||||
// the encoder may fall behind. This is a memory bound, not a
|
||||
// correctness one: encode() queues without limit and does not
|
||||
// drop, so the only cost of an unbounded queue is holding every
|
||||
// pending frame's pixels at once — which at 4K is gigabytes.
|
||||
if (i % 10 === 0) {
|
||||
while (videoEncoder.encodeQueueSize > 30) {
|
||||
await new Promise((r) => setTimeout(r, 4));
|
||||
}
|
||||
onProgress && onProgress({
|
||||
frame: i, total, fraction: i / total, stage: 'rendering',
|
||||
});
|
||||
@ -249,6 +356,20 @@ export class Exporter {
|
||||
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'finishing video' });
|
||||
await videoEncoder.flush();
|
||||
|
||||
this.frameStats = frameStats(emittedAt, framesEncoded, chunksEmitted, fps);
|
||||
// Always report, not just on mismatch: a silent success that lost
|
||||
// frames is exactly the failure this is here to catch.
|
||||
console.info('[export] frameStats', this.frameStats);
|
||||
if (chunksEmitted !== framesEncoded) {
|
||||
const s = this.frameStats;
|
||||
throw new Error(
|
||||
`encoder lost ${framesEncoded - chunksEmitted} of ${framesEncoded} frames ` +
|
||||
`(${chunksEmitted} encoded chunks) — the export would play at ` +
|
||||
`${s.effectiveFps.toFixed(1)} fps instead of ${fps}. ` +
|
||||
`First gap at frame ${s.firstGapAt}; gap sizes ${JSON.stringify(s.gapHistogram)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (hasAudio) {
|
||||
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' });
|
||||
try {
|
||||
|
||||
@ -26,6 +26,7 @@ const dom = {
|
||||
panelBody: document.getElementById('panel-body'),
|
||||
panelTabs: document.getElementById('panel-tabs'),
|
||||
hud: document.getElementById('hud'),
|
||||
toast: document.getElementById('toast'),
|
||||
osd: document.getElementById('btn-osd'),
|
||||
play: document.getElementById('btn-play'),
|
||||
thLabel: document.getElementById('th-label'),
|
||||
@ -490,6 +491,24 @@ function currentPreset() {
|
||||
return el ? el.value : '1080p';
|
||||
}
|
||||
|
||||
/**
|
||||
* Surface a message over the stage.
|
||||
*
|
||||
* Errors stay until dismissed: an export that fails after several minutes of
|
||||
* rendering must not scroll past unnoticed, which is exactly what the status
|
||||
* line inside the export panel allowed.
|
||||
*/
|
||||
function showToast(message, { ok = false, timeout = 0 } = {}) {
|
||||
if (!dom.toast) return;
|
||||
clearTimeout(showToast._timer);
|
||||
dom.toast.textContent = message;
|
||||
dom.toast.classList.toggle('ok', ok);
|
||||
dom.toast.hidden = false;
|
||||
if (timeout) showToast._timer = setTimeout(() => { dom.toast.hidden = true; }, timeout);
|
||||
}
|
||||
|
||||
if (dom.toast) dom.toast.addEventListener('click', () => { dom.toast.hidden = true; });
|
||||
|
||||
function exportProgress(p) {
|
||||
const status = document.getElementById('export-status');
|
||||
const fill = document.getElementById('export-fill');
|
||||
@ -505,6 +524,7 @@ async function runExport(segment) {
|
||||
[...dom.panelTabs.children].forEach((b) => b.classList.toggle('active', b.dataset.tab === 'export'));
|
||||
renderPanel();
|
||||
|
||||
if (dom.toast) dom.toast.hidden = true;
|
||||
try {
|
||||
const preset = currentPreset();
|
||||
const exporter = new Exporter(state.show);
|
||||
@ -523,9 +543,12 @@ async function runExport(segment) {
|
||||
status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` +
|
||||
(warnings ? ` (${warnings})` : '');
|
||||
}
|
||||
showToast(`export done — ${(blob.size / 1e6).toFixed(1)} MB` +
|
||||
(warnings ? `\n${warnings}` : ''), { ok: true, timeout: 6000 });
|
||||
} catch (err) {
|
||||
const status = document.getElementById('export-status');
|
||||
if (status) status.textContent = `failed: ${err.message}`;
|
||||
showToast(`export failed\n${err.message}`);
|
||||
console.error(err);
|
||||
} finally {
|
||||
state.busy = false;
|
||||
|
||||
@ -85,6 +85,29 @@ body {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Export failures used to land only in a status line inside a panel the user
|
||||
may not be looking at — a lost render deserves to interrupt. */
|
||||
#toast {
|
||||
position: absolute; bottom: 16px; left: 50%;
|
||||
transform: translateX(-50%);
|
||||
max-width: min(680px, calc(100% - 32px));
|
||||
background: rgba(7,8,12,0.94);
|
||||
border: 1px solid #7f1d1d;
|
||||
border-left: 3px solid #ef4444;
|
||||
padding: 10px 34px 10px 12px;
|
||||
font-size: 12px; line-height: 1.6;
|
||||
color: var(--text);
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
cursor: pointer;
|
||||
}
|
||||
#toast.ok { border-color: #14532d; border-left-color: var(--accent); }
|
||||
#toast::after {
|
||||
content: '×';
|
||||
position: absolute; top: 6px; right: 10px;
|
||||
color: var(--dim); font-size: 14px;
|
||||
}
|
||||
|
||||
#transport { grid-area: transport; background: var(--panel); }
|
||||
#timeline { height: 46px; }
|
||||
#timeline-canvas { width: 100%; height: 46px; display: block; cursor: pointer; }
|
||||
|
||||
88
flow-state/tools/probe-mp4.js
Normal file
88
flow-state/tools/probe-mp4.js
Normal file
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Report what an exported mp4 actually contains, per track: timescale, sample
|
||||
* count, and the sample-duration table (stts).
|
||||
*
|
||||
* The point is to separate "the exporter dropped frames" from "the player is
|
||||
* stuttering on a file that is fine". A 60 fps, N-second video must show
|
||||
* timescale 60 and one stts row of [N*60, 1]. Several rows — or any row with a
|
||||
* duration of 0 — means the muxed timeline is wrong, and the frames that are
|
||||
* there will not be shown.
|
||||
*
|
||||
* node tools/probe-mp4.js path/to/export.mp4
|
||||
*/
|
||||
import fs from 'fs';
|
||||
|
||||
const path = process.argv[2];
|
||||
if (!path) {
|
||||
console.error('usage: node tools/probe-mp4.js <file.mp4>');
|
||||
process.exit(1);
|
||||
}
|
||||
const buf = fs.readFileSync(path);
|
||||
|
||||
const CONTAINERS = ['moov', 'trak', 'mdia', 'minf', 'stbl'];
|
||||
const tracks = [];
|
||||
let cur = null;
|
||||
|
||||
function walk(start, end) {
|
||||
let off = start;
|
||||
while (off + 8 <= end) {
|
||||
let size = buf.readUInt32BE(off);
|
||||
const type = buf.toString('latin1', off + 4, off + 8);
|
||||
let hdr = 8;
|
||||
if (size === 1) { size = Number(buf.readBigUInt64BE(off + 8)); hdr = 16; }
|
||||
if (size === 0) size = end - off;
|
||||
if (CONTAINERS.includes(type)) {
|
||||
if (type === 'trak') { cur = {}; tracks.push(cur); }
|
||||
walk(off + hdr, off + size);
|
||||
} else if (cur) {
|
||||
box(type, off + hdr);
|
||||
}
|
||||
off += size;
|
||||
}
|
||||
}
|
||||
|
||||
function box(type, s) {
|
||||
if (type === 'mdhd') {
|
||||
cur.timescale = buf.readUInt32BE(s + 12);
|
||||
cur.duration = buf.readUInt32BE(s + 16);
|
||||
} else if (type === 'hdlr') {
|
||||
cur.kind = buf.toString('latin1', s + 8, s + 12);
|
||||
} else if (type === 'stsd') {
|
||||
cur.format = buf.toString('latin1', s + 12, s + 16);
|
||||
} else if (type === 'stsz') {
|
||||
cur.sampleCount = buf.readUInt32BE(s + 8);
|
||||
} else if (type === 'stts') {
|
||||
const n = buf.readUInt32BE(s + 4);
|
||||
const rows = [];
|
||||
let samples = 0;
|
||||
let total = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const count = buf.readUInt32BE(s + 8 + i * 8);
|
||||
const delta = buf.readUInt32BE(s + 12 + i * 8);
|
||||
rows.push({ count, delta });
|
||||
samples += count;
|
||||
total += count * delta;
|
||||
}
|
||||
cur.stts = { rows, samples, total };
|
||||
}
|
||||
}
|
||||
|
||||
walk(0, buf.length);
|
||||
|
||||
for (const t of tracks) {
|
||||
const stts = t.stts || { rows: [], samples: 0, total: 0 };
|
||||
const seconds = t.timescale ? stts.total / t.timescale : 0;
|
||||
console.log(`\n[${t.kind}] ${t.format} · timescale ${t.timescale}`);
|
||||
console.log(` samples: ${t.sampleCount} · media duration: ${seconds.toFixed(3)}s`);
|
||||
if (t.kind === 'vide' && seconds > 0) {
|
||||
console.log(` effective frame rate: ${(stts.samples / seconds).toFixed(3)} fps`);
|
||||
}
|
||||
console.log(` stts rows (${stts.rows.length}):`);
|
||||
for (const r of stts.rows.slice(0, 20)) {
|
||||
console.log(` ${r.count} × ${r.delta}${r.delta === 0 ? ' <-- zero duration: these frames never display' : ''}`);
|
||||
}
|
||||
if (stts.rows.length > 20) console.log(` … ${stts.rows.length - 20} more rows`);
|
||||
const zeros = stts.rows.filter((r) => r.delta === 0).reduce((a, r) => a + r.count, 0);
|
||||
if (zeros) console.log(` ${zeros} samples have zero duration`);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user