diff --git a/flow-state/index.html b/flow-state/index.html
index 51ee63c..78d7966 100644
--- a/flow-state/index.html
+++ b/flow-state/index.html
@@ -20,6 +20,7 @@
+
diff --git a/flow-state/src/export/Exporter.js b/flow-state/src/export/Exporter.js
index 9cd78a8..9cc7e66 100644
--- a/flow-state/src/export/Exporter.js
+++ b/flow-state/src/export/Exporter.js
@@ -25,11 +25,79 @@ export function isSupported() {
return typeof VideoEncoder !== 'undefined' && typeof VideoFrame !== 'undefined';
}
-/** Probe for a codec configuration the browser will actually accept. */
+/**
+ * Frames to run through a candidate encoder when testing it for reordering.
+ *
+ * Hierarchical B-pyramids repeat every 4 frames, so two GOPs is enough to see
+ * the pattern while costing almost nothing even at 4K.
+ */
+const ORDER_PROBE_FRAMES = 12;
+
+/**
+ * Does this configuration emit chunks in presentation order?
+ *
+ * It matters because of what WebCodecs does NOT expose. An encoder that emits
+ * B-frames delivers chunks in *decode* order, but `EncodedVideoChunk` carries
+ * only a presentation timestamp and no decode timestamp — so there is no way to
+ * recover the decode timeline after the fact. mp4-muxer, handed presentation
+ * timestamps, sees DTS run backwards and rejects the chunk; and because that
+ * throw happens inside the encoder's output callback it cannot reach the export
+ * loop at all. The result is a file missing three quarters of its frames with
+ * nothing reported. Writing the correct timeline instead would need negative
+ * composition offsets, which mp4-muxer emits as a version-0 `ctts` box —
+ * unsigned, so it cannot represent them.
+ *
+ * `isConfigSupported` says nothing about reordering, and neither does
+ * `latencyMode: 'realtime'` — measured, Chrome still emits B-frames under it.
+ * So actually encode a few frames and watch what order they come back in.
+ */
+async function emitsInPresentationOrder(config) {
+ const stamps = [];
+ let failed = false;
+ let encoder = null;
+ try {
+ encoder = new VideoEncoder({
+ output: (chunk) => stamps.push(chunk.timestamp),
+ error: () => { failed = true; },
+ });
+ encoder.configure(config);
+
+ const canvas = new OffscreenCanvas(config.width, config.height);
+ const ctx = canvas.getContext('2d');
+ const period = Math.round(1e6 / config.framerate);
+ for (let i = 0; i < ORDER_PROBE_FRAMES; i++) {
+ // Vary the content: an encoder fed identical frames may collapse
+ // them and never exercise its reordering path.
+ ctx.fillStyle = `rgb(${(i * 37) % 256} ${(i * 91) % 256} ${(i * 17) % 256})`;
+ ctx.fillRect(0, 0, config.width, config.height);
+ const frame = new VideoFrame(canvas, { timestamp: i * period, duration: period });
+ encoder.encode(frame, { keyFrame: i === 0 });
+ frame.close();
+ }
+ await encoder.flush();
+ } catch {
+ failed = true;
+ } finally {
+ try { if (encoder) encoder.close(); } catch { /* already closed */ }
+ }
+ if (failed || stamps.length !== ORDER_PROBE_FRAMES) return false;
+ return stamps.every((t, i) => i === 0 || t > stamps[i - 1]);
+}
+
+/**
+ * Probe for a codec configuration the browser will actually accept — and will
+ * encode in presentation order.
+ *
+ * The candidates run high profile first for compression efficiency, down to
+ * baseline last. Baseline forbids B-slices outright, so it is the profile that
+ * cannot reorder; the earlier entries are tried first because when a browser
+ * does not reorder there is no reason to give up their quality.
+ */
async function pickVideoConfig(width, height, bitrate, fps) {
const candidates = [
'avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e',
];
+ const supported = [];
for (const codec of candidates) {
const config = {
codec, width, height, bitrate, framerate: fps,
@@ -37,8 +105,19 @@ async function pickVideoConfig(width, height, bitrate, fps) {
};
try {
const support = await VideoEncoder.isConfigSupported(config);
- if (support.supported) return config;
- } catch { /* try the next one */ }
+ if (!support.supported) continue;
+ } catch { continue; }
+ supported.push(config);
+ if (await emitsInPresentationOrder(config)) return config;
+ }
+ // Every supported profile reorders. Refuse rather than write a file that
+ // silently loses most of its frames — see emitsInPresentationOrder.
+ if (supported.length) {
+ throw new Error(
+ 'every supported H.264 profile emits frames out of order on this browser ' +
+ `(tried ${supported.map((c) => c.codec).join(', ')}), which this exporter ` +
+ 'cannot mux correctly',
+ );
}
return null;
}
@@ -98,30 +177,34 @@ function resampleInterleaved(input, fromRate, toRate, channels) {
}
/**
- * Backpressure: VideoEncoder.encode() silently drops frames once its internal
- * queue is full (encodeQueueSize is capped at an implementation-defined
- * limit). Polling that number is unreliable — the only signal that a frame
- * actually left the queue is the 'dequeue' event, so wait on it before every
- * encode. Keeping the queue small (rather than draining it fully) leaves the
- * hardware encoder a pipeline to chew on while guaranteeing nothing is dropped.
+ * Describe what actually came out of the encoder.
+ *
+ * A frame that goes in and never comes out is otherwise invisible — no error
+ * fires, the muxer simply receives fewer samples and spreads their timestamps
+ * across the full duration, so the file plays at a fraction of the intended
+ * rate. The gap histogram says whether losses were a steady decimation (one
+ * dominant gap size) or bursts (a long tail), which are different bugs.
*/
-const MAX_ENCODER_QUEUE = 8;
-
-function waitForQueue(encoder, errors) {
- if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve();
- return new Promise((resolve, reject) => {
- const cleanup = () => {
- encoder.removeEventListener('dequeue', onDequeue);
- encoder.removeEventListener('error', onError);
- };
- const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); };
- const onDequeue = () => {
- if (errors.length) { cleanup(); reject(errors[0]); return; }
- if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); }
- };
- encoder.addEventListener('dequeue', onDequeue);
- encoder.addEventListener('error', onError);
- });
+function frameStats(emittedAt, encoded, emitted, fps) {
+ const period = 1e6 / fps;
+ // Chunks arrive in decode order. A timestamp that goes backwards means the
+ // encoder reordered — the condition that silently ate three quarters of
+ // every export until latencyMode pinned it down.
+ const reordered = emittedAt.filter((t, i) => i > 0 && t < emittedAt[i - 1]).length;
+ const stamps = emittedAt.slice().sort((a, b) => a - b);
+ const gaps = stamps.slice(1).map((t, i) => Math.round((t - stamps[i]) / period));
+ const gapHistogram = {};
+ for (const g of gaps) gapHistogram[g] = (gapHistogram[g] || 0) + 1;
+ const firstGap = gaps.findIndex((g) => g !== 1);
+ return {
+ encoded,
+ emitted,
+ reordered,
+ effectiveFps: encoded > 0 ? (emitted / encoded) * fps : 0,
+ maxGap: gaps.length ? Math.max(...gaps) : 0,
+ gapHistogram,
+ firstGapAt: firstGap < 0 ? null : Math.round(stamps[firstGap] / period),
+ };
}
export class Exporter {
@@ -187,8 +270,30 @@ export class Exporter {
});
const errors = [];
+ // A frame that goes into the encoder and never comes out is invisible:
+ // no error fires, the muxer just receives fewer samples and spreads
+ // their timestamps over the full duration, so the file plays at a
+ // fraction of the intended rate. Count both ends and refuse to hand
+ // back a video that lost frames.
+ let framesEncoded = 0;
+ let chunksEmitted = 0;
+ const emittedAt = [];
const videoEncoder = new VideoEncoder({
- output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
+ // This callback runs from the encoder, not from the export loop, so
+ // a throw here escapes as an uncaught error and the loop never
+ // learns the chunk was lost. Catch it and route it to `errors`,
+ // which the loop does check. Count only chunks the muxer accepted —
+ // counting them on arrival would report success for frames that
+ // were rejected a line later.
+ output: (chunk, meta) => {
+ try {
+ muxer.addVideoChunk(chunk, meta);
+ chunksEmitted++;
+ emittedAt.push(chunk.timestamp);
+ } catch (e) {
+ errors.push(e);
+ }
+ },
error: (e) => errors.push(e),
});
videoEncoder.configure(videoConfig);
@@ -221,12 +326,6 @@ export class Exporter {
const target = show.renderFrame(frameIndex);
show.present(target); // encode from the canvas, which now holds this frame
- // True backpressure before every encode — VideoEncoder drops
- // frames silently once its queue saturates, which would turn a
- // smooth render into the stutter that first shipped here.
- await waitForQueue(videoEncoder, errors);
- if (errors.length) throw errors[0];
-
const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps);
const videoFrame = new VideoFrame(show.engine.renderer.canvas, {
timestamp,
@@ -235,9 +334,17 @@ export class Exporter {
// Keyframe every two seconds: seekable output without bloating size.
videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 });
videoFrame.close();
+ framesEncoded++;
- // Yield periodically so the progress UI paints.
+ // Yield periodically so the progress UI paints, and cap how far
+ // the encoder may fall behind. This is a memory bound, not a
+ // correctness one: encode() queues without limit and does not
+ // drop, so the only cost of an unbounded queue is holding every
+ // pending frame's pixels at once — which at 4K is gigabytes.
if (i % 10 === 0) {
+ while (videoEncoder.encodeQueueSize > 30) {
+ await new Promise((r) => setTimeout(r, 4));
+ }
onProgress && onProgress({
frame: i, total, fraction: i / total, stage: 'rendering',
});
@@ -249,6 +356,20 @@ export class Exporter {
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'finishing video' });
await videoEncoder.flush();
+ this.frameStats = frameStats(emittedAt, framesEncoded, chunksEmitted, fps);
+ // Always report, not just on mismatch: a silent success that lost
+ // frames is exactly the failure this is here to catch.
+ console.info('[export] frameStats', this.frameStats);
+ if (chunksEmitted !== framesEncoded) {
+ const s = this.frameStats;
+ throw new Error(
+ `encoder lost ${framesEncoded - chunksEmitted} of ${framesEncoded} frames ` +
+ `(${chunksEmitted} encoded chunks) — the export would play at ` +
+ `${s.effectiveFps.toFixed(1)} fps instead of ${fps}. ` +
+ `First gap at frame ${s.firstGapAt}; gap sizes ${JSON.stringify(s.gapHistogram)}`,
+ );
+ }
+
if (hasAudio) {
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' });
try {
diff --git a/flow-state/src/main.js b/flow-state/src/main.js
index 66f3e24..0569bfb 100644
--- a/flow-state/src/main.js
+++ b/flow-state/src/main.js
@@ -26,6 +26,7 @@ const dom = {
panelBody: document.getElementById('panel-body'),
panelTabs: document.getElementById('panel-tabs'),
hud: document.getElementById('hud'),
+ toast: document.getElementById('toast'),
osd: document.getElementById('btn-osd'),
play: document.getElementById('btn-play'),
thLabel: document.getElementById('th-label'),
@@ -490,6 +491,24 @@ function currentPreset() {
return el ? el.value : '1080p';
}
+/**
+ * Surface a message over the stage.
+ *
+ * Errors stay until dismissed: an export that fails after several minutes of
+ * rendering must not scroll past unnoticed, which is exactly what the status
+ * line inside the export panel allowed.
+ */
+function showToast(message, { ok = false, timeout = 0 } = {}) {
+ if (!dom.toast) return;
+ clearTimeout(showToast._timer);
+ dom.toast.textContent = message;
+ dom.toast.classList.toggle('ok', ok);
+ dom.toast.hidden = false;
+ if (timeout) showToast._timer = setTimeout(() => { dom.toast.hidden = true; }, timeout);
+}
+
+if (dom.toast) dom.toast.addEventListener('click', () => { dom.toast.hidden = true; });
+
function exportProgress(p) {
const status = document.getElementById('export-status');
const fill = document.getElementById('export-fill');
@@ -505,6 +524,7 @@ async function runExport(segment) {
[...dom.panelTabs.children].forEach((b) => b.classList.toggle('active', b.dataset.tab === 'export'));
renderPanel();
+ if (dom.toast) dom.toast.hidden = true;
try {
const preset = currentPreset();
const exporter = new Exporter(state.show);
@@ -523,9 +543,12 @@ async function runExport(segment) {
status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` +
(warnings ? ` (${warnings})` : '');
}
+ showToast(`export done — ${(blob.size / 1e6).toFixed(1)} MB` +
+ (warnings ? `\n${warnings}` : ''), { ok: true, timeout: 6000 });
} catch (err) {
const status = document.getElementById('export-status');
if (status) status.textContent = `failed: ${err.message}`;
+ showToast(`export failed\n${err.message}`);
console.error(err);
} finally {
state.busy = false;
diff --git a/flow-state/src/ui/style.css b/flow-state/src/ui/style.css
index ea577c6..0aa8208 100644
--- a/flow-state/src/ui/style.css
+++ b/flow-state/src/ui/style.css
@@ -85,6 +85,29 @@ body {
pointer-events: none;
}
+/* Export failures used to land only in a status line inside a panel the user
+ may not be looking at — a lost render deserves to interrupt. */
+#toast {
+ position: absolute; bottom: 16px; left: 50%;
+ transform: translateX(-50%);
+ max-width: min(680px, calc(100% - 32px));
+ background: rgba(7,8,12,0.94);
+ border: 1px solid #7f1d1d;
+ border-left: 3px solid #ef4444;
+ padding: 10px 34px 10px 12px;
+ font-size: 12px; line-height: 1.6;
+ color: var(--text);
+ white-space: pre-wrap;
+ overflow-wrap: anywhere;
+ cursor: pointer;
+}
+#toast.ok { border-color: #14532d; border-left-color: var(--accent); }
+#toast::after {
+ content: '×';
+ position: absolute; top: 6px; right: 10px;
+ color: var(--dim); font-size: 14px;
+}
+
#transport { grid-area: transport; background: var(--panel); }
#timeline { height: 46px; }
#timeline-canvas { width: 100%; height: 46px; display: block; cursor: pointer; }
diff --git a/flow-state/tools/probe-mp4.js b/flow-state/tools/probe-mp4.js
new file mode 100644
index 0000000..f247b84
--- /dev/null
+++ b/flow-state/tools/probe-mp4.js
@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Report what an exported mp4 actually contains, per track: timescale, sample
+ * count, and the sample-duration table (stts).
+ *
+ * The point is to separate "the exporter dropped frames" from "the player is
+ * stuttering on a file that is fine". A 60 fps, N-second video must show
+ * timescale 60 and one stts row of [N*60, 1]. Several rows — or any row with a
+ * duration of 0 — means the muxed timeline is wrong, and the frames that are
+ * there will not be shown.
+ *
+ * node tools/probe-mp4.js path/to/export.mp4
+ */
+import fs from 'fs';
+
+const path = process.argv[2];
+if (!path) {
+ console.error('usage: node tools/probe-mp4.js ');
+ process.exit(1);
+}
+const buf = fs.readFileSync(path);
+
+const CONTAINERS = ['moov', 'trak', 'mdia', 'minf', 'stbl'];
+const tracks = [];
+let cur = null;
+
+function walk(start, end) {
+ let off = start;
+ while (off + 8 <= end) {
+ let size = buf.readUInt32BE(off);
+ const type = buf.toString('latin1', off + 4, off + 8);
+ let hdr = 8;
+ if (size === 1) { size = Number(buf.readBigUInt64BE(off + 8)); hdr = 16; }
+ if (size === 0) size = end - off;
+ if (CONTAINERS.includes(type)) {
+ if (type === 'trak') { cur = {}; tracks.push(cur); }
+ walk(off + hdr, off + size);
+ } else if (cur) {
+ box(type, off + hdr);
+ }
+ off += size;
+ }
+}
+
+function box(type, s) {
+ if (type === 'mdhd') {
+ cur.timescale = buf.readUInt32BE(s + 12);
+ cur.duration = buf.readUInt32BE(s + 16);
+ } else if (type === 'hdlr') {
+ cur.kind = buf.toString('latin1', s + 8, s + 12);
+ } else if (type === 'stsd') {
+ cur.format = buf.toString('latin1', s + 12, s + 16);
+ } else if (type === 'stsz') {
+ cur.sampleCount = buf.readUInt32BE(s + 8);
+ } else if (type === 'stts') {
+ const n = buf.readUInt32BE(s + 4);
+ const rows = [];
+ let samples = 0;
+ let total = 0;
+ for (let i = 0; i < n; i++) {
+ const count = buf.readUInt32BE(s + 8 + i * 8);
+ const delta = buf.readUInt32BE(s + 12 + i * 8);
+ rows.push({ count, delta });
+ samples += count;
+ total += count * delta;
+ }
+ cur.stts = { rows, samples, total };
+ }
+}
+
+walk(0, buf.length);
+
+for (const t of tracks) {
+ const stts = t.stts || { rows: [], samples: 0, total: 0 };
+ const seconds = t.timescale ? stts.total / t.timescale : 0;
+ console.log(`\n[${t.kind}] ${t.format} · timescale ${t.timescale}`);
+ console.log(` samples: ${t.sampleCount} · media duration: ${seconds.toFixed(3)}s`);
+ if (t.kind === 'vide' && seconds > 0) {
+ console.log(` effective frame rate: ${(stts.samples / seconds).toFixed(3)} fps`);
+ }
+ console.log(` stts rows (${stts.rows.length}):`);
+ for (const r of stts.rows.slice(0, 20)) {
+ console.log(` ${r.count} × ${r.delta}${r.delta === 0 ? ' <-- zero duration: these frames never display' : ''}`);
+ }
+ if (stts.rows.length > 20) console.log(` … ${stts.rows.length - 20} more rows`);
+ const zeros = stts.rows.filter((r) => r.delta === 0).reduce((a, r) => a + r.count, 0);
+ if (zeros) console.log(` ${zeros} samples have zero duration`);
+}