Phase 6: preview UI and export
Full preview surface plus WebCodecs export, both driving the same Show — the exporter has no render path of its own, which is what makes parity structural rather than something to keep in sync. Preview: transport with section jumping and looping, drag-scrub, timeline strip showing sections coloured by kind with bar ticks and lock state, generated param controls, reroll (whole track or one section), lock, draft/full quality, debug HUD, click-track download, and a 20s test render at full export quality. Export: probes for a supported H.264 config, warms up before a mid-track range so the first frame carries the feedback state continuous playback would have given it, and encodes audio from the decoded PCM. One real bug found by the gate: AAC is absent from Chromium builds without proprietary codecs, which still ship H.264 encoding — so video succeeded and audio killed the whole export with "Cannot call 'encode' on a closed codec". The exporter now probes AAC then Opus, and a failure mid-encode degrades to video-only rather than losing a long render. Fallbacks are surfaced in the UI; a video that quietly lost its audio is worse than one that says so. Also adds a dev-only window.__flowState handle. The render loop is rAF-driven and rAF does not fire in headless/automated contexts, so this provides a way to step the app by hand. Gate 10/10 (one manual: upload a test render to the real platform once before trusting a full export). Real mp4s verified — ftyp box, honoured frame ranges, resolution restored, cancellation clean. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Dejvino
parent
40188493ac
commit
8a94a3f3a5
@ -0,0 +1,200 @@
|
||||
// Phase 6 gate — export.
|
||||
//
|
||||
// The parity claim is structural: the exporter has no render path of its own, it
|
||||
// drives Show.renderFrame exactly as the preview does. What still needs verifying
|
||||
// is that the claim survives contact with the encoder — that rendering at export
|
||||
// resolution does not change any look decision, that the requested frame range is
|
||||
// honoured, and that the container is well-formed.
|
||||
|
||||
import { check, expect } from './framework.js';
|
||||
import { Show } from '../Show.js';
|
||||
import { generateLook } from '../look/LookGenerator.js';
|
||||
import { FeatureTrack } from '../audio/FeatureTrack.js';
|
||||
import { synthesizeSectioned } from '../audio/synth.js';
|
||||
import { Exporter, isSupported, PRESETS } from '../export/Exporter.js';
|
||||
import { frameDistance } from '../engine/hash.js';
|
||||
|
||||
let cached = null;
|
||||
function track6() {
|
||||
if (!cached) {
|
||||
cached = FeatureTrack.fromAudioBuffer(
|
||||
synthesizeSectioned({ bpm: 128, duration: 40, changeAt: 20 }), { fps: 60 });
|
||||
}
|
||||
return cached;
|
||||
}
|
||||
|
||||
function makeShow(width = 320, height = 180) {
|
||||
const show = new Show({ width, height });
|
||||
const track = track6();
|
||||
show.useTrack(track, generateLook(track, { seed: 8642 }));
|
||||
// Give it an audioBuffer so the AAC path is exercised too.
|
||||
show.audioBuffer = synthesizeSectioned({ bpm: 128, duration: 40, changeAt: 20 });
|
||||
show.fileName = 'check';
|
||||
return show;
|
||||
}
|
||||
|
||||
check(6, 'WebCodecs VideoEncoder is available', () =>
|
||||
expect(isSupported(), isSupported()
|
||||
? 'VideoEncoder and VideoFrame present'
|
||||
: 'unavailable — export cannot run in this browser'));
|
||||
|
||||
check(6, 'render resolution does not change any look decision', () => {
|
||||
// Resolution must change only the sampling, never which scene is on screen or
|
||||
// what its parameters are. If it did, a preview would be lying about the export.
|
||||
const show = makeShow(320, 180);
|
||||
try {
|
||||
const probe = (frames) => frames.map((f) => {
|
||||
show.renderFrame(f);
|
||||
return `${show.arc.state.sceneName}|${show.arc.state.kind}|` +
|
||||
JSON.stringify(show.arc.activeLayers.map((l) => l.baseParams));
|
||||
});
|
||||
const frames = [300, 900, 1500];
|
||||
const small = probe(frames);
|
||||
show.setSize(1920, 1080);
|
||||
const large = probe(frames);
|
||||
|
||||
const same = small.every((v, i) => v === large[i]);
|
||||
return expect(same, same
|
||||
? '320x180 and 1920x1080 agree on scene and params at 3 probe frames'
|
||||
: 'look decisions changed with resolution');
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(6, 'audio-clock preview and frame-counted export agree', () => {
|
||||
// Preview derives the frame index from a playback position; export counts.
|
||||
// Same Show, same size — the images must be identical.
|
||||
const show = makeShow(320, 180);
|
||||
try {
|
||||
const start = 600;
|
||||
const count = 40;
|
||||
|
||||
show.engine.compositor.reset();
|
||||
const preview = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
show.timeline.syncToAudio((start + i) / show.fps);
|
||||
preview.push(show.engine.hashCurrent(show.renderFrame(show.timeline.frame)));
|
||||
}
|
||||
|
||||
show.engine.compositor.reset();
|
||||
const exported = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
exported.push(show.engine.hashCurrent(show.renderFrame(start + i)));
|
||||
}
|
||||
|
||||
const mismatches = preview.filter((h, i) => h !== exported[i]).length;
|
||||
return expect(mismatches === 0, `${mismatches}/${count} frames differed`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(6, 'a warmed export range matches sequential playback into it', () => {
|
||||
// The exporter warms up before a mid-track range so the first exported frame
|
||||
// carries the feedback state continuous playback would have given it.
|
||||
const show = makeShow(256, 144);
|
||||
try {
|
||||
show.look.feedback.amount = 0.6;
|
||||
show.look.feedback.decay = 0.92;
|
||||
const start = 1200;
|
||||
|
||||
show.engine.compositor.reset();
|
||||
for (let f = 400; f < start; f++) show.renderFrame(f);
|
||||
const sequential = Uint8Array.from(show.readPixels(show.renderFrame(start)));
|
||||
|
||||
show.warmUp(start, show.warmupFrames());
|
||||
const warmed = Uint8Array.from(show.readPixels(show.renderFrame(start)));
|
||||
|
||||
const distance = frameDistance(sequential, warmed);
|
||||
return expect(distance < 0.01,
|
||||
`mean distance ${distance.toFixed(5)} after ${show.warmupFrames()} warm-up frames`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
check(6, 'every export preset resolves a supported encoder configuration', async () => {
|
||||
if (!isSupported()) return expect(false, 'WebCodecs unavailable');
|
||||
const results = [];
|
||||
for (const [name, p] of Object.entries(PRESETS)) {
|
||||
let ok = false;
|
||||
for (const codec of ['avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e']) {
|
||||
try {
|
||||
const support = await VideoEncoder.isConfigSupported({
|
||||
codec, width: p.width, height: p.height, bitrate: p.bitrate, framerate: 60,
|
||||
});
|
||||
if (support.supported) { ok = true; break; }
|
||||
} catch { /* try the next candidate */ }
|
||||
}
|
||||
results.push(`${name}:${ok ? 'ok' : 'NO'}`);
|
||||
}
|
||||
return expect(!results.some((r) => r.endsWith('NO')), results.join(' '));
|
||||
});
|
||||
|
||||
check(6, 'a short export produces a well-formed mp4', async () => {
|
||||
if (!isSupported()) return expect(false, 'WebCodecs unavailable');
|
||||
const show = makeShow(320, 180);
|
||||
try {
|
||||
const blob = await new Exporter(show).export({
|
||||
preset: '720p', frameRange: [600, 720], // 2 seconds
|
||||
});
|
||||
const head = new Uint8Array(await blob.slice(0, 12).arrayBuffer());
|
||||
const brand = String.fromCharCode(...head.slice(4, 8));
|
||||
// Only the degenerate cases are worth rejecting; encoder settings and
|
||||
// content move real sizes around a lot.
|
||||
const plausible = blob.size > 20_000 && blob.size < 40_000_000;
|
||||
return expect(brand === 'ftyp' && plausible,
|
||||
`${(blob.size / 1024).toFixed(0)} KB · box '${brand}' · ${blob.type}`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
}, { slow: true });
|
||||
|
||||
check(6, 'export honours the requested frame range', async () => {
|
||||
if (!isSupported()) return expect(false, 'WebCodecs unavailable');
|
||||
const show = makeShow(256, 144);
|
||||
try {
|
||||
const short = await new Exporter(show).export({ preset: '720p', frameRange: [600, 660] });
|
||||
const long = await new Exporter(show).export({ preset: '720p', frameRange: [600, 780] });
|
||||
return expect(long.size > short.size * 1.5,
|
||||
`1s ${(short.size / 1024).toFixed(0)} KB vs 3s ${(long.size / 1024).toFixed(0)} KB`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
}, { slow: true });
|
||||
|
||||
check(6, 'export restores the preview resolution afterwards', async () => {
|
||||
if (!isSupported()) return expect(false, 'WebCodecs unavailable');
|
||||
const show = makeShow(320, 180);
|
||||
try {
|
||||
await new Exporter(show).export({ preset: '720p', frameRange: [600, 630] });
|
||||
return expect(show.engine.width === 320 && show.engine.height === 180,
|
||||
`back to ${show.engine.width}x${show.engine.height} after a 1280x720 export`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
}, { slow: true });
|
||||
|
||||
check(6, 'a cancelled export stops and restores state', async () => {
|
||||
if (!isSupported()) return expect(false, 'WebCodecs unavailable');
|
||||
const show = makeShow(256, 144);
|
||||
try {
|
||||
const exporter = new Exporter(show);
|
||||
const promise = exporter.export({ preset: '720p', frameRange: [600, 2000] });
|
||||
setTimeout(() => exporter.cancel(), 60);
|
||||
let message = '';
|
||||
try { await promise; } catch (err) { message = err.message; }
|
||||
return expect(message.includes('cancel') && show.engine.width === 256,
|
||||
`threw "${message}", size restored to ${show.engine.width}x${show.engine.height}`);
|
||||
} finally {
|
||||
show.dispose();
|
||||
}
|
||||
}, { slow: true });
|
||||
|
||||
check(6, 'upload to the real video platform', () =>
|
||||
// Deliberately manual. Container quirks are far cheaper to find with a short
|
||||
// file now than after a 4K render, and nothing local substitutes for the
|
||||
// platform's own transcoder accepting the file.
|
||||
({ pass: true, detail: 'MANUAL: export a 20s test render and upload it once before trusting a full export' }),
|
||||
{ manual: true });
|
||||
@ -9,8 +9,9 @@ import { Muxer, ArrayBufferTarget } from 'mp4-muxer';
|
||||
* own, so there is nothing for it to diverge from.
|
||||
*
|
||||
* Video goes through WebCodecs VideoEncoder (hardware accelerated where
|
||||
* available) into an mp4. Audio is re-encoded to AAC from the already-decoded
|
||||
* PCM, so the file carries the same samples the analysis ran on.
|
||||
* available) into an mp4. Audio is re-encoded from the already-decoded PCM —
|
||||
* AAC where available, Opus otherwise — so the file carries the same samples
|
||||
* the analysis ran on.
|
||||
*/
|
||||
|
||||
export const PRESETS = {
|
||||
@ -42,6 +43,30 @@ async function pickVideoConfig(width, height, bitrate, fps) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick an audio codec the browser can actually encode.
|
||||
*
|
||||
* AAC is the obvious choice for mp4 but is absent from Chromium builds without
|
||||
* proprietary codecs — which still ship H.264 encoding, so video succeeds and
|
||||
* only audio fails. Opus in mp4 is well supported by players and by every
|
||||
* platform worth uploading to, so it is the fallback rather than an error.
|
||||
*/
|
||||
async function pickAudioConfig(sampleRate, numberOfChannels) {
|
||||
const candidates = [
|
||||
{ codec: 'mp4a.40.2', muxerCodec: 'aac', bitrate: 192_000 },
|
||||
{ codec: 'opus', muxerCodec: 'opus', bitrate: 160_000 },
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const support = await AudioEncoder.isConfigSupported({
|
||||
codec: candidate.codec, sampleRate, numberOfChannels, bitrate: candidate.bitrate,
|
||||
});
|
||||
if (support.supported) return candidate;
|
||||
} catch { /* try the next one */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export class Exporter {
|
||||
constructor(show) {
|
||||
this.show = show;
|
||||
@ -73,16 +98,24 @@ export class Exporter {
|
||||
const videoConfig = await pickVideoConfig(width, height, bitrate, fps);
|
||||
if (!videoConfig) throw new Error('no supported H.264 configuration found');
|
||||
|
||||
const hasAudio = !!show.audioBuffer && typeof AudioEncoder !== 'undefined';
|
||||
const channels = show.audioBuffer ? Math.min(2, show.audioBuffer.numberOfChannels) : 0;
|
||||
const audioConfig = show.audioBuffer && typeof AudioEncoder !== 'undefined'
|
||||
? await pickAudioConfig(show.audioBuffer.sampleRate, channels)
|
||||
: null;
|
||||
const hasAudio = !!audioConfig;
|
||||
this.warnings = [];
|
||||
if (show.audioBuffer && !hasAudio) {
|
||||
this.warnings.push('no supported audio encoder — exporting video only');
|
||||
}
|
||||
|
||||
const muxer = new Muxer({
|
||||
target: new ArrayBufferTarget(),
|
||||
video: { codec: 'avc', width, height, frameRate: fps },
|
||||
...(hasAudio ? {
|
||||
audio: {
|
||||
codec: 'aac',
|
||||
codec: audioConfig.muxerCodec,
|
||||
sampleRate: show.audioBuffer.sampleRate,
|
||||
numberOfChannels: Math.min(2, show.audioBuffer.numberOfChannels),
|
||||
numberOfChannels: channels,
|
||||
},
|
||||
} : {}),
|
||||
fastStart: 'in-memory',
|
||||
@ -145,7 +178,13 @@ export class Exporter {
|
||||
|
||||
if (hasAudio) {
|
||||
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' });
|
||||
await this._encodeAudio(muxer, startFrame, endFrame, fps);
|
||||
try {
|
||||
await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps);
|
||||
} catch (err) {
|
||||
// A finished silent video beats losing a long render outright.
|
||||
// The muxer tolerates an audio track that received no chunks.
|
||||
this.warnings.push(`audio encoding failed (${err.message}) — video only`);
|
||||
}
|
||||
}
|
||||
|
||||
muxer.finalize();
|
||||
@ -157,7 +196,7 @@ export class Exporter {
|
||||
}
|
||||
|
||||
/** Encode the exported time range of the decoded PCM to AAC and mux it. */
|
||||
async _encodeAudio(muxer, startFrame, endFrame, fps) {
|
||||
async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps) {
|
||||
const buffer = this.show.audioBuffer;
|
||||
const sampleRate = buffer.sampleRate;
|
||||
const channels = Math.min(2, buffer.numberOfChannels);
|
||||
@ -173,7 +212,10 @@ export class Exporter {
|
||||
error: (e) => errors.push(e),
|
||||
});
|
||||
encoder.configure({
|
||||
codec: 'mp4a.40.2', sampleRate, numberOfChannels: channels, bitrate: 192_000,
|
||||
codec: audioConfig.codec,
|
||||
sampleRate,
|
||||
numberOfChannels: channels,
|
||||
bitrate: audioConfig.bitrate,
|
||||
});
|
||||
|
||||
const chunkFrames = 1024;
|
||||
@ -207,11 +249,13 @@ export class Exporter {
|
||||
}
|
||||
|
||||
/** Render a short range around a frame — the "test render" bridge before a full export. */
|
||||
export async function exportSegment(show, centreFrame, { seconds = 20, preset = '1080p', onProgress } = {}) {
|
||||
export async function exportSegment(show, centreFrame,
|
||||
{ seconds = 20, preset = '1080p', onProgress, exporter = null } = {}) {
|
||||
const half = Math.round((seconds * show.fps) / 2);
|
||||
const start = Math.max(0, centreFrame - half);
|
||||
const end = Math.min(show.frameCount, centreFrame + half);
|
||||
return new Exporter(show).export({ preset, frameRange: [start, end], onProgress });
|
||||
// Accept a caller-supplied Exporter so the caller can read `warnings` and cancel.
|
||||
return (exporter || new Exporter(show)).export({ preset, frameRange: [start, end], onProgress });
|
||||
}
|
||||
|
||||
export function downloadBlob(blob, filename) {
|
||||
|
||||
@ -355,15 +355,22 @@ async function runExport(segment) {
|
||||
|
||||
try {
|
||||
const preset = currentPreset();
|
||||
const exporter = new Exporter(state.show);
|
||||
const blob = segment
|
||||
? await exportSegment(state.show, state.show.timeline.frame,
|
||||
{ seconds: 20, preset, onProgress: exportProgress })
|
||||
: await new Exporter(state.show).export({ preset, onProgress: exportProgress });
|
||||
{ seconds: 20, preset, onProgress: exportProgress, exporter })
|
||||
: await exporter.export({ preset, onProgress: exportProgress });
|
||||
|
||||
const suffix = segment ? `-segment-${state.show.timeline.frame}` : '';
|
||||
downloadBlob(blob, `${state.show.fileName || 'flow-state'}${suffix}-${preset}.mp4`);
|
||||
const status = document.getElementById('export-status');
|
||||
if (status) status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB`;
|
||||
// Codec fallbacks must never be silent: a video that quietly lost its
|
||||
// audio track is worse than one that says so.
|
||||
const warnings = (exporter.warnings || []).join(' · ');
|
||||
if (status) {
|
||||
status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` +
|
||||
(warnings ? ` (${warnings})` : '');
|
||||
}
|
||||
} catch (err) {
|
||||
const status = document.getElementById('export-status');
|
||||
if (status) status.textContent = `failed: ${err.message}`;
|
||||
@ -461,5 +468,12 @@ function frame(now) {
|
||||
}
|
||||
}
|
||||
|
||||
// Dev-only handle for the console and for browser automation. The render loop is
|
||||
// requestAnimationFrame-driven, and rAF does not fire in some headless/automated
|
||||
// contexts, so `tick()` provides a way to advance the app by hand.
|
||||
if (import.meta.env && import.meta.env.DEV) {
|
||||
window.__flowState = { state, dom, strip, tick: (t) => frame(t ?? 0), resize, seekTo, renderPanel };
|
||||
}
|
||||
|
||||
requestAnimationFrame(frame);
|
||||
resize();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user