diff --git a/flow-state/src/checks/phase6.js b/flow-state/src/checks/phase6.js index e69de29..05ba404 100644 --- a/flow-state/src/checks/phase6.js +++ b/flow-state/src/checks/phase6.js @@ -0,0 +1,200 @@ +// Phase 6 gate — export. +// +// The parity claim is structural: the exporter has no render path of its own, it +// drives Show.renderFrame exactly as the preview does. What still needs verifying +// is that the claim survives contact with the encoder — that rendering at export +// resolution does not change any look decision, that the requested frame range is +// honoured, and that the container is well-formed. + +import { check, expect } from './framework.js'; +import { Show } from '../Show.js'; +import { generateLook } from '../look/LookGenerator.js'; +import { FeatureTrack } from '../audio/FeatureTrack.js'; +import { synthesizeSectioned } from '../audio/synth.js'; +import { Exporter, isSupported, PRESETS } from '../export/Exporter.js'; +import { frameDistance } from '../engine/hash.js'; + +let cached = null; +function track6() { + if (!cached) { + cached = FeatureTrack.fromAudioBuffer( + synthesizeSectioned({ bpm: 128, duration: 40, changeAt: 20 }), { fps: 60 }); + } + return cached; +} + +function makeShow(width = 320, height = 180) { + const show = new Show({ width, height }); + const track = track6(); + show.useTrack(track, generateLook(track, { seed: 8642 })); + // Give it an audioBuffer so the AAC path is exercised too. + show.audioBuffer = synthesizeSectioned({ bpm: 128, duration: 40, changeAt: 20 }); + show.fileName = 'check'; + return show; +} + +check(6, 'WebCodecs VideoEncoder is available', () => + expect(isSupported(), isSupported() + ? 'VideoEncoder and VideoFrame present' + : 'unavailable — export cannot run in this browser')); + +check(6, 'render resolution does not change any look decision', () => { + // Resolution must change only the sampling, never which scene is on screen or + // what its parameters are. If it did, a preview would be lying about the export. + const show = makeShow(320, 180); + try { + const probe = (frames) => frames.map((f) => { + show.renderFrame(f); + return `${show.arc.state.sceneName}|${show.arc.state.kind}|` + + JSON.stringify(show.arc.activeLayers.map((l) => l.baseParams)); + }); + const frames = [300, 900, 1500]; + const small = probe(frames); + show.setSize(1920, 1080); + const large = probe(frames); + + const same = small.every((v, i) => v === large[i]); + return expect(same, same + ? '320x180 and 1920x1080 agree on scene and params at 3 probe frames' + : 'look decisions changed with resolution'); + } finally { + show.dispose(); + } +}); + +check(6, 'audio-clock preview and frame-counted export agree', () => { + // Preview derives the frame index from a playback position; export counts. + // Same Show, same size — the images must be identical. + const show = makeShow(320, 180); + try { + const start = 600; + const count = 40; + + show.engine.compositor.reset(); + const preview = []; + for (let i = 0; i < count; i++) { + show.timeline.syncToAudio((start + i) / show.fps); + preview.push(show.engine.hashCurrent(show.renderFrame(show.timeline.frame))); + } + + show.engine.compositor.reset(); + const exported = []; + for (let i = 0; i < count; i++) { + exported.push(show.engine.hashCurrent(show.renderFrame(start + i))); + } + + const mismatches = preview.filter((h, i) => h !== exported[i]).length; + return expect(mismatches === 0, `${mismatches}/${count} frames differed`); + } finally { + show.dispose(); + } +}); + +check(6, 'a warmed export range matches sequential playback into it', () => { + // The exporter warms up before a mid-track range so the first exported frame + // carries the feedback state continuous playback would have given it. + const show = makeShow(256, 144); + try { + show.look.feedback.amount = 0.6; + show.look.feedback.decay = 0.92; + const start = 1200; + + show.engine.compositor.reset(); + for (let f = 400; f < start; f++) show.renderFrame(f); + const sequential = Uint8Array.from(show.readPixels(show.renderFrame(start))); + + show.warmUp(start, show.warmupFrames()); + const warmed = Uint8Array.from(show.readPixels(show.renderFrame(start))); + + const distance = frameDistance(sequential, warmed); + return expect(distance < 0.01, + `mean distance ${distance.toFixed(5)} after ${show.warmupFrames()} warm-up frames`); + } finally { + show.dispose(); + } +}); + +check(6, 'every export preset resolves a supported encoder configuration', async () => { + if (!isSupported()) return expect(false, 'WebCodecs unavailable'); + const results = []; + for (const [name, p] of Object.entries(PRESETS)) { + let ok = false; + for (const codec of ['avc1.640034', 'avc1.640033', 'avc1.4d0034', 'avc1.42003e']) { + try { + const support = await VideoEncoder.isConfigSupported({ + codec, width: p.width, height: p.height, bitrate: p.bitrate, framerate: 60, + }); + if (support.supported) { ok = true; break; } + } catch { /* try the next candidate */ } + } + results.push(`${name}:${ok ? 'ok' : 'NO'}`); + } + return expect(!results.some((r) => r.endsWith('NO')), results.join(' ')); +}); + +check(6, 'a short export produces a well-formed mp4', async () => { + if (!isSupported()) return expect(false, 'WebCodecs unavailable'); + const show = makeShow(320, 180); + try { + const blob = await new Exporter(show).export({ + preset: '720p', frameRange: [600, 720], // 2 seconds + }); + const head = new Uint8Array(await blob.slice(0, 12).arrayBuffer()); + const brand = String.fromCharCode(...head.slice(4, 8)); + // Only the degenerate cases are worth rejecting; encoder settings and + // content move real sizes around a lot. + const plausible = blob.size > 20_000 && blob.size < 40_000_000; + return expect(brand === 'ftyp' && plausible, + `${(blob.size / 1024).toFixed(0)} KB · box '${brand}' · ${blob.type}`); + } finally { + show.dispose(); + } +}, { slow: true }); + +check(6, 'export honours the requested frame range', async () => { + if (!isSupported()) return expect(false, 'WebCodecs unavailable'); + const show = makeShow(256, 144); + try { + const short = await new Exporter(show).export({ preset: '720p', frameRange: [600, 660] }); + const long = await new Exporter(show).export({ preset: '720p', frameRange: [600, 780] }); + return expect(long.size > short.size * 1.5, + `1s ${(short.size / 1024).toFixed(0)} KB vs 3s ${(long.size / 1024).toFixed(0)} KB`); + } finally { + show.dispose(); + } +}, { slow: true }); + +check(6, 'export restores the preview resolution afterwards', async () => { + if (!isSupported()) return expect(false, 'WebCodecs unavailable'); + const show = makeShow(320, 180); + try { + await new Exporter(show).export({ preset: '720p', frameRange: [600, 630] }); + return expect(show.engine.width === 320 && show.engine.height === 180, + `back to ${show.engine.width}x${show.engine.height} after a 1280x720 export`); + } finally { + show.dispose(); + } +}, { slow: true }); + +check(6, 'a cancelled export stops and restores state', async () => { + if (!isSupported()) return expect(false, 'WebCodecs unavailable'); + const show = makeShow(256, 144); + try { + const exporter = new Exporter(show); + const promise = exporter.export({ preset: '720p', frameRange: [600, 2000] }); + setTimeout(() => exporter.cancel(), 60); + let message = ''; + try { await promise; } catch (err) { message = err.message; } + return expect(message.includes('cancel') && show.engine.width === 256, + `threw "${message}", size restored to ${show.engine.width}x${show.engine.height}`); + } finally { + show.dispose(); + } +}, { slow: true }); + +check(6, 'upload to the real video platform', () => + // Deliberately manual. Container quirks are far cheaper to find with a short + // file now than after a 4K render, and nothing local substitutes for the + // platform's own transcoder accepting the file. + ({ pass: true, detail: 'MANUAL: export a 20s test render and upload it once before trusting a full export' }), + { manual: true }); diff --git a/flow-state/src/export/Exporter.js b/flow-state/src/export/Exporter.js index 05a0ca7..b67161b 100644 --- a/flow-state/src/export/Exporter.js +++ b/flow-state/src/export/Exporter.js @@ -9,8 +9,9 @@ import { Muxer, ArrayBufferTarget } from 'mp4-muxer'; * own, so there is nothing for it to diverge from. * * Video goes through WebCodecs VideoEncoder (hardware accelerated where - * available) into an mp4. Audio is re-encoded to AAC from the already-decoded - * PCM, so the file carries the same samples the analysis ran on. + * available) into an mp4. Audio is re-encoded from the already-decoded PCM — + * AAC where available, Opus otherwise — so the file carries the same samples + * the analysis ran on. */ export const PRESETS = { @@ -42,6 +43,30 @@ async function pickVideoConfig(width, height, bitrate, fps) { return null; } +/** + * Pick an audio codec the browser can actually encode. + * + * AAC is the obvious choice for mp4 but is absent from Chromium builds without + * proprietary codecs — which still ship H.264 encoding, so video succeeds and + * only audio fails. Opus in mp4 is well supported by players and by every + * platform worth uploading to, so it is the fallback rather than an error. + */ +async function pickAudioConfig(sampleRate, numberOfChannels) { + const candidates = [ + { codec: 'mp4a.40.2', muxerCodec: 'aac', bitrate: 192_000 }, + { codec: 'opus', muxerCodec: 'opus', bitrate: 160_000 }, + ]; + for (const candidate of candidates) { + try { + const support = await AudioEncoder.isConfigSupported({ + codec: candidate.codec, sampleRate, numberOfChannels, bitrate: candidate.bitrate, + }); + if (support.supported) return candidate; + } catch { /* try the next one */ } + } + return null; +} + export class Exporter { constructor(show) { this.show = show; @@ -73,16 +98,24 @@ export class Exporter { const videoConfig = await pickVideoConfig(width, height, bitrate, fps); if (!videoConfig) throw new Error('no supported H.264 configuration found'); - const hasAudio = !!show.audioBuffer && typeof AudioEncoder !== 'undefined'; + const channels = show.audioBuffer ? Math.min(2, show.audioBuffer.numberOfChannels) : 0; + const audioConfig = show.audioBuffer && typeof AudioEncoder !== 'undefined' + ? await pickAudioConfig(show.audioBuffer.sampleRate, channels) + : null; + const hasAudio = !!audioConfig; + this.warnings = []; + if (show.audioBuffer && !hasAudio) { + this.warnings.push('no supported audio encoder — exporting video only'); + } const muxer = new Muxer({ target: new ArrayBufferTarget(), video: { codec: 'avc', width, height, frameRate: fps }, ...(hasAudio ? { audio: { - codec: 'aac', + codec: audioConfig.muxerCodec, sampleRate: show.audioBuffer.sampleRate, - numberOfChannels: Math.min(2, show.audioBuffer.numberOfChannels), + numberOfChannels: channels, }, } : {}), fastStart: 'in-memory', @@ -145,7 +178,13 @@ export class Exporter { if (hasAudio) { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' }); - await this._encodeAudio(muxer, startFrame, endFrame, fps); + try { + await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps); + } catch (err) { + // A finished silent video beats losing a long render outright. + // The muxer tolerates an audio track that received no chunks. + this.warnings.push(`audio encoding failed (${err.message}) — video only`); + } } muxer.finalize(); @@ -157,7 +196,7 @@ export class Exporter { } /** Encode the exported time range of the decoded PCM to AAC and mux it. */ - async _encodeAudio(muxer, startFrame, endFrame, fps) { + async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps) { const buffer = this.show.audioBuffer; const sampleRate = buffer.sampleRate; const channels = Math.min(2, buffer.numberOfChannels); @@ -173,7 +212,10 @@ export class Exporter { error: (e) => errors.push(e), }); encoder.configure({ - codec: 'mp4a.40.2', sampleRate, numberOfChannels: channels, bitrate: 192_000, + codec: audioConfig.codec, + sampleRate, + numberOfChannels: channels, + bitrate: audioConfig.bitrate, }); const chunkFrames = 1024; @@ -207,11 +249,13 @@ export class Exporter { } /** Render a short range around a frame — the "test render" bridge before a full export. */ -export async function exportSegment(show, centreFrame, { seconds = 20, preset = '1080p', onProgress } = {}) { +export async function exportSegment(show, centreFrame, + { seconds = 20, preset = '1080p', onProgress, exporter = null } = {}) { const half = Math.round((seconds * show.fps) / 2); const start = Math.max(0, centreFrame - half); const end = Math.min(show.frameCount, centreFrame + half); - return new Exporter(show).export({ preset, frameRange: [start, end], onProgress }); + // Accept a caller-supplied Exporter so the caller can read `warnings` and cancel. + return (exporter || new Exporter(show)).export({ preset, frameRange: [start, end], onProgress }); } export function downloadBlob(blob, filename) { diff --git a/flow-state/src/main.js b/flow-state/src/main.js index 19721b6..2f8e1f1 100644 --- a/flow-state/src/main.js +++ b/flow-state/src/main.js @@ -355,15 +355,22 @@ async function runExport(segment) { try { const preset = currentPreset(); + const exporter = new Exporter(state.show); const blob = segment ? await exportSegment(state.show, state.show.timeline.frame, - { seconds: 20, preset, onProgress: exportProgress }) - : await new Exporter(state.show).export({ preset, onProgress: exportProgress }); + { seconds: 20, preset, onProgress: exportProgress, exporter }) + : await exporter.export({ preset, onProgress: exportProgress }); const suffix = segment ? `-segment-${state.show.timeline.frame}` : ''; downloadBlob(blob, `${state.show.fileName || 'flow-state'}${suffix}-${preset}.mp4`); const status = document.getElementById('export-status'); - if (status) status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB`; + // Codec fallbacks must never be silent: a video that quietly lost its + // audio track is worse than one that says so. + const warnings = (exporter.warnings || []).join(' · '); + if (status) { + status.textContent = `done — ${(blob.size / 1e6).toFixed(1)} MB` + + (warnings ? ` (${warnings})` : ''); + } } catch (err) { const status = document.getElementById('export-status'); if (status) status.textContent = `failed: ${err.message}`; @@ -461,5 +468,12 @@ function frame(now) { } } +// Dev-only handle for the console and for browser automation. The render loop is +// requestAnimationFrame-driven, and rAF does not fire in some headless/automated +// contexts, so `tick()` provides a way to advance the app by hand. +if (import.meta.env && import.meta.env.DEV) { + window.__flowState = { state, dom, strip, tick: (t) => frame(t ?? 0), resize, seekTo, renderPanel }; +} + requestAnimationFrame(frame); resize();