diff --git a/flow-state/src/export/Exporter.js b/flow-state/src/export/Exporter.js index 6414d03..9cd78a8 100644 --- a/flow-state/src/export/Exporter.js +++ b/flow-state/src/export/Exporter.js @@ -67,6 +67,63 @@ async function pickAudioConfig(sampleRate, numberOfChannels) { return null; } +/** + * Opus only produces a native 48 kHz stream (the input rate is resampled + * inside the encoder), so a 44.1 kHz source would end up in a file whose + * container metadata claims 44100 while the bitstream is 48000. Every stream + * rate — encoder config, chunk timestamps and the muxer timescale — must + * agree, so resample to the codec's native rate instead of trusting the + * encoder to paper over the mismatch. + */ +export const OPUS_NATIVE_RATE = 48000; + +/** Linear-interpolation resampler for interleaved PCM. */ +function resampleInterleaved(input, fromRate, toRate, channels) { + if (fromRate === toRate) return input; + const outFrames = Math.floor((input.length / channels) * (toRate / fromRate)); + const out = new Float32Array(outFrames * channels); + const ratio = fromRate / toRate; + const last = input.length / channels - 1; + for (let s = 0; s < outFrames; s++) { + const pos = s * ratio; + const i0 = Math.floor(pos); + const i1 = Math.min(i0 + 1, last); + const frac = pos - i0; + for (let c = 0; c < channels; c++) { + out[s * channels + c] = + input[i0 * channels + c] * (1 - frac) + input[i1 * channels + c] * frac; + } + } + return out; +} + +/** + * Backpressure: VideoEncoder.encode() silently drops frames once its internal + * queue is full (encodeQueueSize is capped at an implementation-defined + * limit). Polling that number is unreliable — the only signal that a frame + * actually left the queue is the 'dequeue' event, so wait on it before every + * encode. Keeping the queue small (rather than draining it fully) leaves the + * hardware encoder a pipeline to chew on while guaranteeing nothing is dropped. + */ +const MAX_ENCODER_QUEUE = 8; + +function waitForQueue(encoder, errors) { + if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve(); + return new Promise((resolve, reject) => { + const cleanup = () => { + encoder.removeEventListener('dequeue', onDequeue); + encoder.removeEventListener('error', onError); + }; + const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); }; + const onDequeue = () => { + if (errors.length) { cleanup(); reject(errors[0]); return; } + if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); } + }; + encoder.addEventListener('dequeue', onDequeue); + encoder.addEventListener('error', onError); + }); +} + export class Exporter { constructor(show) { this.show = show; @@ -108,13 +165,21 @@ export class Exporter { this.warnings.push('no supported audio encoder — exporting video only'); } + // Every stream rate must agree so container metadata, chunk timestamps + // and the bitstream describe the same timeline. Opus is resampled to + // its native 48 kHz; AAC keeps the decoded PCM's rate. + const sourceRate = show.audioBuffer ? show.audioBuffer.sampleRate : null; + const audioRate = hasAudio && audioConfig.muxerCodec === 'opus' + ? OPUS_NATIVE_RATE + : sourceRate; + const muxer = new Muxer({ target: new ArrayBufferTarget(), video: { codec: 'avc', width, height, frameRate: fps }, ...(hasAudio ? { audio: { codec: audioConfig.muxerCodec, - sampleRate: show.audioBuffer.sampleRate, + sampleRate: audioRate, numberOfChannels: channels, }, } : {}), @@ -156,6 +221,12 @@ export class Exporter { const target = show.renderFrame(frameIndex); show.present(target); // encode from the canvas, which now holds this frame + // True backpressure before every encode — VideoEncoder drops + // frames silently once its queue saturates, which would turn a + // smooth render into the stutter that first shipped here. + await waitForQueue(videoEncoder, errors); + if (errors.length) throw errors[0]; + const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps); const videoFrame = new VideoFrame(show.engine.renderer.canvas, { timestamp, @@ -165,12 +236,8 @@ export class Exporter { videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 }); videoFrame.close(); - // Yield periodically so the progress UI paints and the encoder - // queue drains rather than growing without bound. + // Yield periodically so the progress UI paints. if (i % 10 === 0) { - while (videoEncoder.encodeQueueSize > 30) { - await new Promise((r) => setTimeout(r, 4)); - } onProgress && onProgress({ frame: i, total, fraction: i / total, stage: 'rendering', }); @@ -185,7 +252,7 @@ export class Exporter { if (hasAudio) { onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' }); try { - await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps); + await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate); } catch (err) { // A finished silent video beats losing a long render outright. // The muxer tolerates an audio track that received no chunks. @@ -201,14 +268,14 @@ export class Exporter { } } - /** Encode the exported time range of the decoded PCM to AAC and mux it. */ - async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps) { + /** Encode the exported time range of the decoded PCM and mux it. */ + async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate) { const buffer = this.show.audioBuffer; - const sampleRate = buffer.sampleRate; + const sourceRate = buffer.sampleRate; const channels = Math.min(2, buffer.numberOfChannels); - const startSample = Math.floor((startFrame / fps) * sampleRate); - const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sampleRate)); + const startSample = Math.floor((startFrame / fps) * sourceRate); + const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sourceRate)); const length = Math.max(0, endSample - startSample); if (!length) return; @@ -219,30 +286,40 @@ export class Exporter { }); encoder.configure({ codec: audioConfig.codec, - sampleRate, + sampleRate: audioRate, numberOfChannels: channels, bitrate: audioConfig.bitrate, }); const chunkFrames = 1024; - const interleaved = new Float32Array(chunkFrames * channels); const sources = []; for (let c = 0; c < channels; c++) sources.push(buffer.getChannelData(c)); - for (let offset = 0; offset < length; offset += chunkFrames) { - const count = Math.min(chunkFrames, length - offset); - for (let i = 0; i < count; i++) { - for (let c = 0; c < channels; c++) { - interleaved[i * channels + c] = sources[c][startSample + offset + i]; - } + // Slice the range, resampling to the codec's native rate when needed so + // chunk timestamps and the muxer timescale describe the same timeline. + const interleaved = new Float32Array(length * channels); + for (let i = 0; i < length; i++) { + for (let c = 0; c < channels; c++) { + interleaved[i * channels + c] = sources[c][startSample + i]; } + } + const pcm = audioRate === sourceRate + ? interleaved + : resampleInterleaved(interleaved, sourceRate, audioRate, channels); + + const outFrames = pcm.length / channels; + const buf = new Float32Array(chunkFrames * channels); + + for (let offset = 0; offset < outFrames; offset += chunkFrames) { + const count = Math.min(chunkFrames, outFrames - offset); + buf.set(pcm.subarray(offset * channels, (offset + count) * channels), 0); const data = new AudioData({ format: 'f32', - sampleRate, + sampleRate: audioRate, numberOfFrames: count, numberOfChannels: channels, - timestamp: Math.round((offset / sampleRate) * 1e6), - data: interleaved.slice(0, count * channels), + timestamp: Math.round((offset / audioRate) * 1e6), + data: buf.slice(0, count * channels), }); encoder.encode(data); data.close();