dejvino/music-video-gen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix export stutter: backpressure encoder queue and align opus rate

Browse Source 
VideoEncoder.encode() silently drops frames once its internal queue
saturates; the previous loop only polled encodeQueueSize every 10 frames,
so a lagging hardware encoder lost frames in bursts (the "few frames every
beat" bug). Gate every encode on the dequeue event instead, keeping the
queue small but non-empty.

Also resample PCM to 48 kHz when encoding opus so the muxer timescale,
chunk timestamps and bitstream all agree (opus is natively 48k; a 44.1k
source previously produced a file claiming 44100 while the stream was 48k).
This commit is contained in:
Dejvino 2026-08-06 12:23:29 +02:00
parent 19bfff8651
commit d31d0fc3a1
1 changed files with 100 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
flow-state/src/export/Exporter.js
View File

@ -67,6 +67,63 @@ async function pickAudioConfig(sampleRate, numberOfChannels) {
return null;
}
/**
* Opus only produces a native 48 kHz stream (the input rate is resampled
* inside the encoder), so a 44.1 kHz source would end up in a file whose
* container metadata claims 44100 while the bitstream is 48000. Every stream
* rate — encoder config, chunk timestamps and the muxer timescale — must
* agree, so resample to the codec's native rate instead of trusting the
* encoder to paper over the mismatch.
*/
export const OPUS_NATIVE_RATE = 48000;
/** Linear-interpolation resampler for interleaved PCM. */
function resampleInterleaved(input, fromRate, toRate, channels) {
if (fromRate === toRate) return input;
const outFrames = Math.floor((input.length / channels) * (toRate / fromRate));
const out = new Float32Array(outFrames * channels);
const ratio = fromRate / toRate;
const last = input.length / channels - 1;
for (let s = 0; s < outFrames; s++) {
const pos = s * ratio;
const i0 = Math.floor(pos);
const i1 = Math.min(i0 + 1, last);
const frac = pos - i0;
for (let c = 0; c < channels; c++) {
out[s * channels + c] =
input[i0 * channels + c] * (1 - frac) + input[i1 * channels + c] * frac;
}
}
return out;
}
/**
* Backpressure: VideoEncoder.encode() silently drops frames once its internal
* queue is full (encodeQueueSize is capped at an implementation-defined
* limit). Polling that number is unreliable — the only signal that a frame
* actually left the queue is the 'dequeue' event, so wait on it before every
* encode. Keeping the queue small (rather than draining it fully) leaves the
* hardware encoder a pipeline to chew on while guaranteeing nothing is dropped.
*/
const MAX_ENCODER_QUEUE = 8;
function waitForQueue(encoder, errors) {
if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve();
return new Promise((resolve, reject) => {
const cleanup = () => {
encoder.removeEventListener('dequeue', onDequeue);
encoder.removeEventListener('error', onError);
};
const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); };
const onDequeue = () => {
if (errors.length) { cleanup(); reject(errors[0]); return; }
if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); }
};
encoder.addEventListener('dequeue', onDequeue);
encoder.addEventListener('error', onError);
});
}
export class Exporter {
constructor(show) {
this.show = show;
@ -108,13 +165,21 @@ export class Exporter {
this.warnings.push('no supported audio encoder — exporting video only');
}
// Every stream rate must agree so container metadata, chunk timestamps
// and the bitstream describe the same timeline. Opus is resampled to
// its native 48 kHz; AAC keeps the decoded PCM's rate.
const sourceRate = show.audioBuffer ? show.audioBuffer.sampleRate : null;
const audioRate = hasAudio && audioConfig.muxerCodec === 'opus'
? OPUS_NATIVE_RATE
: sourceRate;
const muxer = new Muxer({
target: new ArrayBufferTarget(),
video: { codec: 'avc', width, height, frameRate: fps },
...(hasAudio ? {
audio: {
codec: audioConfig.muxerCodec,
sampleRate: show.audioBuffer.sampleRate,
sampleRate: audioRate,
numberOfChannels: channels,
},
} : {}),
@ -156,6 +221,12 @@ export class Exporter {
const target = show.renderFrame(frameIndex);
show.present(target); // encode from the canvas, which now holds this frame
// True backpressure before every encode — VideoEncoder drops
// frames silently once its queue saturates, which would turn a
// smooth render into the stutter that first shipped here.
await waitForQueue(videoEncoder, errors);
if (errors.length) throw errors[0];
const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps);
const videoFrame = new VideoFrame(show.engine.renderer.canvas, {
timestamp,
@ -165,12 +236,8 @@ export class Exporter {
videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 });
videoFrame.close();
// Yield periodically so the progress UI paints and the encoder
// queue drains rather than growing without bound.
// Yield periodically so the progress UI paints.
if (i % 10 === 0) {
while (videoEncoder.encodeQueueSize > 30) {
await new Promise((r) => setTimeout(r, 4));
}
onProgress && onProgress({
frame: i, total, fraction: i / total, stage: 'rendering',
});
@ -185,7 +252,7 @@ export class Exporter {
if (hasAudio) {
onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' });
try {
await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps);
await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate);
} catch (err) {
// A finished silent video beats losing a long render outright.
// The muxer tolerates an audio track that received no chunks.
@ -201,14 +268,14 @@ export class Exporter {
}
}
/** Encode the exported time range of the decoded PCM to AAC and mux it. */
async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps) {
/** Encode the exported time range of the decoded PCM and mux it. */
async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate) {
const buffer = this.show.audioBuffer;
const sampleRate = buffer.sampleRate;
const sourceRate = buffer.sampleRate;
const channels = Math.min(2, buffer.numberOfChannels);
const startSample = Math.floor((startFrame / fps) * sampleRate);
const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sampleRate));
const startSample = Math.floor((startFrame / fps) * sourceRate);
const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sourceRate));
const length = Math.max(0, endSample - startSample);
if (!length) return;
@ -219,30 +286,40 @@ export class Exporter {
});
encoder.configure({
codec: audioConfig.codec,
sampleRate,
sampleRate: audioRate,
numberOfChannels: channels,
bitrate: audioConfig.bitrate,
});
const chunkFrames = 1024;
const interleaved = new Float32Array(chunkFrames * channels);
const sources = [];
for (let c = 0; c < channels; c++) sources.push(buffer.getChannelData(c));
for (let offset = 0; offset < length; offset += chunkFrames) {
const count = Math.min(chunkFrames, length - offset);
for (let i = 0; i < count; i++) {
// Slice the range, resampling to the codec's native rate when needed so
// chunk timestamps and the muxer timescale describe the same timeline.
const interleaved = new Float32Array(length * channels);
for (let i = 0; i < length; i++) {
for (let c = 0; c < channels; c++) {
interleaved[i * channels + c] = sources[c][startSample + offset + i];
interleaved[i * channels + c] = sources[c][startSample + i];
}
}
const pcm = audioRate === sourceRate
? interleaved
: resampleInterleaved(interleaved, sourceRate, audioRate, channels);
const outFrames = pcm.length / channels;
const buf = new Float32Array(chunkFrames * channels);
for (let offset = 0; offset < outFrames; offset += chunkFrames) {
const count = Math.min(chunkFrames, outFrames - offset);
buf.set(pcm.subarray(offset * channels, (offset + count) * channels), 0);
const data = new AudioData({
format: 'f32',
sampleRate,
sampleRate: audioRate,
numberOfFrames: count,
numberOfChannels: channels,
timestamp: Math.round((offset / sampleRate) * 1e6),
data: interleaved.slice(0, count * channels),
timestamp: Math.round((offset / audioRate) * 1e6),
data: buf.slice(0, count * channels),
});
encoder.encode(data);
data.close();