Fix export stutter: backpressure encoder queue and align opus rate

VideoEncoder.encode() silently drops frames once its internal queue
saturates; the previous loop only polled encodeQueueSize every 10 frames,
so a lagging hardware encoder lost frames in bursts (the "few frames every
beat" bug). Gate every encode on the dequeue event instead, keeping the
queue small but non-empty.

Also resample PCM to 48 kHz when encoding opus so the muxer timescale,
chunk timestamps and bitstream all agree (opus is natively 48k; a 44.1k
source previously produced a file claiming 44100 while the stream was 48k).

flow-state/src/export/Exporter.js

@@ -67,6 +67,63 @@ async function pickAudioConfig(sampleRate, numberOfChannels) {
     return null;
 }
 
+/**
+ * Opus only produces a native 48 kHz stream (the input rate is resampled
+ * inside the encoder), so a 44.1 kHz source would end up in a file whose
+ * container metadata claims 44100 while the bitstream is 48000. Every stream
+ * rate — encoder config, chunk timestamps and the muxer timescale — must
+ * agree, so resample to the codec's native rate instead of trusting the
+ * encoder to paper over the mismatch.
+ */
+export const OPUS_NATIVE_RATE = 48000;
+
+/** Linear-interpolation resampler for interleaved PCM. */
+function resampleInterleaved(input, fromRate, toRate, channels) {
+    if (fromRate === toRate) return input;
+    const outFrames = Math.floor((input.length / channels) * (toRate / fromRate));
+    const out = new Float32Array(outFrames * channels);
+    const ratio = fromRate / toRate;
+    const last = input.length / channels - 1;
+    for (let s = 0; s < outFrames; s++) {
+        const pos = s * ratio;
+        const i0 = Math.floor(pos);
+        const i1 = Math.min(i0 + 1, last);
+        const frac = pos - i0;
+        for (let c = 0; c < channels; c++) {
+            out[s * channels + c] =
+                input[i0 * channels + c] * (1 - frac) + input[i1 * channels + c] * frac;
+        }
+    }
+    return out;
+}
+
+/**
+ * Backpressure: VideoEncoder.encode() silently drops frames once its internal
+ * queue is full (encodeQueueSize is capped at an implementation-defined
+ * limit). Polling that number is unreliable — the only signal that a frame
+ * actually left the queue is the 'dequeue' event, so wait on it before every
+ * encode. Keeping the queue small (rather than draining it fully) leaves the
+ * hardware encoder a pipeline to chew on while guaranteeing nothing is dropped.
+ */
+const MAX_ENCODER_QUEUE = 8;
+
+function waitForQueue(encoder, errors) {
+    if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) return Promise.resolve();
+    return new Promise((resolve, reject) => {
+        const cleanup = () => {
+            encoder.removeEventListener('dequeue', onDequeue);
+            encoder.removeEventListener('error', onError);
+        };
+        const onError = () => { cleanup(); reject(errors[0] || new Error('encoder error')); };
+        const onDequeue = () => {
+            if (errors.length) { cleanup(); reject(errors[0]); return; }
+            if (encoder.encodeQueueSize < MAX_ENCODER_QUEUE) { cleanup(); resolve(); }
+        };
+        encoder.addEventListener('dequeue', onDequeue);
+        encoder.addEventListener('error', onError);
+    });
+}
+
 export class Exporter {
     constructor(show) {
         this.show = show;
@@ -108,13 +165,21 @@ export class Exporter {
             this.warnings.push('no supported audio encoder — exporting video only');
         }
 
+        // Every stream rate must agree so container metadata, chunk timestamps
+        // and the bitstream describe the same timeline. Opus is resampled to
+        // its native 48 kHz; AAC keeps the decoded PCM's rate.
+        const sourceRate = show.audioBuffer ? show.audioBuffer.sampleRate : null;
+        const audioRate = hasAudio && audioConfig.muxerCodec === 'opus'
+            ? OPUS_NATIVE_RATE
+            : sourceRate;
+
         const muxer = new Muxer({
             target: new ArrayBufferTarget(),
             video: { codec: 'avc', width, height, frameRate: fps },
             ...(hasAudio ? {
                 audio: {
                     codec: audioConfig.muxerCodec,
-                    sampleRate: show.audioBuffer.sampleRate,
+                    sampleRate: audioRate,
                     numberOfChannels: channels,
                 },
             } : {}),
@@ -156,6 +221,12 @@ export class Exporter {
                 const target = show.renderFrame(frameIndex);
                 show.present(target);   // encode from the canvas, which now holds this frame
 
+                // True backpressure before every encode — VideoEncoder drops
+                // frames silently once its queue saturates, which would turn a
+                // smooth render into the stutter that first shipped here.
+                await waitForQueue(videoEncoder, errors);
+                if (errors.length) throw errors[0];
+
                 const timestamp = Math.round(((frameIndex - startFrame) * 1e6) / fps);
                 const videoFrame = new VideoFrame(show.engine.renderer.canvas, {
                     timestamp,
@@ -165,12 +236,8 @@ export class Exporter {
                 videoEncoder.encode(videoFrame, { keyFrame: i % (fps * 2) === 0 });
                 videoFrame.close();
 
-                // Yield periodically so the progress UI paints and the encoder
-                // queue drains rather than growing without bound.
+                // Yield periodically so the progress UI paints.
                 if (i % 10 === 0) {
-                    while (videoEncoder.encodeQueueSize > 30) {
-                        await new Promise((r) => setTimeout(r, 4));
-                    }
                     onProgress && onProgress({
                         frame: i, total, fraction: i / total, stage: 'rendering',
                     });
@@ -185,7 +252,7 @@ export class Exporter {
             if (hasAudio) {
                 onProgress && onProgress({ frame: total, total, fraction: 1, stage: 'encoding audio' });
                 try {
-                    await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps);
+                    await this._encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate);
                 } catch (err) {
                     // A finished silent video beats losing a long render outright.
                     // The muxer tolerates an audio track that received no chunks.
@@ -201,14 +268,14 @@ export class Exporter {
         }
     }
 
-    /** Encode the exported time range of the decoded PCM to AAC and mux it. */
-    async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps) {
+    /** Encode the exported time range of the decoded PCM and mux it. */
+    async _encodeAudio(muxer, audioConfig, startFrame, endFrame, fps, audioRate) {
         const buffer = this.show.audioBuffer;
-        const sampleRate = buffer.sampleRate;
+        const sourceRate = buffer.sampleRate;
         const channels = Math.min(2, buffer.numberOfChannels);
 
-        const startSample = Math.floor((startFrame / fps) * sampleRate);
-        const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sampleRate));
+        const startSample = Math.floor((startFrame / fps) * sourceRate);
+        const endSample = Math.min(buffer.length, Math.ceil((endFrame / fps) * sourceRate));
         const length = Math.max(0, endSample - startSample);
         if (!length) return;
 
@@ -219,30 +286,40 @@ export class Exporter {
         });
         encoder.configure({
             codec: audioConfig.codec,
-            sampleRate,
+            sampleRate: audioRate,
             numberOfChannels: channels,
             bitrate: audioConfig.bitrate,
         });
 
         const chunkFrames = 1024;
-        const interleaved = new Float32Array(chunkFrames * channels);
         const sources = [];
         for (let c = 0; c < channels; c++) sources.push(buffer.getChannelData(c));
 
-        for (let offset = 0; offset < length; offset += chunkFrames) {
-            const count = Math.min(chunkFrames, length - offset);
-            for (let i = 0; i < count; i++) {
-                for (let c = 0; c < channels; c++) {
-                    interleaved[i * channels + c] = sources[c][startSample + offset + i];
-                }
+        // Slice the range, resampling to the codec's native rate when needed so
+        // chunk timestamps and the muxer timescale describe the same timeline.
+        const interleaved = new Float32Array(length * channels);
+        for (let i = 0; i < length; i++) {
+            for (let c = 0; c < channels; c++) {
+                interleaved[i * channels + c] = sources[c][startSample + i];
             }
+        }
+        const pcm = audioRate === sourceRate
+            ? interleaved
+            : resampleInterleaved(interleaved, sourceRate, audioRate, channels);
+
+        const outFrames = pcm.length / channels;
+        const buf = new Float32Array(chunkFrames * channels);
+
+        for (let offset = 0; offset < outFrames; offset += chunkFrames) {
+            const count = Math.min(chunkFrames, outFrames - offset);
+            buf.set(pcm.subarray(offset * channels, (offset + count) * channels), 0);
             const data = new AudioData({
                 format: 'f32',
-                sampleRate,
+                sampleRate: audioRate,
                 numberOfFrames: count,
                 numberOfChannels: channels,
-                timestamp: Math.round((offset / sampleRate) * 1e6),
-                data: interleaved.slice(0, count * channels),
+                timestamp: Math.round((offset / audioRate) * 1e6),
+                data: buf.slice(0, count * channels),
             });
             encoder.encode(data);
             data.close();