diff --git a/bun.lock b/bun.lock index 3648ff2370..655ff91968 100644 --- a/bun.lock +++ b/bun.lock @@ -166,7 +166,7 @@ "name": "@moq/hang", "version": "0.5.2", "dependencies": { - "@kixelated/libavjs-webcodecs-polyfill": "^0.5.5", + "@kixelated/libavjs-webcodecs-polyfill": "^0.5.6", "@moq/clock": "workspace:^", "@moq/json": "workspace:^", "@moq/libav": "workspace:^", @@ -799,7 +799,7 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], - "@kixelated/libavjs-webcodecs-polyfill": ["@kixelated/libavjs-webcodecs-polyfill@0.5.5", "", { "dependencies": { "@libav.js/types": "^6.7.7", "@ungap/global-this": "^0.4.4" } }, "sha512-Q1zgnTMMQ2F7IE9ylx3C1XzVbg5vYN18jiDINO5U3kNPBOHdYuUlJsMhtBoqr1M6ocLtoiqdHmLs7tHFgrw5KA=="], + "@kixelated/libavjs-webcodecs-polyfill": ["@kixelated/libavjs-webcodecs-polyfill@0.5.6", "", { "dependencies": { "@libav.js/types": "^6.7.7", "@ungap/global-this": "^0.4.4" } }, "sha512-A6FBsEAT/t0cDDaxo3h5XabiHX2RtycQbWV+MNKUghmctrXU1ygl05i3TTtnSF/eLmCLLO8xBMrXO+eGnpLWFw=="], "@libav.js/types": ["@libav.js/types@6.10.9", "", {}, "sha512-AdngxD/VO8OI6/w7wlJC77kuFcsSvCUuo2RuC6gGeSW2DEvbzgBo2JtFLV6UWZp/tRNZhKm6jS0Tv8gwCKBHLg=="], diff --git a/js/hang/package.json b/js/hang/package.json index c0802c05ff..bd6cb59f95 100644 --- a/js/hang/package.json +++ b/js/hang/package.json @@ -24,7 +24,7 @@ "release": "bun ../common/release.ts" }, "dependencies": { - "@kixelated/libavjs-webcodecs-polyfill": "^0.5.5", + "@kixelated/libavjs-webcodecs-polyfill": "^0.5.6", "@moq/clock": "workspace:^", "@moq/json": "workspace:^", "@moq/libav": "workspace:^", diff --git a/js/hang/src/util/libav.test.ts b/js/hang/src/util/libav.test.ts index 8458db93ba..340d030a13 100644 --- a/js/hang/src/util/libav.test.ts +++ b/js/hang/src/util/libav.test.ts @@ -114,7 +114,7 @@ test("the polyfill deinterleaves every channel to f32-planar", async () => { } const tail = new Float32Array(2); - audio.copyTo(tail, { planeIndex: 1, format: "f32-planar", frameOffset: 1 }); + audio.copyTo(tail, { planeIndex: 1, format: "f32-planar", frameOffset: 1, frameCount: 2 }); expect(Array.from(tail)).toEqual([1, 2].map((frame) => data[frame * 2 + 1] / scale)); // A view smaller than the copy is refused, even with room in the buffer behind it. const backing = new Float32Array(8); diff --git a/js/hang/src/util/libav.ts b/js/hang/src/util/libav.ts index 0cf5e26227..3e22899116 100644 --- a/js/hang/src/util/libav.ts +++ b/js/hang/src/util/libav.ts @@ -14,12 +14,11 @@ export async function polyfill(): Promise { console.warn("using libav WebCodecs polyfill; performance may be degraded"); // @moq/libav is libav.js's opus-af variant plus the AAC decoder, which no published variant has. - // I forked libavjs-webcodecs-polyfill to avoid Typescript errors; there's no changes otherwise. + // I forked libavjs-webcodecs-polyfill to fix its TypeScript packaging and planar audio. loading = Promise.all([import("@moq/libav"), import("@kixelated/libavjs-webcodecs-polyfill")]).then( async ([{ default: LibAV }, libav]) => { const native = globalThis.AudioDecoder; await libav.load({ LibAV, polyfill: true }); - fixPlanarCopy(libav.AudioData as unknown as typeof AudioData); if (native) return true; type Config = Parameters["configure"]>[0]; @@ -28,14 +27,7 @@ export async function polyfill(): Promise { aac(config) ? { ...config, codec: { libavjs: { codec: "aac" } } } : config; class AudioDecoder extends libav.AudioDecoder { - #planar = false; - - constructor(init: ConstructorParameters[0]) { - super({ ...init, output: (data) => init.output(this.#planar ? planar(data) : data) }); - } - override configure(config: Config): void { - this.#planar = aac(config); super.configure(route(config)); } @@ -51,67 +43,3 @@ export async function polyfill(): Promise { } return await loading; } - -// libav's AAC decoder always emits planar float, but the polyfill concatenates the planes and labels -// the result interleaved "f32". Relabel it so channels read back from the right plane. -function planar(data: T): T { - if (data.format !== "f32") return data; - const samples = new Float32Array(data.numberOfFrames * data.numberOfChannels); - data.copyTo(samples, { planeIndex: 0 }); - const relabeled = new (data.constructor as typeof AudioData)({ - format: "f32-planar", - sampleRate: data.sampleRate, - numberOfFrames: data.numberOfFrames, - numberOfChannels: data.numberOfChannels, - timestamp: data.timestamp, - data: samples, - }); - data.close(); - return relabeled as T; -} - -// Each interleaved sample format, with the offset and scale that map it to [-1, 1]. -const INTERLEAVED = { - u8: [Uint8Array, 0x80, 0x80], - s16: [Int16Array, 0, 0x8000], - s32: [Int32Array, 0, 0x80000000], - f32: [Float32Array, 0, 1], -} as const; - -// The polyfill converts interleaved samples to "f32-planar" from the wrong offset, so every channel -// after the first reads past its frames (NaN) instead of its own samples. libopus decodes -// interleaved, so stereo Opus lost its right channel. Deinterleave from a same-format copy instead, -// which the polyfill gets right. -function fixPlanarCopy(Data: typeof AudioData): void { - const copyTo = Data.prototype.copyTo; - Data.prototype.copyTo = function (this: AudioData, destination: AllowSharedBufferSource, options) { - const format = this.format ?? ""; - if (options.format !== "f32-planar" || !(format in INTERLEAVED)) return copyTo.call(this, destination, options); - const [View, offset, scale] = INTERLEAVED[format as keyof typeof INTERLEAVED]; - const channels = this.numberOfChannels; - const index = (value: number) => Number.isSafeInteger(value) && value >= 0; - if (!index(options.planeIndex) || options.planeIndex >= channels) { - throw new RangeError(`no plane ${options.planeIndex}`); - } - - const start = options.frameOffset ?? 0; - const count = options.frameCount ?? this.numberOfFrames - start; - if (!index(start) || !index(count) || start >= this.numberOfFrames || start + count > this.numberOfFrames) { - throw new RangeError("frames out of range"); - } - // Bounded by the caller's view, not the buffer behind it. - if (count * Float32Array.BYTES_PER_ELEMENT > destination.byteLength) { - throw new RangeError("destination too small"); - } - const samples = new View(this.numberOfFrames * channels); - copyTo.call(this, samples, { planeIndex: 0 }); - const out = new Float32Array(count); - for (let i = 0; i < count; i++) - out[i] = (samples[(start + i) * channels + options.planeIndex] - offset) / scale; - // Copied as bytes: a destination need not be 4-byte aligned. - const bytes = ArrayBuffer.isView(destination) - ? new Uint8Array(destination.buffer, destination.byteOffset, out.byteLength) - : new Uint8Array(destination, 0, out.byteLength); - bytes.set(new Uint8Array(out.buffer)); - }; -} diff --git a/quest/m1/README.md b/quest/m1/README.md index cf219ef29c..14ecfee183 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -124,7 +124,6 @@ tracks, so per-group stats stop costing cluster announces. - [JSON encoder options without a downcast](/quest/m1/json-encoder-options.md) - moq-mux's `json_snapshot` takes its `deltas` switch through a typed path any custom catalog entry can use, not an `Any` downcast - [Tooling tests gate every merge](/quest/m1/gh-test-gate.md) - every queue Check runs `just gh test`, so a change to the CI selector or release tooling cannot merge without its own tests - [CI housekeeping](/quest/m1/ci-housekeeping.md) - validate queue nextest at 24 threads and remove the actionlint queue ignore after its release -- [libav polyfill fork fix](/quest/m1/libav-polyfill-fork.md) - our polyfill fork copies planar audio correctly and `@moq/hang` drops its patch - [Interop AAC](/quest/m1/interop-aac.md) - the Rust interop publisher adds an AAC tone and every subscriber checks it beside Opus - [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured diff --git a/quest/m1/libav-polyfill-fork.md b/quest/m1/libav-polyfill-fork.md deleted file mode 100644 index 2f65d9dfeb..0000000000 --- a/quest/m1/libav-polyfill-fork.md +++ /dev/null @@ -1,19 +0,0 @@ -# [XS] Fix the libav polyfill's planar copy in our fork - -## Goal - -The `@kixelated` fork of libavjs-webcodecs-polyfill converts interleaved -samples to planar from the right offset, a release of it is published, and -`@moq/hang` drops the copy patch that works around it. - -## Plan - -Found by [moq-dev/moq#6284](https://github.com/moq-dev/moq/pull/6284): the -polyfill garbles every channel after the first, so stereo Opus under Node, Bun, -or a browser without `AudioDecoder` lost its right channel. #6284 patches the -conversion in `Util.Libav.polyfill()` with a regression test; keep that test -when the patch goes. - -Decided 2026-10-22 (maintainer): fix it in the fork only; post nothing upstream. - -Public API: none. Wire: none.