Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions js/hang/src/util/libav.copy.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { expect, test } from "bun:test";
import { polyfill } from "./libav";

// Bun has no WebCodecs, so this runs on the polyfill's AudioData (another test file may have loaded it first),
// which libopus decodes into.
test("the polyfill deinterleaves every channel to f32-planar", async () => {
expect(await polyfill()).toBe(true);

const cases = [
{ format: "f32", data: Float32Array.of(0.5, -0.5, 0.25, -0.25, 0, 1), scale: 1 },
{ format: "s16", data: Int16Array.of(0x4000, -0x4000, 0x2000, -0x2000, 0, 0x7fff), scale: 0x8000 },
] as const;
for (const { format, data, scale } of cases) {
const audio = new AudioData({
format,
sampleRate: 48_000,
numberOfFrames: 3,
numberOfChannels: 2,
timestamp: 0,
data,
});
for (const channel of [0, 1]) {
const plane = new Float32Array(3);
audio.copyTo(plane, { planeIndex: channel, format: "f32-planar" });
expect(Array.from(plane)).toEqual([0, 1, 2].map((frame) => data[frame * 2 + channel] / scale));
}

const tail = new Float32Array(2);
audio.copyTo(tail, { planeIndex: 1, format: "f32-planar", frameOffset: 1 });
expect(Array.from(tail)).toEqual([1, 2].map((frame) => data[frame * 2 + 1] / scale));
// A view smaller than the copy is refused, even with room in the buffer behind it.
const backing = new Float32Array(8);
expect(() =>
audio.copyTo(backing.subarray(0, 1), { planeIndex: 1, format: "f32-planar", frameCount: 3 }),
).toThrow(RangeError);
expect(Array.from(backing)).toEqual(new Array(8).fill(0));
// An unaligned byte view takes the floats too.
const unaligned = new Uint8Array(13);
audio.copyTo(unaligned.subarray(1), { planeIndex: 1, format: "f32-planar" });
expect(Array.from(new Float32Array(unaligned.slice(1).buffer))).toEqual(
[0, 1, 2].map((frame) => Math.fround(data[frame * 2 + 1] / scale)),
);
for (const bad of [
{ planeIndex: -1 },
{ planeIndex: 0.5 },
{ planeIndex: 0, frameOffset: -1 },
// WebCodecs refuses an offset at the end even though nothing would be copied.
{ planeIndex: 0, frameOffset: 3 },
]) {
expect(() => audio.copyTo(backing, { ...bad, format: "f32-planar" })).toThrow(RangeError);
}
audio.close();
}
});
47 changes: 47 additions & 0 deletions js/hang/src/util/libav.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export async function polyfill(): Promise<boolean> {
async ([{ default: LibAV }, libav]) => {
const native = globalThis.AudioDecoder;
await libav.load({ LibAV, polyfill: true });
fixPlanarCopy(libav.AudioData as unknown as typeof AudioData);
if (native) return true;

type Config = Parameters<InstanceType<typeof libav.AudioDecoder>["configure"]>[0];
Expand Down Expand Up @@ -68,3 +69,49 @@ function planar<T extends AudioData>(data: T): T {
data.close();
return relabeled as T;
}

// Each interleaved sample format, with the offset and scale that map it to [-1, 1].
const INTERLEAVED = {
u8: [Uint8Array, 0x80, 0x80],
s16: [Int16Array, 0, 0x8000],
s32: [Int32Array, 0, 0x80000000],
f32: [Float32Array, 0, 1],
} as const;

// The polyfill converts interleaved samples to "f32-planar" from the wrong offset, so every channel
// after the first reads past its frames (NaN) instead of its own samples. libopus decodes
// interleaved, so stereo Opus lost its right channel. Deinterleave from a same-format copy instead,
// which the polyfill gets right.
function fixPlanarCopy(Data: typeof AudioData): void {
const copyTo = Data.prototype.copyTo;
Data.prototype.copyTo = function (this: AudioData, destination: AllowSharedBufferSource, options) {
const format = this.format ?? "";
if (options.format !== "f32-planar" || !(format in INTERLEAVED)) return copyTo.call(this, destination, options);
const [View, offset, scale] = INTERLEAVED[format as keyof typeof INTERLEAVED];
const channels = this.numberOfChannels;
const index = (value: number) => Number.isSafeInteger(value) && value >= 0;
if (!index(options.planeIndex) || options.planeIndex >= channels) {
throw new RangeError(`no plane ${options.planeIndex}`);
}

const start = options.frameOffset ?? 0;
const count = options.frameCount ?? this.numberOfFrames - start;
if (!index(start) || !index(count) || start >= this.numberOfFrames || start + count > this.numberOfFrames) {
throw new RangeError("frames out of range");
}
// Bounded by the caller's view, not the buffer behind it.
if (count * Float32Array.BYTES_PER_ELEMENT > destination.byteLength) {
throw new RangeError("destination too small");
}
const samples = new View(this.numberOfFrames * channels);
copyTo.call(this, samples, { planeIndex: 0 });
const out = new Float32Array(count);
for (let i = 0; i < count; i++)
out[i] = (samples[(start + i) * channels + options.planeIndex] - offset) / scale;
// Copied as bytes: a destination need not be 4-byte aligned.
const bytes = ArrayBuffer.isView(destination)
? new Uint8Array(destination.buffer, destination.byteOffset, out.byteLength)
: new Uint8Array(destination, 0, out.byteLength);
bytes.set(new Uint8Array(out.buffer));
};
}
4 changes: 3 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,9 @@ lite-07 stream type here.
- [moq-net compiles once](/quest/m1/net-erase-transport.md) - the session drivers stop monomorphizing per transport and per crate, saving several MiB of the relay, FFI library, `libmoq` and CLI
- [Audio devices on ARM](/quest/m1/audio-devices-arm.md) - `just rs audio-devices` passes on the hosted ARM runner
- [CI housekeeping](/quest/m1/ci-housekeeping.md) - validate queue nextest at 24 threads and remove the actionlint queue ignore after its release
- [Interop audio delivery](/quest/m1/interop-audio.md) - every interop subscriber decodes and checks the publishers' 440 Hz tone, and the Rust publisher sends Opus and AAC
- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the wire-compat media lanes decode and check the interop tone across releases
- [Interop AAC](/quest/m1/interop-aac.md) - the Rust interop publisher adds an AAC tone and every subscriber checks it beside Opus
- [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails
- [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - Opus and AAC decode through every moq-audio backend, moq-c, and moq-ffi is checked against known tones and clips, like moq-video's suite
- [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured
- [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut
Expand Down
2 changes: 1 addition & 1 deletion quest/m1/audio-decode-conformance.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,6 @@ Public API: none expected. Wire: none.

## Related

- [Interop audio delivery](/quest/m1/interop-audio.md) - the same decoders, end to end across clients
- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the same decoders, end to end across releases
- [AudioToolbox decode](/quest/m2/audio-decode-audiotoolbox.md) - a new backend that should join this suite when it lands
- [Opus backends](/quest/m2/audio-opus-backend.md) - compares Opus implementations; this suite is its correctness floor
28 changes: 28 additions & 0 deletions quest/m1/interop-aac.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# [M] Interop subscribers check an AAC tone too

## Goal

The Rust CLI interop publisher sends an AAC tone beside its Opus one, and every
subscriber decodes and checks both renditions with `test/interop/tone.py` (the
browser with its in-page check), so AAC import and decode are proven across
clients the way Opus is.

## Plan

Split from the interop audio checks, which landed Opus only. The libav fallback
now decodes AAC, so the Node and Bun subscribers can follow.

Things to look out for:

- Every subscriber picks the first audio rendition today; checking both means
each client decodes every rendition it is offered, or the harness asks for one
by name per pass.
- ffmpeg's AAC encoder adds priming samples; a decoder that ignores them shifts
the tone without a gap, which the check does not see (it does not anchor
phase to timestamps).

Public API: none expected. Wire: none.

## Related

- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the release lanes that would follow an AAC track
52 changes: 0 additions & 52 deletions quest/m1/interop-audio.md

This file was deleted.

34 changes: 34 additions & 0 deletions quest/m1/interop-preskip.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# [S] Interop tone check catches a misapplied pre-skip

## Goal

`test/interop/tone.py` fails a subscriber whose decoder drops or keeps the
wrong number of priming samples (Opus pre-skip, AAC priming), the way it
already fails a channel swap or a gap. Today a wrong pre-skip only shifts a
steady tone's phase, which the energy-and-frequency check cannot see.

## Plan

Found in review of the interop audio checks. The publishers' tone phase is a
function of the publisher's timestamps (the FFI clients and ffmpeg's `aevalsrc`
both start at zero), so the decoded sample at timestamp `t` should carry phase
`2πft`: anchor the Goertzel phase to the frame timestamps.

Things to look out for:

- Precision differs per client: the libav fallback stamps milliseconds (about
1.4 rad at 440 Hz), and the browser publisher's capture clock is not tied to
its WAV's phase. Check only where the timestamps are precise enough, and say
which clients are left out.
- Add a negative control that misapplies the pre-skip on purpose, so the check
is proven able to fail.
- The GStreamer subscriber likely fails it today: `opusdec` trims pre-skip only
from a buffer's `GstAudioClippingMeta`, never from the OpusHead, and `moqsrc`
attaches none.

Public API: none expected. Wire: none.

## Related

- [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - pre-skip continuity in unit tests
- [Interop AAC](/quest/m1/interop-aac.md) - AAC priming is the same question
33 changes: 33 additions & 0 deletions quest/m1/wire-compat-audio.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# [M] Wire-compat media lanes check the audio tone

## Goal

`just test wire-compat`'s media lanes prove audio crosses releases the way
`just test interop` proves it within the checkout: the current and released
Rust CLIs publish the interop tone, and the released and current Rust and JS
readers decode it and pass `test/interop/tone.py`. A garbled decode, a missing
audio rendition, or a timestamp gap between versions fails the lane.

## Plan

Split from the interop audio work, which checks the tone in every in-tree cell
but skips audio when `interop.sh` runs released binaries or JS clients
(`AUDIO=0`): the lanes keep the video-only Annex-B publisher and readers.

Things to look out for:

- The Rust readers export fMP4. Decoding its audio with ffmpeg, as the lanes
already decode video, still owes tone.py the frame timestamps it checks for
gaps.
- `compat/client.ts` runs against both package sets, so its audio decode can
only use API the released `@moq/hang` has.
- Released `@moq/hang`'s libav fallback garbles every channel past the first
when it converts interleaved PCM to planar (fixed alongside the interop audio
checks). Until a release carries that fix, a released JS reader fails a stereo
tone: decide between a planned break and starting mono.

Public API: none. Wire: none.

## Related

- [Interop AAC](/quest/m1/interop-aac.md) - adds an AAC track to the in-tree matrix; these lanes would follow
4 changes: 2 additions & 2 deletions quest/m2/media-qa-audio.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ Decided in planning (2026-10-20):
draw the media QA frame-counter pattern and generate the stepped tone, and
`moq` publishes them through moq-mux. The browser grades pitch and sync,
which covers native importer timestamps. The FFI publishers' tone is
graded for pitch and continuity by [interop audio
delivery](/quest/m1/interop-audio.md), not here.
graded for pitch and continuity by the interop matrix
(`test/interop/tone.py`), not here.
- Browser AAC publish runs end to end for the first time in the AAC case.
- Watch mute and volume are verified on the output (muted is silence, volume
scales the level). The media QA probe taps `audio.out.root`, upstream of the
Expand Down
6 changes: 6 additions & 0 deletions rs/moq-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ winit = { version = "0.30.13", optional = true }
sd-notify = { workspace = true }

[dev-dependencies]
# Decodes the interop matrix's audio tone (the `interop-audio` example).
moq-audio = { path = "../moq-audio" }
# `test_relay` stands up a real relay for the `fetch` and `announced` tests to read through.
moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] }
# The scripted RTSP server `import rtsp` pulls from.
Expand All @@ -176,3 +178,7 @@ harness = false
[[example]]
name = "interop-tail"
path = "../../test/interop/clients/rust/tail.rs"

[[example]]
name = "interop-audio"
path = "../../test/interop/clients/rust/audio.rs"
5 changes: 5 additions & 0 deletions rs/moq-gst/MIGRATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ How to update your pipelines for each breaking change, newest first.

## Unreleased

### `moqsrc` audio caps are the standard ones

`moqsrc` audio pads now carry `rate` and `channels` as `int`, and Opus pads the mapping fields (`channel-mapping-family`, `stream-count`, `coupled-count`, plus `channel-mapping` beyond stereo) with the OpusHead and an empty OpusTags as `streamheader`, so `decodebin` and `opusdec` accept them and `opusdec` applies the output gain.
Opus pads no longer carry the nonstandard `codec_data` and `stream-format=ogg`; a capsfilter naming either, or a `uint` rate or channel count, must drop it.

### Rust embedders move to gstreamer 0.25 (#6169)

Only Rust code that links `gstmoq` as a library is affected; pipelines and the loadable plugin are not.
Expand Down
Loading
Loading