diff --git a/js/hang/src/util/libav.copy.test.ts b/js/hang/src/util/libav.copy.test.ts new file mode 100644 index 00000000000..ed8fb1155e2 --- /dev/null +++ b/js/hang/src/util/libav.copy.test.ts @@ -0,0 +1,54 @@ +import { expect, test } from "bun:test"; +import { polyfill } from "./libav"; + +// Bun has no WebCodecs, so this runs on the polyfill's AudioData (another test file may have loaded it first), +// which libopus decodes into. +test("the polyfill deinterleaves every channel to f32-planar", async () => { + expect(await polyfill()).toBe(true); + + const cases = [ + { format: "f32", data: Float32Array.of(0.5, -0.5, 0.25, -0.25, 0, 1), scale: 1 }, + { format: "s16", data: Int16Array.of(0x4000, -0x4000, 0x2000, -0x2000, 0, 0x7fff), scale: 0x8000 }, + ] as const; + for (const { format, data, scale } of cases) { + const audio = new AudioData({ + format, + sampleRate: 48_000, + numberOfFrames: 3, + numberOfChannels: 2, + timestamp: 0, + data, + }); + for (const channel of [0, 1]) { + const plane = new Float32Array(3); + audio.copyTo(plane, { planeIndex: channel, format: "f32-planar" }); + expect(Array.from(plane)).toEqual([0, 1, 2].map((frame) => data[frame * 2 + channel] / scale)); + } + + const tail = new Float32Array(2); + audio.copyTo(tail, { planeIndex: 1, format: "f32-planar", frameOffset: 1 }); + expect(Array.from(tail)).toEqual([1, 2].map((frame) => data[frame * 2 + 1] / scale)); + // A view smaller than the copy is refused, even with room in the buffer behind it. + const backing = new Float32Array(8); + expect(() => + audio.copyTo(backing.subarray(0, 1), { planeIndex: 1, format: "f32-planar", frameCount: 3 }), + ).toThrow(RangeError); + expect(Array.from(backing)).toEqual(new Array(8).fill(0)); + // An unaligned byte view takes the floats too. + const unaligned = new Uint8Array(13); + audio.copyTo(unaligned.subarray(1), { planeIndex: 1, format: "f32-planar" }); + expect(Array.from(new Float32Array(unaligned.slice(1).buffer))).toEqual( + [0, 1, 2].map((frame) => Math.fround(data[frame * 2 + 1] / scale)), + ); + for (const bad of [ + { planeIndex: -1 }, + { planeIndex: 0.5 }, + { planeIndex: 0, frameOffset: -1 }, + // WebCodecs refuses an offset at the end even though nothing would be copied. + { planeIndex: 0, frameOffset: 3 }, + ]) { + expect(() => audio.copyTo(backing, { ...bad, format: "f32-planar" })).toThrow(RangeError); + } + audio.close(); + } +}); diff --git a/js/hang/src/util/libav.ts b/js/hang/src/util/libav.ts index 1c91df8f7ee..0cf5e26227c 100644 --- a/js/hang/src/util/libav.ts +++ b/js/hang/src/util/libav.ts @@ -19,6 +19,7 @@ export async function polyfill(): Promise { async ([{ default: LibAV }, libav]) => { const native = globalThis.AudioDecoder; await libav.load({ LibAV, polyfill: true }); + fixPlanarCopy(libav.AudioData as unknown as typeof AudioData); if (native) return true; type Config = Parameters["configure"]>[0]; @@ -68,3 +69,49 @@ function planar(data: T): T { data.close(); return relabeled as T; } + +// Each interleaved sample format, with the offset and scale that map it to [-1, 1]. +const INTERLEAVED = { + u8: [Uint8Array, 0x80, 0x80], + s16: [Int16Array, 0, 0x8000], + s32: [Int32Array, 0, 0x80000000], + f32: [Float32Array, 0, 1], +} as const; + +// The polyfill converts interleaved samples to "f32-planar" from the wrong offset, so every channel +// after the first reads past its frames (NaN) instead of its own samples. libopus decodes +// interleaved, so stereo Opus lost its right channel. Deinterleave from a same-format copy instead, +// which the polyfill gets right. +function fixPlanarCopy(Data: typeof AudioData): void { + const copyTo = Data.prototype.copyTo; + Data.prototype.copyTo = function (this: AudioData, destination: AllowSharedBufferSource, options) { + const format = this.format ?? ""; + if (options.format !== "f32-planar" || !(format in INTERLEAVED)) return copyTo.call(this, destination, options); + const [View, offset, scale] = INTERLEAVED[format as keyof typeof INTERLEAVED]; + const channels = this.numberOfChannels; + const index = (value: number) => Number.isSafeInteger(value) && value >= 0; + if (!index(options.planeIndex) || options.planeIndex >= channels) { + throw new RangeError(`no plane ${options.planeIndex}`); + } + + const start = options.frameOffset ?? 0; + const count = options.frameCount ?? this.numberOfFrames - start; + if (!index(start) || !index(count) || start >= this.numberOfFrames || start + count > this.numberOfFrames) { + throw new RangeError("frames out of range"); + } + // Bounded by the caller's view, not the buffer behind it. + if (count * Float32Array.BYTES_PER_ELEMENT > destination.byteLength) { + throw new RangeError("destination too small"); + } + const samples = new View(this.numberOfFrames * channels); + copyTo.call(this, samples, { planeIndex: 0 }); + const out = new Float32Array(count); + for (let i = 0; i < count; i++) + out[i] = (samples[(start + i) * channels + options.planeIndex] - offset) / scale; + // Copied as bytes: a destination need not be 4-byte aligned. + const bytes = ArrayBuffer.isView(destination) + ? new Uint8Array(destination.buffer, destination.byteOffset, out.byteLength) + : new Uint8Array(destination, 0, out.byteLength); + bytes.set(new Uint8Array(out.buffer)); + }; +} diff --git a/quest/m1/README.md b/quest/m1/README.md index e89770e38f3..33d6a0e1a0e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -107,7 +107,9 @@ lite-07 stream type here. - [moq-net compiles once](/quest/m1/net-erase-transport.md) - the session drivers stop monomorphizing per transport and per crate, saving several MiB of the relay, FFI library, `libmoq` and CLI - [Audio devices on ARM](/quest/m1/audio-devices-arm.md) - `just rs audio-devices` passes on the hosted ARM runner - [CI housekeeping](/quest/m1/ci-housekeeping.md) - validate queue nextest at 24 threads and remove the actionlint queue ignore after its release -- [Interop audio delivery](/quest/m1/interop-audio.md) - every interop subscriber decodes and checks the publishers' 440 Hz tone, and the Rust publisher sends Opus and AAC +- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the wire-compat media lanes decode and check the interop tone across releases +- [Interop AAC](/quest/m1/interop-aac.md) - the Rust interop publisher adds an AAC tone and every subscriber checks it beside Opus +- [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - Opus and AAC decode through every moq-audio backend, moq-c, and moq-ffi is checked against known tones and clips, like moq-video's suite - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured - [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut diff --git a/quest/m1/audio-decode-conformance.md b/quest/m1/audio-decode-conformance.md index 2cecde4b48e..d3cd65542b8 100644 --- a/quest/m1/audio-decode-conformance.md +++ b/quest/m1/audio-decode-conformance.md @@ -55,6 +55,6 @@ Public API: none expected. Wire: none. ## Related -- [Interop audio delivery](/quest/m1/interop-audio.md) - the same decoders, end to end across clients +- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the same decoders, end to end across releases - [AudioToolbox decode](/quest/m2/audio-decode-audiotoolbox.md) - a new backend that should join this suite when it lands - [Opus backends](/quest/m2/audio-opus-backend.md) - compares Opus implementations; this suite is its correctness floor diff --git a/quest/m1/interop-aac.md b/quest/m1/interop-aac.md new file mode 100644 index 00000000000..22b8edea50b --- /dev/null +++ b/quest/m1/interop-aac.md @@ -0,0 +1,28 @@ +# [M] Interop subscribers check an AAC tone too + +## Goal + +The Rust CLI interop publisher sends an AAC tone beside its Opus one, and every +subscriber decodes and checks both renditions with `test/interop/tone.py` (the +browser with its in-page check), so AAC import and decode are proven across +clients the way Opus is. + +## Plan + +Split from the interop audio checks, which landed Opus only. The libav fallback +now decodes AAC, so the Node and Bun subscribers can follow. + +Things to look out for: + +- Every subscriber picks the first audio rendition today; checking both means + each client decodes every rendition it is offered, or the harness asks for one + by name per pass. +- ffmpeg's AAC encoder adds priming samples; a decoder that ignores them shifts + the tone without a gap, which the check does not see (it does not anchor + phase to timestamps). + +Public API: none expected. Wire: none. + +## Related + +- [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the release lanes that would follow an AAC track diff --git a/quest/m1/interop-audio.md b/quest/m1/interop-audio.md deleted file mode 100644 index 570649348a0..00000000000 --- a/quest/m1/interop-audio.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] Every interop subscriber decodes the audio tone - -## Goal - -`just test interop --all` proves audio delivery the way it proves video: every -subscriber decodes the audio track and checks the publisher's 440 Hz tone, and -every publisher sends one. A garbled decode (wrong channel order, pre-skip, or -sample rate), a missing audio rendition, or a timestamp gap fails the matrix. - -Today only the browser subscriber looks at audio, and only for -`audioBytes > 0`. The Rust, Python, Go, C, C++, native Node/Bun, and GStreamer -subscribers all select the video track, and the Rust CLI publisher sends no -audio, so native audio delivery is untested end to end. - -## Plan - -Decided in planning (2026-10-20): - -- Depth: decode and verify the tone, not just count frames. A Goertzel filter - at 440 Hz and timestamp continuity per subscriber; bytes arriving is the bar - that already lets a broken decode pass. A multichannel publisher sends a - distinct frequency per channel, so a channel swap fails too. -- The Rust CLI publisher adds an Opus track in fMP4 (ffmpeg `sine` at - 440 Hz), which also covers moq-mux's Opus import, and an AAC track beside - it. The Node and Bun subscribers decode AAC through `@moq/hang`'s libav - fallback, which `js/hang/src/util/libav.test.ts` covers under Bun. -- Folded in from the same gap analysis: the `just test wire-compat` media lanes - gain an audio check (released vs current, decoded), and moq-transcode gains - a test that an audio rendition passes through untouched. - -Recommendations: - -- Decode with what each client already has: moq-ffi's audio decode for Python, - Go, C, and C++, moq-audio for the Rust subscriber, `decodebin` on `audio_0` - for GStreamer, WebCodecs for native JS. A subscriber whose binding cannot - decode audio is a binding gap to report, not a reason to skip the check. -- The browser subscriber upgrades from `audioBytes > 0` to the same tone check; - the media QA probe (`test/interop/clients/js/src/probe.ts`) already detects - tones. -- Add a negative control like media QA's silent-audio and audio-offset faults: - a publisher sending silence or the wrong frequency must fail every - subscriber. -- Keep the expected tone in one place the clients share, so a future publisher - change does not desync the checks. - -Public API: none. Wire: none. - -## Related - -- [Bindings in the interop matrix](/quest/m2/interop-bindings.md) - Kotlin, Swift, and Dart clients; whichever lands second carries the audio check to them -- [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - the same decode paths, checked in unit tests -- [Media QA audio cases](/quest/m2/media-qa-audio.md) - grades A/V sync and codec changes in the browser diff --git a/quest/m1/interop-preskip.md b/quest/m1/interop-preskip.md new file mode 100644 index 00000000000..c7a7f6df7b6 --- /dev/null +++ b/quest/m1/interop-preskip.md @@ -0,0 +1,34 @@ +# [S] Interop tone check catches a misapplied pre-skip + +## Goal + +`test/interop/tone.py` fails a subscriber whose decoder drops or keeps the +wrong number of priming samples (Opus pre-skip, AAC priming), the way it +already fails a channel swap or a gap. Today a wrong pre-skip only shifts a +steady tone's phase, which the energy-and-frequency check cannot see. + +## Plan + +Found in review of the interop audio checks. The publishers' tone phase is a +function of the publisher's timestamps (the FFI clients and ffmpeg's `aevalsrc` +both start at zero), so the decoded sample at timestamp `t` should carry phase +`2πft`: anchor the Goertzel phase to the frame timestamps. + +Things to look out for: + +- Precision differs per client: the libav fallback stamps milliseconds (about + 1.4 rad at 440 Hz), and the browser publisher's capture clock is not tied to + its WAV's phase. Check only where the timestamps are precise enough, and say + which clients are left out. +- Add a negative control that misapplies the pre-skip on purpose, so the check + is proven able to fail. +- The GStreamer subscriber likely fails it today: `opusdec` trims pre-skip only + from a buffer's `GstAudioClippingMeta`, never from the OpusHead, and `moqsrc` + attaches none. + +Public API: none expected. Wire: none. + +## Related + +- [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - pre-skip continuity in unit tests +- [Interop AAC](/quest/m1/interop-aac.md) - AAC priming is the same question diff --git a/quest/m1/wire-compat-audio.md b/quest/m1/wire-compat-audio.md new file mode 100644 index 00000000000..3ecab30157a --- /dev/null +++ b/quest/m1/wire-compat-audio.md @@ -0,0 +1,33 @@ +# [M] Wire-compat media lanes check the audio tone + +## Goal + +`just test wire-compat`'s media lanes prove audio crosses releases the way +`just test interop` proves it within the checkout: the current and released +Rust CLIs publish the interop tone, and the released and current Rust and JS +readers decode it and pass `test/interop/tone.py`. A garbled decode, a missing +audio rendition, or a timestamp gap between versions fails the lane. + +## Plan + +Split from the interop audio work, which checks the tone in every in-tree cell +but skips audio when `interop.sh` runs released binaries or JS clients +(`AUDIO=0`): the lanes keep the video-only Annex-B publisher and readers. + +Things to look out for: + +- The Rust readers export fMP4. Decoding its audio with ffmpeg, as the lanes + already decode video, still owes tone.py the frame timestamps it checks for + gaps. +- `compat/client.ts` runs against both package sets, so its audio decode can + only use API the released `@moq/hang` has. +- Released `@moq/hang`'s libav fallback garbles every channel past the first + when it converts interleaved PCM to planar (fixed alongside the interop audio + checks). Until a release carries that fix, a released JS reader fails a stereo + tone: decide between a planned break and starting mono. + +Public API: none. Wire: none. + +## Related + +- [Interop AAC](/quest/m1/interop-aac.md) - adds an AAC track to the in-tree matrix; these lanes would follow diff --git a/quest/m2/media-qa-audio.md b/quest/m2/media-qa-audio.md index a4bd89318ba..03d12b3510c 100644 --- a/quest/m2/media-qa-audio.md +++ b/quest/m2/media-qa-audio.md @@ -18,8 +18,8 @@ Decided in planning (2026-10-20): draw the media QA frame-counter pattern and generate the stepped tone, and `moq` publishes them through moq-mux. The browser grades pitch and sync, which covers native importer timestamps. The FFI publishers' tone is - graded for pitch and continuity by [interop audio - delivery](/quest/m1/interop-audio.md), not here. + graded for pitch and continuity by the interop matrix + (`test/interop/tone.py`), not here. - Browser AAC publish runs end to end for the first time in the AAC case. - Watch mute and volume are verified on the output (muted is silence, volume scales the level). The media QA probe taps `audio.out.root`, upstream of the diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 2970c6dec35..354199d790d 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -150,6 +150,8 @@ winit = { version = "0.30.13", optional = true } sd-notify = { workspace = true } [dev-dependencies] +# Decodes the interop matrix's audio tone (the `interop-audio` example). +moq-audio = { path = "../moq-audio" } # `test_relay` stands up a real relay for the `fetch` and `announced` tests to read through. moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } # The scripted RTSP server `import rtsp` pulls from. @@ -176,3 +178,7 @@ harness = false [[example]] name = "interop-tail" path = "../../test/interop/clients/rust/tail.rs" + +[[example]] +name = "interop-audio" +path = "../../test/interop/clients/rust/audio.rs" diff --git a/rs/moq-gst/MIGRATE.md b/rs/moq-gst/MIGRATE.md index dfaf1ae1f81..8da134ac585 100644 --- a/rs/moq-gst/MIGRATE.md +++ b/rs/moq-gst/MIGRATE.md @@ -4,6 +4,11 @@ How to update your pipelines for each breaking change, newest first. ## Unreleased +### `moqsrc` audio caps are the standard ones + +`moqsrc` audio pads now carry `rate` and `channels` as `int`, and Opus pads the mapping fields (`channel-mapping-family`, `stream-count`, `coupled-count`, plus `channel-mapping` beyond stereo) with the OpusHead and an empty OpusTags as `streamheader`, so `decodebin` and `opusdec` accept them and `opusdec` applies the output gain. +Opus pads no longer carry the nonstandard `codec_data` and `stream-format=ogg`; a capsfilter naming either, or a `uint` rate or channel count, must drop it. + ### Rust embedders move to gstreamer 0.25 (#6169) Only Rust code that links `gstmoq` as a library is affected; pipelines and the loadable plugin are not. diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 277eb11abe3..9316007c684 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1382,13 +1382,17 @@ fn video_caps(config: &hang::catalog::VideoConfig) -> Result { Ok(caps) } +// GStreamer's audio caps carry rate and channels as `int`: an unsigned field never intersects a +// decoder's template, so decodebin finds no decoder. fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { + let rate = i32::try_from(config.sample_rate).context("sample rate out of range")?; + let channels = i32::try_from(config.channel_count).context("channel count out of range")?; let caps = match &config.codec { hang::catalog::AudioCodec::AAC(_) => { let mut builder = gst::Caps::builder("audio/mpeg") .field("mpegversion", 4) - .field("rate", config.sample_rate) - .field("channels", config.channel_count); + .field("rate", rate) + .field("channels", channels); if let Some(description) = &config.description { builder = builder .field("codec_data", gst::Buffer::from_slice(description.clone())) @@ -1398,28 +1402,72 @@ fn audio_caps(config: &hang::catalog::AudioConfig) -> Result { } builder.build() } - hang::catalog::AudioCodec::Opus => { - let mut builder = gst::Caps::builder("audio/x-opus") - .field("rate", config.sample_rate) - .field("channels", config.channel_count); - if let Some(description) = &config.description { - builder = builder - .field("codec_data", gst::Buffer::from_slice(description.clone())) - .field("stream-format", "ogg"); - } - builder.build() - } + hang::catalog::AudioCodec::Opus => opus_caps(config, rate, channels)?, hang::catalog::AudioCodec::Mp3 => gst::Caps::builder("audio/mpeg") .field("mpegversion", 1) .field("layer", 3) - .field("rate", config.sample_rate) - .field("channels", config.channel_count) + .field("rate", rate) + .field("channels", channels) .build(), other => bail!("unsupported audio codec: {other:?}"), }; Ok(caps) } +/// An OpusTags packet (RFC 7845 §5.2) with no vendor and no comments. +const OPUS_TAGS: &[u8] = b"OpusTags\0\0\0\0\0\0\0\0"; + +/// The `audio/x-opus` caps opusdec and moqsink negotiate on: the stream layout as mapping fields, +/// which decodebin needs to pick a decoder, plus the catalog's OpusHead and an empty OpusTags as the +/// streamheader. opusdec reads the head, and so its output gain, only from a two-buffer streamheader. +fn opus_caps(config: &hang::catalog::AudioConfig, rate: i32, channels: i32) -> Result { + let head = match &config.description { + Some(description) => moq_mux::codec::opus::Config::parse(&mut description.as_ref()) + .context("Opus description is not an OpusHead")?, + None => moq_mux::codec::opus::Config::new(config.sample_rate, config.channel_count), + }; + anyhow::ensure!( + head.channel_count == config.channel_count, + "the catalog's {} Opus channels contradict its OpusHead's {}", + config.channel_count, + head.channel_count + ); + let mut builder = gst::Caps::builder("audio/x-opus") + .field("rate", rate) + .field("channels", channels); + builder = match &head.mapping { + Some(mapping) => builder + .field("channel-mapping-family", i32::from(mapping.family())) + .field("stream-count", i32::from(mapping.streams())) + .field("coupled-count", i32::from(mapping.coupled())) + .field( + "channel-mapping", + gst::Array::new(mapping.table().iter().map(|&c| i32::from(c))), + ), + // Family 0 is mono or stereo: one stream, coupled when stereo. + None => { + anyhow::ensure!( + (1..=2).contains(&channels), + "Opus family 0 carries 1 or 2 channels, not {channels}" + ); + builder + .field("channel-mapping-family", 0i32) + .field("stream-count", 1i32) + .field("coupled-count", channels - 1) + } + }; + if let Some(description) = &config.description { + builder = builder.field( + "streamheader", + gst::Array::new([ + gst::Buffer::from_slice(description.clone()), + gst::Buffer::from_slice(OPUS_TAGS), + ]), + ); + } + Ok(builder.build()) +} + #[cfg(test)] mod tests { use super::{PumpState, Timeline, ended, plan_reconcile}; @@ -1475,6 +1523,40 @@ mod tests { } } + /// opusdec's templates require the mapping fields as ints, so decodebin finds no decoder without them. + #[test] + fn opus_caps_carry_the_stream_layout() { + gst::init().unwrap(); + for channels in [1, 2] { + let head = moq_mux::codec::opus::Config::new(48_000, channels).with_pre_skip(312); + for description in [None, Some(head.encode().unwrap())] { + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, channels); + config.description = description.clone(); + let caps = super::audio_caps(&config).unwrap(); + let structure = caps.structure(0).unwrap(); + assert_eq!(structure.get::("rate").unwrap(), 48_000); + assert_eq!(structure.get::("channels").unwrap(), channels as i32); + assert_eq!(structure.get::("channel-mapping-family").unwrap(), 0); + assert_eq!(structure.get::("stream-count").unwrap(), 1); + assert_eq!(structure.get::("coupled-count").unwrap(), channels as i32 - 1); + let headers = structure.get_optional::("streamheader").unwrap(); + let headers: Option>> = headers.map(|headers| { + headers + .iter() + .map(|header| header.get::().unwrap().map_readable().unwrap().to_vec()) + .collect() + }); + assert_eq!( + headers, + description + .as_ref() + .map(|head| vec![head.to_vec(), super::OPUS_TAGS.to_vec()]), + "opusdec ignores a streamheader without the OpusTags after the OpusHead" + ); + } + } + } + // The shape type is generic, so the set math can be exercised with a plain integer standing // in for (caps, container): equal value == unchanged rendition, different value == reshape. fn renditions(pairs: &[(&str, u32)]) -> HashMap { diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 19d4057ab49..f21c0f5d6e9 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -966,6 +966,33 @@ mod tests { assert_eq!(decoders.probed, [(Codec::H265, false)]); } + /// The ladder transcodes video alone: audio is referenced in the source, with every field kept, + /// so a player decodes exactly what the source published. + #[test] + fn populate_passes_audio_through_untouched() { + let mut child = moq_mux::catalog::hang::Catalog::<()>::default(); + let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + audio.description = Some( + moq_mux::codec::opus::Config::new(48_000, 2) + .with_pre_skip(312) + .encode() + .unwrap(), + ); + audio.bitrate = Some(128_000); + audio.track = Some("1.m4s".to_string()); + audio.jitter = Some(std::time::Duration::from_millis(20)); + child.audio.insert("audio", audio.clone()).unwrap(); + + let rel = RelativeOwned::from(".".to_string()); + let mut out = moq_mux::catalog::hang::Catalog::<()>::default(); + populate(&mut out, &child, &[], Some(&rel)).unwrap(); + + let mut expected = audio; + expected.broadcast = Some(rel); + assert_eq!(out.audio.renditions.get("audio"), Some(&expected)); + assert_eq!(out.audio.renditions.len(), 1); + } + #[test] fn populate_preserves_the_child_archive() { let mut child = moq_mux::catalog::hang::Catalog::<()>::default(); diff --git a/test/interop/README.md b/test/interop/README.md index 2405da75e56..fa3e2104940 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -12,12 +12,27 @@ distribution-mechanism matrix. It stands up a `moq-relay`, then for each publisher language publishes an H.264 broadcast and confirms every subscriber sees data flowing before the timeout. -Most subscribers check for a non-empty frame. The browser additionally verifies +Most subscribers check for a non-empty video frame. The browser additionally verifies WebCodecs output painted to a canvas and drives the player's pause/resume controls. -Every publisher but the Rust CLI also carries an Opus track, which the browser -subscriber checks end-to-end: the browser encodes fake microphone audio, and the -Python, Go, and C++ clients encode a synthetic tone through `moq-ffi` at a 2.5 ms frame -duration, so the matrix covers the FFI audio path with a non-default codec config. + +Every publisher also sends an Opus tone, one sine per channel, and every subscriber +decodes it. Python and C++ send a mono 440 Hz tone; the rest send 440 Hz left and +660 Hz right, so a channel swap fails too. The Rust CLI imports it from ffmpeg in +fragmented MP4, the browser encodes its fake microphone playing the tone from a WAV +file, and the Python, Go, and C++ clients encode it through `moq-ffi` at a 2.5 ms +frame duration, so the matrix covers the FFI audio path with a non-default codec config. +Each subscriber decodes with what it already has: moq-audio for Rust (the +`interop-audio` example, since the CLI has no PCM output), `moq-ffi` for Python, +Go, and C++, `moq-c` for C, `decodebin` on `audio_0` for GStreamer, and +`@moq/hang`'s libav WebCodecs fallback under Node and Bun. Each writes its first +600 ms of PCM and frame timestamps for [`tone.py`](tone.py), which requires the +publisher's channel count, contiguous timestamps, and each channel's tone carrying +most of its energy by a Goertzel filter. A garbled decode (wrong channel order or +sample rate), silence, or a timestamp gap fails the cell; a misapplied pre-skip only +shifts the tone's phase, which this does not check. The browser +runs the same filter on each channel of the player's audio graph. A negative control +then has the Rust CLI swap its channels' tones, and every subscriber must refuse it +as the wrong tone, not fail some other way. `interop.sh` is the one place the tones are chosen. The normal run also checks a finite raw track through the relay. Its publisher announces end 4 before writing four 256 KiB groups; the reader verifies every byte, @@ -56,7 +71,7 @@ player survive the publication lifecycle. See [Media QA](#media-qa). | Client | Source under test | Built with | Roles | |---|---|---|---| -| Rust | `rs/moq-relay` + `rs/moq-cli` | `cargo build` | publish (video) + subscribe | +| Rust | `rs/moq-relay` + `rs/moq-cli` (+ `rs/moq-audio` for the `interop-audio` example) | `cargo build` | publish (video + audio) + subscribe | | Python | `py/moq-rs` (+ `rs/moq-ffi`, import `moq`) | `uv build` wheels (maturin + hatchling), installed into a venv in the run directory | publish (video + audio) + subscribe | | Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `sh/go/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | | C++ | `cpp/moq` (+ `rs/moq-ffi`, `find_package(moq-cpp)`) | `cmake` build + install of `cpp/moq` (uniffi-bindgen-cpp), then `cmake` for the client | publish (video + audio) + subscribe | @@ -85,8 +100,9 @@ marks the cell unavailable. The GStreamer client builds the `moqsrc` plugin from `rs/moq-gst` and points `GST_PLUGIN_PATH` at it, then reads a broadcast with -`gst-launch-1.0 moqsrc ... ! filesink`. The plugin dynamic-links the host's -GStreamer, so this cell needs `gst-launch-1.0` + the core plugins on the system. +`gst-launch-1.0 moqsrc ... ! filesink`, decoding `audio_0` through `decodebin`. +The plugin dynamic-links the host's GStreamer, so this cell needs `gst-launch-1.0` +plus the core plugins and `opusdec` on the system. The `nix develop` shell ships them; a bare shell without GStreamer marks the cell unavailable rather than failing it. @@ -382,9 +398,12 @@ contract](../README.md). interop.sh orchestrator: build clients, run the relay + matrix or media checks interop.toml relay config (token auth via `moq auth serve`, self-signed localhost) report.py the CI report: case records to a static pass/fail page +tone.py judges a subscriber's decoded audio against its publisher's tones bare-fin.ts the JS side of `just test bare-fin`, driven by moq-net's tests varint.ts the JS side of the varint check, driven by moq-net's tests clients/ + rust/audio.rs the Rust subscriber's audio decode via rs/moq-audio (`interop-audio`) + rust/tail.rs the finite raw-track client (`interop-tail`) python/interop.py publish/subscribe via py/moq-rs (import moq) go/main.go publish/subscribe via go/wrapper (import moq-go/moq) cpp/main.cpp publish/subscribe via cpp/moq (find_package(moq-cpp)) diff --git a/test/interop/clients/c/subscribe.c b/test/interop/clients/c/subscribe.c index 8de8dd52616..0b6bc57eea5 100644 --- a/test/interop/clients/c/subscribe.c +++ b/test/interop/clients/c/subscribe.c @@ -2,13 +2,15 @@ // against the workspace moq-c (the C bindings, built by `cargo build -p moq-c`). // // moq-c is a handle + callback API: connect, consume a broadcast, get a -// catalog snapshot via callback, start the video track, and a frame callback -// fires as frames arrive. Once a non-empty frame lands we close and drain every -// registration, then return 0; a timeout exits 1. Publishing isn't wired up: the -// raw-stream importer that the other clients use to publish isn't part of this -// subscribe-only client. +// catalog snapshot via callback, start the video track and the audio decoder, +// and a callback fires as each frame arrives. Once a non-empty video frame has +// landed and --audio-ms of audio has been decoded to --audio-out (for tone.py to +// judge) we close and drain every registration, then return 0; a timeout exits 1. +// Publishing isn't wired up: the raw-stream importer that the other clients use +// to publish isn't part of this subscribe-only client. // -// c-interop subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 +// c-interop subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 \ +// --audio-out /tmp/audio --audio-ms 600 // // --version offers only that version, through moq_client_config. #include @@ -33,11 +35,19 @@ typedef struct { int32_t catalog; int32_t video_track; // handle from moq_consume_video (0 until on_catalog starts it) + // The audio decoder (0 until on_catalog starts it), where it writes, and how + // much it has written. `audio_got` is set once `audio_samples` reaches `audio_want`. + int32_t audio_track; + FILE *audio_log, *audio_pcm; + uint64_t audio_ms, audio_samples, audio_want; + uint32_t audio_frame_bytes; + int audio_got; + // One flag per registration handed &ctx, set by that registration's terminal - // (<= 0) callback. main returns only once all four are set: that terminal is + // (<= 0) callback. main returns only once all five are set: that terminal is // moq-c's documented last touch of user_data, so it is what makes main's // frame safe to destroy. - int done_status, done_broadcast, done_catalog, done_frame; + int done_status, done_broadcast, done_catalog, done_frame, done_audio; } ctx_t; // Callbacks run on moq-c's runtime thread; main waits on the condvar. ctx lives @@ -76,6 +86,34 @@ static void on_frame(void *ud, int32_t frame) { moq_consume_frame_free((uint32_t)frame); } +// Writes each decoded audio frame to the files tone.py reads, until enough has arrived. +// Runs on moq-c's runtime thread, the only writer, so the files need no lock. +static void on_audio(void *ud, int32_t frame) { + ctx_t *c = (ctx_t *)ud; + if (frame <= 0) { // 0 = ended, negative = error + if (frame < 0) fprintf(stderr, "audio decode error: %d\n", frame); + done(c, &c->done_audio); + return; + } + moq_audio_frame f; + memset(&f, 0, sizeof(f)); + if (moq_decode_audio_frame((uint32_t)frame, &f) == 0 && c->audio_samples < c->audio_want) { + uint64_t samples = f.data_size / c->audio_frame_bytes; + fprintf(c->audio_log, "%llu %llu\n", (unsigned long long)f.timestamp_us, (unsigned long long)samples); + fwrite(f.data, 1, f.data_size, c->audio_pcm); + c->audio_samples += samples; + if (c->audio_samples >= c->audio_want) { + fflush(c->audio_log); + fflush(c->audio_pcm); + pthread_mutex_lock(&c->mu); + c->audio_got = 1; + pthread_cond_broadcast(&c->cv); + pthread_mutex_unlock(&c->mu); + } + } + moq_decode_audio_frame_free((uint32_t)frame); +} + // Delivers a broadcast handle for each publisher instance serving the path, then // a terminal code (<= 0). Store the first positive handle and wake main; one frame // is all this client reads, so a later instance's handle is freed unread. @@ -104,8 +142,33 @@ static void on_catalog(void *ud, int32_t catalog) { pthread_mutex_lock(&c->mu); int start = c->video_track <= 0; + int start_audio = c->audio_track <= 0; pthread_mutex_unlock(&c->mu); + if (start_audio) { + // Like video, a lazy publisher may add audio in a later update. Decode at + // the codec's own rate and layout, so the catalog's are what tone.py checks. + moq_audio_config acfg; + memset(&acfg, 0, sizeof(acfg)); + if (moq_consume_audio_config((uint32_t)catalog, 0, &acfg) == 0 && acfg.channel_count > 0) { + c->audio_frame_bytes = 4 * acfg.channel_count; + c->audio_want = c->audio_ms * acfg.sample_rate / 1000; + fprintf(c->audio_log, "%u %u\n", acfg.sample_rate, acfg.channel_count); + moq_audio_decoder_output out; + memset(&out, 0, sizeof(out)); + out.format = MOQ_AUDIO_SAMPLE_FORMAT_F32; + out.max_delay_us = 1000000; + int32_t track = moq_decode_audio((uint32_t)catalog, 0, &out, on_audio, ud); + if (track > 0) { + pthread_mutex_lock(&c->mu); + c->audio_track = track; + pthread_mutex_unlock(&c->mu); + } else { + fprintf(stderr, "moq_decode_audio failed: %d\n", track); + } + } + } + if (start) { // A lazy publisher may announce video in a later catalog update, so this // just no-ops (config returns < 0) until a video track exists at index 0. @@ -168,11 +231,14 @@ static _Noreturn void fail(const char *fmt, ...) { static void drain(ctx_t *c) { pthread_mutex_lock(&c->mu); int32_t track = c->video_track; - // A video track that was never started registered nothing, so it owes no terminal. + int32_t audio = c->audio_track; + // A track that was never started registered nothing, so it owes no terminal. if (track <= 0) c->done_frame = 1; + if (audio <= 0) c->done_audio = 1; pthread_mutex_unlock(&c->mu); if (track > 0) moq_consume_video_cancel((uint32_t)track); + if (audio > 0) moq_decode_audio_cancel((uint32_t)audio); moq_consume_catalog_cancel((uint32_t)c->catalog); moq_origin_announced_broadcast_cancel((uint32_t)c->broadcast_wait); moq_session_close((uint32_t)c->session); @@ -182,30 +248,34 @@ static void drain(ctx_t *c) { deadline.tv_sec += 10; pthread_mutex_lock(&c->mu); - while (!(c->done_frame && c->done_catalog && c->done_broadcast && c->done_status)) { + while (!(c->done_frame && c->done_audio && c->done_catalog && c->done_broadcast && c->done_status)) { if (pthread_cond_timedwait(&c->cv, &c->mu, &deadline) != 0) break; // timed out } - int ok = c->done_frame && c->done_catalog && c->done_broadcast && c->done_status; - int f = c->done_frame, ct = c->done_catalog, b = c->done_broadcast, s = c->done_status; + int ok = c->done_frame && c->done_audio && c->done_catalog && c->done_broadcast && c->done_status; + int f = c->done_frame, a = c->done_audio, ct = c->done_catalog, b = c->done_broadcast, s = c->done_status; pthread_mutex_unlock(&c->mu); if (!ok) { - fail("error: teardown never quiesced (frame=%d catalog=%d broadcast=%d status=%d)\n", f, ct, b, s); + fail("error: teardown never quiesced (frame=%d audio=%d catalog=%d broadcast=%d status=%d)\n", f, a, ct, b, + s); } } int main(int argc, char **argv) { - const char *url = NULL, *broadcast = NULL, *version = NULL; + const char *url = NULL, *broadcast = NULL, *version = NULL, *audio_out = NULL; double timeout_s = 20.0; + long audio_ms = 0; for (int i = 1; i < argc; i++) { if (!strcmp(argv[i], "--url") && i + 1 < argc) url = argv[++i]; else if (!strcmp(argv[i], "--broadcast") && i + 1 < argc) broadcast = argv[++i]; else if (!strcmp(argv[i], "--timeout") && i + 1 < argc) timeout_s = atof(argv[++i]); else if (!strcmp(argv[i], "--version") && i + 1 < argc) version = argv[++i]; + else if (!strcmp(argv[i], "--audio-out") && i + 1 < argc) audio_out = argv[++i]; + else if (!strcmp(argv[i], "--audio-ms") && i + 1 < argc) audio_ms = atol(argv[++i]); // a leading "subscribe" positional (and anything else) is ignored. } - if (!url || !broadcast) { - fprintf(stderr, "usage: c-interop subscribe --url U --broadcast B [--timeout S] [--version V]\n"); + if (!url || !broadcast || !audio_out || audio_ms <= 0) { + fprintf(stderr, "usage: c-interop subscribe --url U --broadcast B --audio-out PREFIX --audio-ms MS [--timeout S] [--version V]\n"); return 2; } @@ -214,6 +284,17 @@ int main(int argc, char **argv) { pthread_mutex_init(&c.mu, NULL); pthread_cond_init(&c.cv, NULL); + char path[4096]; + snprintf(path, sizeof(path), "%s.txt", audio_out); + c.audio_log = fopen(path, "w"); + snprintf(path, sizeof(path), "%s.pcm", audio_out); + c.audio_pcm = fopen(path, "wb"); + if (!c.audio_log || !c.audio_pcm) { + fprintf(stderr, "error: cannot write %s.{txt,pcm}\n", audio_out); + return 1; + } + c.audio_ms = (uint64_t)audio_ms; + c.origin = moq_origin_create(); if (c.origin <= 0) { fprintf(stderr, "error: moq_origin_create failed: %d\n", c.origin); @@ -270,23 +351,26 @@ int main(int argc, char **argv) { } pthread_mutex_lock(&c.mu); - while (!c.got) { + while (!c.got || !c.audio_got) { if (pthread_cond_timedwait(&c.cv, &c.mu, &deadline) != 0) break; // timed out } - int got = c.got; + int got = c.got, audio_got = c.audio_got; pthread_mutex_unlock(&c.mu); - if (!got) { - fail("error: timed out waiting for data\n"); + if (!got || !audio_got) { + fail("error: timed out waiting for data (video=%d audio=%d)\n", got, audio_got); } - fprintf(stderr, "received a frame from %s\n", broadcast); + fprintf(stderr, "received a frame and %llu audio samples from %s\n", (unsigned long long)c.audio_samples, + broadcast); // The data path succeeded, which is all this interop client verifies. Returning // (rather than _exit) is the other half of what it verifies: an embedder that // closes and drains gets a clean process exit, with moq-c's runtime thread // still live behind its LazyLock. drain(&c); + fclose(c.audio_log); + fclose(c.audio_pcm); moq_consume_close((uint32_t)bc); moq_origin_close((uint32_t)c.origin); fflush(stderr); diff --git a/test/interop/clients/cpp/main.cpp b/test/interop/clients/cpp/main.cpp index 8343138ebc9..64831d6f13d 100644 --- a/test/interop/clients/cpp/main.cpp +++ b/test/interop/clients/cpp/main.cpp @@ -2,13 +2,15 @@ // package with find_package(moq-cpp). // // publish reads raw Annex-B H.264 from stdin (e.g. piped from ffmpeg) and feeds it to a -// streaming importer, which infers frame boundaries. Alongside it, a synthetic tone is -// encoded through libopus so the matrix exercises the FFI audio path, not only the video -// one. subscribe connects, finds the video track in the catalog, and exits 0 as soon as any -// non-empty frame arrives (exit 1 on timeout or no data). +// streaming importer, which infers frame boundaries. Alongside it, a tone per channel +// (--tone) is encoded through libopus so the matrix exercises the FFI audio path, not only +// the video one. subscribe connects, waits for a catalog with video and audio, decodes +// --audio-ms of audio to --audio-out for tone.py to judge, and exits 0 once a non-empty +// video frame arrives too (exit 1 on timeout or no data). // -// ffmpeg ... -f h264 - | cpp-interop publish --url http://localhost:4443 --broadcast b.hang -// cpp-interop subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 +// ffmpeg ... -f h264 - | cpp-interop publish --url http://localhost:4443 --broadcast b.hang --tone 440 +// cpp-interop subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 \ +// --audio-out /tmp/audio --audio-ms 600 // // --version offers only that version. --grant waits for the grant the relay sends over AUTH // and prints it as an `auth granted` line before the client starts. @@ -22,7 +24,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -45,10 +49,10 @@ constexpr size_t READ_CHUNK = 64 * 1024; // SubscribeMedia max delay: how much reordering the jitter buffer tolerates. constexpr uint64_t MAX_DELAY_US = 1'000'000; -// Synthetic audio: a 48 kHz mono tone, encoded as Opus. +// Synthetic audio: a 48 kHz tone per channel at 0.5 amplitude, encoded as Opus. constexpr const char *AUDIO_TRACK = "tone"; constexpr uint32_t AUDIO_RATE = 48'000; -constexpr double AUDIO_TONE_HZ = 440.0; +constexpr double AUDIO_AMPLITUDE = 0.5; constexpr double PI = 3.14159265358979323846; // A non-default frame duration, and the shortest Opus offers. 20 ms would pass even if the // microsecond field were truncated to milliseconds somewhere. @@ -150,17 +154,23 @@ void close_session(const std::shared_ptr &session) { until(session->shutdown(), Clock::now() + std::chrono::seconds(5), "session shutdown"); } -// Feeds the encoder a real-time tone until `stop` is set. -void publish_tone(const std::shared_ptr &audio, const std::atomic &stop) { +// Feeds the encoder a real-time tone per channel, interleaved, until `stop` is set. +void publish_tone( + const std::shared_ptr &audio, const std::vector &tones, const std::atomic &stop +) { const auto started = Clock::now(); - std::vector data(AUDIO_BATCH_SAMPLES * sizeof(float)); + std::vector data(AUDIO_BATCH_SAMPLES * tones.size() * sizeof(float)); uint64_t timestamp_us = 0; size_t phase = 0; while (!stop.load()) { for (size_t i = 0; i < AUDIO_BATCH_SAMPLES; i++) { - auto sample = static_cast(std::sin(2 * PI * AUDIO_TONE_HZ * static_cast(phase + i) / AUDIO_RATE)); - // Little-endian f32, which is every target this runs on. - std::memcpy(&data[i * sizeof(float)], &sample, sizeof(float)); + for (size_t c = 0; c < tones.size(); c++) { + auto sample = static_cast( + AUDIO_AMPLITUDE * std::sin(2 * PI * tones[c] * static_cast(phase + i) / AUDIO_RATE) + ); + // Little-endian f32, which is every target this runs on. + std::memcpy(&data[(i * tones.size() + c) * sizeof(float)], &sample, sizeof(float)); + } } ok(audio->write({timestamp_us, data}), "tone write"); phase += AUDIO_BATCH_SAMPLES; @@ -170,7 +180,7 @@ void publish_tone(const std::shared_ptr &audio, const std::a } } -int publish(const std::string &url, const std::string &path, const Dial &dial) { +int publish(const std::string &url, const std::string &path, const Dial &dial, const std::vector &tones) { // No deadline to publish: the harness stops the publisher when the round ends. const auto forever = Clock::now() + std::chrono::hours(24); auto session = connect(url, dial, forever); @@ -186,18 +196,20 @@ int publish(const std::string &url, const std::string &path, const Dial &dial) { output.track = AUDIO_TRACK; output.frame_duration_us = AUDIO_FRAME_DURATION_US; auto audio = ok( - moq::AudioProducer::init(catalog, {moq::AudioSampleFormat::kF32, AUDIO_RATE, 1}, output, nullptr), + moq::AudioProducer::init( + catalog, {moq::AudioSampleFormat::kF32, AUDIO_RATE, static_cast(tones.size())}, output, nullptr + ), "audio producer" ); ok(broadcast->announce({}), "announce"); std::printf( - "publishing \"%s\" (Annex-B H.264 from stdin + a %.0f Hz tone) to %s\n", path.c_str(), AUDIO_TONE_HZ, + "publishing \"%s\" (Annex-B H.264 from stdin + a %zu-channel tone) to %s\n", path.c_str(), tones.size(), url.c_str() ); std::fflush(stdout); std::atomic stop{false}; - std::thread tone([&] { publish_tone(audio, stop); }); + std::thread tone([&] { publish_tone(audio, tones, stop); }); // read returns as soon as any bytes are available, so ffmpeg's real-time output is // forwarded rather than batched into full chunks. @@ -224,7 +236,41 @@ int publish(const std::string &url, const std::string &path, const Dial &dial) { return 0; } -int subscribe(const std::string &url, const std::string &path, const Dial &dial, double timeout) { +// Decodes the first `audio_ms` of the rendition to `out`.{txt,pcm}, the layout tone.py reads. +void decode_audio( + const std::shared_ptr &consumer, const std::string &id, const moq::Audio &audio, + const std::string &out, uint64_t audio_ms, Clock::time_point deadline +) { + moq::AudioDecoderOutput output{}; + output.format = moq::AudioSampleFormat::kF32; + output.max_delay_us = MAX_DELAY_US; + auto decoder = until(moq::AudioConsumer::subscribe(consumer, id, audio, output), deadline, "audio consumer"); + + std::ofstream log(out + ".txt"); + std::ofstream pcm(out + ".pcm", std::ios::binary); + log << audio.sample_rate << " " << audio.channel_count << "\n"; + const size_t frame_bytes = sizeof(float) * audio.channel_count; + uint64_t decoded = 0; + while (decoded * 1000 < audio_ms * audio.sample_rate) { + auto frame = until(decoder->next(), deadline, "audio frame"); + if (!frame) { + fail("audio track ended early"); + } + const size_t samples = frame->data.size() / frame_bytes; + log << frame->timestamp_us << " " << samples << "\n"; + pcm.write(reinterpret_cast(frame->data.data()), static_cast(frame->data.size())); + decoded += samples; + } + decoder->cancel(); + if (!log || !pcm) { + fail("writing the decoded audio"); + } +} + +int subscribe( + const std::string &url, const std::string &path, const Dial &dial, double timeout, + const std::string &audio_out, uint64_t audio_ms +) { const auto deadline = Clock::now() + std::chrono::duration_cast(std::chrono::duration(timeout)); auto session = connect(url, dial, deadline); @@ -233,14 +279,14 @@ int subscribe(const std::string &url, const std::string &path, const Dial &dial, auto consumer = until(announced->available(), deadline, "available"); // The catalog is a live track. A lazy publisher (e.g. the browser, which only encodes on - // demand) may announce video in a later update rather than the first snapshot, so wait - // for a catalog that actually has a video track. + // demand) may announce a rendition in a later update rather than the first snapshot, so + // wait for a catalog that has both video and audio. auto catalogs = until(moq::MediaCatalogConsumer::subscribe(consumer), deadline, "catalog consumer"); std::optional catalog; - while (!catalog || catalog->video.empty()) { + while (!catalog || catalog->video.empty() || catalog->audio.empty()) { catalog = until(catalogs->next(), deadline, "catalog"); if (!catalog) { - fail("catalog stream ended without a video track"); + fail("catalog stream ended without a video and an audio track"); } } const auto &[id, video] = *catalog->video.begin(); @@ -252,6 +298,11 @@ int subscribe(const std::string &url, const std::string &path, const Dial &dial, "container consumer" ); + // Audio first: it has to arrive without a gap, and the video consumer keeps its first frame + // (or skips to a newer one) while this reads. + const auto &[audio_id, audio] = *catalog->audio.begin(); + decode_audio(consumer, audio_id, audio, audio_out, audio_ms, deadline); + size_t total = 0; while (total == 0) { auto frame = until(media->next(), deadline, "next frame"); @@ -273,12 +324,18 @@ int subscribe(const std::string &url, const std::string &path, const Dial &dial, int main(int argc, char **argv) { if (argc < 2) { - fail("usage: cpp-interop publish|subscribe --url URL --broadcast PATH [--timeout SECONDS] [--version V] [--grant]"); + fail( + "usage: cpp-interop publish|subscribe --url URL --broadcast PATH [--timeout SECONDS] [--version V] " + "[--grant] [--tone HZ,...] [--audio-out PREFIX --audio-ms MS]" + ); } const std::string role = argv[1]; std::string url; std::string broadcast; Dial dial; + std::vector tones; + std::string audio_out; + uint64_t audio_ms = 0; double timeout = 20; for (int i = 2; i < argc; i++) { const std::string flag = argv[i]; @@ -298,6 +355,15 @@ int main(int argc, char **argv) { timeout = std::atof(value.c_str()); } else if (flag == "--version") { dial.version = value; + } else if (flag == "--tone") { + std::stringstream list(value); + for (std::string hz; std::getline(list, hz, ',');) { + tones.push_back(std::atof(hz.c_str())); + } + } else if (flag == "--audio-out") { + audio_out = value; + } else if (flag == "--audio-ms") { + audio_ms = std::strtoull(value.c_str(), nullptr, 10); } else { fail(("unknown flag " + flag).c_str()); } @@ -309,10 +375,16 @@ int main(int argc, char **argv) { ok(moq::moq_log_level("warn"), "log level"); if (role == "publish") { - return publish(url, broadcast, dial); + if (tones.empty()) { + fail("publish needs --tone"); + } + return publish(url, broadcast, dial, tones); } if (role == "subscribe") { - return subscribe(url, broadcast, dial, timeout); + if (audio_out.empty() || audio_ms == 0) { + fail("subscribe needs --audio-out and --audio-ms"); + } + return subscribe(url, broadcast, dial, timeout, audio_out, audio_ms); } fail(("unknown role " + role).c_str()); } diff --git a/test/interop/clients/go/main.go b/test/interop/clients/go/main.go index f9e89de4089..b6ab1333db4 100644 --- a/test/interop/clients/go/main.go +++ b/test/interop/clients/go/main.go @@ -3,13 +3,14 @@ // // publish reads raw Annex-B H.264 from stdin (e.g. piped from ffmpeg) and feeds // it to a streaming importer, which infers frame boundaries. Alongside it, a -// synthetic tone is encoded through libopus so the matrix exercises the FFI -// audio path, not only the video one. subscribe connects, finds the video track -// in the catalog, and exits 0 as soon as any non-empty frame arrives (exit 1 on -// timeout or no data). +// tone per channel (--tone) is encoded through libopus so the matrix exercises +// the FFI audio path, not only the video one. subscribe connects, waits for a +// catalog with video and audio, and exits 0 once a non-empty video frame has +// arrived and --audio-ms of audio has been decoded to --audio-out for tone.py to +// judge (exit 1 on timeout or no data). // -// ffmpeg ... -f h264 - | go-interop publish --url http://localhost:4443 --broadcast b.hang -// go-interop subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 +// ffmpeg ... -f h264 - | go-interop publish --url http://localhost:4443 --broadcast b.hang --tone 440,660 +// go-interop subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 --audio-out /tmp/audio --audio-ms 600 // // --version offers only that version. --grant waits for the grant the relay sends // over AUTH and prints it as an `auth granted` line before the client starts. @@ -25,6 +26,8 @@ import ( "io" "math" "os" + "strconv" + "strings" "time" "moq.dev/moq" @@ -37,11 +40,11 @@ const readChunk = 64 * 1024 // media.TrackConsumer max delay: how much reordering the jitter buffer tolerates. const maxDelay = time.Second -// Synthetic audio: a 48 kHz mono tone, encoded as Opus. +// Synthetic audio: a 48 kHz tone per channel at 0.5 amplitude, encoded as Opus. const ( - audioTrack = "tone" - audioRate = 48_000 - audioToneHz = 440.0 + audioTrack = "tone" + audioRate = 48_000 + audioAmplitude = 0.5 // A non-default frame duration, and the shortest Opus offers. 20 ms would // pass even if the microsecond field were truncated to milliseconds // somewhere. @@ -51,12 +54,12 @@ const ( audioBatchSamples = audioRate * audioBatchUs / 1_000_000 ) -// publishTone feeds the encoder a real-time tone until ctx is cancelled. -func publishTone(ctx context.Context, audio *moqaudio.Producer) { +// publishTone feeds the encoder a real-time tone per channel, interleaved, until ctx is cancelled. +func publishTone(ctx context.Context, audio *moqaudio.Producer, tones []float64) { ticker := time.NewTicker(audioBatchUs * time.Microsecond) defer ticker.Stop() - data := make([]byte, audioBatchSamples*4) + data := make([]byte, audioBatchSamples*len(tones)*4) timestampUs := uint64(0) phase := 0 for { @@ -66,8 +69,10 @@ func publishTone(ctx context.Context, audio *moqaudio.Producer) { case <-ticker.C: } for i := 0; i < audioBatchSamples; i++ { - sample := math.Sin(2 * math.Pi * audioToneHz * float64(phase+i) / audioRate) - binary.LittleEndian.PutUint32(data[i*4:], math.Float32bits(float32(sample))) + for c, hz := range tones { + sample := audioAmplitude * math.Sin(2*math.Pi*hz*float64(phase+i)/audioRate) + binary.LittleEndian.PutUint32(data[(i*len(tones)+c)*4:], math.Float32bits(float32(sample))) + } } if err := audio.Write(moqaudio.Frame{Timestamp: time.Duration(timestampUs) * time.Microsecond, Data: data}); err != nil { fmt.Fprintf(os.Stderr, "tone: %v\n", err) @@ -121,7 +126,7 @@ func patterns(list []string) string { return string(out) } -func publish(ctx context.Context, url, broadcast string, opts dialOptions, timeout time.Duration) error { +func publish(ctx context.Context, url, broadcast string, opts dialOptions, timeout time.Duration, tones []float64) error { client, err := dial(ctx, url, opts, timeout) if err != nil { return err @@ -146,7 +151,7 @@ func publish(ctx context.Context, url, broadcast string, opts dialOptions, timeo audio, err := moqaudio.NewProducer(catalog, moqaudio.EncoderInput{ Format: moq.AudioSampleFormatF32, SampleRate: audioRate, - Channels: 1, + Channels: uint32(len(tones)), }, moqaudio.EncoderOutput{ Codec: moq.OpusAudioCodec(), Track: func() *string { track := audioTrack; return &track }(), @@ -158,13 +163,13 @@ func publish(ctx context.Context, url, broadcast string, opts dialOptions, timeo if err := producer.Announce(moq.Route{}); err != nil { return err } - fmt.Printf("publishing %q (Annex-B H.264 from stdin + a %.0f Hz tone) to %s\n", broadcast, audioToneHz, url) + fmt.Printf("publishing %q (Annex-B H.264 from stdin + a %v Hz tone) to %s\n", broadcast, tones, url) toneCtx, stopTone := context.WithCancel(ctx) toneDone := make(chan struct{}) go func() { defer close(toneDone) - publishTone(toneCtx, audio) + publishTone(toneCtx, audio, tones) }() // Let the tone unwind before any Finish, so no write races a finished // producer. Runs before the deferred producer.Close, and is a no-op once @@ -200,9 +205,9 @@ func publish(ctx context.Context, url, broadcast string, opts dialOptions, timeo } // The catalog is a live track. A lazy publisher (e.g. the browser, which only -// encodes on demand) may announce video in a later update rather than the first -// snapshot, so wait for a catalog that actually has a video track. -func catalogWithVideo(ctx context.Context, consumer *moq.BroadcastConsumer) (*moqmedia.Catalog, error) { +// encodes on demand) may announce a rendition in a later update rather than the +// first snapshot, so wait for a catalog that has both video and audio. +func catalogWithMedia(ctx context.Context, consumer *moq.BroadcastConsumer) (*moqmedia.Catalog, error) { catalogs, err := moqmedia.NewCatalogConsumer(ctx, consumer) if err != nil { return nil, err @@ -215,15 +220,58 @@ func catalogWithVideo(ctx context.Context, consumer *moq.BroadcastConsumer) (*mo return nil, err } if catalog == nil { - return nil, errors.New("catalog stream ended without a video track") + return nil, errors.New("catalog stream ended without a video and an audio track") } - if len(catalog.Video) > 0 { + if len(catalog.Video) > 0 && len(catalog.Audio) > 0 { return catalog, nil } } } -func subscribe(ctx context.Context, url, broadcast string, opts dialOptions, timeout time.Duration) error { +// decodeAudio decodes the first audioMs of the rendition to out.{txt,pcm}, the layout tone.py reads. +func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string, rendition moqmedia.Audio, out string, audioMs int) error { + decoder, err := moqaudio.Subscribe(ctx, consumer, id, rendition, moqaudio.DecoderOutput{ + Format: moq.AudioSampleFormatF32, + MaxDelay: func() *time.Duration { d := maxDelay; return &d }(), + }) + if err != nil { + return err + } + defer decoder.Cancel() + + log, err := os.Create(out + ".txt") + if err != nil { + return err + } + defer log.Close() + pcm, err := os.Create(out + ".pcm") + if err != nil { + return err + } + defer pcm.Close() + + fmt.Fprintf(log, "%d %d\n", rendition.SampleRate, rendition.ChannelCount) + frameBytes := 4 * int(rendition.ChannelCount) + decoded := 0 + for decoded*1000 < audioMs*int(rendition.SampleRate) { + frame, err := decoder.Next(ctx) + if err != nil { + return err + } + if frame == nil { + return errors.New("audio track ended early") + } + samples := len(frame.Data) / frameBytes + fmt.Fprintf(log, "%d %d\n", frame.Timestamp.Microseconds(), samples) + if _, err := pcm.Write(frame.Data); err != nil { + return err + } + decoded += samples + } + return nil +} + +func subscribe(ctx context.Context, url, broadcast string, opts dialOptions, timeout time.Duration, audioOut string, audioMs int) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -244,11 +292,19 @@ func subscribe(ctx context.Context, url, broadcast string, opts dialOptions, tim return err } - catalog, err := catalogWithVideo(ctx, consumer) + catalog, err := catalogWithMedia(ctx, consumer) if err != nil { return err } + var audioID string + var audio moqmedia.Audio + for audioID, audio = range catalog.Audio { + break + } + audioDone := make(chan error, 1) + go func() { audioDone <- decodeAudio(ctx, consumer, audioID, audio, audioOut, audioMs) }() + var id string var video moqmedia.Video for id, video = range catalog.Video { @@ -275,13 +331,16 @@ func subscribe(ctx context.Context, url, broadcast string, opts dialOptions, tim if total == 0 { return errors.New("no frame data received") } + if err := <-audioDone; err != nil { + return fmt.Errorf("audio: %w", err) + } fmt.Printf("received %d bytes from %q\n", total, broadcast) return nil } func run() error { if len(os.Args) < 2 { - return errors.New("usage: go-interop publish|subscribe --url URL --broadcast PATH [--timeout SECONDS] [--version V] [--grant]") + return errors.New("usage: go-interop publish|subscribe --url URL --broadcast PATH [--timeout SECONDS] [--version V] [--grant] [--tone HZ,...] [--audio-out PREFIX --audio-ms MS]") } role := os.Args[1] @@ -291,6 +350,9 @@ func run() error { timeout := flags.Float64("timeout", 20, "seconds to wait for data") version := flags.String("version", "", "offer only this version") grant := flags.Bool("grant", false, "wait for the grant the relay sends over AUTH and print it") + tone := flags.String("tone", "", "publish: comma-separated frequency of each channel, in Hz") + audioOut := flags.String("audio-out", "", "subscribe: decode audio to this prefix's .txt and .pcm") + audioMs := flags.Int("audio-ms", 0, "subscribe: how much audio to decode, in milliseconds") if err := flags.Parse(os.Args[2:]); err != nil { return err } @@ -307,9 +369,20 @@ func run() error { opts := dialOptions{version: *version, grant: *grant} switch role { case "publish": - return publish(ctx, *url, *broadcast, opts, wait) + var tones []float64 + for _, field := range strings.Split(*tone, ",") { + hz, err := strconv.ParseFloat(field, 64) + if err != nil { + return fmt.Errorf("--tone: %w", err) + } + tones = append(tones, hz) + } + return publish(ctx, *url, *broadcast, opts, wait, tones) case "subscribe": - return subscribe(ctx, *url, *broadcast, opts, wait) + if *audioOut == "" || *audioMs <= 0 { + return errors.New("subscribe needs --audio-out and --audio-ms") + } + return subscribe(ctx, *url, *broadcast, opts, wait, *audioOut, *audioMs) default: return fmt.Errorf("unknown role %q", role) } diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index b06b151c77e..2fac8b1f230 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -3,22 +3,29 @@ * `@moq/hang` under a runtime with no native WebTransport, via moq's own * `@moq/web-transport` polyfill (a prebuilt NAPI QUIC/HTTP3 addon, the one piece * that comes from npm rather than this checkout). Runs under both node and bun. - * Connect, find the video track in the .hang catalog, read it through the same - * container consumer the browser player uses, and exit 0 as soon as a non-empty - * frame arrives (1 on timeout). Subscribe-only: publishing media needs a - * WebCodecs encoder a native JS runtime lacks. + * Connect, wait for a .hang catalog with video and audio, read video through the + * same container consumer the browser player uses until a non-empty frame arrives, + * and decode the first --audio-ms of audio to --audio-out for tone.py to judge. + * Neither runtime has WebCodecs, so audio decodes through `@moq/hang`'s libav + * fallback, as a player under them would. Exits 0 once both are done (1 on + * timeout). Subscribe-only: publishing media needs a WebCodecs encoder a native + * JS runtime lacks. * - * node --import tsx subscribe.ts subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 + * node --import tsx subscribe.ts subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 \ + * --audio-out /tmp/audio --audio-ms 600 * * `--version` offers only that version. Any grant the relay sends over AUTH is printed as an * `auth granted` line. * * @module */ +import { writeFileSync } from "node:fs"; import { parseArgs } from "node:util"; import { Milli } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; +import * as Cmaf from "@moq/hang/cmaf"; import * as Container from "@moq/hang/container"; +import { Hex, Libav } from "@moq/hang/util"; import * as Json from "@moq/json"; import * as Moq from "@moq/net"; import { install } from "@moq/web-transport"; @@ -34,6 +41,11 @@ install(); // subscriber has to move on to the next group instead of failing. const MAX_DELAY = Milli(1000); +// Audio is checked for gaps, which a reader skipping behind live would cause itself: the matrix +// starts every subscriber at once, and a runtime starved by the others can fall a second behind. +// That is this reader's speed, not the delivery under test, so it gets a wider budget. +const AUDIO_MAX_DELAY = Milli(5000); + const { positionals, values } = parseArgs({ allowPositionals: true, options: { @@ -41,33 +53,164 @@ const { positionals, values } = parseArgs({ broadcast: { type: "string" }, timeout: { type: "string", default: "20" }, version: { type: "string" }, + "audio-out": { type: "string" }, + "audio-ms": { type: "string" }, }, }); const role = positionals[0]; const url = values.url; const broadcast = values.broadcast; +const audioOut = values["audio-out"]; +const audioMs = Number.parseInt(values["audio-ms"] ?? "", 10); const timeoutMs = Number.parseFloat(values.timeout ?? "20") * 1000; -if (role !== "subscribe" || !url || !broadcast || !Number.isFinite(timeoutMs) || timeoutMs <= 0) { - console.error("usage: subscribe.ts subscribe --url U --broadcast B [--timeout S>0] [--version V]"); +if ( + role !== "subscribe" || + !url || + !broadcast || + !audioOut || + !(audioMs > 0) || + !Number.isFinite(timeoutMs) || + timeoutMs <= 0 +) { + console.error( + "usage: subscribe.ts subscribe --url U --broadcast B --audio-out PREFIX --audio-ms MS [--timeout S>0] [--version V]", + ); process.exit(2); } // The .hang catalog lives on the "catalog.json" track. It's a @moq/json snapshot+delta -// value, reconstructed by Json.Snapshot.Consumer. A lazy publisher may announce video in a -// later update, so keep reading until one has it. -async function video(bc: Moq.Broadcast.Consumer): Promise<[string, Catalog.VideoConfig]> { +// value, reconstructed by Json.Snapshot.Consumer. A lazy publisher may announce a +// rendition in a later update, so keep reading until one has both video and audio. +async function catalog( + bc: Moq.Broadcast.Consumer, +): Promise<{ video: [string, Catalog.VideoConfig]; audio: [string, Catalog.AudioConfig] }> { const track = bc.track("catalog.json").subscribe({ priority: Catalog.PRIORITY.catalog }); const catalog = new Json.Snapshot.Consumer({ track, schema: Catalog.RootSchema }); for (;;) { const root = (await catalog.latest())?.value; - if (!root) throw new Error("catalog ended without a video track"); - const first = Object.entries(root.video?.renditions ?? {})[0]; - if (first) return first; + if (!root) throw new Error("catalog ended without a video and an audio track"); + const video = Object.entries(root.video?.renditions ?? {})[0]; + const audio = Object.entries(root.audio?.renditions ?? {})[0]; + if (video && audio) return { video, audio }; + } +} + +function containerFormat(config: Catalog.VideoConfig | Catalog.AudioConfig, kind: "video" | "audio"): Container.Format { + switch (config.container.kind) { + case "legacy": + return new Container.Legacy.Format(config); + case "loc": + return new Container.Loc.Format(kind); + case "cmaf": + return new Cmaf.Format(Cmaf.decodeInitSegment(Buffer.from(config.container.init, "base64"))); + default: + throw new Error(`unsupported ${kind} container: ${JSON.stringify(config.container)}`); + } +} + +async function video(bc: Moq.Broadcast.Consumer, name: string, config: Catalog.VideoConfig): Promise { + const sub = bc.track(name).subscribe({ priority: 0, maxDelay: MAX_DELAY }); + const consumer = new Container.Consumer(sub, { format: containerFormat(config, "video"), maxDelay: MAX_DELAY }); + try { + for (;;) { + const next = await consumer.next(); + if (!next) break; + const bytes = next.kind === "frame" ? next.frame.payload.byteLength : 0; + if (bytes > 0) return bytes; + } + } finally { + consumer.close(); + } + throw new Error("no frame data received"); +} + +/** Decode the first {@link audioMs} of the rendition to `{audioOut}.{txt,pcm}`, the layout tone.py reads. */ +async function audio(bc: Moq.Broadcast.Consumer, name: string, config: Catalog.AudioConfig): Promise { + const channels = config.numberOfChannels; + const want = (audioMs * config.sampleRate) / 1000; + const log = [`${config.sampleRate} ${channels}`]; + const pcm: Float32Array[] = []; + let decoded = 0; + let failed: Error | undefined; + + const { promise: done, resolve } = Promise.withResolvers(); + const decoder = new AudioDecoder({ + output: (data) => { + if (decoded < want) { + // Every AudioData converts to f32-planar; interleave it the way the other clients write. + const frames = data.numberOfFrames; + const planes = Array.from({ length: data.numberOfChannels }, (_, plane) => { + const samples = new Float32Array(frames); + data.copyTo(samples, { planeIndex: plane, format: "f32-planar" }); + return samples; + }); + const interleaved = new Float32Array(frames * planes.length); + for (let i = 0; i < frames; i++) { + for (const [c, plane] of planes.entries()) interleaved[i * planes.length + c] = plane[i]; + } + log.push(`${data.timestamp} ${frames}`); + pcm.push(interleaved); + decoded += frames; + if (decoded >= want) resolve(); + } + data.close(); + }, + error: (error) => { + failed = error; + resolve(); + }, + }); + // Opus in every container is raw packets; its description (an OpusHead or dOps) is not + // what WebCodecs wants, which is why the browser player leaves it out too. + decoder.configure({ + codec: config.codec, + sampleRate: config.sampleRate, + numberOfChannels: channels, + description: config.codec === "opus" || !config.description ? undefined : Hex.toBytes(config.description), + }); + + const sub = bc.track(name).subscribe({ priority: Catalog.PRIORITY.audio, maxDelay: AUDIO_MAX_DELAY }); + const consumer = new Container.Consumer(sub, { + format: containerFormat(config, "audio"), + maxDelay: AUDIO_MAX_DELAY, + }); + const feed = (async () => { + // The consumer also hands back the backlog the relay served behind the live edge, older + // groups after newer ones. A player places those by timestamp; a stateful Opus decoder can + // only move forward, so skip them, as video does. + let last = Number.NEGATIVE_INFINITY; + for (;;) { + const next = await consumer.next(); + if (!next) throw new Error("audio track ended early"); + if (next.kind !== "frame" || next.frame.timestamp <= last) continue; + last = next.frame.timestamp; + if (decoder.state === "closed") return; + decoder.decode( + new EncodedAudioChunk({ type: "key", data: next.frame.payload, timestamp: next.frame.timestamp }), + ); + } + })(); + try { + await Promise.race([done, feed]); + } finally { + consumer.close(); + if (decoder.state !== "closed") decoder.close(); } + if (failed) throw failed; + + writeFileSync(`${audioOut}.txt`, `${log.join("\n")}\n`); + writeFileSync( + `${audioOut}.pcm`, + Buffer.concat(pcm.map((samples) => Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength))), + ); } async function run(): Promise { + // Before subscribing: compiling libav's wasm blocks the event loop long enough that a + // subscription already open falls a group behind and skips it. + if (!(await Libav.polyfill())) throw new Error("no AudioDecoder"); + const origin = new Moq.Origin.Producer(); const connection = await Moq.Connection.connect({ url: new URL(url as string), @@ -95,36 +238,13 @@ async function run(): Promise { bc = requested.active.peek(); } - const [name, config] = await video(bc); - let format: Container.Format; - if (config.container.kind === "legacy") { - format = new Container.Legacy.Format(config); - } else if (config.container.kind === "loc") { - format = new Container.Loc.Format("video"); - } else { - throw new Error(`unsupported video container: ${JSON.stringify(config.container)}`); - } - - const sub = bc.track(name).subscribe({ priority: 0, maxDelay: MAX_DELAY }); - const consumer = new Container.Consumer(sub, { format, maxDelay: MAX_DELAY }); - try { - for (;;) { - const next = await consumer.next(); - if (!next) break; - const bytes = next.kind === "frame" ? next.frame.payload.byteLength : 0; - if (bytes > 0) { - // The harness judges success by this marker, not the exit code: the - // @moq/web-transport NAPI addon can segfault during the runtime's exit - // teardown after a frame has arrived (an upstream bug, seen under bun), - // which would turn a real success into a signal exit. - console.error(`received ${bytes} bytes from ${broadcast}`); - return; - } - } - } finally { - consumer.close(); - } - throw new Error("no frame data received"); + const renditions = await catalog(bc); + const [bytes] = await Promise.all([video(bc, ...renditions.video), audio(bc, ...renditions.audio)]); + // The harness judges success by this marker, not the exit code: the + // @moq/web-transport NAPI addon can segfault during the runtime's exit + // teardown after a frame has arrived (an upstream bug, seen under bun), + // which would turn a real success into a signal exit. + console.error(`received ${bytes} bytes and ${audioMs}ms of audio from ${broadcast}`); } finally { unwatch(); requested?.close(); diff --git a/test/interop/clients/js/driver.ts b/test/interop/clients/js/driver.ts index b2f2030d690..88072dfdb79 100644 --- a/test/interop/clients/js/driver.ts +++ b/test/interop/clients/js/driver.ts @@ -1,14 +1,15 @@ /** * Drives a headless Chromium against the vite-built page (see `serve`) for the interop matrix. publish - * streams fake camera/microphone input until killed; subscribe verifies rendered playback, - * pause/resume, and optionally browser-to-browser audio. + * streams a fake camera and a microphone playing `--audio-file` until killed; subscribe verifies + * rendered playback, that each audio channel the player outputs carries its `--tone`, and + * pause/resume. * * The publishers on the other side of the matrix are separate processes with no readiness signal, * so this driver tolerates a slow announcement with one reload. `media.ts` is the strict path: it * waits on an explicit publisher-ready state and never reloads. * - * bun driver.ts publish --url http://127.0.0.1:4443 --broadcast b.hang [--version V] - * bun driver.ts subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 [--version V] [--expect-audio] + * bun driver.ts publish --url http://127.0.0.1:4443 --broadcast b.hang --audio-file tone.wav [--version V] + * bun driver.ts subscribe --url http://127.0.0.1:4443 --broadcast b.hang --timeout 20 --tone 440,660 [--version V] * * `--version` makes the element offer that protocol version alone. * @@ -18,6 +19,7 @@ import { parseArgs } from "node:util"; import type { Page } from "playwright"; import { type BrowserErrors, + Failure, finishTraces, launch, open, @@ -33,7 +35,7 @@ import { waitForState, waitForWatch, } from "./harness"; -import type { Sample } from "./src/contract"; +import { type Sample, TONE_CHECK } from "./src/contract"; /** Remove the page's players, which closes their sessions, and wait until every transport has. */ async function closeSessions(page: Page): Promise { @@ -58,7 +60,8 @@ const { positionals, values } = parseArgs({ broadcast: { type: "string" }, version: { type: "string" }, timeout: { type: "string", default: "20" }, - "expect-audio": { type: "boolean", default: false }, + "audio-file": { type: "string" }, + tone: { type: "string" }, }, }); @@ -66,23 +69,38 @@ const role = positionals[0]; const url = values.url; const broadcast = values.broadcast; const timeoutMs = Number.parseFloat(values.timeout ?? "20") * 1000; -const expectAudio = values["expect-audio"] ?? false; +const audioFile = values["audio-file"]; +const tones = (values.tone ?? "").split(",").filter(Boolean).map(Number); if ( (role !== "publish" && role !== "subscribe") || !url || !broadcast || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || - (expectAudio && role !== "subscribe") + (role === "publish" && !audioFile) || + (role === "subscribe" && (!tones.length || tones.some((hz) => !(hz > 0)))) ) { console.error( - "usage: driver.ts publish|subscribe --url U --broadcast B [--version V] [--timeout S>0] [--expect-audio]", + "usage: driver.ts publish --url U --broadcast B --audio-file WAV | subscribe --url U --broadcast B --tone HZ,... [--version V] [--timeout S>0]", ); process.exit(2); } +/** Whether every channel the player outputs carries its own tone, judged as tone.py judges a native subscriber. */ +function toneHeard(state: Sample): boolean { + return ( + state.audioContext === "running" && + state.audioChannels === tones.length && + state.tones?.length === tones.length && + state.tones.every(({ rms, share }) => rms >= TONE_CHECK.minRms && share >= TONE_CHECK.minShare) + ); +} + const PAUSE_STABILITY_MS = 750; +/** How long audible audio may play before it has to carry the tone: the analyser's window, several times over. */ +const TONE_GRACE_MS = 3000; + async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { let previous = await readPlayerState(page); let stableSince = Date.now(); @@ -93,7 +111,7 @@ async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: n const stable = current.videoFrames === previous.videoFrames && current.videoTimestamp === previous.videoTimestamp && - (!expectAudio || current.audioBytes === previous.audioBytes); + current.audioBytes === previous.audioBytes; if (!stable) stableSince = Date.now(); if (stable && Date.now() - stableSince >= PAUSE_STABILITY_MS) return current; previous = current; @@ -107,20 +125,27 @@ const browser = await launch([ "--use-fake-device-for-media-stream", "--use-fake-ui-for-media-stream", "--autoplay-policy=no-user-gesture-required", + // The fake microphone loops this file instead of its default beeps. + ...(audioFile ? [`--use-file-for-fake-audio-capture=${audioFile}`] : []), ]); let code = 1; try { const [page, errors] = await open( browser, - pageUrl(server.origin, role, { url, broadcast, ...(values.version ? { versions: values.version } : {}) }), + pageUrl(server.origin, role, { + url, + broadcast, + ...(values.version ? { versions: values.version } : {}), + ...(role === "subscribe" ? { tone: tones.join(",") } : {}), + }), role, role === "subscribe", ); if (role === "subscribe") await waitForWatch(page); if (role === "publish") { - console.error(`publishing ${broadcast} (fake camera + microphone) to ${url}`); + console.error(`publishing ${broadcast} (fake camera + ${audioFile} as the microphone) to ${url}`); // Stream until the orchestrator stops us. await new Promise((resolve) => process.once("SIGTERM", resolve)); code = 0; @@ -151,12 +176,35 @@ try { } const interactionDeadline = Date.now() + timeoutMs; - if (expectAudio) { - playing = await waitForState(page, errors, { + try { + // Once audio plays, the tone gets a bounded grace rather than the whole timeout, so a + // wrong tone fails as fast as it does in the native subscribers. + await waitForState(page, errors, { deadline: interactionDeadline, - description: "browser audio", - predicate: (state) => state.hasAudio && state.audioBytes > 0 && state.audioContext === "running", + description: "audible audio", + predicate: (state) => state.audioContext === "running" && (state.rms ?? 0) >= TONE_CHECK.minRms, + }); + playing = await waitForState(page, errors, { + deadline: Math.min(interactionDeadline, Date.now() + TONE_GRACE_MS), + description: "the audio tone", + predicate: toneHeard, }); + } catch (err) { + // Only a tone wait that ran out is an audio check failure; a page error reports as itself. + if (!(err instanceof Failure)) throw err; + // `wrong tone` is the marker the harness's control requires: audible audio in the publisher's + // layout that does not carry its tones. Anything else failing is a plain audio failure. + const state = await readPlayerState(page); + const wrongTone = + state.audioContext === "running" && + state.audioChannels === tones.length && + state.tones?.length === tones.length && + state.tones.every(({ rms }) => rms >= TONE_CHECK.minRms); + console.error( + `audio check failed: ${wrongTone ? "wrong tone: " : ""}${state.audioChannels} catalog channels, ` + + `per channel ${JSON.stringify(state.tones)}, want ${tones} Hz, loudest ${state.toneHz?.toFixed(0)} Hz`, + ); + throw err; } await pause(page); @@ -181,13 +229,13 @@ try { state.videoFrames > paused.videoFrames && state.videoTimestamp !== undefined && state.videoTimestamp > (paused.videoTimestamp ?? -1) && - (!expectAudio || state.audioBytes > paused.audioBytes), + state.audioBytes > paused.audioBytes, }); throwPageErrors(errors); + const heard = playing.tones?.map(({ share }, channel) => `${tones[channel]} Hz ${share.toFixed(2)}`).join(", "); console.error( - `rendered, paused, and resumed ${broadcast}: video=${resumed.videoFrames} frames` + - (expectAudio ? ` audio=${resumed.audioBytes} bytes` : ""), + `rendered, paused, and resumed ${broadcast}: video=${resumed.videoFrames} frames audio=${resumed.audioBytes} bytes (${heard})`, ); code = 0; } diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 937bd08bdfe..01e095c7272 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -77,6 +77,12 @@ export const TONE_FLOOR_DB = 15; */ export const AUDIBLE_RMS = 0.02; +/** + * The interop matrix's tone check, the in-page twin of `test/interop/tone.py`: each channel must + * carry at least this level and this share of its energy at its own tone. Keep the two in step. + */ +export const TONE_CHECK = { minRms: 0.05, minShare: 0.8 }; + /** Live instances of each resource the page's wrappers count. */ export type Resources = { /** Open `WebTransport` sessions, i.e. connections to the relay. */ @@ -156,6 +162,13 @@ export type Sample = { noiseDb?: number; /** Root mean square of the graph root's output over the analyser window. */ rms?: number; + /** The catalog's audio channel count. */ + audioChannels?: number; + /** + * Per channel of the graph root, when the page was given the expected tones: the channel's RMS + * and the share of its energy at its own tone, over the last ~170ms it played. + */ + tones?: { rms: number; share: number }[]; /** `aria-label` of the primary control button: "Pause" while playing, "Play" while paused. */ controlLabel?: string; diff --git a/test/interop/clients/js/src/probe.ts b/test/interop/clients/js/src/probe.ts index de7ea20f21d..6a8fde656e9 100644 --- a/test/interop/clients/js/src/probe.ts +++ b/test/interop/clients/js/src/probe.ts @@ -65,6 +65,59 @@ function analyze(analyser: AnalyserNode, spectrum: Float32Array, wa }; } +/** + * Samples per channel the tone check judges: ~170ms at 48kHz. Long enough that a tone 9% off + * frequency (a 44.1kHz stream played as 48kHz) has drifted well out of phase with the filter, and + * short enough that a window falls between the playout hiccups two busy headless browsers cause. + */ +const TONE_WINDOW = 8192; + +/** Fraction of the samples' energy at `hz`, by a Goertzel filter: 1.0 for a pure tone at that frequency. */ +function goertzelShare(samples: Float32Array, rate: number, hz: number): number { + const coeff = 2 * Math.cos((2 * Math.PI * hz) / rate); + let s1 = 0; + let s2 = 0; + let energy = 0; + for (const x of samples) { + const s0 = x + coeff * s1 - s2; + s2 = s1; + s1 = s0; + energy += x * x; + } + if (energy === 0) return 0; + return (2 * (s1 * s1 + s2 * s2 - coeff * s1 * s2)) / (samples.length * energy); +} + +/** One analyser per channel of the graph root, each judged against that channel's tone. */ +class ToneCheck { + #analysers: AnalyserNode[]; + #tones: number[]; + #wave = new Float32Array(TONE_WINDOW); + + constructor(root: AudioNode, tones: number[]) { + this.#tones = tones; + const splitter = new ChannelSplitterNode(root.context, { numberOfOutputs: tones.length }); + root.connect(splitter); + this.#analysers = tones.map((_, channel) => { + const analyser = new AnalyserNode(root.context, { fftSize: TONE_WINDOW }); + splitter.connect(analyser, channel); + return analyser; + }); + } + + read(): { rms: number; share: number }[] { + return this.#analysers.map((analyser, channel) => { + analyser.getFloatTimeDomainData(this.#wave); + let energy = 0; + for (const x of this.#wave) energy += x * x; + return { + rms: Math.sqrt(energy / this.#wave.length), + share: goertzelShare(this.#wave, analyser.context.sampleRate, this.#tones[channel]), + }; + }); + } +} + // Read both the fixture counter and the "is anything painted at all" check off one pixel snapshot. function readCanvas(canvas: HTMLCanvasElement | null) { if (!canvas || canvas.width === 0 || canvas.height === 0) return { painted: false }; @@ -107,12 +160,16 @@ export function watchResources(): void { /** * Start sampling a player to the console, beginning immediately. * + * With `tones`, the frequency each channel should carry, every sample also judges each channel + * against its tone. + * * Returns a function that stops sampling. */ -export function attach(el: MoqWatch): () => void { +export function attach(el: MoqWatch, tones: number[] = []): () => void { const spectrum = new Float32Array(FFT_SIZE / 2); const wave = new Float32Array(FFT_SIZE); let analyser: AnalyserNode | undefined; + let toneCheck: ToneCheck | undefined; let root: AudioNode | undefined; let seq = 0; @@ -123,11 +180,14 @@ export function attach(el: MoqWatch): () => void { if (current !== root) { root = current; analyser = undefined; + toneCheck = undefined; if (current) { analyser = new AnalyserNode(current.context, { fftSize: FFT_SIZE, smoothingTimeConstant: 0 }); current.connect(analyser); + if (tones.length) toneCheck = new ToneCheck(current, tones); } } + const audioRendition = Object.values(el.catalog?.audio?.renditions ?? {})[0]; const ui = document.querySelector("moq-watch-ui")?.shadowRoot; const control = ui?.querySelector("button.control[aria-label]"); @@ -152,6 +212,8 @@ export function attach(el: MoqWatch): () => void { audioStalled: el.audio.out.stalled.peek(), delay: el.sync.out.delay.peek(), ...(analyser ? analyze(analyser, spectrum, wave) : {}), + audioChannels: audioRendition?.numberOfChannels, + tones: toneCheck?.read(), controlLabel: control?.getAttribute("aria-label") ?? undefined, centerPlayVisible: centerPlay ? getComputedStyle(centerPlay).display !== "none" : false, paused: el.paused, diff --git a/test/interop/clients/js/src/setup.ts b/test/interop/clients/js/src/setup.ts index b01321c739f..7976c6c7fa9 100644 --- a/test/interop/clients/js/src/setup.ts +++ b/test/interop/clients/js/src/setup.ts @@ -31,6 +31,19 @@ if (role === "publish") { // Chromium's --use-fake-device-for-media-stream feeds getUserMedia fake // camera and microphone input. Audio is encoded lazily when a player asks. el.setAttribute("source", "camera"); + // The microphone plays the driver's test tone. Keep Chromium's voice processing off it: that + // mixes stereo to mono and suppresses a steady tone as noise. + el.signals.run((effect) => { + const mic = effect.get(el.sources.audio); + if (mic && "constraints" in mic) { + mic.constraints.set({ + channelCount: 2, + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + }); + } + }); document.body.appendChild(el); // So the driver can wait for the session to close before closing the browser. watchResources(); @@ -101,7 +114,9 @@ if (role === "publish") { document.body.appendChild(player); watchResources(); - let stop = attach(el); + // The interop matrix names the tone each channel should carry; the media checks do not. + const tones = (params.get("tone") ?? "").split(",").filter(Boolean).map(Number); + let stop = attach(el, tones); // Where the leaked-session control parks the player it refuses to tear down. const leak = document.createElement("div"); @@ -132,7 +147,7 @@ if (role === "publish") { const canvas = el.querySelector("canvas"); canvas?.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height); player.appendChild(el); - stop = attach(el); + stop = attach(el, tones); }, }); } else if (role === "close") { diff --git a/test/interop/clients/python/interop.py b/test/interop/clients/python/interop.py index 29259f24e50..1e83fa3c643 100644 --- a/test/interop/clients/python/interop.py +++ b/test/interop/clients/python/interop.py @@ -2,16 +2,18 @@ publish: read raw Annex-B H.264 from stdin (e.g. piped from ffmpeg) and feed it to a streaming importer, which infers frame boundaries. Alongside - it, encode a synthetic tone through libopus so the matrix exercises - the FFI audio path, not only the video one. -subscribe: connect, find the video track in the catalog, and exit 0 as soon as - any non-empty frame arrives (exit 1 on timeout / no data). + it, encode a tone per channel (--tone) through libopus so the matrix + exercises the FFI audio path, not only the video one. +subscribe: connect, wait for a catalog with video and audio, and exit 0 once a + non-empty video frame has arrived and --audio-ms of audio has been + decoded to --audio-out for tone.py to judge (exit 1 on timeout / no data). --version offers only that version. --grant waits for the grant the relay sends over AUTH and prints it as an `auth granted` line before the client starts. - ffmpeg ... -f h264 - | python interop.py publish --url http://localhost:4443 --broadcast b.hang - python interop.py subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 + ffmpeg ... -f h264 - | python interop.py publish --url http://localhost:4443 --broadcast b.hang --tone 440 + python interop.py subscribe --url http://localhost:4443 --broadcast b.hang --timeout 20 \ + --audio-out /tmp/audio --audio-ms 600 """ import argparse @@ -28,10 +30,10 @@ READ_CHUNK = 64 * 1024 MAX_DELAY = timedelta(seconds=1) # container consumer congestion-control / lookahead window -# Synthetic audio: a 48 kHz mono tone, encoded as Opus. +# Synthetic audio: a 48 kHz tone per channel at 0.5 amplitude, encoded as Opus. AUDIO_TRACK = "tone" AUDIO_RATE = 48_000 -AUDIO_TONE_HZ = 440.0 +AUDIO_AMPLITUDE = 0.5 # A non-default frame duration, and the shortest Opus offers. 20 ms would pass # even if the microsecond field were truncated to milliseconds somewhere. AUDIO_FRAME_DURATION_US = 2_500 @@ -40,14 +42,18 @@ AUDIO_BATCH_SAMPLES = AUDIO_RATE * AUDIO_BATCH_US // 1_000_000 -async def _publish_tone(audio: moq.audio.Producer) -> None: - """Feed the encoder a real-time tone until the task is cancelled.""" +async def _publish_tone(audio: moq.audio.Producer, tones: list[int]) -> None: + """Feed the encoder a real-time tone per channel, interleaved, until the task is cancelled.""" timestamp_us = 0 phase = 0 loop = asyncio.get_running_loop() started = loop.time() while True: - samples = [math.sin(2 * math.pi * AUDIO_TONE_HZ * (phase + i) / AUDIO_RATE) for i in range(AUDIO_BATCH_SAMPLES)] + samples = [ + AUDIO_AMPLITUDE * math.sin(2 * math.pi * hz * (phase + i) / AUDIO_RATE) + for i in range(AUDIO_BATCH_SAMPLES) + for hz in tones + ] audio.write( moq.audio.Frame( timestamp=timedelta(microseconds=timestamp_us), data=struct.pack(f"<{len(samples)}f", *samples) @@ -81,7 +87,9 @@ async def first() -> moq.Grant: ) -async def publish(url: str, broadcast: str, version: str | None, grant: bool, timeout: float) -> None: +async def publish( + url: str, broadcast: str, version: str | None, grant: bool, timeout: float, tones: list[int] +) -> None: async with _client(url, version) as client: if grant: await _print_grant(client, timeout) @@ -93,7 +101,7 @@ async def publish(url: str, broadcast: str, version: str | None, grant: bool, ti ) audio = moq.audio.Producer( catalog, - moq.audio.EncoderInput(format=moq.audio.SampleFormat.F32, sample_rate=AUDIO_RATE, channels=1), + moq.audio.EncoderInput(format=moq.audio.SampleFormat.F32, sample_rate=AUDIO_RATE, channels=len(tones)), moq.audio.EncoderOutput( codec=moq.audio.Codec.opus(), track=AUDIO_TRACK, @@ -101,9 +109,9 @@ async def publish(url: str, broadcast: str, version: str | None, grant: bool, ti ), ) producer.announce() - print(f"publishing {broadcast!r} (Annex-B H.264 from stdin + a {AUDIO_TONE_HZ:.0f} Hz tone) to {url}") + print(f"publishing {broadcast!r} (Annex-B H.264 from stdin + a {tones} Hz tone) to {url}") - tone = asyncio.create_task(_publish_tone(audio)) + tone = asyncio.create_task(_publish_tone(audio, tones)) loop = asyncio.get_running_loop() stdin = sys.stdin.buffer # read1 returns as soon as any bytes are available (read() would block @@ -125,23 +133,47 @@ async def publish(url: str, broadcast: str, version: str | None, grant: bool, ti media.finish() -async def _catalog_with_video(consumer: moq.BroadcastConsumer) -> moq.media.Catalog: +async def _catalog_with_media(consumer: moq.BroadcastConsumer) -> moq.media.Catalog: # The catalog is a live track. A lazy publisher (e.g. the browser, which only - # encodes on demand) may announce video in a *later* update, not the first - # snapshot, so wait for a catalog that actually has a video track. + # encodes on demand) may announce a rendition in a *later* update, not the first + # snapshot, so wait for a catalog that has both video and audio. catalog_consumer = await moq.media.CatalogConsumer.subscribe(consumer) async for catalog in catalog_consumer: - if catalog.video: + if catalog.video and catalog.audio: return catalog - raise RuntimeError("catalog stream ended without a video track") + raise RuntimeError("catalog stream ended without a video and an audio track") -async def subscribe(url: str, broadcast: str, version: str | None, grant: bool, timeout: float) -> None: +async def _decode_audio( + consumer: moq.BroadcastConsumer, id: str, audio: moq.media.Audio, out: str, audio_ms: int +) -> None: + """Decode the first `audio_ms` of the rendition to `out`.{txt,pcm}, the layout tone.py reads.""" + decoder = await moq.audio.Consumer.subscribe( + consumer, id, audio, moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, max_delay=MAX_DELAY) + ) + frame_bytes = 4 * audio.channel_count + decoded = 0 + with open(f"{out}.txt", "w") as log, open(f"{out}.pcm", "wb") as pcm: + log.write(f"{audio.sample_rate} {audio.channel_count}\n") + async for frame in decoder: + samples = len(frame.data) // frame_bytes + log.write(f"{frame.timestamp // timedelta(microseconds=1)} {samples}\n") + pcm.write(frame.data) + decoded += samples + if decoded * 1000 >= audio_ms * audio.sample_rate: + decoder.cancel() + return + raise RuntimeError("audio track ended early") + + +async def subscribe( + url: str, broadcast: str, version: str | None, grant: bool, timeout: float, audio_out: str, audio_ms: int +) -> None: async with _client(url, version) as client: if grant: await _print_grant(client, timeout) consumer = await asyncio.wait_for(client.announced_broadcast(broadcast), timeout) - catalog = await asyncio.wait_for(_catalog_with_video(consumer), timeout) + catalog = await asyncio.wait_for(_catalog_with_media(consumer), timeout) id, video = next(iter(catalog.video.items())) @@ -158,7 +190,10 @@ async def drain() -> None: if total > 0: return - await asyncio.wait_for(drain(), timeout) + audio_id, audio = next(iter(catalog.audio.items())) + await asyncio.wait_for( + asyncio.gather(drain(), _decode_audio(consumer, audio_id, audio, audio_out, audio_ms)), timeout + ) if total <= 0: raise RuntimeError("no frame data received") @@ -173,15 +208,27 @@ def main() -> None: parser.add_argument("--timeout", type=float, default=20.0) parser.add_argument("--version") parser.add_argument("--grant", action="store_true") + parser.add_argument("--tone", help="publish: comma-separated frequency of each channel, in Hz") + parser.add_argument("--audio-out", help="subscribe: decode audio to this prefix's .txt and .pcm") + parser.add_argument("--audio-ms", type=int, help="subscribe: how much audio to decode") args = parser.parse_args() + if args.role == "publish" and not args.tone: + parser.error("publish needs --tone") + if args.role == "subscribe" and (not args.audio_out or not args.audio_ms): + parser.error("subscribe needs --audio-out and --audio-ms") # Warnings and errors from the Rust layer, such as a publication outside the grant. moq.log_level("warn") try: if args.role == "publish": - asyncio.run(publish(args.url, args.broadcast, args.version, args.grant, args.timeout)) + tones = [int(hz) for hz in args.tone.split(",")] + asyncio.run(publish(args.url, args.broadcast, args.version, args.grant, args.timeout, tones)) else: - asyncio.run(subscribe(args.url, args.broadcast, args.version, args.grant, args.timeout)) + asyncio.run( + subscribe( + args.url, args.broadcast, args.version, args.grant, args.timeout, args.audio_out, args.audio_ms + ) + ) except KeyboardInterrupt: pass except (TimeoutError, asyncio.TimeoutError): diff --git a/test/interop/clients/rust/audio.rs b/test/interop/clients/rust/audio.rs new file mode 100644 index 00000000000..21beb76abf8 --- /dev/null +++ b/test/interop/clients/rust/audio.rs @@ -0,0 +1,97 @@ +//! Audio interop subscriber: decode a broadcast's audio with moq-audio for tone.py to judge. +//! +//! Waits for a catalog with an enabled audio rendition, decodes its first AUDIO_MS at the codec's +//! own rate and layout, and writes PREFIX.txt (` `, then ` ` +//! per frame) and PREFIX.pcm (f32le interleaved). The CLI's `export` covers the video side. +//! +//! interop-audio URL BROADCAST PREFIX AUDIO_MS [VERSION] +use std::io::Write; +use std::time::Duration; + +use anyhow::{Context, ensure}; +use moq_mux::catalog::{CatalogFormat, Stream}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args: Vec<_> = std::env::args().collect(); + ensure!( + args.len() == 5 || args.len() == 6, + "usage: interop-audio URL BROADCAST PREFIX AUDIO_MS [VERSION]" + ); + let url: url::Url = args[1].parse()?; + let prefix = &args[3]; + let audio_ms: u64 = args[4].parse()?; + + let mut config = moq_tokio::connect::Config::default(); + // An empty version offers the defaults. + if let Some(version) = args.get(5).filter(|version| !version.is_empty()) { + config.version.push(version.parse().map_err(anyhow::Error::msg)?); + } + config.websocket.enabled = Some(false); + let client = config.init(Default::default())?.with_reconnect(false); + let origin = moq_tokio::origin::spawn(); + let consumer = origin.consume(); + let session = client + .clone() + .with_subscriber(origin) + .connect(url) + .established() + .await?; + + let broadcast = consumer.routed_broadcast(&args[2]).await?; + let source = moq_mux::Source::new(consumer, broadcast.clone()); + let mut catalogs = moq_mux::catalog::Consumer::<()>::new(&broadcast, CatalogFormat::Hang).await?; + // A lazy publisher (the browser) may add audio in a later catalog update. + let (name, audio) = loop { + let catalog = catalogs.next().await?.context("catalog ended without audio")?; + if let Some((name, audio)) = catalog.audio.enabled().next() { + break (name.clone(), audio.clone()); + } + }; + let rendition = source.resolve(audio.broadcast.as_ref(), audio.epoch.as_ref()).await?; + + let mut options = moq_audio::decode::Options::new(); + options.output.format = moq_audio::Format::F32; + options.max_delay = Duration::from_secs(1); + let mut decoder = moq_audio::decode::Consumer::new(&rendition, &audio, &name, options).await?; + let rate = decoder.sample_rate(); + let channels = decoder.layout().channels(); + + let mut log = std::fs::File::create(format!("{prefix}.txt"))?; + let mut pcm = std::fs::File::create(format!("{prefix}.pcm"))?; + // The catalog's rate and layout, not the decoder's: a decoder that disagrees with the + // catalog is exactly what tone.py should catch. + writeln!(log, "{} {}", audio.sample_rate, audio.channel_count)?; + ensure!( + rate == audio.sample_rate && channels as u32 == audio.channel_count, + "decoder emits {channels} channels at {rate} Hz, the catalog says {} at {}", + audio.channel_count, + audio.sample_rate + ); + + let want = audio_ms * u64::from(rate) / 1000; + let mut decoded = 0; + let mut timeline = None; + while decoded < want { + let read = decoder.read().await?.context("audio track ended early")?; + let moq_audio::decode::Read::Frame { discontinuity, frame } = read else { + anyhow::bail!("audio track ended early"); + }; + // moq-audio reports a hole it saw directly; tone.py checks the timestamps too. + ensure!( + *timeline.get_or_insert(discontinuity) == discontinuity, + "audio discontinuity after {decoded} samples" + ); + let samples = frame.data.len() as u64 / (4 * channels as u64); + writeln!(log, "{} {samples}", frame.timestamp.as_micros())?; + pcm.write_all(&frame.data)?; + decoded += samples; + } + eprintln!("decoded {decoded} samples of {name} with {}", decoder.name()); + + drop(decoder); + session.close().await?; + // Dropping only queues the QUIC close; send it before the runtime stops. + client.close().await; + Ok(()) +} diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 25d2402af71..d84ffb4930c 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -10,10 +10,12 @@ # cargo/bun/uv/go/cmake/cc. # # It stands up a moq-relay, then for each publisher language publishes an H.264 -# broadcast and confirms every subscriber sees data flowing before the timeout. -# Every publisher but the Rust CLI also carries audio. The browser subscriber -# verifies rendered WebCodecs output, player pause/resume, and that audio, then -# that a session the relay refuses hands Chromium the close code and reason. +# broadcast with an Opus tone per audio channel, and confirms every subscriber sees +# video and decodes that tone, contiguous, before the timeout (tone.py judges the +# decode; the browser judges its own playback). A publisher with its channels' +# tones swapped must then fail every subscriber's check. The browser subscriber +# also verifies rendered WebCodecs output and player pause/resume, then that a +# session the relay refuses hands Chromium the close code and reason. # # Every client dials with a token minted for its cell and verified by # `moq auth serve`. Clients that print the grant the relay sent back over AUTH @@ -304,6 +306,37 @@ DROP_VERSIONS=(moq-lite-05 moq-lite-06 moq-lite-07-wip) # The lite drafts the compressed-track lanes cross: lite-07 carries the flag, lite-06 gets inflated bytes. COMPRESS_VERSIONS=(moq-lite-07-wip moq-lite-06) +# Every publisher in this checkout sends a tone per audio channel, and every subscriber decodes +# AUDIO_MS of it for tone.py (the browser: its page) to check against the publisher's list. This +# is the one place the tones are chosen. A stereo publisher sends a different tone per channel, so a +# channel swap fails too. A run that swaps in released CLIs or JS clients (the wire-compat lanes) +# skips audio, and the bindings' clients, which always check it, refuse to run there. +AUDIO=1 +[[ -z "${MOQ_BIN:-}${INTEROP_SUB_MOQ:-}${INTEROP_COMPAT_CLIENT:-}" ]] || AUDIO=0 +if [[ "$AUDIO" -eq 0 ]]; then + for sub in "${SUB_LIST[@]}"; do + case "$sub" in + rust) ;; + js-native-node | js-native-bun) [[ -n "${INTEROP_COMPAT_CLIENT:-}" ]] || sub_needs_audio="$sub" ;; + *) sub_needs_audio="$sub" ;; + esac + if [[ -n "${sub_needs_audio:-}" ]]; then + echo "error: the $sub subscriber always checks audio, which a run with MOQ_BIN, INTEROP_SUB_MOQ, or INTEROP_COMPAT_CLIENT turns off" >&2 + exit 2 + fi + done +fi +AUDIO_MS=600 +STEREO_TONE="440,660" +publisher_tone() { + case "$1" in + python | cpp) echo 440 ;; + *) echo "$STEREO_TONE" ;; + esac +} +# The wrong-tone control's publisher sends the stereo tone with its channels swapped. +SWAPPED_TONE="660,440" + # Record a case for the CI report: `record case=C [field=value...]` (see # harness_result). Every session in a case dials the same relay with each client's own # transport choice, and the version of its pass, unless compat.sh pins one for the whole run. @@ -363,6 +396,9 @@ plan_pass() { for sub in "${SUB_LIST[@]}"; do record "not run" case=unauthorized publisher="$pub" subscriber="$sub"; done record "not run" case="unauthorized publisher" publisher="$pub" done + if [[ "$AUDIO" -eq 1 && "$VERSION" == auto ]]; then + for sub in "${SUB_LIST[@]}"; do record "not run" case="wrong tone" publisher=rust subscriber="$sub"; done + fi [[ "$VERSION" != auto ]] || ! needs js || record "not run" case="refused session" subscriber=js fi [[ "$IN_TREE" -eq 1 ]] || return 0 @@ -456,8 +492,8 @@ build_relay_cli() { echo "error: failed to build moq-relay / moq-cli" >&2 exit 1 } - if [[ "$NEGATIVE" -eq 0 && "$MEDIA" -eq 0 ]]; then - (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-cli --example interop-tail) + if [[ "$MEDIA" -eq 0 ]]; then + (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-cli --example interop-tail --example interop-audio) fi [[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" # The `moq-cli` crate ships its binary as `moq` (a `[[bin]]` override). @@ -681,7 +717,16 @@ prepare_gst() { GST_REGISTRY_1_0="$HARNESS_RUN/gst-registry.bin" \ gst-inspect-1.0 moq 2>/dev/null | grep -qE '^[[:space:]]+moqsrc:'; then mark_broken gst "moqsrc not exposed (plugin failed to load against this GStreamer)" + return fi + # The audio check decodes through these, from the system's plugins. + local factory + for factory in decodebin opusdec audioconvert identity filesink; do + GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" gst-inspect-1.0 "$factory" >/dev/null 2>&1 || { + mark_broken gst "GStreamer has no $factory" + return + } + done } # ── setup ─────────────────────────────────────────────────────────────────── @@ -837,40 +882,82 @@ ffmpeg_h264() { -f h264 - } +# An ffmpeg audio source playing : one 0.5-amplitude sine per channel, at 48 kHz. +# shellcheck disable=SC2329 # reached from a function 'harness_spawn' invokes +tone_source() { + local hz tones exprs=() layout=mono + IFS=',' read -r -a tones <<<"$1" + for hz in "${tones[@]}"; do exprs+=("0.5*sin(2*PI*$hz*t)"); done + [[ "${#exprs[@]}" -eq 1 ]] || layout=stereo + local IFS='|' + printf 'aevalsrc=%s:s=48000:c=%s' "${exprs[*]}" "$layout" +} + +# The Rust CLI's input: the same picture plus as Opus, both in fragmented MP4, so its +# cells also cover moq-mux's fMP4 Opus import. +# shellcheck disable=SC2329 # reached from a function 'harness_spawn' invokes +ffmpeg_fmp4() { + ffmpeg -hide_banner -loglevel error -re -f lavfi -i "testsrc=size=${SIZE}:rate=${FPS}" \ + -re -f lavfi -i "$(tone_source "$1")" \ + -c:v libx264 -profile:v baseline -preset ultrafast -pix_fmt yuv420p \ + -x264-params "keyint=${FPS}:min-keyint=${FPS}:scenecut=0" \ + -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - +} + +# The browser's microphone: as a WAV Chromium loops. Ten seconds is a whole number of +# cycles of any whole-Hz tone, so the loop is seamless. +# shellcheck disable=SC2329 # reached from a function 'harness_spawn' invokes +tone_wav() { + local wav + wav="$HARNESS_RUN/tone-${1//,/-}.wav" + [[ -f "$wav" ]] || ffmpeg -hide_banner -loglevel error -f lavfi -i "$(tone_source "$1"):d=10" -c:a pcm_s16le "$wav" + printf '%s' "$wav" +} + # The publisher pipeline, run in a process group of its own by `harness_spawn` # so reaping it takes ffmpeg with it. Every non-browser publisher consumes the # same ffmpeg Annex-B stream on stdin; the client frames it (moq-cli / the FFI # importers only frame-and-forward). # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' run_publisher() { - local lang="$1" broadcast="$2" url="$3" + local lang="$1" broadcast="$2" url="$3" tone="$4" client_flags "$lang" case "$lang" in rust) # A finished import stays up for its max-age; `stop_publisher` allows 10 s to close. - ffmpeg_h264 | RUST_LOG="$CLI_LOG" "$MOQ" --connect "$url" --connect-version "$(cell_version rust)" \ - --broadcast "$broadcast" import --max-age 1s avc3 + # Released CLIs (the wire-compat lanes) keep the video-only Annex-B input. + if [[ "$AUDIO" -eq 1 ]]; then + ffmpeg_fmp4 "$tone" | RUST_LOG="$CLI_LOG" "$MOQ" --connect "$url" --connect-version "$(cell_version rust)" \ + --broadcast "$broadcast" import --max-age 1s fmp4 + else + ffmpeg_h264 | RUST_LOG="$CLI_LOG" "$MOQ" --connect "$url" --connect-version "$(cell_version rust)" \ + --broadcast "$broadcast" import --max-age 1s avc3 + fi ;; python) ffmpeg_h264 | "$PY" "$CLIENTS/python/interop.py" \ - publish --url "$url" --broadcast "$broadcast" ${FLAGS[@]+"${FLAGS[@]}"} + publish --url "$url" --broadcast "$broadcast" --tone "$tone" ${FLAGS[@]+"${FLAGS[@]}"} ;; go) - ffmpeg_h264 | "$GO_INTEROP" publish --url "$url" --broadcast "$broadcast" ${FLAGS[@]+"${FLAGS[@]}"} + ffmpeg_h264 | "$GO_INTEROP" publish --url "$url" --broadcast "$broadcast" --tone "$tone" ${FLAGS[@]+"${FLAGS[@]}"} ;; cpp) - ffmpeg_h264 | "$CPP_INTEROP" publish --url "$url" --broadcast "$broadcast" ${FLAGS[@]+"${FLAGS[@]}"} + ffmpeg_h264 | "$CPP_INTEROP" publish --url "$url" --broadcast "$broadcast" --tone "$tone" ${FLAGS[@]+"${FLAGS[@]}"} ;; js-native) # exec, so the client leads the group and `stop_publisher` waits for its own close. exec bun "$INTEROP_COMPAT_CLIENT/client.ts" publish "$url" "$broadcast" ;; js) - # Headless Chromium encodes its own H.264 from a fake camera via - # WebCodecs (lazily, once a subscriber creates demand). exec, so the - # driver leads the group and `stop_publisher` waits for its own close. + # Headless Chromium encodes its own H.264 from a fake camera, and Opus from a + # fake microphone playing the tone, via WebCodecs (lazily, once a subscriber + # creates demand). exec, so the driver leads the group and `stop_publisher` + # waits for its own close. + local wav + wav=$(tone_wav "$tone") cd "$CLIENTS/js" && exec bun driver.ts publish \ - --url "$url" --broadcast "$broadcast" ${FLAGS[@]+"${FLAGS[@]}"} + --url "$url" --broadcast "$broadcast" --audio-file "$wav" ${FLAGS[@]+"${FLAGS[@]}"} ;; *) echo "unknown publisher: $lang" >&2 @@ -879,12 +966,13 @@ run_publisher() { esac } -# start_publisher , logging to pub-.log. -# Sets global PUB_PID to the publisher's process group leader. +# start_publisher [tone], logging to pub-.log. The tone +# defaults to the one subscribers expect from . Sets global PUB_PID to the publisher's +# process group leader. PUB_PID="" start_publisher() { - local round="$1" lang="$2" broadcast="$3" url="$4" - harness_spawn "pub-$round" "$LOGS/pub-$round.log" run_publisher "$lang" "$broadcast" "$url" + local round="$1" lang="$2" broadcast="$3" url="$4" tone="${5:-$(publisher_tone "$2")}" + harness_spawn "pub-$round" "$LOGS/pub-$round.log" run_publisher "$lang" "$broadcast" "$url" "$tone" PUB_PID="$HARNESS_PID" } @@ -937,10 +1025,75 @@ observe_group() { --timeout "$TIMEOUT" --track-file "$HARNESS_RUN/$broadcast.track" "$@") } +# The GStreamer subscriber with audio: moqsrc's video_0 must deliver a byte, and audio_0 decodes +# through decodebin to .pcm while an identity logs each decoded buffer's timestamp and size. +# Once enough of both has landed, an interrupt makes gst-launch stop the pipeline and close its +# session. Not with -e: moqsrc does not end its pads on an EOS sent into the pipeline. The log, +# trimmed to the buffers the file holds, becomes .txt, the layout tone.py reads. +# shellcheck disable=SC2329 # reached from run_subscriber, which 'harness_spawn' invokes +run_gst_audio() { + local url="$1" broadcast="$2" out="$3" tone="$4" tones pid want deadline version + version=$(cell_version gst) + IFS=',' read -r -a tones <<<"$tone" + # At 48 kHz, f32: a stop condition only. tone.py judges whatever rate and layout decoded. + want=$((AUDIO_MS * 48 * 4 * ${#tones[@]})) + # SECONDS counts whole seconds, so a fractional timeout rounds up rather than down to nothing. + deadline=$((SECONDS + ${TIMEOUT%.*})) + [[ "$TIMEOUT" != *.* ]] || deadline=$((deadline + 1)) + GST_PLUGIN_PATH_1_0="$GST_PLUGIN_DIR" GST_REGISTRY_1_0="$HARNESS_RUN/gst-run-registry.bin" \ + gst-launch-1.0 -v moqsrc name=s url="$url" broadcast="$broadcast" ${version:+versions="$version"} \ + s.video_0 ! filesink location="$out.video" buffer-mode=2 \ + s.audio_0 ! decodebin ! audioconvert ! audio/x-raw,format=F32LE,layout=interleaved \ + ! identity name=frames silent=false ! filesink location="$out.pcm" buffer-mode=2 >"$out.gst" 2>&1 & + pid=$! + until [[ -s "$out.video" && "$(wc -c <"$out.pcm" 2>/dev/null || echo 0)" -ge "$want" ]]; do + if ! kill -0 "$pid" 2>/dev/null || ((SECONDS >= deadline)); then + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + echo "gst: no video or not enough audio within ${TIMEOUT}s" + tail -n 20 "$out.gst" + return 1 + fi + sleep 0.1 + done + kill -INT "$pid" + # Shutdown shares the cell's budget, with the same 3 s grace `timeout -k 3` gives the others. + while kill -0 "$pid" 2>/dev/null && ((SECONDS < deadline + 3)); do sleep 0.1; done + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" || { + echo "gst-launch did not stop cleanly on an interrupt" + tail -n 20 "$out.gst" + return 1 + } + python3 - "$out" <<'PYCODE' +import os, re, sys +out = sys.argv[1] +text = open(f"{out}.gst").read() +caps = re.search(r"GstIdentity:frames\.GstPad:src: caps = audio/x-raw, (.*)", text) +assert caps, "no decoded audio caps in the gst log" +rate = int(re.search(r"rate=\(int\)(\d+)", caps[1])[1]) +channels = int(re.search(r"channels=\(int\)(\d+)", caps[1])[1]) +# The pipeline stopped mid-stream, so the file may lack the last buffers the identity logged. +held, kept, lines = os.path.getsize(f"{out}.pcm"), 0, [f"{rate} {channels}"] +for size, h, m, s in re.findall(r"last-message = chain.*?\((\d+) bytes, dts: [^,]*, pts: (\d+):(\d+):([\d.]+)", text): + if kept + int(size) > held: + break + kept += int(size) + pts = round((int(h) * 3600 + int(m) * 60 + float(s)) * 1_000_000) + lines.append(f"{pts} {int(size) // (4 * channels)}") +os.truncate(f"{out}.pcm", kept) +open(f"{out}.txt", "w").write("\n".join(lines) + "\n") +PYCODE +} + +# run_subscriber [audio prefix] [tone]: with $AUDIO, every subscriber +# but the browser also decodes $AUDIO_MS of audio to