diff --git a/Cargo.lock b/Cargo.lock index 220f432804f..d496f4ae12e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6214,9 +6214,9 @@ dependencies = [ [[package]] name = "mp4-atom" -version = "0.16.2" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f6638ef4f165c7eb423d7ab9666fff423c8ab53f0dc67ec523aa49530729a7" +checksum = "718b5227fa90c52ed2e72d4d553560130d9e128acb6217d9fe0ea57312ee7063" dependencies = [ "bytes", "derive_more 2.1.1", diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 5fd84d3208c..c5316b207a3 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -54,7 +54,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts/README.md). ## Required -- [Counted field-less `trun` entries](/quest/m1/cut-audit/fmp4-sample-resolver.md) - a moof of field-less runs cannot materialize millions of sample entries - [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a route flip A to B to A no longer panics a peeked reader - [A description-less AAC rendition means one thing](/quest/m1/cut-audit/aac-description-less.md) - every player and exporter agrees on the frame format - [The export clock keeps only the rules admission still needs](/quest/m1/cut-audit/jitter-admission.md) - the clock rules the time and byte admission bound made redundant are measured and removed diff --git a/quest/m1/cut-audit/fmp4-sample-resolver.md b/quest/m1/cut-audit/fmp4-sample-resolver.md deleted file mode 100644 index 2d871671ee3..00000000000 --- a/quest/m1/cut-audit/fmp4-sample-resolver.md +++ /dev/null @@ -1,43 +0,0 @@ -# [M] Counted field-less trun entries - -## Goal - -Parsing a moof allocates in proportion to its bytes: a `trun` whose samples -inherit every field from `tfhd`/`trex` decodes as a count, not one -materialized entry per sample. Land that counted representation in -kixelated/mp4-atom, release it, and adopt it in moq-mux so field-less runs -cannot amplify before the sample resolver sees the mdat. - -## Plan - -[moq-dev/moq#6864](https://github.com/moq-dev/moq/pull/6864) put fMP4 -import, subtitle import, and CMAF decode on one lazy `sample::Resolver`. It -refuses empty samples, out-of-bounds or overlapping ranges, and absolute -`base_data_offset`, so output is bounded by the mdat. The parser is not: -mp4-atom 0.16.2 pushes one `TrunEntry` per sample even when the run carries -no per-sample field, capped only at 4096 per run. A 16 KB moof of 1000 such -runs materializes 4.1 million entries before the resolver refuses anything. - -Settled (2026-10-26, maintainer): a counted representation, not a refusal. -[kixelated/mp4-atom#248](https://github.com/kixelated/mp4-atom/pull/248) -refuses nonempty field-less runs and breaks the AV1, BBB, and ESDS fixtures, -which are valid files; close or rework it rather than weaken those tests. Do -not add another sample-count cap or duplicate the atom parser in moq-mux. - -- mp4-atom: replace `Trun.entries: Vec` with a shape that stores a - field-less run (including a first-sample-flags-only run) as a count and - yields entries through an iterator, without allocating from - `sample_count`. Encode must round-trip both forms. The API shape is the - maintainer's call; propose it in the mp4-atom PR. This is breaking, so - aim for the 0.17.0 release that - [VR fields](/quest/m2/mux-vr-fields.md) is waiting on. -- moq-mux re-exports `mp4_atom`, so the bump is a breaking change to its - public API. Migrate `sample::Resolver::resolve`, the segment-start scan, - and `kept`/`fragment` in `import.rs`, plus the tests that build `Trun` - literals. -- Verify with the fMP4 tests and the interleaved container benchmark, - including its 1-byte samples, and a test that a moof of many field-less - runs decodes without a per-sample allocation. - -Public API: breaking `mp4_atom::Trun` through moq-mux's re-export. MoQ wire: -none. diff --git a/quest/m2/README.md b/quest/m2/README.md index 3bfec57c227..5bea3494ff0 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -151,7 +151,6 @@ lite-07; bitrate enforcement is not part of this release. - [Shared fullscreen, formatters, and stats cards in @moq/ui](/quest/m2/ui-shared-controls.md) - watch and publish stop keeping copies of their shared controls - [moqsrc A/V sync](/quest/m2/moqsrc-av-sync.md) - a nightly test measures A/V skew from `moqsrc` through clock-synced sinks - [MP4 export](/quest/m2/mp4-export.md) - `moq export mp4 --output` records crash-safe fragments, then finishes a regular MP4 with moov at the end -- [Importers fill the VR fields](/quest/m2/mux-vr-fields.md) - MP4, Matroska, and Opus imports set the catalog's projection, stereo, and ambisonic layout - [fMP4 ambisonic layout](/quest/m2/mux-vr-ambisonic-fmp4.md) - fMP4 Opus import fills `ambisonic` from the `dOps` mapping table - [Linux decoded frames](/quest/m2/obs-decode-linux.md) - present supported native decoded surfaces with visible CPU fallback - [Windows decoded frames](/quest/m2/obs-decode-windows.md) - present decoded D3D11 surfaces in OBS without CPU readback diff --git a/quest/m2/cmaf-opus-surround.md b/quest/m2/cmaf-opus-surround.md index f7e264f6ba1..80303afef44 100644 --- a/quest/m2/cmaf-opus-surround.md +++ b/quest/m2/cmaf-opus-surround.md @@ -16,11 +16,14 @@ family 0), the shape drafted in kixelated/mp4-atom#246. That is a breaking Decided 2026-10-17: the `mp4-atom` change is not worth landing on its own. Alone it only swaps `mp4-atom`'s generic decode error for `UnsupportedMappingFamily`; the payoff is surround Opus. So the former -`mp4-atom-dops-mapping` quest (draft moq-dev/moq#5819) folded in here, and -the breaking `mp4-atom` release lands only when this quest does. - -Order: land kixelated/mp4-atom#246, release `mp4-atom` 0.17, then one moq PR -that bumps `rs/moq-mux` to `mp4-atom = "0.17"` and does the import and export. +`mp4-atom-dops-mapping` quest (draft moq-dev/moq#5819) folded in here. + +Decided 2026-10-28 (maintainer): kixelated/mp4-atom#246 ships in 0.17.0 +beside kixelated/mp4-atom#254, so downstream takes one breaking bump. +0.17.0 is released (2026-10-30), and moq-mux takes it for the counted +trun (moq-dev/moq#7043, carried by moq-dev/moq#7308), still refusing a +nonzero family explicitly, since `mp4-atom` no longer does. +This quest then only does the import and export. A git patch to the branch cannot stand in for the release, since `cargo deny` refuses unknown git sources. diff --git a/quest/m2/mux-vr-ambisonic-fmp4.md b/quest/m2/mux-vr-ambisonic-fmp4.md index 24d54bbc01a..8fdb89c6eaa 100644 --- a/quest/m2/mux-vr-ambisonic-fmp4.md +++ b/quest/m2/mux-vr-ambisonic-fmp4.md @@ -4,15 +4,17 @@ An fMP4 Opus track with channel mapping family 2 imports with the catalog's `ambisonic` field, under the same RFC 8486 rules the Matroska and raw Opus -importers apply in [VR fields](/quest/m2/mux-vr-fields.md). +importers apply (`opus::Mapping::new` refuses the family 2 channel counts the +catalog cannot carry, and the OpusHead's `AudioConfig` sets the layout). ## Plan -Split from VR fields on 2026-10-22 (maintainer): only this part needs the -`dOps` channel mapping table, so the rest of VR fields does not wait on it. +Split from the VR fields quest on 2026-10-22 (maintainer), which landed the +Matroska and raw Opus side: only this part needs the `dOps` channel mapping +table. Read the mapping from the `dOps` that [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) carries through the -OpusHead, and reuse VR fields' family 2 validation. Test with an fMP4 +OpusHead, and reuse that family 2 validation. Test with an fMP4 first-order ambisonic fixture and a refused non-diegetic stereo one. Public API: none. Wire: none. @@ -20,4 +22,3 @@ Public API: none. Wire: none. ## Required - [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) - the `dOps` mapping table an ambisonic fMP4 track needs -- [VR fields](/quest/m2/mux-vr-fields.md) - the ambisonic validation and catalog plumbing this reuses diff --git a/quest/m2/mux-vr-fields.md b/quest/m2/mux-vr-fields.md deleted file mode 100644 index ebca62a2487..00000000000 --- a/quest/m2/mux-vr-fields.md +++ /dev/null @@ -1,58 +0,0 @@ -# [S] moq-mux importers fill the VR catalog fields - -## Goal - -The `moq-mux` importers fill the hang catalog's spatial fields from -[#5543](https://github.com/moq-dev/moq/pull/5543): a video rendition's -`projection` and `stereo` from MP4 `st3d`/`sv3d` (Spherical Video V2) and -Matroska `Projection`/`StereoMode`, and an audio rendition's -`layout: "ambisonic"` from Opus channel mapping family 2. Media without these -boxes or elements imports as today. - -## Plan - -Decided with the maintainer on 2026-10-20, from #5543's follow-ups. #5543 has -landed the fields. - -- The fields are live state like `color` (#5543), so fill them the way the - importers fill color (`rs/moq-mux/src/codec/color.rs`) and update them when - the source's config changes. -- Derive `fov` from the equirectangular bounds. The catalog's `fov` is a - longitude span centred straight ahead over the full 180 degrees of latitude - (`rs/hang/src/catalog/video/projection.rs`), so refuse asymmetric left and - right bounds, any top or bottom crop, and a nonzero pose (`sv3d`'s `prhd`, - Matroska `ProjectionPoseYaw`/`Pitch`/`Roll`): importing them would silently - recenter or rotate the video. -- Refuse what the catalog cannot describe, since an absent field claims flat - video or plain channels: a cubemap or mesh projection, so Google's - mesh-based VR180 is refused until the catalog has a mesh kind. The same - goes for any stereo packing or eye order beyond mono, left-right, and - top-bottom (such as `st3d` right-left, or a Matroska right-eye-first, - interleaved, or anaglyph `StereoMode`), each with a fixture. For Opus - family 2, RFC 8486 allows `(1 + n)^2 + 2j` channels with `j` 0 or 1; only - `j = 0` with `n` of at least 1 (4, 9, 16, ... channels) maps to - `ambisonic`, and zeroth order or a non-diegetic stereo pair (1, 3, 6, 11, - ... channels) is refused. A count RFC 8486 forbids is malformed and refused - the same way. -- kixelated/mp4-atom#244 added `st3d`/`sv3d` (plus Apple `vexu`/`hfov`); - take mp4-atom 0.17.0 (kixelated/mp4-atom#247, not yet released), whose - sample entries gain a `spatial` field. Importing Apple's `vexu`/`hfov` - stays out of scope. `webm-iterable` already names the Matroska elements. -- Ambisonic fMP4 needs the `dOps` mapping table, so it moved to - [fMP4 ambisonic layout](/quest/m2/mux-vr-ambisonic-fmp4.md) (2026-10-22, - maintainer); Matroska and raw Opus land here. Before taking 0.17.0, - settle with [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) whether - the release also carries kixelated/mp4-atom#246. -- Exporters writing the fields back are out of scope. - -Tests: a fixture per source (MP4 360, a 180-degree equirectangular MP4 with -`equi` bounds, Matroska stereo, Opus first-order ambisonics) imports with the -expected fields, and a cubemap, a mesh VR180, an equirectangular source with -asymmetric bounds, a vertical crop, or a nonzero pose, an ambisonic stream -with non-diegetic stereo, and a malformed one are refused. - -Public API: none beyond the catalog fields #5543 adds. Wire: none. - -## Related - -- [WebXR playback](/quest/m3/watch-webxr.md) - `` renders these fields diff --git a/quest/m3/watch-webxr.md b/quest/m3/watch-webxr.md index b3ce89065ca..8d9a1e885ce 100644 --- a/quest/m3/watch-webxr.md +++ b/quest/m3/watch-webxr.md @@ -28,7 +28,3 @@ into CI; a manual run on a headset browser proves playback. Public API: possibly a `` attribute to enter or disable immersive playback. Wire: none. - -## Related - -- [moq-mux importers fill the VR fields](/quest/m2/mux-vr-fields.md) - native sources that carry these fields diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 207de7dc38a..0d4e38dd6f1 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -597,7 +597,7 @@ mod tests { while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).unwrap() { if let mp4_atom::Any::Moof(moof) = atom { return moof.traf[0].trun[0] - .entries + .samples .iter() .map(|entry| entry.duration) .collect(); diff --git a/rs/moq-mux/CHANGELOG.md b/rs/moq-mux/CHANGELOG.md index 520d6a4860e..304f9752987 100644 --- a/rs/moq-mux/CHANGELOG.md +++ b/rs/moq-mux/CHANGELOG.md @@ -7,9 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- MP4 and Matroska import fill a video rendition's `projection` and `stereo` from Spherical Video V2's `sv3d`/`st3d` boxes and Matroska's `Projection`/`StereoMode`, and Opus channel mapping family 2 (raw Opus, Matroska, and other OpusHead sources) imports with `layout: "ambisonic"`. What the catalog cannot describe is refused with `fmp4::Error::UnsupportedSpatial`, `mkv::Error::UnsupportedSpatial`, or `opus::Error::UnsupportedChannelCount`: a cubemap or mesh projection, a nonzero pose, uncentred or vertically cropped equirectangular bounds, a stereo packing other than mono, left-right, or top-bottom with the left eye first, and family 2 at zeroth order or with RFC 8486's non-diegetic stereo pair. + ### Changed - [**breaking**] Declare per-track archive durations from GOP hints or the first completed record. +- [**breaking**] Update the re-exported `mp4_atom` to 0.17, whose `Trun.entries` is now `Trun.samples`: a run without per-sample fields decodes as a count, so a moof of field-less runs no longer materializes an entry per sample before import refuses it. Read entries with `trun.samples.iter()` and build them with `samples: entries.into()`. `TrunEntry::cts` widens to `i64` (a version 0 offset is unsigned), `EsDescriptor` gains its optional fields, and `Dref.urls` becomes typed `entries`. `Dops` gains `channel_mapping: Option` (`None` for family 0); the fMP4 importer still refuses a nonzero family. ### Fixed diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index ceaabe8420b..129f159a31c 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -28,7 +28,7 @@ moq-json = { workspace = true } moq-loc = { workspace = true } moq-msf = { workspace = true } moq-net = { workspace = true } -mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } +mp4-atom = { version = "0.17.0", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" num_enum = { workspace = true } roxmltree = "0.21.1" diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs index 2f598f66563..08fec84acfd 100644 --- a/rs/moq-mux/benches/containers.rs +++ b/rs/moq-mux/benches/containers.rs @@ -207,12 +207,13 @@ fn interleaved(init: &[u8], tracks: usize, runs: usize, fragments: usize, payloa trun: (0..runs) .map(|run| mp4_atom::Trun { data_offset: Some(0), - entries: vec![mp4_atom::TrunEntry { + samples: vec![mp4_atom::TrunEntry { duration: Some(trak.mdia.mdhd.timescale / 50), size: Some(payload.len() as u32), flags: Some(if run == 0 { 0 } else { 0x0001_0000 }), cts: None, - }], + }] + .into(), }) .collect(), ..Default::default() diff --git a/rs/moq-mux/src/codec/mod.rs b/rs/moq-mux/src/codec/mod.rs index d11a5895f80..d885cebc0fc 100644 --- a/rs/moq-mux/src/codec/mod.rs +++ b/rs/moq-mux/src/codec/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod legacy; pub(crate) mod mp2; pub mod mp3; pub mod opus; +pub(crate) mod spatial; pub(crate) mod video; pub mod vp8; pub mod vp9; diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index a897c7cb4c6..f5c77cc604a 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -118,6 +118,11 @@ impl From for hang::catalog::AudioConfig { fn from(config: Config) -> Self { let mut audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, config.channel_count); audio.warmup = Some(super::WARMUP); + // Family 2 is ambisonics in AmbiX order (RFC 8486 §3.1); its mapping refuses the channel + // counts the catalog's layout cannot carry. + if config.mapping.is_some_and(|mapping| mapping.family() == 2) { + audio.layout = Some(hang::catalog::AudioLayout::Ambisonic); + } audio.description = config.encode().ok(); audio.container = hang::catalog::Container::Legacy; audio @@ -143,6 +148,30 @@ mod tests { assert_eq!(config.sample_rate, 48_000); } + #[test] + fn family_two_is_ambisonic() { + // First order ambisonics, one stream per channel, in ACN order. + let mapping = crate::codec::opus::Mapping::new(crate::codec::opus::mapping::Config { + family: 2, + streams: 4, + coupled: 0, + table: &[0, 1, 2, 3], + }) + .unwrap(); + let mut head = super::Config::new(48_000, 4); + head.mapping = Some(mapping); + let config = super::config(&head.encode().unwrap()).unwrap(); + assert_eq!(config.layout, Some(hang::catalog::AudioLayout::Ambisonic)); + assert_eq!(config.channel_count, 4); + + // Family 1 surround and family 0 stereo stay plain channels. + let mut head = super::Config::new(48_000, 6); + head.mapping = Some(crate::codec::opus::Mapping::vorbis(6).unwrap()); + assert_eq!(super::config(&head.encode().unwrap()).unwrap().layout, None); + let head = super::Config::new(48_000, 2).encode().unwrap(); + assert_eq!(super::config(&head).unwrap().layout, None); + } + #[tokio::test(start_paused = true)] async fn a_loc_reservation_reaches_the_wire_and_the_catalog() { let mut broadcast = moq_net::broadcast::Info::new().produce(); diff --git a/rs/moq-mux/src/codec/opus/mapping.rs b/rs/moq-mux/src/codec/opus/mapping.rs index 4157dd5303f..672bf7f673b 100644 --- a/rs/moq-mux/src/codec/opus/mapping.rs +++ b/rs/moq-mux/src/codec/opus/mapping.rs @@ -59,7 +59,9 @@ impl Mapping { /// Check `config` into a mapping with one output channel per table entry. /// - /// Refuses family 0, a channel count the family does not allow, and a table + /// Refuses family 0, a channel count the family does not allow (family 2 + /// allows only the 4, 9, 16, ... channels of first order ambisonics and up, + /// without RFC 8486's non-diegetic stereo pair), and a table /// that names no streams, more coupled streams than streams, or a channel /// past the decoded ones. pub fn new(config: Config<'_>) -> Result { @@ -77,6 +79,13 @@ impl Mapping { if table.is_empty() || table.len() > max_channels { return Err(Error::UnsupportedChannelCount(table.len() as u32)); } + // RFC 8486 §3.1 gives family 2 (1 + n)² ambisonic channels for n in 0..=14, plus 0 or 2 + // non-diegetic stereo ones. The catalog's ambisonic layout carries neither zeroth order nor + // the stereo pair, so only first order and up without the pair is accepted. + let order = (table.len() as u32).isqrt(); + if family == 2 && (order < 2 || order * order != table.len() as u32) { + return Err(Error::UnsupportedChannelCount(table.len() as u32)); + } let decoded = streams as u32 + coupled as u32; if streams == 0 || coupled > streams || decoded > 255 { @@ -169,6 +178,16 @@ mod tests { assert!(matches!(new(0, 1, 1, &[0, 1]), Err(Error::UnsupportedMappingFamily(0)))); assert!(matches!(new(1, 1, 0, &[]), Err(Error::UnsupportedChannelCount(0)))); assert!(matches!(new(1, 9, 0, &[0; 9]), Err(Error::UnsupportedChannelCount(9)))); + assert_eq!(new(2, 4, 0, &[0, 1, 2, 3]).unwrap().family(), 2); + assert_eq!(new(2, 9, 0, &[0, 1, 2, 3, 4, 5, 6, 7, 8]).unwrap().family(), 2); + // Zeroth order, first order with RFC 8486's non-diegetic stereo pair, and counts it forbids. + for channels in [1u8, 3, 6, 11, 2, 5, 8, 10] { + let table: Vec = (0..channels).collect(); + assert!( + matches!(new(2, channels, 0, &table), Err(Error::UnsupportedChannelCount(c)) if c == channels as u32), + "{channels} channels" + ); + } for (streams, coupled, table) in [(0, 0, &[0][..]), (1, 2, &[0]), (1, 1, &[2]), (200, 100, &[0])] { assert!( matches!(new(1, streams, coupled, table), Err(Error::InvalidMappingTable)), diff --git a/rs/moq-mux/src/codec/opus/mod.rs b/rs/moq-mux/src/codec/opus/mod.rs index a38d1772219..bc57f408e80 100644 --- a/rs/moq-mux/src/codec/opus/mod.rs +++ b/rs/moq-mux/src/codec/opus/mod.rs @@ -33,8 +33,9 @@ pub enum Error { #[error("unsupported OpusHead version {0}")] UnsupportedVersion(u8), - /// The channel count is zero, or more than the channel mapping family allows: - /// family 0 covers mono/stereo and family 1 up to eight channels. + /// The channel count is zero, or one the channel mapping family does not allow: + /// family 0 covers mono/stereo, family 1 up to eight channels, and family 2 the + /// square counts of first order ambisonics and up. #[error("channel mapping family does not allow {0} channels")] UnsupportedChannelCount(u32), diff --git a/rs/moq-mux/src/codec/spatial.rs b/rs/moq-mux/src/codec/spatial.rs new file mode 100644 index 00000000000..714553bf920 --- /dev/null +++ b/rs/moq-mux/src/codec/spatial.rs @@ -0,0 +1,62 @@ +//! The catalog projection every spherical video source resolves to. +//! +//! The codec bitstream never signals a projection or stereo packing; containers do (ISOBMFF's +//! Spherical Video V2 `sv3d` and `st3d` boxes, Matroska's `Projection` and `StereoMode`). An absent +//! catalog field claims flat, mono video, so an importer refuses what the catalog cannot describe +//! rather than dropping it. + +use hang::catalog::Projection; + +/// The equirectangular projection a Spherical Video V2 `equi` box's bounds describe. +/// +/// Each bound is the 0.32 fixed-point fraction of the projection its edge leaves uncovered. The +/// catalog's `fov` is a longitude span centred straight ahead over the full 180 degrees of +/// latitude, so asymmetric left and right bounds, which would recentre the picture, and any top or +/// bottom crop are refused. Full coverage leaves `fov` absent, which means 360 degrees. +pub(crate) fn equirectangular(top: u32, bottom: u32, left: u32, right: u32) -> Result { + if top != 0 || bottom != 0 { + return Err("equirectangular latitude crop"); + } + if left != right { + return Err("asymmetric equirectangular bounds"); + } + // Both sides together must leave some longitude. + if left >= 1 << 31 { + return Err("equirectangular bounds leave no picture"); + } + let fov = (left != 0).then(|| 360.0 * (1.0 - f64::from(left) / f64::from(1u32 << 31))); + Ok(Projection::Equirectangular { fov }) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn bounds_set_the_field_of_view() { + assert_eq!( + equirectangular(0, 0, 0, 0), + Ok(Projection::Equirectangular { fov: None }) + ); + let quarter = 1 << 30; + assert_eq!( + equirectangular(0, 0, quarter, quarter), + Ok(Projection::Equirectangular { fov: Some(180.0) }) + ); + let Ok(Projection::Equirectangular { fov: Some(fov) }) = equirectangular(0, 0, (1 << 31) - 1, (1 << 31) - 1) + else { + panic!("expected a sliver"); + }; + assert!(fov > 0.0, "{fov}"); + } + + #[test] + fn uncentred_or_cropped_bounds_are_refused() { + let quarter = 1 << 30; + equirectangular(0, 0, quarter, 0).unwrap_err(); + equirectangular(1, 0, 0, 0).unwrap_err(); + equirectangular(0, quarter, quarter, quarter).unwrap_err(); + equirectangular(0, 0, 1 << 31, 1 << 31).unwrap_err(); + equirectangular(0, 0, u32::MAX, u32::MAX).unwrap_err(); + } +} diff --git a/rs/moq-mux/src/container/fmp4/edit.rs b/rs/moq-mux/src/container/fmp4/edit.rs index f477338adb8..87e9b590f5b 100644 --- a/rs/moq-mux/src/container/fmp4/edit.rs +++ b/rs/moq-mux/src/container/fmp4/edit.rs @@ -112,6 +112,7 @@ mod tests { pre_skip: 312, input_sample_rate: 48_000, output_gain: 0, + channel_mapping: None, }, btrt: None, } diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index d9ee346a920..3506eae4296 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -1183,7 +1183,7 @@ fn traf_samples(fragment: &Bytes) -> Vec<(u32, usize)> { continue; }; for traf in moof.traf { - let samples = traf.trun.iter().map(|trun| trun.entries.len()).sum(); + let samples = traf.trun.iter().map(|trun| trun.samples.len()).sum(); trafs.push((traf.tfhd.track_id, samples)); } } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index aed4ba9db89..7a4011ecbbd 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -652,26 +652,30 @@ impl Import { _ => return Err(Error::MultipleCodecs.into()), }; - let (mut config, colr, pasp) = match codec { + let (mut config, colr, pasp, spatial) = match codec { mp4_atom::Codec::Avc1(avc1) => ( self.init_h264(false, &avc1.avcc, &avc1.visual, container)?, avc1.colr.as_ref(), avc1.pasp.as_ref(), + avc1.spatial.as_deref(), ), mp4_atom::Codec::Avc3(avc3) => ( self.init_h264(true, &avc3.avcc, &avc3.visual, container)?, avc3.colr.as_ref(), avc3.pasp.as_ref(), + avc3.spatial.as_deref(), ), mp4_atom::Codec::Hev1(hev1) => ( self.init_h265(true, &hev1.hvcc, &hev1.visual, container)?, hev1.colr.as_ref(), hev1.pasp.as_ref(), + hev1.spatial.as_deref(), ), mp4_atom::Codec::Hvc1(hvc1) => ( self.init_h265(false, &hvc1.hvcc, &hvc1.visual, container)?, hvc1.colr.as_ref(), hvc1.pasp.as_ref(), + hvc1.spatial.as_deref(), ), mp4_atom::Codec::Vp08(vp08) => { let mut config = VideoConfig::new(VideoCodec::VP8); @@ -686,7 +690,7 @@ impl Import { config.coded_width = Some(vp08.visual.width as _); config.coded_height = Some(vp08.visual.height as _); config.container = container; - (config, vp08.colr.as_ref(), vp08.pasp.as_ref()) + (config, vp08.colr.as_ref(), vp08.pasp.as_ref(), vp08.spatial.as_deref()) } mp4_atom::Codec::Vp09(vp09) => { // https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L238 @@ -707,7 +711,7 @@ impl Import { config.coded_width = Some(vp09.visual.width as _); config.coded_height = Some(vp09.visual.height as _); config.container = container; - (config, vp09.colr.as_ref(), vp09.pasp.as_ref()) + (config, vp09.colr.as_ref(), vp09.pasp.as_ref(), vp09.spatial.as_deref()) } mp4_atom::Codec::Av01(av01) => { let mut config = VideoConfig::new(crate::codec::av1::av1_from_av1c(&av01.av1c)); @@ -715,7 +719,7 @@ impl Import { config.coded_width = Some(av01.visual.width as _); config.coded_height = Some(av01.visual.height as _); config.container = container; - (config, av01.colr.as_ref(), av01.pasp.as_ref()) + (config, av01.colr.as_ref(), av01.pasp.as_ref(), av01.spatial.as_deref()) } mp4_atom::Codec::Unknown(unknown, _) => return Err(Error::UnknownCodec(*unknown).into()), unsupported => return Err(Error::UnsupportedCodec(Box::new(unsupported.clone())).into()), @@ -723,6 +727,7 @@ impl Import { // The codec config outranks the colr box a muxer added, which only fills its gaps. let colr = colr.map(super::colr_color).transpose()?.flatten(); crate::codec::color::fill(&mut config.color, colr.as_ref(), "colr"); + (config.projection, config.stereo) = super::spatial(spatial)?; if let Some(pasp) = pasp { let width = config.coded_width.ok_or(Error::InvalidAspect("missing width"))?; let height = config.coded_height.ok_or(Error::InvalidAspect("missing height"))?; @@ -836,9 +841,14 @@ impl Import { } mp4_atom::Codec::Opus(opus) => { // dOps carries the OpusHead fields in ISOBMFF byte order; republish them as the - // OpusHead description so decoders apply its pre-skip and gain. mp4-atom already - // refuses a nonzero channel mapping family, so the table is never dropped here. + // OpusHead description so decoders apply its pre-skip and gain. The head built here + // is family 0, so refuse a mapping table rather than drop it. let dops = &opus.dops; + if let Some(mapping) = &dops.channel_mapping { + return Err( + Error::Opus(crate::codec::opus::Error::UnsupportedMappingFamily(mapping.family)).into(), + ); + } let mut head = crate::codec::opus::Config::new(dops.input_sample_rate, dops.output_channel_count as u32) .with_pre_skip(dops.pre_skip); @@ -904,7 +914,11 @@ impl Import { /// A segment is the unit a group carries, which is what lets audio (whose samples are all /// independently decodable, so it has no boundary of its own) follow the same segmentation as /// video instead of opening a group per fragment. - fn starts_segment(&self, moof: &Moof, has_video: bool) -> bool { + /// + /// `mdat_len` bounds the scan across the whole moof: each sample takes at least one byte of the + /// mdat, and the resolver refuses any past it, so forged field-less runs of billions of samples, + /// however many trafs repeat them, aren't walked here. + fn starts_segment(&self, moof: &Moof, has_video: bool, mdat_len: usize) -> bool { if self.pending_cut { return true; } @@ -923,6 +937,7 @@ impl Import { return false; } + let mut budget = mdat_len; moof.traf.iter().any(|traf| { let Some(track) = self.tracks.get(&traf.tfhd.track_id) else { return false; @@ -931,8 +946,13 @@ impl Import { && traf .trun .iter() - .flat_map(|trun| trun.entries.iter()) - .any(|entry| track.defaults.resolve(&traf.tfhd, entry).is_sync()) + .flat_map(|trun| trun.samples.iter()) + .take_while(|_| { + let more = budget > 0; + budget = budget.saturating_sub(1); + more + }) + .any(|entry| track.defaults.resolve(&traf.tfhd, &entry).is_sync()) }) } @@ -955,7 +975,7 @@ impl Import { // Resolve the segment boundary BEFORE the per-track loop. A moof can carry both an audio // and a video traf, and whichever comes first must see the same answer, or the other rolls // a fragment late and the two tracks stop agreeing on where segments fall. - if self.starts_segment(&moof, has_video) { + if self.starts_segment(&moof, has_video, payload.len()) { self.segment += 1; // A boundary the source declared is also an HLS segment boundary; one merely inferred // from a keyframe is not, since the default pacing already sees that group open. @@ -1087,7 +1107,7 @@ impl Fmp4Track { let duration = sample.duration; // Checked: a negative composition offset must not wrap into a huge u64 PTS. - let pts = dts.checked_add_signed(cts as i64).ok_or(Error::PtsOverflow)?; + let pts = dts.checked_add_signed(cts).ok_or(Error::PtsOverflow)?; let duration_ts = duration .map(|duration| Timestamp::new(duration as u64, timescale)) .transpose()?; @@ -1175,9 +1195,12 @@ fn fragment( for trun_mut in &mut traf_mut.trun { // Reserve the data_offset field; the real value is filled in below. trun_mut.data_offset = Some(0); - for entry in &mut trun_mut.entries { - if entry.duration == Some(0) { - entry.duration = None; + // Only explicit entries carry a duration; an inherited run takes the tfhd's, cleared above. + if let mp4_atom::TrunSamples::Explicit(entries) = &mut trun_mut.samples { + for entry in entries { + if entry.duration == Some(0) { + entry.duration = None; + } } } } @@ -1197,9 +1220,9 @@ fn fragment( // Advance past this trun's sample data let trun_data_size: u64 = trun_mut - .entries + .samples .iter() - .map(|entry| u64::from(defaults.resolve(&traf_mut.tfhd, entry).size)) + .map(|entry| u64::from(defaults.resolve(&traf_mut.tfhd, &entry).size)) .sum(); cumulative_offset += trun_data_size; } @@ -1247,7 +1270,7 @@ fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { duration, size: Some(sample.range.len() as u32), flags: Some(sample.flags), - cts: Some(i32::try_from(cts).map_err(|_| Error::PtsOverflow)?), + cts: Some(i64::try_from(cts).map_err(|_| Error::PtsOverflow)?), }); } let first = run.samples.first().ok_or(Error::MissingTrun)?; @@ -1258,7 +1281,7 @@ fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries, + samples: entries.into(), }], ..Default::default() }) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 5a3c6c1835a..d722276ab0e 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -373,6 +373,160 @@ fn container_aspect_pasp_rejects_invalid_ratios() { ); } +fn with_spatial(mut codec: mp4_atom::Codec, spatial: mp4_atom::Spatial) -> mp4_atom::Codec { + let spatial = Some(Box::new(spatial)); + match &mut codec { + mp4_atom::Codec::Avc1(c) => c.spatial = spatial, + mp4_atom::Codec::Avc3(c) => c.spatial = spatial, + mp4_atom::Codec::Hvc1(c) => c.spatial = spatial, + mp4_atom::Codec::Hev1(c) => c.spatial = spatial, + mp4_atom::Codec::Av01(c) => c.spatial = spatial, + mp4_atom::Codec::Vp08(c) => c.spatial = spatial, + mp4_atom::Codec::Vp09(c) => c.spatial = spatial, + _ => unreachable!(), + } + codec +} + +/// Spherical Video V2 boxes: an `st3d` with `stereo_mode`, and an `sv3d` projecting with `kind`. +fn spherical(stereo_mode: Option, kind: Option) -> mp4_atom::Spatial { + mp4_atom::Spatial { + st3d: stereo_mode.map(|stereo_mode| mp4_atom::St3d { stereo_mode }), + sv3d: kind.map(|kind| mp4_atom::Sv3d { + svhd: Some(mp4_atom::Svhd { + metadata_source: "moq".into(), + }), + proj: Some(mp4_atom::Proj { + prhd: Some(mp4_atom::Prhd::default()), + kind: Some(kind), + unknown: vec![], + }), + unknown: vec![], + }), + ..Default::default() + } +} + +fn equi(top: u32, bottom: u32, left: u32, right: u32) -> Option { + Some(mp4_atom::ProjKind::Equi(mp4_atom::Equi { + projection_bounds_top: top, + projection_bounds_bottom: bottom, + projection_bounds_left: left, + projection_bounds_right: right, + })) +} + +#[test] +fn spherical_boxes_fill_projection_and_stereo() { + use hang::catalog::{Projection, Stereo}; + let quarter = 1 << 30; + for (spatial, projection, stereo) in [ + // Monoscopic 360. + ( + spherical(Some(mp4_atom::St3d::MONO), equi(0, 0, 0, 0)), + Some(Projection::Equirectangular { fov: None }), + None, + ), + // VR180: half the longitude, side by side. + ( + spherical(Some(mp4_atom::St3d::LEFT_RIGHT), equi(0, 0, quarter, quarter)), + Some(Projection::Equirectangular { fov: Some(180.0) }), + Some(Stereo::LeftRight), + ), + // Flat stereo, with no sv3d. + ( + spherical(Some(mp4_atom::St3d::TOP_BOTTOM), None), + None, + Some(Stereo::TopBottom), + ), + ] { + for codec in aspect_entries(false) { + let config = video_config(with_spatial(codec, spatial.clone())); + assert_eq!( + (&config.projection, &config.stereo), + (&projection, &stereo), + "{}", + config.codec + ); + } + } + // Without the boxes, the video stays flat and mono. + let config = video_config(aspect_entries(false).remove(0)); + assert_eq!((config.projection, config.stereo), (None, None)); +} + +#[test] +fn spherical_boxes_the_catalog_cannot_describe_are_refused() { + let quarter = 1 << 30; + let posed = |pose: mp4_atom::Prhd| { + let mut spatial = spherical(None, equi(0, 0, 0, 0)); + spatial.sv3d.as_mut().unwrap().proj.as_mut().unwrap().prhd = Some(pose); + spatial + }; + let mut repeated = spherical(Some(mp4_atom::St3d::MONO), None); + let mut st3d = Vec::new(); + mp4_atom::St3d { + stereo_mode: mp4_atom::St3d::LEFT_RIGHT, + } + .encode(&mut st3d) + .unwrap(); + // The body after the 8-byte header; mp4-atom keeps the second st3d it decodes as unknown. + repeated.unknown = vec![mp4_atom::Any::Unknown(mp4_atom::St3d::KIND, st3d[8..].to_vec())]; + let mut unprojected = spherical(None, equi(0, 0, 0, 0)); + unprojected.sv3d.as_mut().unwrap().proj = None; + + for (name, spatial) in [ + ( + "cubemap", + spherical(None, Some(mp4_atom::ProjKind::Cbmp(mp4_atom::Cbmp::default()))), + ), + ( + "mesh VR180", + spherical( + Some(mp4_atom::St3d::LEFT_RIGHT), + Some(mp4_atom::ProjKind::Mshp(mp4_atom::Mshp { + crc: 0, + encoding_four_cc: b"raw ".into(), + data: vec![], + })), + ), + ), + ("asymmetric bounds", spherical(None, equi(0, 0, quarter, 0))), + ("vertical crop", spherical(None, equi(quarter, quarter, 0, 0))), + ("no picture left", spherical(None, equi(0, 0, 1 << 31, 1 << 31))), + ( + "yawed", + posed(mp4_atom::Prhd { + pose_yaw_degrees: 90 << 16, + ..Default::default() + }), + ), + ( + "rolled", + posed(mp4_atom::Prhd { + pose_roll_degrees: 1, + ..Default::default() + }), + ), + ("right-left", spherical(Some(mp4_atom::St3d::RIGHT_LEFT), None)), + ("custom stereo", spherical(Some(mp4_atom::St3d::STEREO_CUSTOM), None)), + ("unknown stereo", spherical(Some(5), None)), + ("repeated st3d", repeated), + ("sv3d without proj", unprojected), + ] { + for codec in aspect_entries(false) { + let err = video_config_result(with_spatial(codec, spatial.clone())).unwrap_err(); + assert!( + matches!( + err, + crate::Error::Cmaf(crate::container::fmp4::Error::UnsupportedSpatial(_)) + ), + "{name}: {err:?}" + ); + } + } +} + #[test] fn test_bbb_catalog() { let data = include_bytes!("test_data/bbb.mp4"); @@ -597,16 +751,16 @@ async fn separated_runs_keep_their_samples() { trun: vec![ mp4_atom::Trun { data_offset: Some(0), - entries: vec![video_entry(0), video_entry(non_sync)], + samples: vec![video_entry(0), video_entry(non_sync)].into(), }, mp4_atom::Trun { data_offset: Some(0), - entries: vec![video_entry(non_sync)], + samples: vec![video_entry(non_sync)].into(), }, // No data_offset: it starts where the run before it ended. mp4_atom::Trun { data_offset: None, - entries: vec![video_entry(non_sync), video_entry(non_sync)], + samples: vec![video_entry(non_sync), video_entry(non_sync)].into(), }, ], ..Default::default() @@ -621,7 +775,7 @@ async fn separated_runs_keep_their_samples() { }), trun: vec![mp4_atom::Trun { data_offset: Some(0), - entries: vec![audio_entry(2), audio_entry(4)], + samples: vec![audio_entry(2), audio_entry(4)].into(), }], ..Default::default() }, @@ -698,13 +852,14 @@ fn empty_samples_are_refused() { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: vec![ + samples: vec![ mp4_atom::TrunEntry { size: explicit.then_some(0), ..Default::default() }; 4 - ], + ] + .into(), }], ..Default::default() }], @@ -750,7 +905,7 @@ fn absolute_sample_bases_are_refused() { base_media_decode_time: 0, }), trun: vec![mp4_atom::Trun { - entries: vec![Default::default()], + samples: vec![Default::default()].into(), ..Default::default() }], ..Default::default() @@ -795,7 +950,7 @@ async fn shared_sample_bytes_are_refused() { trun: (0..runs) .map(|_| mp4_atom::Trun { data_offset: Some(0), - entries: vec![Default::default()], + samples: vec![Default::default()].into(), }) .collect(), ..Default::default() @@ -877,15 +1032,16 @@ async fn unwrapped_samples_present_at_their_composition_time() { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: entries + samples: entries .iter() .map(|&(cts, flags, _)| mp4_atom::TrunEntry { duration: Some(tick as u32), size: Some(2), flags: Some(flags), - cts: Some(cts as i32), + cts: Some(cts as i64), }) - .collect(), + .collect::>() + .into(), }], ..Default::default() }], @@ -923,6 +1079,66 @@ async fn unwrapped_samples_present_at_their_composition_time() { } /// bbb's init with `entries` as the edit list of track `track_id`. +/// Once the video group is open, the keyframe scan for a segment boundary stops at the mdat +/// rather than walking a forged moof's field-less runs of `u32::MAX` non-sync samples. +#[test] +fn fieldless_runs_do_not_stall_the_segment_scan() { + let (_, (video_id, video_scale), _) = bbb_init(); + let init = bbb_init_with_edits(video_id, vec![media_edit(0)]); + let tick = video_scale.as_u64() / 24; + + let traf = |runs: usize| mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id: video_id, + default_sample_duration: Some(tick as u32), + default_sample_size: Some(1), + default_sample_flags: Some(0x0001_0000), + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: tick, + }), + trun: vec![ + mp4_atom::Trun { + data_offset: None, + samples: mp4_atom::TrunSamples::Inherited { + count: u32::MAX, + first_flags: None, + }, + }; + runs + ], + ..Default::default() + }; + // Many runs in one traf, and many trafs over a larger mdat: the budget spans the whole moof, so + // repeating the traf can't multiply it. + for (trafs, runs, mdat) in [(1, 1000, 6), (1000, 1, 1 << 16)] { + let moof = mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: 2 }, + traf: vec![traf(runs); trafs], + }; + let mut forged = Vec::new(); + moof.encode(&mut forged).unwrap(); + mp4_atom::Mdat { data: vec![0; mdat] }.encode(&mut forged).unwrap(); + + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = + crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + fmp4.decode(&video_fragment(video_id, 0, tick, &[(0, 0, 0xA)])).unwrap(); + assert!( + matches!( + fmp4.decode(&forged), + Err(crate::Error::Cmaf(super::Error::SampleRangeOutOfBounds { .. })) + ), + "{trafs} trafs, passthrough {passthrough}" + ); + } + } +} + fn bbb_init_with_edits(track_id: u32, entries: Vec) -> Vec { let (init, _, _) = bbb_init(); let (ftyp, mut moov) = decode_init(&init); @@ -1013,15 +1229,16 @@ fn video_fragment(track_id: u32, dts: u64, tick: u64, entries: &[(u64, u32, u8)] }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: entries + samples: entries .iter() .map(|&(cts, flags, _)| mp4_atom::TrunEntry { duration: Some(tick as u32), size: Some(2), flags: Some(flags), - cts: Some(cts as i32), + cts: Some(cts as i64), }) - .collect(), + .collect::>() + .into(), }], ..Default::default() }], @@ -1605,6 +1822,7 @@ fn opus_init(entry_rate: u16) -> (mp4_atom::Dops, Vec) { pre_skip: 312, input_sample_rate: 44_100, output_gain: -1536, + channel_mapping: None, }; let opus = mp4_atom::Opus { audio: mp4_atom::Audio { @@ -1660,17 +1878,30 @@ fn opus_catalog_uses_the_decode_rate() { /// A dOps with a channel mapping table is refused rather than imported without it. #[test] fn opus_dops_mapping_family_is_refused() { - let (_, mut data) = opus_init(48_000); - - // The mapping family byte follows version, channels, pre-skip, rate, and gain. - let at = data.windows(4).position(|w| w == b"dOps").unwrap() + 4 + 10; - assert_eq!(data[at], 0); - data[at] = 1; + let (_, data) = opus_init(48_000); + let (ftyp, mut moov) = decode_init(&data); + let mp4_atom::Codec::Opus(opus) = &mut moov.trak[0].mdia.minf.stbl.stsd.codecs[0] else { + panic!("expected Opus sample entry"); + }; + opus.dops.channel_mapping = Some(mp4_atom::OpusChannelMapping { + family: 1, + stream_count: 1, + coupled_count: 1, + channel_mapping: vec![0, 1], + }); + let mut data = Vec::new(); + ftyp.encode(&mut data).unwrap(); + moov.encode(&mut data).unwrap(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); - assert!(fmp4.decode(&bytes::BytesMut::from(data.as_slice())).is_err()); + assert!(matches!( + fmp4.decode(&bytes::BytesMut::from(data.as_slice())), + Err(crate::Error::Cmaf(super::Error::Opus( + crate::codec::opus::Error::UnsupportedMappingFamily(1) + ))) + )); assert!(catalog.snapshot().audio.renditions.is_empty()); } @@ -2056,7 +2287,7 @@ fn audio_fragment(base_media_decode_time: u64, duration: u32, size: u32) -> Vec< tfdt: Some(mp4_atom::Tfdt { base_media_decode_time }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: vec![mp4_atom::TrunEntry::default()], + samples: vec![mp4_atom::TrunEntry::default()].into(), }], ..Default::default() }], @@ -2145,7 +2376,7 @@ fn audio_fragment_samples(base_media_decode_time: u64, duration: u32, size: u32, tfdt: Some(mp4_atom::Tfdt { base_media_decode_time }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: vec![mp4_atom::TrunEntry::default(); samples], + samples: vec![mp4_atom::TrunEntry::default(); samples].into(), }], ..Default::default() }], @@ -2251,14 +2482,15 @@ fn fragment_jitter_uses_sample_endpoints() { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: samples + samples: samples .iter() .map(|(duration, cts)| mp4_atom::TrunEntry { duration: *duration, cts: Some(*cts), ..Default::default() }) - .collect(), + .collect::>() + .into(), }], ..Default::default() }], @@ -2811,13 +3043,14 @@ fn webvtt_fragment(decode_time: u64, samples: &[Vec], duration: u32) -> Vec< }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: samples + samples: samples .iter() .map(|sample| mp4_atom::TrunEntry { size: Some(sample.len() as u32), ..Default::default() }) - .collect(), + .collect::>() + .into(), }], ..Default::default() }], diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 11b55d2afa6..ef5d8730fb2 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -143,6 +143,10 @@ pub enum Error { #[error("invalid colr code point: {0}")] InvalidColr(u16), + /// A Spherical Video V2 `sv3d` or `st3d` box the catalog cannot describe, or a malformed one. + #[error("unsupported spherical video: {0}")] + UnsupportedSpatial(&'static str), + /// The sample entry declares an unsupported codec. #[error("unsupported codec: {0:?}")] UnsupportedCodec(Box), @@ -693,7 +697,7 @@ fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> Result>()?; @@ -712,7 +716,7 @@ fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> Result Result, +) -> Result<(Option, Option)> { + use mp4_atom::{Atom, Cbmp, Equi, Mshp, Prhd, Proj, ProjKind, St3d, Sv3d}; + + let Some(spatial) = spatial else { + return Ok((None, None)); + }; + let unsupported = |reason| Err(Error::UnsupportedSpatial(reason)); + let kept = + |unknown: &[mp4_atom::Any], kinds: &[mp4_atom::FourCC]| unknown.iter().any(|atom| kinds.contains(&atom.kind())); + if kept(&spatial.unknown, &[St3d::KIND, Sv3d::KIND]) { + return unsupported("repeated or malformed st3d or sv3d"); + } + + let stereo = match spatial.st3d.map(|st3d| st3d.stereo_mode) { + None | Some(St3d::MONO) => None, + Some(St3d::LEFT_RIGHT) => Some(hang::catalog::Stereo::LeftRight), + Some(St3d::TOP_BOTTOM) => Some(hang::catalog::Stereo::TopBottom), + Some(_) => return unsupported("stereo mode"), + }; + + let Some(sv3d) = &spatial.sv3d else { + return Ok((None, stereo)); + }; + if kept(&sv3d.unknown, &[Proj::KIND]) { + return unsupported("repeated or malformed proj"); + } + let proj = sv3d + .proj + .as_ref() + .ok_or(Error::UnsupportedSpatial("sv3d without proj"))?; + if kept(&proj.unknown, &[Prhd::KIND, Equi::KIND, Cbmp::KIND, Mshp::KIND]) { + return unsupported("repeated or malformed projection"); + } + if let Some(prhd) = proj.prhd + && (prhd.pose_yaw_degrees, prhd.pose_pitch_degrees, prhd.pose_roll_degrees) != (0, 0, 0) + { + return unsupported("nonzero pose"); + } + let projection = match &proj.kind { + Some(ProjKind::Equi(equi)) => crate::codec::spatial::equirectangular( + equi.projection_bounds_top, + equi.projection_bounds_bottom, + equi.projection_bounds_left, + equi.projection_bounds_right, + ) + .map_err(Error::UnsupportedSpatial)?, + Some(ProjKind::Cbmp(_)) => return unsupported("cubemap projection"), + Some(ProjKind::Mshp(_)) => return unsupported("mesh projection"), + _ => return unsupported("unknown projection"), + }; + Ok((Some(projection), stereo)) +} + /// The `nclx` box for a catalog colour, or `None` when it has no code point or range to carry. /// An absent code point is written as unspecified (2) and an absent range as limited. fn color_colr(color: Option<&hang::catalog::Color>) -> Option { @@ -965,6 +1030,7 @@ pub(crate) fn synthesize_audio_trak(track_id: u32, timescale: u64, config: &Audi pre_skip: head.pre_skip, input_sample_rate: head.sample_rate, output_gain: head.output_gain, + channel_mapping: None, }, btrt: None, }) @@ -1009,7 +1075,7 @@ pub(crate) fn synthesize_audio_trak(track_id: u32, timescale: u64, config: &Audi avg_bitrate, dec_specific: Some(dec_specific), }, - sl_config: Default::default(), + ..Default::default() }, }, btrt: None, @@ -1186,7 +1252,7 @@ fn build_mdia(timescale: u32, handler: &[u8; 4], is_video: bool, sample_entry: m smhd: (!is_video).then(mp4_atom::Smhd::default), dinf: mp4_atom::Dinf { dref: mp4_atom::Dref { - urls: vec![mp4_atom::Url::default()], + entries: vec![mp4_atom::DrefEntry::Url(mp4_atom::Url::default())], }, }, stbl: mp4_atom::Stbl { @@ -1320,9 +1386,9 @@ fn gcd(mut a: u128, mut b: u128) -> u128 { /// The decode timeline a fragment actually carries: its `tfdt` and each sample's composition /// offset. This is what a player reads, as opposed to the PTS [`decode`] reconstructs from it. #[cfg(test)] -pub(crate) fn timeline(fragment: &Bytes) -> (u64, Vec) { +pub(crate) fn timeline(fragment: &Bytes) -> (u64, Vec) { let traf = first_traf(fragment); - let cts = traf.trun[0].entries.iter().map(|e| e.cts.unwrap_or_default()).collect(); + let cts = traf.trun[0].samples.iter().map(|e| e.cts.unwrap_or_default()).collect(); (traf.tfdt.as_ref().unwrap().base_media_decode_time, cts) } @@ -1330,7 +1396,7 @@ pub(crate) fn timeline(fragment: &Bytes) -> (u64, Vec) { #[cfg(test)] pub(crate) fn sample_durations(fragment: &Bytes) -> Vec> { first_traf(fragment).trun[0] - .entries + .samples .iter() .map(|e| e.duration) .collect() @@ -1640,7 +1706,7 @@ mod tests { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: vec![ + samples: vec![ mp4_atom::TrunEntry { size: Some(2), duration: Some(33_333), @@ -1651,7 +1717,8 @@ mod tests { duration: Some(33_333), ..Default::default() }, - ], + ] + .into(), }], ..Default::default() }], @@ -1835,7 +1902,7 @@ mod tests { // On the wire every sample is a sync sample, whatever the keyframe bit said. let flags: Vec = first_traf(&fragment).trun[0] - .entries + .samples .iter() .map(|entry| entry.flags.unwrap()) .collect(); @@ -2064,11 +2131,12 @@ mod tests { }), trun: vec![mp4_atom::Trun { data_offset: None, - entries: vec![mp4_atom::TrunEntry { + samples: vec![mp4_atom::TrunEntry { size: None, duration: None, ..Default::default() - }], + }] + .into(), }], ..Default::default() }], diff --git a/rs/moq-mux/src/container/fmp4/sample.rs b/rs/moq-mux/src/container/fmp4/sample.rs index 261114c4090..80588c1ccf4 100644 --- a/rs/moq-mux/src/container/fmp4/sample.rs +++ b/rs/moq-mux/src/container/fmp4/sample.rs @@ -84,7 +84,7 @@ pub(crate) struct Resolver { /// A sample whose byte range and duration have been validated. pub(crate) struct Resolved { pub sample: Sample, - pub cts: i32, + pub cts: i64, pub range: std::ops::Range, } @@ -106,12 +106,12 @@ impl Resolver { if traf.tfhd.base_data_offset.is_some() { return Err(super::Error::UnsupportedBaseDataOffset); } - let mut remaining: usize = traf.trun.iter().map(|run| run.entries.len()).sum(); + let mut remaining: u64 = traf.trun.iter().map(|run| run.samples.len() as u64).sum(); if remaining == 0 { return Err(super::Error::MissingTrun); } let mut runs = traf.trun.iter(); - let mut entries = [].iter(); + let mut entries = mp4_atom::TrunSamplesIter::default(); let mut offset = 0usize; Ok(std::iter::from_fn(move || { if remaining == 0 { @@ -129,9 +129,9 @@ impl Resolver { .and_then(|start| start.checked_sub(self.data_start)) .ok_or(super::Error::InvalidDataOffset)?; } - entries = run.entries.iter(); + entries = run.samples.iter(); }; - let sample = defaults.resolve(&traf.tfhd, entry); + let sample = defaults.resolve(&traf.tfhd, &entry); if sample.size == 0 { return Err(super::Error::EmptySample); } @@ -282,7 +282,10 @@ mod tests { .iter() .map(|&data_offset| mp4_atom::Trun { data_offset, - entries: vec![Default::default()], + samples: mp4_atom::TrunSamples::Inherited { + count: 1, + first_flags: None, + }, }) .collect(), ..Default::default() @@ -336,11 +339,49 @@ mod tests { } } + /// A moof of many field-less runs decodes as counts rather than one entry per sample, and + /// resolving it stops at the end of the mdat. + #[test] + fn fieldless_runs_do_not_amplify() { + use mp4_atom::{Decode, Encode}; + + let samples = mp4_atom::TrunSamples::Inherited { + count: u32::MAX, + first_flags: None, + }; + let mut traf = traf(&[None; 1000]); + for run in &mut traf.trun { + run.samples = samples.clone(); + } + let moof = mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: 1 }, + traf: vec![traf], + }; + let mut buf = Vec::new(); + moof.encode(&mut buf).unwrap(); + assert!(buf.len() < 16 * 1024); + + let decoded = mp4_atom::Moof::decode(&mut buf.as_slice()).unwrap(); + assert!(decoded.traf[0].trun.iter().all(|run| run.samples == samples)); + + let mut resolver = Resolver::new(100, 6); + let resolved: Vec<_> = resolver + .resolve(&decoded.traf[0], Defaults::default()) + .unwrap() + .collect(); + assert_eq!(resolved.len(), 4); + assert!(resolved[..3].iter().all(Result::is_ok)); + assert!(matches!( + resolved[3], + Err(super::super::Error::SampleRangeOutOfBounds { .. }) + )); + } + #[test] fn only_the_final_sample_may_lack_a_duration() { use super::super::Error; let mut traf = traf(&[None, None]); - traf.trun[1].entries[0].duration = Some(0); + traf.trun[1].samples = vec![entry(None, Some(0), None)].into(); // Empty trailing runs don't make the last sample non-final. traf.trun.push(Default::default()); let mut resolver = Resolver::new(100, 4); @@ -350,7 +391,7 @@ mod tests { .map(|sample| sample.unwrap().sample.duration) .collect(); assert_eq!(durations, [Some(1), None]); - traf.trun[0].entries[0].duration = Some(0); + traf.trun[0].samples = vec![entry(None, Some(0), None)].into(); let mut resolver = Resolver::new(100, 4); assert!(matches!( resolver.resolve(&traf, Defaults::default()).unwrap().next().unwrap(), diff --git a/rs/moq-mux/src/container/fmp4/text.rs b/rs/moq-mux/src/container/fmp4/text.rs index 26f76c4bfdc..45858382a6f 100644 --- a/rs/moq-mux/src/container/fmp4/text.rs +++ b/rs/moq-mux/src/container/fmp4/text.rs @@ -72,7 +72,7 @@ impl Track { for resolved in input.resolver.resolve(traf, self.defaults)? { let sample::Resolved { sample, cts, range } = resolved?; let duration = sample.duration.ok_or(Error::MissingSampleDuration)?; - let pts = dts.checked_add_signed(cts as i64).ok_or(Error::PtsOverflow)?; + let pts = dts.checked_add_signed(cts).ok_or(Error::PtsOverflow)?; let duration = Timestamp::new(duration as u64, self.timescale)?; let data = &input.data[range]; dts = dts.checked_add(duration.value()).ok_or(Error::PtsOverflow)?; diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index a31b9f13589..5bb44b04a26 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -718,6 +718,7 @@ fn build_video_config( .transpose()? .flatten(); crate::codec::color::fill(&mut config.color, colour.as_ref(), "Colour"); + (config.projection, config.stereo) = matroska_spatial(video_children.unwrap_or_default())?; if let Some(children) = video_children && let Some(aspect) = matroska_aspect(children, (width, height))? { @@ -796,6 +797,74 @@ fn matroska_aspect(children: &[MatroskaSpec], pixels: (Option, Option) Ok(Some(aspect)) } +/// The projection and stereo packing a Matroska `Projection` and `StereoMode` signal, each `None` +/// for flat or mono video. +/// +/// The catalog has no cubemap, mesh, pose, or eye order beyond left-first side by side and top-bottom, +/// and an absent field claims flat, mono video, so those are refused rather than dropped. +fn matroska_spatial( + children: &[MatroskaSpec], +) -> Result<(Option, Option)> { + let mut stereo = None; + let mut projection = None; + for child in children { + match child { + // 0 is mono, 1 side by side left eye first, 3 top-bottom left eye first. The rest put the + // right eye first, interleave, checkerboard, or anaglyph, or lace the eyes in one block. + MatroskaSpec::StereoMode(mode) => { + stereo = match mode { + 0 => None, + 1 => Some(hang::catalog::Stereo::LeftRight), + 3 => Some(hang::catalog::Stereo::TopBottom), + _ => return Err(Error::UnsupportedSpatial("StereoMode").into()), + } + } + MatroskaSpec::Projection(Master::Full(fields)) => projection = matroska_projection(fields)?, + _ => {} + } + } + Ok((projection, stereo)) +} + +/// The projection a Matroska `Projection` element signals, `None` for a rectangular one. +fn matroska_projection(fields: &[MatroskaSpec]) -> Result> { + let unsupported = Error::UnsupportedSpatial; + let (mut kind, mut private) = (0, None); + for field in fields { + match field { + MatroskaSpec::ProjectionType(v) => kind = *v, + MatroskaSpec::ProjectionPrivate(v) => private = Some(v.as_slice()), + MatroskaSpec::ProjectionPoseYaw(v) + | MatroskaSpec::ProjectionPosePitch(v) + | MatroskaSpec::ProjectionPoseRoll(v) + if *v != 0.0 => + { + return Err(unsupported("nonzero ProjectionPose").into()); + } + _ => {} + } + } + match kind { + 0 => Ok(None), + // The private data is an `equi` box's body: version and flags, then the top, bottom, left, + // and right bounds. Without it the picture covers the whole sphere, as FFmpeg reads it. + 1 => { + let [top, bottom, left, right] = match private { + None => [0; 4], + Some(&[0, _, _, _, ref bounds @ ..]) if bounds.len() == 16 => { + std::array::from_fn(|i| u32::from_be_bytes(bounds[i * 4..][..4].try_into().unwrap())) + } + Some(_) => return Err(unsupported("equirectangular ProjectionPrivate").into()), + }; + let projection = crate::codec::spatial::equirectangular(top, bottom, left, right).map_err(unsupported)?; + Ok(Some(projection)) + } + 2 => Err(unsupported("cubemap projection").into()), + 3 => Err(unsupported("mesh projection").into()), + _ => Err(unsupported("ProjectionType").into()), + } +} + /// The colour a Matroska `Colour` element signals. Its code points are H.273's, `Range` counts 0 /// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. A code /// point past 255 or a light level past 65535 has no representation, so it is refused. diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index ebf4544f1b7..b7a98080573 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -2052,3 +2052,155 @@ async fn real_live_webm_imports_across_chunk_boundaries() { assert!(timestamps.windows(2).all(|pair| pair[0] < pair[1])); } } + +/// A WebM with one VP9 track whose `Video` element holds `children` beside its dimensions. +fn spatial_webm(children: Vec) -> Vec { + use MatroskaSpec::*; + let mut video = vec![PixelWidth(320), PixelHeight(240)]; + video.extend(children); + MkvBuilder::new() + .header("webm") + .segment_start() + .info(1_000_000) + .tracks(vec![TrackEntry(Master::Full(vec![ + TrackNumber(1), + TrackType(1), + CodecID("V_VP9".into()), + Video(Master::Full(video)), + ]))]) + .segment_end() + .build() +} + +/// An equirectangular `Projection` whose private data is an `equi` body with these bounds. +fn equirectangular(bounds: [u32; 4], pose: [f64; 3]) -> MatroskaSpec { + use MatroskaSpec::*; + let mut private = vec![0; 4]; + for bound in bounds { + private.extend(bound.to_be_bytes()); + } + Projection(Master::Full(vec![ + ProjectionType(1), + ProjectionPrivate(private), + ProjectionPoseYaw(pose[0]), + ProjectionPosePitch(pose[1]), + ProjectionPoseRoll(pose[2]), + ])) +} + +#[test] +fn projection_and_stereo_mode_fill_the_catalog() { + use MatroskaSpec::*; + use hang::catalog::{Projection as Proj, Stereo}; + let quarter = 1 << 30; + for (children, projection, stereo) in [ + (vec![], None, None), + ( + vec![StereoMode(0), equirectangular([0; 4], [0.0; 3])], + Some(Proj::Equirectangular { fov: None }), + None, + ), + // VR180, side by side. + ( + vec![StereoMode(1), equirectangular([0, 0, quarter, quarter], [0.0; 3])], + Some(Proj::Equirectangular { fov: Some(180.0) }), + Some(Stereo::LeftRight), + ), + // Equirectangular without private data covers the whole sphere. + ( + vec![StereoMode(3), Projection(Master::Full(vec![ProjectionType(1)]))], + Some(Proj::Equirectangular { fov: None }), + Some(Stereo::TopBottom), + ), + (vec![Projection(Master::Full(vec![ProjectionType(0)]))], None, None), + ] { + let catalog = run(&spatial_webm(children)); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!((&video.projection, &video.stereo), (&projection, &stereo)); + } +} + +#[test] +fn projection_and_stereo_mode_the_catalog_cannot_describe_are_refused() { + use MatroskaSpec::*; + let quarter = 1 << 30; + let mut short = vec![0; 4]; + short.extend([0; 12]); + for (name, child) in [ + ("cubemap", Projection(Master::Full(vec![ProjectionType(2)]))), + ("mesh", Projection(Master::Full(vec![ProjectionType(3)]))), + ("unknown projection", Projection(Master::Full(vec![ProjectionType(4)]))), + ("asymmetric bounds", equirectangular([0, 0, quarter, 0], [0.0; 3])), + ("vertical crop", equirectangular([quarter, 0, 0, 0], [0.0; 3])), + ("yawed", equirectangular([0; 4], [90.0, 0.0, 0.0])), + ("pitched", equirectangular([0; 4], [0.0, -10.0, 0.0])), + ("rolled", equirectangular([0; 4], [0.0, 0.0, f64::NAN])), + ( + "short private data", + Projection(Master::Full(vec![ProjectionType(1), ProjectionPrivate(short)])), + ), + ("top-bottom right eye first", StereoMode(2)), + ("checkerboard", StereoMode(5)), + ("row interleaved", StereoMode(7)), + ("anaglyph", StereoMode(10)), + ("side by side right eye first", StereoMode(11)), + ("laced eyes", StereoMode(13)), + ] { + let data = spatial_webm(vec![child]); + for initial in [false, true] { + assert!( + matches!( + import_result(&data, initial), + Err(crate::Error::Mkv(super::Error::UnsupportedSpatial(_))) + ), + "{name}" + ); + } + } +} + +/// An OpusHead in channel mapping family 2 with `channels` channels, each its own stream. +fn ambisonic_head(channels: u8) -> Vec { + let mut head = b"OpusHead".to_vec(); + head.extend([1, channels, 0, 0, 0x80, 0xBB, 0, 0, 0, 0, 2, channels, 0]); + head.extend(0..channels); + head +} + +fn opus_webm(head: Vec) -> Vec { + use MatroskaSpec::*; + MkvBuilder::new() + .header("webm") + .segment_start() + .info(1_000_000) + .tracks(vec![TrackEntry(Master::Full(vec![ + TrackNumber(1), + TrackType(2), + CodecID("A_OPUS".into()), + CodecPrivate(head), + ]))]) + .segment_end() + .build() +} + +#[test] +fn opus_family_two_is_ambisonic() { + let catalog = run(&opus_webm(ambisonic_head(4))); + let audio = catalog.audio.renditions.values().next().unwrap(); + assert_eq!(audio.layout, Some(hang::catalog::AudioLayout::Ambisonic)); + assert_eq!(audio.channel_count, 4); + + // Zeroth order, first order with a non-diegetic stereo pair, and a count RFC 8486 forbids. + for channels in [1, 6, 5] { + let data = opus_webm(ambisonic_head(channels)); + for initial in [false, true] { + assert!( + matches!( + import_result(&data, initial), + Err(crate::Error::Opus(crate::codec::opus::Error::UnsupportedChannelCount(c))) if c == channels as u32 + ), + "{channels} channels" + ); + } + } +} diff --git a/rs/moq-mux/src/container/mkv/mod.rs b/rs/moq-mux/src/container/mkv/mod.rs index 5d9b8acfb54..1657222c78f 100644 --- a/rs/moq-mux/src/container/mkv/mod.rs +++ b/rs/moq-mux/src/container/mkv/mod.rs @@ -100,6 +100,10 @@ pub enum Error { #[error("invalid Colour {0}")] InvalidColour(&'static str), + /// A `StereoMode` or `Projection` element the catalog cannot describe, or a malformed one. + #[error("unsupported spherical video: {0}")] + UnsupportedSpatial(&'static str), + /// The named track joined after the MKV header was emitted. #[error("MKV track layout changed after header was emitted: track '{0}' added")] HeaderAddedTrack(String),