From 85248eed3e07eecd782c57a100b36622225790b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 10:12:29 -0800 Subject: [PATCH 01/10] chore(quest): claim quest/m2/compose/layout.md Quest-Claim: e97694e335dd9933 From 5f85208d562a77f232e11ede73a5d513deca2a6d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 12 Nov 2026 10:32:09 -0800 Subject: [PATCH 02/10] wip: the studio publishes a composition --- rs/moq-scene/Cargo.toml | 2 +- rs/moq-scene/src/collection.rs | 62 ++-- rs/moq-scene/src/composition.rs | 610 ++++++++++++++++++++++++++++++++ rs/moq-scene/src/compositor.rs | 16 +- rs/moq-scene/src/layout.rs | 454 ------------------------ rs/moq-scene/src/lib.rs | 9 +- rs/moq-scene/src/publish.rs | 231 +++++++----- rs/moq-scene/src/source.rs | 357 ++++++++++++++----- rs/moq-studio/src/engine.rs | 241 +++++++++---- rs/moq-studio/src/main.rs | 17 +- rs/moq-studio/src/ui.rs | 27 +- rs/moq-studio/src/watch.rs | 531 +++++++++++++++++---------- 12 files changed, 1593 insertions(+), 964 deletions(-) create mode 100644 rs/moq-scene/src/composition.rs delete mode 100644 rs/moq-scene/src/layout.rs diff --git a/rs/moq-scene/Cargo.toml b/rs/moq-scene/Cargo.toml index 999c472..9091494 100644 --- a/rs/moq-scene/Cargo.toml +++ b/rs/moq-scene/Cargo.toml @@ -99,7 +99,7 @@ x11rb = { version = "0.13", features = ["dri3"] } wayland-client = "0.31" [dev-dependencies] -# Reads the layer layout track back as a player would. +# Reads the composition track back as a player would. moq-json = { workspace = true, features = ["async"] } # An encoder on every machine, for the source track and replay tests. moq-video = { workspace = true, features = ["openh264"] } diff --git a/rs/moq-scene/src/collection.rs b/rs/moq-scene/src/collection.rs index 56784c7..0775098 100644 --- a/rs/moq-scene/src/collection.rs +++ b/rs/moq-scene/src/collection.rs @@ -40,13 +40,14 @@ pub struct Collection { /// wherever the source is placed. A source without an entry is unfiltered. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub video: BTreeMap, - /// The sources published independently, by name: each one's own track - /// goes to the relay beside the program, and the - /// [layer layout](crate::layout) says where viewers show it over the + /// The sources published independently, by name: each reaches viewers + /// as a broadcast of its own, a local source's track in one the studio + /// publishes and a broadcast source as its publisher's, and the + /// [composition](crate::composition) says where viewers show it over the /// program and how they play its sound. The program and every audio - /// scene's mix leave it out. Only a source with a track of its own can - /// be, and not one with its background removed, since its track would - /// show the background. Wherever it is placed, it takes only what every + /// scene's mix leave it out. Only a broadcast or a source with a track of + /// its own can be, and not one with its background removed, since its + /// track would show the background. Wherever it is placed, it takes only what every /// player can reproduce, and no scene that another scene draws holds it. #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] pub published: BTreeSet, @@ -469,11 +470,11 @@ impl Collection { "`{name}` draws scene `{scene}`, which is not a scene" ))); } - if let Source::Layer { broadcast, .. } = source - && !matches!(self.sources.get(broadcast), Some(Source::Broadcast { .. })) + if let Source::Layer { clock, .. } = source + && !matches!(self.sources.get(clock), Some(Source::Broadcast { .. })) { return Err(Error::Invalid(format!( - "`{name}` plays on `{broadcast}`, which is not a broadcast source" + "`{name}` plays on `{clock}`, which is not a broadcast source" ))); } let Source::Clone { source } = source else { continue }; @@ -518,10 +519,17 @@ impl Collection { "`{name}` is published, but there is no such source" ))); }; - if !source.publishes() { + if !source.independent() { return Err(Error::Invalid(format!( - "`{name}` cannot be published independently: only a camera, display, window, microphone, \ - system audio, color, image, media, or replay source has a track of its own" + "`{name}` cannot be published independently: only a broadcast, or a camera, display, window, \ + microphone, system audio, color, image, media, or replay source with a track of its own, can be" + ))); + } + let segment = !name.contains('/') && moq_net::Path::new(name).is_ok() && name != "." && name != ".."; + if source.publishes() && !segment { + return Err(Error::Invalid(format!( + "`{name}` is published independently, so its name is one segment of its broadcast's path: no \ + `/`, not `.` or `..`, and a `\\` only before another" ))); } if self @@ -638,9 +646,9 @@ impl Collection { } /// Refuse an item showing a source published independently that the - /// layer layout cannot carry: one blended or filtered past opacity, which + /// composition cannot carry: one blended or filtered past opacity, which /// not every player can reproduce, or one in a scene another scene draws, - /// which the layout cannot place. + /// which the composition cannot place. fn validate_published(&self) -> Result<(), Error> { let nested: BTreeSet<&str> = self .sources @@ -1254,7 +1262,7 @@ fn is_true(value: &bool) -> bool { /// What a source or item is drawn through, in order. /// /// Every filter is a pure function of the picture, so every host that composes -/// a scene with moq-scene draws it the same way, and it travels in a layout as +/// a scene with moq-scene draws it the same way, and it travels in a composition as /// its name and settings. The CSS ones mean what the CSS Filter Effects /// function of the same name does, in the gamma-encoded sRGB the canvas holds, /// with lengths in canvas pixels. A filter naming another source uses that @@ -1855,18 +1863,22 @@ pub enum Source { /// same frame. A scene drawing it cannot go on program, and the top layer /// cannot draw it, since either would draw the program into itself. Program, - /// A source a studio broadcast publishes independently, played from the - /// broadcast the [`Source::Broadcast`] called `broadcast` plays, on that - /// source's playout clock, so it lands with the program as the - /// [layer layout](crate::layout) places it. Never in a scene file: only - /// a player of the layout makes one. + /// A broadcast a [composition](crate::composition) lists, played on the + /// playout clock of the [`Source::Broadcast`] called `clock`, its + /// timestamps moved `offset` onto that clock's, so it lands with the + /// program as the composition places it. Never in a scene file: only a + /// player of the composition makes one. #[serde(skip)] Layer { - /// The broadcast source whose broadcast and clock it plays on. - broadcast: String, - /// Its entry under the broadcast catalog's - /// [`SECTION`](crate::publish::SECTION) `sources`. - source: String, + /// The broadcast source whose clock it plays on: the program. + clock: String, + /// Its broadcast's path. + path: String, + /// The stream it must be, or `None` for whichever serves the path. + instance: Option, + /// Microseconds added to its timestamps to put them on `clock`'s + /// timeline. A change plays on without opening the broadcast again. + offset: i64, }, } diff --git a/rs/moq-scene/src/composition.rs b/rs/moq-scene/src/composition.rs new file mode 100644 index 0000000..1f8acf8 --- /dev/null +++ b/rs/moq-scene/src/composition.rs @@ -0,0 +1,610 @@ +//! The composition: a broadcast of the studio's own whose catalog lists other +//! broadcasts, the program and each source published independently, local or +//! a guest's, with where and when each shows and how its sound mixes, so a +//! player places each as the studio would have. +//! +//! The program leaves a published source out, so a player draws each +//! [picture](Composition::pictures) over the program, back to front, and mixes +//! each [sound](Child::sound) with the program's audio. A composition carries +//! only what every player can reproduce: a picture's place, crop, opacity, +//! visibility, and keyframes, and a sound's gain, mute, pan, and delay. +//! [`Collection::validate`] refuses a published source placed with anything +//! more, such as a blur or a blend mode. +//! +//! Each listed broadcast keeps its own timeline. The composition's media clock +//! is the program's, and each [`Child`] carries the offset that puts its +//! broadcast's timestamps on it: zero for a local source, which the studio +//! stamps on the program's clock, and for a guest, how far its stamps sat from +//! the program's when the studio played them. +//! +//! It travels as JSON on [`TRACK`] in snapshot mode, in a broadcast of its own +//! [beneath the program's](PATH), so a viewer joining late reads the +//! composition in force. The format is the studio's own, provisional until +//! hang specifies a composition. + +use std::collections::BTreeMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, DurationMicroSeconds, serde_as}; + +use crate::Error; +use crate::collection::{Audio, Collection, Item, Layer, Size}; + +/// The composition broadcast's track carrying the composition, listed in its +/// catalog's `json` section. +pub const TRACK: &str = "live.kixel.studio.composition"; + +/// Where the composition's broadcast is, beneath the program's. +pub const PATH: &str = "composition"; + +/// Where each local source published independently has a broadcast of its +/// own, beneath the program's, under its name: `/source/`. +pub const SOURCES: &str = "source"; + +/// How far a child's offset may wander before a new composition carries it. +/// +/// The offset is measured from a playout clock that re-anchors on every write +/// of a guest's sound, so it jitters by a millisecond or so while the guest +/// plays steadily. Well under the 45 ms of sound leading the picture a viewer +/// can detect (ITU-R BT.1359), it keeps a viewer's guest in step with the +/// studio's without a group per frame. +pub const DRIFT: Duration = Duration::from_millis(10); + +/// Where and when each broadcast the composition lists shows or plays. +#[serde_as] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Composition { + /// The format's version, first so a reader of another one fails on it. + pub version: Version, + /// The program's canvas, which pictures are placed on in pixels. + pub canvas: Size, + /// When the scene on program went on, on the composition's media clock: + /// the moment its pictures' keyframes play from. In microseconds on the + /// wire. + #[serde_as(as = "DurationMicroSeconds")] + pub start: Duration, + /// The program, whose clock the composition's is. + pub program: Program, + /// Each source published independently that a picture shows or the + /// program's audio plays, by source name. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub sources: BTreeMap, + /// Each picture over the program, back to front: the scene on program's, + /// then the top layer's. Each is a scene file item whose `source` names a + /// key of [`sources`](Self::sources), whose filters are only opacity, and + /// whose keyframes play from [`start`](Self::start). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub pictures: Vec, +} + +impl Composition { + /// The composition of `collection` with the scene called `scene` on + /// program since `start` on the media clock, or no scene for `None`, + /// drawn over `program`. Each source published independently is listed + /// as `children` locates it, its sound filled in; one `children` leaves + /// out, such as a guest offline, is left out with its pictures, as the + /// studio shows nothing of it either. + pub fn new( + collection: &Collection, + scene: Option<&str>, + start: Duration, + program: Program, + mut children: BTreeMap, + ) -> Self { + let pictures: Vec = pictures(collection, scene, true) + .into_iter() + .filter(|item| children.contains_key(&item.source)) + .collect(); + let heard = collection.program_audio().map(|audio| collection.heard(audio)); + for source in heard.into_iter().flatten() { + if let Some(child) = children.get_mut(source) { + child.sound = Some(collection.audio.get(source).cloned().unwrap_or_default()); + } + } + children.retain(|name, child| child.sound.is_some() || pictures.iter().any(|item| item.source == *name)); + Self { + version: Version, + canvas: Size { + width: collection.canvas.width, + height: collection.canvas.height, + }, + start, + program, + sources: children, + pictures, + } + } +} + +/// Each item of the scene called `scene`, then of the top layer if `top`, +/// showing a source published independently, back to front, shown or hidden: +/// its `source` names the published source, and its crop and filters fold in +/// its source's, as the compositor draws them. +pub(crate) fn pictures(collection: &Collection, scene: Option<&str>, top: bool) -> Vec { + let scene = scene.filter(|scene| collection.scene(scene).is_some()); + let places = scene.map(Layer::scene).into_iter().chain(top.then(Layer::top)); + let mut pictures = Vec::new(); + for place in places { + for item in collection.items(&place).unwrap_or_default() { + let source = collection.capture(&item.source); + if !collection.published.contains(source) { + continue; + } + pictures.push(Item { + source: source.to_string(), + crop: collection.crop(item), + filters: collection.filters(item).cloned().collect(), + ..item.clone() + }); + } + } + pictures +} + +/// The format's version: deserializing refuses any but +/// [`Version::CURRENT`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "u32", try_from = "u32")] +pub struct Version; + +impl Version { + /// The version this build reads and writes. Version 1 was the layer + /// layout, which named tracks of the program's broadcast. + pub const CURRENT: u32 = 2; +} + +impl From for u32 { + fn from(_: Version) -> Self { + Version::CURRENT + } +} + +impl TryFrom for Version { + type Error = String; + + fn try_from(found: u32) -> Result { + match found { + Self::CURRENT => Ok(Self), + found => Err(format!( + "composition version {found} is not supported; this build reads version {}", + Self::CURRENT + )), + } + } +} + +/// The program's broadcast. +#[serde_as] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Program { + /// Its path, relative to the composition's, as hang's references are. + pub broadcast: moq_net::path::Relative, + /// The stream whose timeline the composition's clock is, so a player + /// waits for it rather than placing anything on another's. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde_as(as = "Option")] + pub instance: Option, +} + +/// One broadcast the composition places over the program or mixes with it. +#[serde_as] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Child { + /// Its path, relative to the composition's, as hang's references are. + pub broadcast: moq_net::path::Relative, + /// The stream [`offset`](Self::offset) holds for, or `None` for whichever + /// serves the path, as for a source the studio stamps on the program's + /// clock. A relay-minted instance is never named, since it cannot match + /// another relay's. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde_as(as = "Option")] + pub instance: Option, + /// Microseconds added to its timestamps to put them on the composition's + /// clock, negative for a broadcast stamped ahead of it. + #[serde(default, skip_serializing_if = "is_zero")] + pub offset: i64, + /// Its gain, mute, pan, and delay in the program's audio, or `None` when + /// the program's audio does not play it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sound: Option