From 29bdf02f968645cc6a86876a66e095a5c8bb36e3 Mon Sep 17 00:00:00 2001 From: Alban Gicquel Date: Thu, 15 Oct 2026 10:43:17 -0400 Subject: [PATCH] tecs.audio: load a clip from the bytes of an encoded file in memory Audio:loadBytes(name, bytes, options?) decodes a clip from a whole encoded file a game already holds, container and all, through the same symphonia probe a file goes through, so every format load reads from disk reads from memory too. A game holding raw samples writes the container itself; for PCM that is a 44-byte WAV header. The name is the clip's identity on the same terms as a path is for load, in the same namespace: loading a name again returns the clip already held without reading the new bytes, a failure is cached the same way, and a Sound row and a snapshot carry the name. A memory clip is always resident and keeps no copy of the bytes, so options.stream = true raises and reload reports false with a reason. Clip.memory tells the two kinds apart in clips() and clip(id). The backend contract gains an optional loadClipBytes, which the silent backend and the native binding implement; a backend without it makes loadBytes return nil and a reason. tecs-audio adds decode::decode_bytes, Engine::load_clip_bytes and tecs_audio_load_clip_bytes, which takes a pointer and a length because an encoded file is full of zeroes. --- README.md | 18 +++ native/rust/tecs-audio/src/decode.rs | 131 +++++++++++++++++-- native/rust/tecs-audio/src/engine.rs | 187 ++++++++++++++++++++++++--- native/rust/tecs-audio/src/lib.rs | 123 ++++++++++++++++++ src/tecs/audio.nupp | 136 ++++++++++++++++++- src/tecs/platform/audiobackend.nupp | 13 ++ src/tecs/platform/audionative.nupp | 25 ++++ tests/audionativetest.nupp | 55 ++++++++ tests/audiotest.nupp | 174 ++++++++++++++++++++++++- 9 files changed, 829 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 894c65fa..7ac7e39d 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ test suite reach the same world without starting a graphics stack. layers, object factories, runtime tile edits and authored collision outlines. - [Retained UI backed by Taffy](docs/ui/index.md), intrinsic text and images, scrolling, clipping, pointer capture, keyboard focus and controller activation. +- Audio clips decoded from a file or from the bytes of a whole encoded file held + in memory, for a game that unpacks its own archives. The 2D shape materials include rectangles, circles, rings, rounded rectangles, triangles and stars. 3D rendering, custom post-processing and multi-camera @@ -465,6 +467,22 @@ decode graph, so adapting either meant writing custom sources anyway and taking the dependency. What remained was roughly 450 lines: resampling, a gain envelope, a speaker pair, and a loop wrap. +A clip from memory is the bytes of a whole encoded file, container and all, +rather than raw samples. `loadBytes` hands them to the same `symphonia` probe a +file goes through, so every format `load` reads from disk reads from memory too, +and there is one decode path rather than two. The alternative was a +`loadSamples` taking floats with a rate and a channel count, and it lost on +three counts: it needs a sample-format vocabulary (width, signedness, endianness, +interleaving) that a WAV header already is, it copies a Nupp table across the +seam one float at a time where a string crosses as one pointer and a length, +and a game format that is not raw PCM would still need the decoders anyway. A +game holding raw samples writes the 44-byte header itself, which is the cost +this pushes onto it. The clip is always resident, because streaming would mean +Rust keeping the bytes alive for every voice; the bytes are copied once for the +decode and dropped. Its name shares the clip namespace with paths, because a +`Sound` row and a snapshot carry a clip as a string, and so `reload` reports +false for it rather than having a file to re-read. + The capture buffer is bounded rather than growing without limit as the SDL stream did. It holds one second by default, drops the oldest, and reports the loss through `Microphone.overruns`. An unbounded allocation driven by a diff --git a/native/rust/tecs-audio/src/decode.rs b/native/rust/tecs-audio/src/decode.rs index 42e359a7..aebe325b 100644 --- a/native/rust/tecs-audio/src/decode.rs +++ b/native/rust/tecs-audio/src/decode.rs @@ -1,9 +1,11 @@ //! Symphonia-backed audio decoding for the Tecs audio runtime. //! -//! This module turns a file on disk into interleaved f32 samples at the source -//! layout. It never downmixes and never resamples: the mixer above owns both. +//! This module turns an encoded file, on disk or held in memory, into +//! interleaved f32 samples at the source layout. It never downmixes and never +//! resamples: the mixer above owns both. use std::fs::File; +use std::io::Cursor; use std::path::Path; use symphonia::core::audio::GenericAudioBufferRef; @@ -22,7 +24,7 @@ use symphonia::core::codecs::CodecParameters; use symphonia::core::errors::Error as SymphoniaError; use symphonia::core::formats::probe::Hint; use symphonia::core::formats::{FormatOptions, FormatReader, SeekMode, SeekTo, TrackType}; -use symphonia::core::io::{MediaSourceStream, MediaSourceStreamOptions}; +use symphonia::core::io::{MediaSource, MediaSourceStream, MediaSourceStreamOptions}; use symphonia::core::meta::MetadataOptions; use symphonia::core::units::{Duration, Time, TimeBase}; @@ -77,8 +79,32 @@ pub struct Decoded { /// cannot be identified, it carries no audio track, or no linked decoder /// handles its codec. pub fn decode_all(path: &str) -> Result { - let mut streamer = Streamer::open(path)?; + read_to_end(Streamer::open(path)?) +} +/// Fully decodes the bytes of a whole encoded file held in memory. +/// +/// The bytes are what a file on disk would hold, container and all, in any +/// format a linked decoder reads, so a caller that has raw samples builds the +/// container itself. `name` stands where a path would: every error names it, +/// and its extension, when it carries one, hints the probe the way a path's +/// does. The probe reads the content either way. +/// +/// The bytes are copied once for the length of the decode, because the reader +/// takes an owned source, and nothing of them is kept afterwards. Returns an +/// error naming `name` when the bytes are empty, their container cannot be +/// identified, they carry no audio track, or no linked decoder handles their +/// codec. +pub fn decode_bytes(name: &str, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(format!("tecs-audio: \"{name}\" holds no bytes to decode")); + } + let reader = probe(name, Box::new(Cursor::new(bytes.to_vec())))?; + read_to_end(Streamer::from_reader(name, reader)?) +} + +/// Reads a streamer to its end and returns everything it decoded. +fn read_to_end(mut streamer: Streamer) -> Result { let mut samples = Vec::new(); while streamer.read(&mut samples, usize::MAX)? {} @@ -151,8 +177,14 @@ impl Streamer { /// cannot be opened, its container cannot be identified, it carries no audio /// track, or no linked decoder handles its codec. pub fn open(path: &str) -> Result { - let reader = open_reader(path)?; + Streamer::from_reader(path, open_reader(path)?) + } + /// Builds a streamer over a reader whose container is already identified. + /// + /// `path` labels every error the streamer reports, and is the name a + /// memory input was given when there is no path. + fn from_reader(path: &str, reader: Box) -> Result { let track = reader .default_track(TrackType::Audio) .ok_or_else(|| no_audio_track(path))?; @@ -390,10 +422,19 @@ fn open_reader(path: &str) -> Result, String> { let file = File::open(path).map_err(|err| format!("tecs-audio: failed to open \"{path}\": {err}"))?; - let source = MediaSourceStream::new(Box::new(file), MediaSourceStreamOptions::default()); + probe(path, Box::new(file)) +} + +/// Identifies the container format of an opened input. +/// +/// Passes the extension of `label`, a path or a memory input's name, to the +/// probe as a hint when it carries a usable one, and probes on content alone +/// otherwise. +fn probe(label: &str, input: Box) -> Result, String> { + let source = MediaSourceStream::new(input, MediaSourceStreamOptions::default()); let mut hint = Hint::new(); - if let Some(extension) = Path::new(path).extension().and_then(|ext| ext.to_str()) { + if let Some(extension) = Path::new(label).extension().and_then(|ext| ext.to_str()) { if !extension.is_empty() { hint.with_extension(extension); } @@ -406,7 +447,7 @@ fn open_reader(path: &str) -> Result, String> { FormatOptions::default(), MetadataOptions::default(), ) - .map_err(|err| format!("tecs-audio: failed to identify the format of \"{path}\": {err}")) + .map_err(|err| format!("tecs-audio: failed to identify the format of \"{label}\": {err}")) } /// Converts a container-stated duration into seconds. @@ -776,6 +817,80 @@ mod tests { assert!(!error.is_empty()); } + #[test] + fn decode_bytes_matches_decoding_the_same_file_sample_for_sample() { + let expected = stereo_tone(4800); + let wav = TempWav::write(&expected, 44_100, 2, false); + let bytes = fs::read(&wav.path).expect("the generated WAV reads back"); + + let from_file = decode_all(wav.as_str()).expect("the generated WAV decodes"); + let from_memory = + decode_bytes("music/tone.wav", &bytes).expect("the same bytes decode from memory"); + + assert_eq!(from_memory.sample_rate, from_file.sample_rate); + assert_eq!(from_memory.channels, from_file.channels); + assert_eq!(from_memory.samples, from_file.samples); + } + + #[test] + fn decode_bytes_reads_a_name_with_no_extension_by_content() { + let expected = stereo_tone(480); + let bytes = encode_wav(&expected, 48_000, 2, true); + + let decoded = decode_bytes("generated tone", &bytes).expect("the probe reads the content"); + + assert_eq!(decoded.channels, 2); + assert_eq!(decoded.samples, expected); + } + + #[test] + fn decode_bytes_reads_a_mono_8_bit_unsigned_wav_built_by_hand() { + // RIFF/WAVE, format tag 1 (PCM), one channel, 11025 Hz, 8 bits, five + // frames. Unsigned 8-bit PCM centres on 128, so these five bytes are + // the bottom of the range, halfway down, silence, halfway up and the + // top of the range. + let data: [u8; 5] = [0, 64, 128, 192, 255]; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36u32 + data.len() as u32).to_le_bytes()); + bytes.extend_from_slice(b"WAVE"); + bytes.extend_from_slice(b"fmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&11_025u32.to_le_bytes()); + bytes.extend_from_slice(&11_025u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&data); + + let decoded = decode_bytes("SOUND.BIN", &bytes).expect("the hand-built WAV decodes"); + + assert_eq!(decoded.sample_rate, 11_025); + assert_eq!(decoded.channels, 1); + assert_eq!(decoded.samples, vec![-1.0, -0.5, 0.0, 0.5, 127.0 / 128.0]); + } + + #[test] + fn decode_bytes_reports_bytes_that_are_not_audio_by_name() { + let error = expect_error( + decode_bytes("junk.wav", b"these bytes are not audio at all"), + "text decoded as audio", + ); + assert!( + error.contains("junk.wav"), + "the error does not name the input: {error}" + ); + + let error = expect_error(decode_bytes("empty.wav", &[]), "no bytes decoded"); + assert!( + error.contains("empty.wav") && error.contains("no bytes"), + "the error does not say the input was empty: {error}" + ); + } + #[test] fn decoder_names_reports_the_linked_decoders() { let names = decoder_names(); diff --git a/native/rust/tecs-audio/src/engine.rs b/native/rust/tecs-audio/src/engine.rs index 482b73ea..b17ac449 100644 --- a/native/rust/tecs-audio/src/engine.rs +++ b/native/rust/tecs-audio/src/engine.rs @@ -201,24 +201,7 @@ impl Engine { let entry = if resident { let decoded = decode::decode_all(path).map_err(|reason| self.fail(reason))?; - if decoded.channels == 0 || decoded.channels as usize > MAX_CHANNELS { - return Err(self.fail(format!( - "'{path}' has {} channels, and the mixer carries at most {MAX_CHANNELS}", - decoded.channels - ))); - } - let frames = decoded.samples.len() / decoded.channels.max(1) as usize; - let seconds = if decoded.sample_rate == 0 { - 0.0 - } else { - frames as f64 / decoded.sample_rate as f64 - }; - Clip::Resident { - samples: decoded.samples.into(), - channels: decoded.channels, - sample_rate: decoded.sample_rate, - duration: seconds, - } + self.resident(path, decoded)? } else { // Opened and dropped, so a path that cannot be read fails here // rather than silently producing a voice that never sounds. @@ -235,6 +218,50 @@ impl Engine { } }; + self.hold(clip, entry) + } + + /// Decodes the bytes of a whole encoded file held in memory and holds the + /// result under `clip`. + /// + /// A clip from memory is always resident: there is no file for a voice to + /// stream from, and the bytes are not kept. `name` stands where a path + /// would, in every reason this reports and as the extension the probe is + /// hinted with. + /// + /// @return what the load settled on, or the reason nothing was loaded + pub fn load_clip_bytes(&self, name: &str, bytes: &[u8], clip: u32) -> Result { + let decoded = decode::decode_bytes(name, bytes).map_err(|reason| self.fail(reason))?; + let entry = self.resident(name, decoded)?; + self.hold(clip, entry) + } + + /// Turns decoded samples into a resident clip, refusing a channel count the + /// mixer cannot carry. + fn resident(&self, label: &str, decoded: decode::Decoded) -> Result { + if decoded.channels == 0 || decoded.channels as usize > MAX_CHANNELS { + return Err(self.fail(format!( + "'{label}' has {} channels, and the mixer carries at most {MAX_CHANNELS}", + decoded.channels + ))); + } + let frames = decoded.samples.len() / decoded.channels.max(1) as usize; + let seconds = if decoded.sample_rate == 0 { + 0.0 + } else { + frames as f64 / decoded.sample_rate as f64 + }; + Ok(Clip::Resident { + samples: decoded.samples.into(), + channels: decoded.channels, + sample_rate: decoded.sample_rate, + duration: seconds, + }) + } + + /// Puts a clip under its index, replacing whatever the index held, and + /// reports what it settled on. + fn hold(&self, clip: u32, entry: Clip) -> Result { let settled = match &entry { Clip::Resident { duration, .. } => Loaded { duration: *duration, @@ -434,6 +461,130 @@ mod tests { assert!(!engine.last_error().is_empty()); } + #[test] + fn plays_a_clip_decoded_from_memory_like_one_read_from_a_file() { + // One second of mono 16-bit PCM at the engine's own rate, a constant + // half-scale level, so the render needs no resampling to compare. + let frames: u32 = 48_000; + let level: i16 = 16_384; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + frames * 2).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&48_000u32.to_le_bytes()); + bytes.extend_from_slice(&96_000u32.to_le_bytes()); + bytes.extend_from_slice(&2u16.to_le_bytes()); + bytes.extend_from_slice(&16u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&(frames * 2).to_le_bytes()); + for _ in 0..frames { + bytes.extend_from_slice(&level.to_le_bytes()); + } + + let engine = Engine::open_offline(48_000, 2, 8); + let loaded = engine + .load_clip_bytes("level.wav", &bytes, 3) + .expect("the bytes decode"); + assert!(loaded.resident); + assert!((loaded.duration - 1.0).abs() < 1.0e-9); + + engine.submit(&[play(65_537, 3)]); + assert_eq!(engine.voice_count(), 1); + let mut out = vec![0.0f32; 2 * 256]; + assert_eq!(engine.render(&mut out), 256); + assert!( + out.iter().skip(64).any(|value| *value != 0.0), + "a voice on a memory clip renders silence" + ); + } + + /// Builds a mono 8-bit unsigned WAV whose data chunk has an odd length and + /// so carries the RIFF pad byte after it. + fn odd_unsigned_wav(rate: u32, data: &[u8]) -> Vec { + assert!(data.len() % 2 == 1, "the case under test is an odd chunk"); + let padded = data.len() as u32 + 1; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + padded).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&rate.to_le_bytes()); + bytes.extend_from_slice(&rate.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + // The chunk size states the data alone; the pad byte is not counted. + bytes.extend_from_slice(&(data.len() as u32).to_le_bytes()); + bytes.extend_from_slice(data); + bytes.push(0); + bytes + } + + #[test] + fn resamples_an_odd_8_bit_clip_at_a_nonstandard_rate_from_memory() { + // A ramp, because linear interpolation reproduces a ramp exactly, so + // every resampled frame has a value to be checked against. + let data: Vec = (0..201u32).map(|index| 28 + index as u8).collect(); + let source_value = |position: f64| (28.0 + position) / 128.0 - 1.0; + + for rate in [10_989u32, 18_868] { + let bytes = odd_unsigned_wav(rate, &data); + + let decoded = decode::decode_bytes("SOUND.WAV", &bytes).expect("the WAV decodes"); + assert_eq!(decoded.sample_rate, rate); + assert_eq!(decoded.channels, 1); + // The pad byte is not a sample. + assert_eq!(decoded.samples.len(), data.len()); + for (index, sample) in decoded.samples.iter().enumerate() { + assert_eq!(*sample, data[index] as f32 / 128.0 - 1.0); + } + + let engine = Engine::open_offline(48_000, 2, 8); + let loaded = engine + .load_clip_bytes("SOUND.WAV", &bytes, 1) + .expect("the clip loads"); + assert!(loaded.resident); + assert!((loaded.duration - data.len() as f64 / rate as f64).abs() < 1.0e-12); + + engine.submit(&[play(65_537, 1)]); + let mut out = vec![0.0f32; 2 * 4096]; + engine.render(&mut out); + assert_eq!(engine.voice_count(), 0, "the clip outlasted 4096 frames"); + + // Output frame k reads source position k * rate / 48000, up to the + // last pair of source frames the interpolator can blend. + let step = rate as f64 / 48_000.0; + let sounded = ((data.len() - 1) as f64 / step).ceil() as usize; + for frame in 0..sounded { + let want = source_value(frame as f64 * step) as f32; + assert!( + (out[frame * 2] - want).abs() < 1.0e-4 && out[frame * 2 + 1] == out[frame * 2], + "frame {frame} at {rate} Hz rendered {}, expected {want}", + out[frame * 2] + ); + } + assert!( + out[(sounded + 1) * 2..].iter().all(|value| *value == 0.0), + "the voice at {rate} Hz sounded past its {sounded} resampled frames" + ); + } + } + + #[test] + fn reports_bytes_that_are_not_audio_and_keeps_the_index_empty() { + let engine = Engine::open_offline(48_000, 2, 8); + let failure = engine.load_clip_bytes("junk.ogg", b"not audio", 4); + assert!(failure.is_err()); + assert!(engine.last_error().contains("junk.ogg")); + engine.submit(&[play(65_537, 4)]); + assert_eq!(engine.voice_count(), 0); + } + #[test] fn answers_a_position_query_for_a_handle_naming_nothing() { let engine = Engine::open_offline(48_000, 2, 8); diff --git a/native/rust/tecs-audio/src/lib.rs b/native/rust/tecs-audio/src/lib.rs index 0e4e5578..4b89a5a0 100644 --- a/native/rust/tecs-audio/src/lib.rs +++ b/native/rust/tecs-audio/src/lib.rs @@ -287,6 +287,59 @@ pub unsafe extern "C" fn tecs_audio_load_clip( } } +/// Decodes the bytes of a whole encoded file held in memory and holds the +/// result under `clip`, always resident. +/// +/// `name` stands where a path would for `tecs_audio_load_clip`: every reason +/// names it, and its extension hints the probe. The bytes are borrowed for the +/// call only and nothing of them is kept. +/// +/// # Safety +/// +/// `name` must be a NUL-terminated string, `bytes` must address `length` +/// readable bytes when `length` is nonzero, `info` must be writable, and the +/// device must be live. +/// +/// @return zero when the clip loaded, and one with the reason left for +/// `tecs_audio_last_error` when it did not +#[no_mangle] +pub unsafe extern "C" fn tecs_audio_load_clip_bytes( + device: *mut Engine, + name: *const c_char, + bytes: *const u8, + length: u64, + clip: u32, + info: *mut ClipInfo, +) -> i32 { + let value = match unsafe { engine(device) } { + None => return STATUS_FAILED, + Some(value) => value, + }; + if name.is_null() || info.is_null() { + return STATUS_FAILED; + } + let text = match unsafe { CStr::from_ptr(name) }.to_str() { + Err(_) => return STATUS_FAILED, + Ok(text) => text, + }; + let span: &[u8] = if bytes.is_null() || length == 0 { + &[] + } else { + // SAFETY: the caller guarantees this readable range for the call. + unsafe { std::slice::from_raw_parts(bytes, length as usize) } + }; + match value.load_clip_bytes(text, span, clip) { + Err(_) => STATUS_FAILED, + Ok(loaded) => { + unsafe { + (*info).duration = loaded.duration as f32; + (*info).resident = i32::from(loaded.resident); + } + STATUS_OK + } + } +} + /// Drops whatever a clip index holds. /// /// # Safety @@ -877,6 +930,76 @@ mod tests { } } + #[test] + fn crosses_a_clip_held_in_memory_and_reports_one_that_is_not_audio() { + let device = tecs_audio_open_offline(48_000, 2, 8); + let name = CString::new("sound.wav").expect("the literal has no NUL"); + let mut info = ClipInfo { + duration: 0.0, + resident: 0, + }; + // A mono 8-bit WAV of 4800 frames at 48 kHz: a tenth of a second. + let frames: u32 = 4_800; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + frames).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&48_000u32.to_le_bytes()); + bytes.extend_from_slice(&48_000u32.to_le_bytes()); + bytes.extend_from_slice(&1u16.to_le_bytes()); + bytes.extend_from_slice(&8u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&frames.to_le_bytes()); + bytes.resize(bytes.len() + frames as usize, 128); + unsafe { + let status = tecs_audio_load_clip_bytes( + device, + name.as_ptr(), + bytes.as_ptr(), + bytes.len() as u64, + 1, + &mut info, + ); + assert_eq!(status, STATUS_OK); + assert_eq!(info.resident, 1); + assert!((info.duration - 0.1).abs() < 1.0e-6); + + let junk = b"not audio"; + let status = tecs_audio_load_clip_bytes( + device, + name.as_ptr(), + junk.as_ptr(), + junk.len() as u64, + 2, + &mut info, + ); + assert_eq!(status, STATUS_FAILED); + let reason = CStr::from_ptr(tecs_audio_last_error(device)) + .to_str() + .expect("the reason is UTF-8"); + assert!(reason.contains("sound.wav"), "got {reason}"); + + let status = tecs_audio_load_clip_bytes( + device, + name.as_ptr(), + std::ptr::null(), + 0, + 3, + &mut info, + ); + assert_eq!(status, STATUS_FAILED); + + let null: *mut Engine = std::ptr::null_mut(); + let status = + tecs_audio_load_clip_bytes(null, name.as_ptr(), bytes.as_ptr(), 1, 4, &mut info); + assert_eq!(status, STATUS_FAILED); + tecs_audio_close(device); + } + } + #[test] fn tolerates_a_null_device_on_every_entry_point() { let null: *mut Engine = std::ptr::null_mut(); diff --git a/src/tecs/audio.nupp b/src/tecs/audio.nupp index 6e894bab..4a40da19 100644 --- a/src/tecs/audio.nupp +++ b/src/tecs/audio.nupp @@ -28,6 +28,25 @@ if voice ~= 0 then end ``` +## Clips from memory + +`loadBytes` decodes a clip from the bytes of a whole encoded file a game already +holds, in any format `load` reads from disk. A game that unpacks its own +archives hands over each file as it found it; one that holds raw samples puts a +container around them first, which for PCM is a 44-byte WAV header. + +```nupp +local bytes = assert(nupp.io.files.read("assets/sounds.pak")):sub(1025, 9216) +local chime = assert(mixer:loadBytes("sounds.pak/chime.wav", bytes)) +mixer:play(chime) +``` + +The name is the clip's identity exactly as a path is for `load`, and the two +share one namespace: a [`Sound`](tecs.audio.Sound) row and a snapshot carry the +name, and loading it again returns the clip already held. A memory clip is +always resident and keeps no copy of the bytes, so it has no file for `reload` +to re-read; a game that wants new bytes loads them under a new name. + Units, in one place, because none of them is in a signature. Time is seconds everywhere on this surface: a fade, a seek, a loop point, a cooldown, a clip's duration. Every fade is a duration to run for rather than a moment to finish @@ -463,10 +482,12 @@ export type Config = { backend: audiobackend.Backend? } ---- Configures `Audio.load`. +--- Configures `Audio.load` and `Audio.loadBytes`. export type LoadOptions = { --- Caller-writable. Forces streaming when true and residency when false. --- Left unset, the clip's duration decides against `streamSeconds`. + --- `loadBytes` raises on true, because a clip from memory has no file to + --- stream from, and is resident either way. stream: boolean? } @@ -550,8 +571,9 @@ export type Limit = { --- Reports one shared playable sound after its load. export record Clip - --- Read-only. Reports the path the clip was loaded from, exactly as given. - --- This is a clip's identity, so two spellings of one file are two clips. + --- Read-only. Reports the path the clip was loaded from, or the name + --- `loadBytes` was given, exactly as given. This is a clip's identity, so + --- two spellings of one file are two clips. path: string --- Read-only. Reports the index a [`Sound`](tecs.audio.Sound) carries. @@ -574,6 +596,11 @@ export record Clip --- Read-only. Reports whether decoded audio stays in memory. False --- identifies a clip each voice reads from the file for itself. resident: boolean + + --- Read-only. Reports whether the clip was decoded from bytes `loadBytes` + --- was given rather than read from a file. Such a clip has no file, so + --- `reload` reports false for it. + memory: boolean end --- Describes one sounding voice for inspection. @@ -794,6 +821,7 @@ export record Audio destroy: function(exclusive self: Audio): nil decoders: function(borrows self: Audio): {string} load: function(exclusive self: Audio, path: string, options: LoadOptions?): (Clip?, string?) + loadBytes: function(exclusive self: Audio, name: string, bytes: string, options: LoadOptions?): (Clip?, string?) clip: function(borrows self: Audio, id: integer): Clip? clips: function(borrows self: Audio): {Clip} reload: function(exclusive self: Audio, path: string): (boolean, string?) @@ -1349,7 +1377,15 @@ function Audio.load(exclusive self: Audio, path: string, options: LoadOptions?): local reason = failure or ("cannot load " .. path) self.clipsById[ id - ] = new Clip(path = path, id = id, status = "failed", error = reason, duration = 0, resident = false) + ] = new Clip( + path = path, + id = id, + status = "failed", + error = reason, + duration = 0, + resident = false, + memory = false + ) LOGGER:error("cannot load '%s': %s", path, reason) return nil, reason @@ -1361,7 +1397,90 @@ function Audio.load(exclusive self: Audio, path: string, options: LoadOptions?): status = "ready", error = nil, duration = info.duration, - resident = info.resident + resident = info.resident, + memory = false + ) + self.clipsById[id] = clip + + return clip +end + +--- Decodes a sound from the bytes of a whole encoded file and returns its +--- cached clip. +--- +--- The bytes are what the file would hold on disk, container and all, in any +--- format `load` reads. A caller holding raw samples builds the container +--- itself. The clip is always resident and keeps decoded samples rather than +--- the bytes, which the caller may drop once this returns. +--- +--- `name` is the clip's identity on the same terms as a path is for `load`, +--- and shares its namespace: loading a name already loaded, from bytes or from +--- a file, returns that clip without reading `bytes`, and a failure is cached +--- the same way. A [`Sound`](tecs.audio.Sound) row and a snapshot carry the +--- name, so a restored row starts once this mixer has loaded it again. +--- +--- @param self the mixer that holds the clip +--- @param name the non-empty identity, which is not a path and touches no file. +--- Its extension, when it has one, may hint the format to the decoder, +--- which reads the content either way. +--- @param bytes the whole encoded file, which may contain zero bytes +--- @param options the residency override, where only an unset or false +--- `stream` is accepted +--- @return the cached clip, or nil when the bytes could not be decoded +--- @return the reason, when the first return is nil +--- @raises string when the name is empty +--- @raises string when `options.stream` is true +function Audio.loadBytes(exclusive self: Audio, name: string, bytes: string, options: LoadOptions?): (Clip?, string?) + if name == "" then + error("tecs.audio.loadBytes: a clip name cannot be empty", 2) + end + if options ~= nil and options.stream == true then + error("tecs.audio.loadBytes: a clip from memory cannot stream, because there is no file to read from", 2) + end + local id = clipId(name) + local existing = self.clipsById[id] + if existing ~= nil then + if existing.status == "ready" then + return existing + end + + return nil, existing.error or ("cannot decode " .. name) + end + + local info: audiobackend.ClipInfo? = nil + local failure: string? = nil + local decode = self.backend.loadClipBytes + if decode == nil then + failure = "this audio backend cannot decode a clip from memory" + else + info, failure = decode(name, bytes, id) + end + if info == nil then + local reason = failure or ("cannot decode " .. name) + self.clipsById[ + id + ] = new Clip( + path = name, + id = id, + status = "failed", + error = reason, + duration = 0, + resident = false, + memory = true + ) + LOGGER:error("cannot decode '%s': %s", name, reason) + + return nil, reason + end + + local clip = new Clip( + path = name, + id = id, + status = "ready", + error = nil, + duration = info.duration, + resident = info.resident, + memory = true ) self.clipsById[id] = clip @@ -1405,7 +1524,9 @@ end --- A clip's index is its path's, so an edited file comes back under the index --- every [`Sound`](tecs.audio.Sound) row already carries and nothing in the --- world is touched. A streamed clip holds nothing to replace, so this reports ---- success and the next voice to start reads what is on disk now. +--- success and the next voice to start reads what is on disk now. A clip +--- `loadBytes` decoded has no file and kept no bytes, so this reports false +--- for it and leaves it as it was. --- --- Blocking, like every other reload: it is a debug operation, and answering --- before the file has been read would report a success that had not happened. @@ -1420,6 +1541,9 @@ function Audio.reload(exclusive self: Audio, path: string): (boolean, string?) if clip == nil then return false, "nothing has loaded '" .. path .. "', so there is no clip to re-read" end + if clip.memory then + return false, "'" .. path .. "' was decoded from bytes in memory, so there is no file to re-read" + end if not clip.resident then return true end diff --git a/src/tecs/platform/audiobackend.nupp b/src/tecs/platform/audiobackend.nupp index 8c2ac24d..688391d3 100644 --- a/src/tecs/platform/audiobackend.nupp +++ b/src/tecs/platform/audiobackend.nupp @@ -131,6 +131,16 @@ export record Backend --- load settled on, or nil and a reason. loadClip: function(path: string, clip: integer, mode: LoadMode, streamSeconds: number): (ClipInfo?, string?) + --- Read-only. Decodes `bytes`, the whole of an encoded file held in memory, + --- container and all, and holds the result under `clip`, returning what the + --- load settled on, or nil and a reason. The result is always resident, + --- because there is no file for a voice to stream from. `name` stands where + --- a path would: a reason names it, and its extension may hint the format. + --- The bytes are borrowed for the call and not kept. Omitted by a backend + --- that cannot decode from memory, which `tecs.audio` answers with a + --- reason rather than a raise. + loadClipBytes: (function(name: string, bytes: string, clip: integer): (ClipInfo?, string?))? + --- Read-only. Drops whatever a clip index holds. Voices already reading it --- finish on the samples they started with. releaseClip: function(clip: integer): nil @@ -356,6 +366,9 @@ export function silent(): Backend ): (ClipInfo?, string?) return new ClipInfo(duration = 0, resident = false) end, + loadClipBytes = function(_name: string, _bytes: string, _clip: integer): (ClipInfo?, string?) + return new ClipInfo(duration = 0, resident = false) + end, releaseClip = function(_clip: integer): nil end, position = function(_handle: integer): number? diff --git a/src/tecs/platform/audionative.nupp b/src/tecs/platform/audionative.nupp index 5e32cd18..23f8f55c 100644 --- a/src/tecs/platform/audionative.nupp +++ b/src/tecs/platform/audionative.nupp @@ -115,6 +115,18 @@ cdef function tecs_audio_load_clip( borrows info: tecsAudioClipInfo* ): int32 +-- The bytes cross as a pointer and a length rather than as a C string: an +-- encoded file is full of zeroes, so its length cannot come from a terminator. + +cdef function tecs_audio_load_clip_bytes( + borrows device: voidptr?, + name: cstring, + borrows bytes: cstring, + length: uint64, + clip: uint32, + borrows info: tecsAudioClipInfo* +): int32 + cdef function tecs_audio_release_clip(borrows device: voidptr?, clip: uint32) cdef function tecs_audio_position(borrows device: voidptr?, handle: uint32): float @@ -343,6 +355,18 @@ local function wrap(openDevice: function(): voidptr?, absent: string): (audiobac return written end + local function loadClipBytes(name: string, bytes: string, clip: integer): (audiobackend.ClipInfo?, string?) + if closed[1] then + return nil, "the audio output is closed" + end + local status = tecs_audio_load_clip_bytes(device, name, bytes, #bytes as uint64, clip as uint32, info) + if status ~= 0 then + return nil, reason("tecsaudio could not decode the bytes of '" .. name .. "'") + end + + return new audiobackend.ClipInfo(duration = info.duration as number, resident = info.resident ~= 0) + end + local function loadClip( path: string, clip: integer, @@ -377,6 +401,7 @@ local function wrap(openDevice: function(): voidptr?, absent: string): (audiobac submit = submit, drain = drain, loadClip = loadClip, + loadClipBytes = loadClipBytes, releaseClip = function(clip: integer): nil tecs_audio_release_clip(device, clip as uint32) end, diff --git a/tests/audionativetest.nupp b/tests/audionativetest.nupp index c7139c73..377f69c1 100644 --- a/tests/audionativetest.nupp +++ b/tests/audionativetest.nupp @@ -76,6 +76,61 @@ function suite.reportsAMissingFileWithTheLibraryReason(): nil backend.close() end +--- Returns `value` as `width` little-endian bytes. +local function littleEndian(value: integer, width: integer): string + local bytes: {string} = {} + local rest = value + for index = 1, width do + bytes[index] = string.char(rest % 256) + rest = rest // 256 + end + + return table.concat(bytes) +end + +--- Builds a mono 8-bit unsigned WAV holding `data`, padded to an even chunk. +local function unsignedWav(rate: integer, data: string): string + local pad = #data % 2 == 1 and "\0" or "" + local format = "fmt " .. littleEndian( + 16, + 4 + ) .. littleEndian( + 1, + 2 + ) .. littleEndian( + 1, + 2 + ) .. littleEndian(rate, 4) .. littleEndian(rate, 4) .. littleEndian(1, 2) .. littleEndian(8, 2) + local chunk = "data" .. littleEndian(#data, 4) .. data .. pad + + return "RIFF" .. littleEndian(4 + #format + #chunk, 4) .. "WAVE" .. format .. chunk +end + +function suite.decodesAClipFromBytesThroughTheLibrary(): nil + local backend = opened() + -- 1099 frames at 10989 Hz, an odd chunk with its pad byte, of silence, + -- which unsigned 8-bit PCM writes as 128. + local bytes = unsignedWav(10989, string.rep(string.char(128), 1099)) + local seconds = 1099 / 10989 + local info, reason = assert(backend.loadClipBytes)("SOUND.WAV", bytes, 1) + test.equal(reason, nil) + test.assert(assert(info).resident) + test.assert(math.abs(assert(info).duration - seconds) < 1e-6) + + local mixer = audio.newAudio({backend = backend}) + local clip = assert(mixer:loadBytes("tests/memory/silence.wav", bytes)) + test.assert(clip.memory and clip.resident) + test.assert(math.abs(clip.duration - seconds) < 1e-6) + -- The offline backend reports no output, so nothing starts; the clip is + -- what was under test. + test.equal(mixer:play(clip), 0) + + local junk, junkReason = mixer:loadBytes("tests/memory/junk.wav", "these bytes are not audio") + test.equal(junk, nil) + test.matches(assert(junkReason), "junk.wav") + mixer:destroy() +end + function suite.namesTheDecodersTheLibraryLinked(): nil local backend = opened() test.assert(#backend.decoders() > 0) diff --git a/tests/audiotest.nupp b/tests/audiotest.nupp index 31963654..b3d7a4fc 100644 --- a/tests/audiotest.nupp +++ b/tests/audiotest.nupp @@ -14,6 +14,8 @@ local record Recorder failures: {[string]: string} released: {integer} closed: {boolean} + decoded: {[string]: string} + decodes: {integer} end local function newRecorder(): Recorder @@ -23,6 +25,8 @@ local function newRecorder(): Recorder local failures: {[string]: string} = {} local released: {integer} = {} local closed = {false} + local decoded: {[string]: string} = {} + local decodes = {0} local backend = new audiobackend.Backend( available = true, @@ -74,6 +78,16 @@ local function newRecorder(): Recorder return new audiobackend.ClipInfo(duration = duration, resident = resident) end, + loadClipBytes = function(name: string, bytes: string, _clip: integer): (audiobackend.ClipInfo?, string?) + decodes[1] = decodes[1] + 1 + local failure = failures[name] + if failure ~= nil then + return nil, failure + end + decoded[name] = bytes + + return new audiobackend.ClipInfo(duration = durations[name] or 1, resident = true) + end, releaseClip = function(clip: integer): nil released[#released + 1] = clip end, @@ -98,7 +112,9 @@ local function newRecorder(): Recorder durations = durations, failures = failures, released = released, - closed = closed + closed = closed, + decoded = decoded, + decodes = decodes ) end @@ -218,6 +234,162 @@ function suite.cachesOneClipPerPathAndReportsFailures(): nil test.equal(secondReason, "no such file") end +--- Returns a clip name nothing else in this suite has interned. +local function freshName(): string + nextPath[1] = nextPath[1] + 1 + + return "test/memory/clip" .. tostring(nextPath[1]) .. ".wav" +end + +--- Stands for the bytes of an encoded file. The recorder never decodes them, +--- so all that matters is that they carry zeroes and arrive whole. +local FILE_BYTES: string = "RIFF\0\0\0\0WAVE" + +function suite.decodesAClipFromBytesAndPlaysItResident(): nil + local recorder = newRecorder() + local mixer = audio.newAudio({backend = recorder.backend, streamSeconds = 3}) + local name = freshName() + -- Longer than the streaming threshold, and resident regardless. + recorder.durations[name] = 30 + + local clip = assert(mixer:loadBytes(name, FILE_BYTES)) + test.equal(recorder.decoded[name], FILE_BYTES) + test.equal(clip.path, name) + test.equal(clip.id, audio.clipId(name)) + test.equal(audio.clipPath(clip.id), name) + test.equal(clip.status, "ready") + test.equal(clip.duration, 30) + test.assert(clip.resident) + test.assert(clip.memory) + test.equal(mixer:clip(clip.id), clip) + test.equal(mixer:clips()[1], clip) + + local handle = mixer:play(clip, {gain = 0.5}) + test.assert(handle > 0) + mixer:flush() + local command = assert(lastOf(recorder, audiobackend.PLAY)) + test.equal(command.handle, handle) + test.equal(command.clip, clip.id) + test.equal(command.gain, 0.5) + test.equal(command.flags // audiobackend.STREAM % 2, 0) + + -- A clip from a file says it is not one from memory. + test.assert(not assert(mixer:load(freshPath())).memory) +end + +function suite.keepsTheFirstClipLoadedUnderAName(): nil + local recorder = newRecorder() + local mixer = audio.newAudio({backend = recorder.backend}) + local name = freshName() + + local clip = assert(mixer:loadBytes(name, FILE_BYTES)) + -- A second load of the name returns the clip it already holds, without + -- decoding the new bytes, which is what `load` does for a path. + test.equal(mixer:loadBytes(name, "other bytes"), clip) + test.equal(recorder.decodes[1], 1) + test.equal(recorder.decoded[name], FILE_BYTES) + -- Names and paths share one namespace, in both directions. + test.equal(mixer:load(name), clip) + local path = freshPath() + local fromFile = assert(mixer:load(path)) + test.equal(mixer:loadBytes(path, FILE_BYTES), fromFile) + test.equal(recorder.decodes[1], 1) + -- Stating residency outright is accepted, since it is what happens anyway. + test.equal(mixer:loadBytes(name, FILE_BYTES, {stream = false}), clip) +end + +function suite.refusesToReloadAClipThatHasNoFile(): nil + local recorder = newRecorder() + local mixer = audio.newAudio({backend = recorder.backend}) + local name = freshName() + recorder.durations[name] = 2 + local clip = assert(mixer:loadBytes(name, FILE_BYTES)) + + local ok, reason = mixer:reload(name) + test.assert(not ok) + test.matches(assert(reason), "from bytes in memory") + test.equal(clip.status, "ready") + test.equal(clip.duration, 2) + test.equal(recorder.decodes[1], 1) +end + +function suite.returnsAndCachesAFailedDecode(): nil + local recorder = newRecorder() + local mixer = audio.newAudio({backend = recorder.backend}) + local name = freshName() + recorder.failures[name] = "the bytes are not audio" + + local clip, reason = mixer:loadBytes(name, "junk") + test.equal(clip, nil) + test.equal(reason, "the bytes are not audio") + local failed = assert(mixer:clip(audio.clipId(name))) + test.equal(failed.status, "failed") + test.assert(failed.memory) + + recorder.failures[name] = nil + local again, secondReason = mixer:loadBytes(name, FILE_BYTES) + test.equal(again, nil) + test.equal(secondReason, "the bytes are not audio") + test.equal(recorder.decodes[1], 1) +end + +function suite.reportsABackendThatCannotDecodeFromMemory(): nil + local recorder = newRecorder() + recorder.backend.loadClipBytes = nil + local mixer = audio.newAudio({backend = recorder.backend}) + + local clip, reason = mixer:loadBytes(freshName(), FILE_BYTES) + test.equal(clip, nil) + test.matches(assert(reason), "cannot decode a clip from memory") +end + +function suite.raisesOnAnEmptyNameAndOnAStreamRequest(): nil + local mixer = audio.newAudio({backend = newRecorder().backend}) + test.raises( + function(): nil + mixer:loadBytes("", FILE_BYTES) + end, + "clip name cannot be empty" + ) + test.raises( + function(): nil + mixer:loadBytes(freshName(), FILE_BYTES, {stream = true}) + end, + "cannot stream" + ) +end + +function suite.loadsAClipFromBytesOnTheSilentBackend(): nil + local mixer = audio.newAudio({backend = audiobackend.silent()}) + local clip = assert(mixer:loadBytes(freshName(), FILE_BYTES)) + test.assert(clip.memory) + test.equal(mixer:play(clip), 0) +end + +function suite.carriesAMemoryClipThroughASnapshotByName(): nil + local name = freshName() + local recorder = newRecorder() + local world = ecs.newWorld() + local mixer = audio.install(world, {backend = recorder.backend}) + local clip = assert(mixer:loadBytes(name, FILE_BYTES)) + world:spawn(audio.SoundComponent(clip.id)) + world:commit() + local snapshot = world:saveSnapshot() + + -- Another world and another mixer, which has not loaded the name yet. + local otherRecorder = newRecorder() + local other = ecs.newWorld() + local otherMixer = audio.install(other, {backend = otherRecorder.backend}) + other:loadSnapshot(snapshot) + other:update(0) + test.equal(countOf(otherRecorder, audiobackend.PLAY), 0) + + -- The row names the clip, so it starts once this mixer holds it. + assert(otherMixer:loadBytes(name, FILE_BYTES)) + other:update(0) + test.equal(assert(lastOf(otherRecorder, audiobackend.PLAY)).clip, audio.clipId(name)) +end + function suite.streamsAClipPastTheThreshold(): nil local recorder = newRecorder() local mixer = audio.newAudio({backend = recorder.backend, streamSeconds = 3})