diff --git a/crates/coreav-hevc/src/reconstruction/inter.rs b/crates/coreav-hevc/src/reconstruction/inter.rs index 49ce81ab..f271d4e4 100644 --- a/crates/coreav-hevc/src/reconstruction/inter.rs +++ b/crates/coreav-hevc/src/reconstruction/inter.rs @@ -959,7 +959,7 @@ pub(super) fn predict( .planes .get(component) .ok_or(Error::InvalidData("inter_reference_plane"))?; - kernels::interpolate_inter( + kernels::interpolate_inter_with_rows( kernels::InterPrediction { depth: walker.frame.depth, component: if component == 0 { @@ -983,6 +983,7 @@ pub(super) fn predict( width, height, )?, + &mut walker.frame.scratch.residual, )?; if let Some(parameters) = inter.weights { let denom = *parameters diff --git a/crates/coreav-kernels-hevc/src/inter.rs b/crates/coreav-kernels-hevc/src/inter.rs index 2bc7fe2a..1d5ee8ec 100644 --- a/crates/coreav-kernels-hevc/src/inter.rs +++ b/crates/coreav-kernels-hevc/src/inter.rs @@ -124,7 +124,21 @@ pub fn interpolate_inter( reference: RefPlane<'_>, output: RectMut<'_, i32>, ) -> Result<(), Error> { - interpolate(config, reference, output, true) + interpolate(config, reference, output, true, None) +} + +/// Uses the first 512 caller-owned i32 samples as an eight-row interpolation ring. +/// +/// The samples may contain arbitrary previous values. Every active temporary is +/// overwritten before use; workspace beyond 512 samples and output padding stay unchanged. +pub fn interpolate_inter_with_rows( + config: InterPrediction, + reference: RefPlane<'_>, + output: RectMut<'_, i32>, + rows: &mut [i32], +) -> Result<(), Error> { + let rows = rows.get_mut(..512).ok_or(Error::Buffer)?; + interpolate(config, reference, output, true, Some(rows)) } #[cfg(any(test, feature = "test-support"))] @@ -134,7 +148,7 @@ pub fn interpolate_inter_scalar( reference: RefPlane<'_>, output: RectMut<'_, i32>, ) -> Result<(), Error> { - interpolate(config, reference, output, false) + interpolate(config, reference, output, false, None) } fn interpolate( @@ -142,6 +156,7 @@ fn interpolate( reference: RefPlane<'_>, output: RectMut<'_, i32>, _automatic: bool, + row_storage: Option<&mut [i32]>, ) -> Result<(), Error> { #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] let vector = _automatic.then(crate::simd::Avx2::detect).flatten(); @@ -190,15 +205,29 @@ fn interpolate( } else { coefficients(config.component, phase_y)? }; - let mut scratch = InterScratch { - columns: [0; 71], - source: [0; 71], - rows: [[0; 64]; 8], + let mut owned; + let mut columns; + let mut source; + let (columns, source_buffer, intermediate) = if let Some(rows) = row_storage { + columns = [0; 71]; + source = [0; 71]; + (&mut columns, &mut source, rows) + } else { + owned = InterScratch { + columns: [0; 71], + source: [0; 71], + rows: [[0; 64]; 8], + }; + ( + &mut owned.columns, + &mut owned.source, + owned.rows.as_flattened_mut(), + ) }; let source_width = output.width + x_filter.len().saturating_sub(1); let consecutive = x_start >= 0 && x_start + source_width as i64 <= reference.width as i64; if !consecutive { - for (x, column) in scratch.columns.iter_mut().take(source_width).enumerate() { + for (x, column) in columns.iter_mut().take(source_width).enumerate() { *column = (x_start + x as i64).clamp(0, reference.width as i64 - 1) as usize; } } @@ -218,17 +247,16 @@ fn interpolate( .get(x_start as usize..x_start as usize + source_width) .ok_or(Error::Buffer)? } else { - let input = scratch - .source - .get_mut(..source_width) - .ok_or(Error::Buffer)?; - for (value, &column) in input.iter_mut().zip(&scratch.columns) { + let input = source_buffer.get_mut(..source_width).ok_or(Error::Buffer)?; + for (value, &column) in input.iter_mut().zip(columns.iter()) { *value = *source.get(column).ok_or(Error::Buffer)?; } &*input }; - let row = scratch.rows.get_mut(next_row % 8).ok_or(Error::Buffer)?; - let row = row.get_mut(..output.width).ok_or(Error::Buffer)?; + let row_start = (next_row % 8) * 64; + let row = intermediate + .get_mut(row_start..row_start + output.width) + .ok_or(Error::Buffer)?; if x_filter.is_empty() { for (value, &sample) in row.iter_mut().zip(input.iter()) { *value = i32::from(sample) << internal_shift; @@ -252,16 +280,15 @@ fn interpolate( .ok_or(Error::Buffer)?; if y_filter.is_empty() { destination.copy_from_slice( - scratch - .rows - .get(y % 8) - .and_then(|row| row.get(..output.width)) + intermediate + .get((y % 8) * 64..(y % 8) * 64 + output.width) .ok_or(Error::Buffer)?, ); } else { let mut rows: [&[i32]; 8] = [&[]; 8]; for (j, row) in rows.iter_mut().take(row_count).enumerate() { - *row = scratch.rows.get((y + j) % 8).ok_or(Error::Buffer)?; + let begin = ((y + j) % 8) * 64; + *row = intermediate.get(begin..begin + 64).ok_or(Error::Buffer)?; } #[cfg(all(not(feature = "scalar"), target_arch = "x86_64"))] if let Some(vector) = vector { diff --git a/crates/coreav-kernels-hevc/src/inter_tests.rs b/crates/coreav-kernels-hevc/src/inter_tests.rs index f2b6fcf8..d8111415 100644 --- a/crates/coreav-kernels-hevc/src/inter_tests.rs +++ b/crates/coreav-kernels-hevc/src/inter_tests.rs @@ -150,6 +150,112 @@ fn automatic_and_scalar_match_all_widths_phases_and_tails() { } } +#[test] +fn reused_dirty_interpolation_rows_match_equations_across_successive_shapes() { + let mut workspace = [i32::MIN; 1025]; + for bits in [8, 10] { + let mut source = vec![u16::MAX; 87 * 28 + 83]; + for y in 0..29 { + for x in 0..83 { + source[y * 87 + x] = ((47 * x + 91 * y + 13 * x * y) % (1 << bits)) as u16; + } + } + for component in [Component::Luma, Component::Chroma] { + let phases = if component == Component::Luma { 4 } else { 8 }; + let limit = if component == Component::Luma { 64 } else { 32 }; + for width in [64, 1, 31, 8, 63, 7, 32, 9] { + if width > limit { + continue; + } + for phase_y in 0..phases { + for phase_x in 0..phases { + for (x, y, motion_x, motion_y) in + [(8, 8, 0, 0), (0, 0, 0, 0), (82, 28, 32000, -32000)] + { + let request = config( + bits, + component, + x, + y, + motion_x + phase_x, + motion_y + phase_y, + ) + .unwrap(); + let height = + [1, 3, 9][(width + phase_x as usize + phase_y as usize) % 3]; + let stride = width + 3; + let mut actual = vec![i32::MAX; 1 + stride * height]; + let mut scalar = actual.clone(); + let mut expected = actual.clone(); + for y in 0..height { + for x in 0..width { + expected[1 + y * stride + x] = + equation_sample(request, (&source, 87, 83, 29), (x, y)) + .unwrap(); + } + } + interpolate_inter_with_rows( + request, + RefPlane::new(&source, 87, 83, 29).unwrap(), + RectMut::new(&mut actual[1..], stride, width, height).unwrap(), + &mut workspace[1..], + ) + .unwrap(); + inter::interpolate_inter_scalar( + request, + RefPlane::new(&source, 87, 83, 29).unwrap(), + RectMut::new(&mut scalar[1..], stride, width, height).unwrap(), + ) + .unwrap(); + assert_eq!( + actual, expected, + "{bits}/{component:?}/{width}/{request:?}" + ); + assert_eq!(actual, scalar); + assert_eq!(workspace[0], i32::MIN); + assert!(workspace[513..].iter().all(|&v| v == i32::MIN)); + } + } + } + } + } + } +} + +#[test] +fn reused_interpolation_rows_reject_short_storage_and_invalid_support_before_writes() { + for bits in [8, 10] { + let mut source = [0; 64]; + let mut workspace = [i32::MIN; 513]; + let mut output = [777; 4]; + for length in [0, 1, 511] { + assert_eq!( + interpolate_inter_with_rows( + config(bits, Component::Luma, 3, 3, 2, 2).unwrap(), + RefPlane::new(&source, 8, 8, 8).unwrap(), + RectMut::new(&mut output, 2, 2, 2).unwrap(), + &mut workspace[..length], + ), + Err(Error::Buffer) + ); + assert_eq!(output, [777; 4]); + assert_eq!(workspace, [i32::MIN; 513]); + } + source[63] = 1 << bits; + assert_eq!( + interpolate_inter_with_rows( + config(bits, Component::Luma, 3, 3, 2, 2).unwrap(), + RefPlane::new(&source, 8, 8, 8).unwrap(), + RectMut::new(&mut output, 2, 2, 2).unwrap(), + &mut workspace, + ), + Err(Error::Sample) + ); + assert_eq!(output, [777; 4]); + assert_eq!(workspace, [i32::MIN; 513]); + } +} + #[test] fn inter_scratch_fits_existing_reconstruction_envelope() { assert!(core::mem::size_of::() <= SCRATCH_BYTES); diff --git a/crates/coreav-kernels-hevc/src/lib.rs b/crates/coreav-kernels-hevc/src/lib.rs index caa4d5c4..70736098 100644 --- a/crates/coreav-kernels-hevc/src/lib.rs +++ b/crates/coreav-kernels-hevc/src/lib.rs @@ -57,8 +57,8 @@ pub use filter::{ pub use filter::{deblock_chroma_scalar, deblock_luma_scalar}; #[cfg(feature = "native-hevc")] pub use inter::{ - InterPrediction, MotionVector, PredictionWeight, RefPlane, interpolate_inter, predict_bi, - predict_uni, weight_bi, weight_uni, + InterPrediction, MotionVector, PredictionWeight, RefPlane, interpolate_inter, + interpolate_inter_with_rows, predict_bi, predict_uni, weight_bi, weight_uni, }; #[cfg(all(feature = "native-hevc", feature = "test-support"))] #[doc(hidden)] diff --git a/docs/design/hevc-interpolation-row-scratch.md b/docs/design/hevc-interpolation-row-scratch.md new file mode 100644 index 00000000..0141ea0b --- /dev/null +++ b/docs/design/hevc-interpolation-row-scratch.md @@ -0,0 +1,23 @@ +# HEVC interpolation row scratch + +The interpolator initializes an eight-by-64 `i32` row ring for each prediction block, then overwrites every active horizontal result before reading it. Keep the same ring and filters, but allow its backing samples to come from the decoder's existing residual workspace. No frame, worker or heap allocation is added. + +## Ownership and lifetime + +An additive checked kernel entry point accepts at least 512 mutable `i32` samples. The existing entry points keep their local initialized scratch and their error behavior. The new entry point uses only the first 512 samples; column indices and boundary source samples remain local. A flat ring row starts at `(logical_row % 8) * 64` and its active extent is the prediction width. + +The decoder's existing residual vector has 1,024 samples. Inter prediction runs before transform reconstruction and borrows that vector only during one synchronous kernel call. Before any later residual read, coded transforms write the full active residual through `inverse_residual`, and uncoded transforms fill that active extent with zero. Interpolation therefore cannot replace a live residual value. A mutable borrow prevents simultaneous predictor or worker use of the same storage. + +For every output row, the existing `next_row < y + row_count` loop writes every required horizontal row before the vertical pass reads it. A reused ring may contain arbitrary values elsewhere: filtering reads only the current prediction width, and each row is replaced before its slot is reused. Integer and horizontal-only paths keep their existing row copy. No numerical operation is deleted. + +## Preserved behavior + +H.265 clause 8.5.3.3.3 and Tables 8-8 and 8-9 still determine the same taps, coefficients, intermediate shifts and operation order at both supported depths. Full clipped-support depth validation still precedes scratch or output writes. Boundary replication, checked source extents, output padding, invalid-input rejection and the signed intermediate range are unchanged. No unsafe code is needed. + +Keep the existing 9,224-byte kernel scratch envelope and decoder admission calculations unchanged. `Frame`, its scratch owner and worker refresh gain no fields. Only existing residual samples are borrowed, so cold and retained memory do not grow. + +## Verification + +Compare reused, deliberately dirty scratch against the existing forced-scalar path and literal equations across both depths, luma/chroma phases, widths and tails, interior and edge motion, and successive calls with different shapes. Reject short workspace and invalid support before output writes, and retain unused workspace and output padding. Run complete automatic and forced-scalar conformance before shipping. + +Instruction evidence must count this row-only reuse in the real decoder against its exact baseline, with oracle samples and completed work checked in both arms. An earlier whole-scratch reuse probe is only motivation: it also removed column/source initialization and added fixed frame storage, neither of which this design does. Do not treat that probe's reduction as this implementation's result. Count the actual code before any timing comparison and disclose every selected timing cell.