From d9badf8a5bc8a2a56093dcf32da9001fc58fecf4 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Mon, 9 Nov 2026 05:04:44 +0900 Subject: [PATCH 1/3] perf(h264): gate the P intra 4x4 trial on inter rank Signed-off-by: Gunwoo Kim --- crates/coreav-h264-encoder/src/codec.rs | 78 ++++++++++++++++++++++--- crates/coreav-h264-encoder/src/inter.rs | 16 ++++- docs/design/h264-encoder-pipeline.md | 4 +- 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/crates/coreav-h264-encoder/src/codec.rs b/crates/coreav-h264-encoder/src/codec.rs index bd4c80b30..d1efef029 100644 --- a/crates/coreav-h264-encoder/src/codec.rs +++ b/crates/coreav-h264-encoder/src/codec.rs @@ -23,6 +23,7 @@ const STRIDE: usize = 21; const ORIGIN: usize = STRIDE + 1; const P_INTRA_TYPE_BASE: u32 = 5; const FINALISTS: usize = 2; +const INTRA4_INTER_MARGIN: u64 = 4; const UNRANKED: (u64, u8) = (u64::MAX, u8::MAX); type Ranked = [(u64, u8); 4]; @@ -901,7 +902,7 @@ impl Decision<'_> { base: &Tile, cbases: &[Tile; 2], a: IntraAvail, - four: bool, + scope: IntraScope, ) -> Result { let (lumas, chromas) = self.ranked_modes(base, cbases, a)?; let luma = *lumas @@ -914,7 +915,13 @@ impl Decision<'_> { .filter(|r| **r != UNRANKED) .ok_or(Error::Internal)? .1; - let four = if four { + let trial = match scope { + IntraScope::Quick(rank) => { + rank.is_none_or(|rank| rank.saturating_mul(INTRA4_INTER_MARGIN) > luma.0) + } + _ => false, + }; + let four = if trial { Some(intra4( base, self.source, @@ -969,7 +976,7 @@ impl Decision<'_> { } } if !exhaustive { - match self.quick_intra(base, cbases, a, scope == IntraScope::Quick) { + match self.quick_intra(base, cbases, a, scope) { Ok(candidate) => { consider(best, candidate); return Ok(()); @@ -1085,7 +1092,7 @@ impl Decision<'_> { pub(crate) enum IntraScope { None, Sixteen, - Quick, + Quick(Option), Exhaustive, } @@ -1481,10 +1488,14 @@ mod tests { pending_skip: 0, }; INTRA_PAIRS.with(|count| count.set(0)); - let candidate = decision.quick_intra(&base, &csources, a, true).unwrap(); + let candidate = decision + .quick_intra(&base, &csources, a, IntraScope::Quick(None)) + .unwrap(); selected_four |= candidate.luma.mode.is_none(); assert_eq!(INTRA_PAIRS.with(|count| count.get()), 1); - let sixteen = decision.quick_intra(&base, &csources, a, false).unwrap(); + let sixteen = decision + .quick_intra(&base, &csources, a, IntraScope::Sixteen) + .unwrap(); assert!(sixteen.luma.mode.is_some_and(|mode| legal(mode, 16, a))); if candidate.luma.mode.is_some() { assert!(sixteen.key == candidate.key); @@ -1493,6 +1504,50 @@ mod tests { assert!(selected_four); } + #[test] + fn p_intra_four_trial_needs_the_inter_rank_margin_over_intra16_satd() { + let base = Tile::empty(); + let csources = [Tile::empty(), Tile::empty()]; + let a = intra_availability(Border::default()); + let mut gated_four = false; + for pattern in 0..8 { + let mut source = Tile::empty(); + for y in 0..16 { + for x in 0..16 { + source.0[Tile::at(x, y)] = + ((x * (17 + pattern) + y * (7 + pattern * 3)) % 256) as u8; + } + } + let decision = Decision { + source: &source, + csources: &csources, + context: MbContext { + top: None, + left: None, + p_slice: true, + }, + qp: 28, + pending_skip: 0, + }; + let satd = decision.ranked_modes(&base, &csources, a).unwrap().0[0].0; + let quick = |scope| { + decision + .quick_intra(&base, &csources, a, scope) + .unwrap() + .key + }; + let open = decision + .quick_intra(&base, &csources, a, IntraScope::Quick(None)) + .unwrap(); + gated_four |= open.luma.mode.is_none(); + let sixteen = quick(IntraScope::Sixteen); + assert!(quick(IntraScope::Quick(Some(satd / INTRA4_INTER_MARGIN))) == sixteen); + assert!(quick(IntraScope::Quick(Some(satd / INTRA4_INTER_MARGIN + 1))) == open.key); + assert!(quick(IntraScope::Quick(Some(u64::MAX))) == open.key); + } + assert!(gated_four); + } + #[test] fn cheap_intra_completes_one_legal_pair_for_every_availability_and_qp_boundary() { let base = Tile::empty(); @@ -1523,7 +1578,7 @@ mod tests { &base, &csources, a, - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut Error::Coefficient, ) @@ -1568,7 +1623,12 @@ mod tests { }; let a = intra_availability(Border::default()); assert!(matches!( - decision.quick_intra(&base, &[base.clone(), base.clone()], a, true), + decision.quick_intra( + &base, + &[base.clone(), base.clone()], + a, + IntraScope::Quick(None) + ), Err(Error::MacroblockLimit) )); INTRA_PAIRS.with(|count| count.set(0)); @@ -1579,7 +1639,7 @@ mod tests { &base, &[base.clone(), base.clone()], a, - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut refusal, ) diff --git a/crates/coreav-h264-encoder/src/inter.rs b/crates/coreav-h264-encoder/src/inter.rs index f9818c954..b4fcf0ac3 100644 --- a/crates/coreav-h264-encoder/src/inter.rs +++ b/crates/coreav-h264-encoder/src/inter.rs @@ -877,6 +877,11 @@ impl Search<'_> { } else { None }; + let rank = [whole.as_ref(), split.as_ref()] + .into_iter() + .flatten() + .map(|plan| plan.2) + .min(); let mut plans = [whole, split]; plans.sort_by_key(|plan| plan.as_ref().map_or(usize::MAX, |plan| plan.0)); let [first, second] = &mut plans; @@ -912,7 +917,7 @@ impl Search<'_> { Ok(if settled { IntraScope::Sixteen } else { - IntraScope::Quick + IntraScope::Quick(rank.filter(|_| best.is_some())) }) } } @@ -1271,7 +1276,12 @@ mod tests { consider( &mut expected, decision - .quick_intra(&base, &csources, intra_availability(search.border), true) + .quick_intra( + &base, + &csources, + intra_availability(search.border), + IntraScope::Quick(None), + ) .unwrap(), ); let mut best = Some(search.evaluate(skip).unwrap()); @@ -1281,7 +1291,7 @@ mod tests { &base, &csources, intra_availability(search.border), - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut Error::Coefficient, ) diff --git a/docs/design/h264-encoder-pipeline.md b/docs/design/h264-encoder-pipeline.md index 38c87191d..b1cadfd35 100644 --- a/docs/design/h264-encoder-pipeline.md +++ b/docs/design/h264-encoder-pipeline.md @@ -26,7 +26,7 @@ Each step is one pull request with one outcome, in this order. Before a step is 1. **IDR cost ladder (output change).** For each I4x4 block, rank the legal modes by SATD plus mode bits and keep the exact trials of the best two that complete; a trial whose levels leave the coefficient range is not a finalist. Rank the I16x16 and chroma modes by SATD and complete only the best two of each, so an IDR macroblock normally builds at most six complete candidates. When none of them passes macroblock admission, the macroblock falls back to the complete search, which can build more, so the shortlist never refuses a macroblock the complete search accepts. P pictures keep the complete fallback. 2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. -3. **P intra gating (output change, revised).** Comparing the best I16x16 SATD with the inter SATD cost proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The revised step gates on an I4x4-aware estimate (for example SATD from neighbouring source samples, without reconstruction), or keeps the cheap proposal and skips only the exact build that follows it. The gate applies only once an inter candidate has been admitted; without one, the complete intra search runs as now, so admission never depends on SATD. It is measured again before it is built. +3. **P intra gating (output change, margin gate).** Comparing the best I16x16 SATD with the inter SATD cost directly proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The shipped gate keeps the cheap I16x16 proposal and skips only the cheap I4x4 proposal, and only when four times the best inter plan's SATD rank (16x16 or 8x8, motion cost included) is at most the best I16x16 SATD. With a factor of one or two, an I16x16-based gate still lost I4x4 decisions on one input; with four, every input and plane stayed within +0.5% PSNR BD-rate of main. The gate applies only once an inter candidate has been admitted; without one, the I4x4 proposal runs as before, and a rejected cheap proposal still falls back to the complete search, so admission never depends on SATD. 4. **Integer search early termination (dropped).** Stopping the integer refinement once the best integer cost falls below a QP-scaled bound saved little time, because integer probes are SAD-only and cheap next to fractional probes and exact evaluations, and bounds high enough to matter cost rate on real motion. Fractional-probe cost is addressed by step 8 instead. 5. **Narrow transform path (byte-identical).** Forward transform, quantization, scaling and inverse transform on 32-bit fixed arrays, with the residual taken in the same pass and range checks at the block boundary instead of per call. 6. **Edge-extended reference planes (byte-identical).** Pad each reference picture once so motion compensation and SAD read rows without clamping, and keep candidate samples in reusable per-macroblock scratch instead of moved structures. @@ -48,7 +48,7 @@ The stage attribution suggests, without establishing a majority for any one stag |---|---|---|---|---|---| | 1 | Vector six-tap filter and quarter-sample averages over the half-sample planes | Identical | Step 8 | Largest on real motion: fractional refinement is the biggest part of motion search, and the plane pass becomes one streaming filter per picture | Obvious, under the [encoder kernel policy](encoder-policy.md#0044-simd-kernels-for-the-opt-in-h264-encoder) | | 2 | Step 10, P finalist shortlist | Changes | Steps 2 and 6 | Large on every input: complete P inter evaluation is the second-largest P stage | Obvious | -| 3 | Revised step 3, I4x4-aware P intra gate | Changes | None | Large on low motion, where the P intra proposal costs about as much as motion search | Obvious; measured before it is built | +| 3 | An I4x4-aware P intra gate that also covers the macroblocks step 3's margin gate leaves open | Changes | Step 3 | Medium on low motion, where the remaining P intra proposals cost about as much as motion search | Obvious; measured before it is built | | 4 | Vector SATD for 4x4 and 8x8 Hadamard sums | Identical | None | Medium, and growing as steps 1, 2, 3 and 10 move decisions to SATD ranking | Obvious, under the kernel policy | | 5 | Vector forward transform, quantization, inverse scaling and inverse transform on the 32-bit path | Identical | Step 5 | Medium on every input: these leaves run in every exact evaluation | Obvious, under the kernel policy | | 6 | Step 7, exact early quiet test | Identical | Step 5 | Medium on low motion, small on real motion | Obvious | From 09f3dd9c1c694ce05dc0690065d4debd8b80bd22 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Mon, 9 Nov 2026 05:28:56 +0900 Subject: [PATCH 2/3] perf(h264): raise the split bound for settled macroblocks Signed-off-by: Gunwoo Kim --- crates/coreav-h264-encoder/src/inter.rs | 77 ++++++++++++++++++++++++- docs/design/h264-encoder-pipeline.md | 2 +- 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/crates/coreav-h264-encoder/src/inter.rs b/crates/coreav-h264-encoder/src/inter.rs index b4fcf0ac3..c105fed96 100644 --- a/crates/coreav-h264-encoder/src/inter.rs +++ b/crates/coreav-h264-encoder/src/inter.rs @@ -5,6 +5,7 @@ use crate::motion::{Layout, Part}; use coreav_kernels_h264::Block; const SPLIT_SATD: u64 = 64; +const SETTLED_SPLIT_SATD: u64 = 2 * SPLIT_SATD; #[derive(Clone, PartialEq, Eq)] pub(super) struct Inter { @@ -869,9 +870,14 @@ impl Search<'_> { } let ceiling = best.as_ref().map(|candidate| candidate.key.cost); let whole = self.propose(Layout::Whole, ceiling)?; + let bound = if settled { + SETTLED_SPLIT_SATD + } else { + SPLIT_SATD + }; let split = if whole .as_ref() - .is_none_or(|plan| plan.2 > SPLIT_SATD << (self.decision.qp / 6)) + .is_none_or(|plan| plan.2 > bound << (self.decision.qp / 6)) { self.propose(Layout::Sub([0; 4]), ceiling)? } else { @@ -1148,6 +1154,75 @@ mod tests { } } + #[test] + fn settled_macroblocks_need_twice_the_satd_before_the_split_search() { + let mut reference = [ + FramePlane::new(16, 16).unwrap(), + FramePlane::new(8, 8).unwrap(), + FramePlane::new(8, 8).unwrap(), + ]; + for plane in &mut reference { + plane.data.fill(128); + } + let csources = [Tile::empty(), Tile::empty()]; + let mut settled_between_bounds = false; + for qp in [22, 28] { + for (noise, spike) in (0..8).flat_map(|noise| [0, 4, 8, 12].map(|spike| (noise, spike))) + { + let mut state = 99u32; + let mut source = Tile::empty(); + for y in 0..16 { + for x in 0..16 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let sample = (state % (2 * noise + 1)) as i32 - noise as i32; + let peak = if x < 4 && y < 4 && (x + y) % 2 == 0 { + spike + } else { + 0 + }; + source.0[Tile::at(x, y)] = (128 + sample + peak) as u8; + } + } + let decision = Decision { + source: &source, + csources: &csources, + context: MbContext { + top: None, + left: None, + p_slice: true, + }, + qp, + pending_skip: 0, + }; + let search = Search { + reference: &reference, + decision: &decision, + border: Border::default(), + mx: 0, + my: 0, + previous: 0, + }; + let whole = search.propose(Layout::Whole, None).unwrap().unwrap(); + SEARCHES.with(|count| count.set(0)); + let scope = search + .candidates(&mut None, &mut Error::Coefficient) + .unwrap(); + let settled = scope == IntraScope::Sixteen; + let bound = if settled { 2 * SPLIT_SATD } else { SPLIT_SATD } << (qp / 6); + let expected = match scope { + IntraScope::None => 0, + _ => 1 + usize::from(whole.2 > bound), + }; + assert_eq!(SEARCHES.with(|count| count.get()), expected); + settled_between_bounds |= + settled && whole.2 > SPLIT_SATD << (qp / 6) && whole.2 <= bound; + } + } + assert!(settled_between_bounds); + } + #[test] fn inter_syntax_floors_preserve_equal_bounds_and_skip_noncompetitive_trials() { let mut reference = [ diff --git a/docs/design/h264-encoder-pipeline.md b/docs/design/h264-encoder-pipeline.md index b1cadfd35..727bc63cf 100644 --- a/docs/design/h264-encoder-pipeline.md +++ b/docs/design/h264-encoder-pipeline.md @@ -25,7 +25,7 @@ Each P macroblock passes a cost ladder: a cheap exact test settles most skips wi Each step is one pull request with one outcome, in this order. Before a step is built, its ceiling is counted in a throwaway build that deletes the work it removes; a step whose ceiling is negligible is dropped. Byte-identical steps must reproduce the current main stream and reconstruction in every case of both identity sets (`scripts/h264_g1_simd_identity.py` and the 57-case set), in dispatched and scalar builds. Output-changing steps report PSNR BD-rate against current main over QP 22, 28, 34 and 40 on A, B, C and D (PCHIP on log rate, common interval, no extrapolation), regenerate any byte fixture they change in the same pull request, and decode with the pinned FFmpeg to the encoder reconstruction. Throughput is an encoder-only reference figure from one GitHub-hosted job that runs current main and the candidate alternately, with instruction counts beside it. 1. **IDR cost ladder (output change).** For each I4x4 block, rank the legal modes by SATD plus mode bits and keep the exact trials of the best two that complete; a trial whose levels leave the coefficient range is not a finalist. Rank the I16x16 and chroma modes by SATD and complete only the best two of each, so an IDR macroblock normally builds at most six complete candidates. When none of them passes macroblock admission, the macroblock falls back to the complete search, which can build more, so the shortlist never refuses a macroblock the complete search accepts. P pictures keep the complete fallback. -2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. +2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. When coding the residual at the skip vector did not beat skip, the macroblock is already well predicted, so the bound doubles. Raising the bound for every macroblock instead cost one real-motion input more than +1.5% PSNR BD-rate per plane even at 1.25 times the bound, while the doubled bound for these macroblocks alone kept every input and plane within +0.5% of main. 3. **P intra gating (output change, margin gate).** Comparing the best I16x16 SATD with the inter SATD cost directly proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The shipped gate keeps the cheap I16x16 proposal and skips only the cheap I4x4 proposal, and only when four times the best inter plan's SATD rank (16x16 or 8x8, motion cost included) is at most the best I16x16 SATD. With a factor of one or two, an I16x16-based gate still lost I4x4 decisions on one input; with four, every input and plane stayed within +0.5% PSNR BD-rate of main. The gate applies only once an inter candidate has been admitted; without one, the I4x4 proposal runs as before, and a rejected cheap proposal still falls back to the complete search, so admission never depends on SATD. 4. **Integer search early termination (dropped).** Stopping the integer refinement once the best integer cost falls below a QP-scaled bound saved little time, because integer probes are SAD-only and cheap next to fractional probes and exact evaluations, and bounds high enough to matter cost rate on real motion. Fractional-probe cost is addressed by step 8 instead. 5. **Narrow transform path (byte-identical).** Forward transform, quantization, scaling and inverse transform on 32-bit fixed arrays, with the residual taken in the same pass and range checks at the block boundary instead of per call. From 4550cf9ce14c98500f759b46aeea65d3c6cb1963 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Mon, 9 Nov 2026 05:29:07 +0900 Subject: [PATCH 3/3] docs(h264): state the measured split-bound cost Signed-off-by: Gunwoo Kim --- docs/design/h264-encoder-pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/h264-encoder-pipeline.md b/docs/design/h264-encoder-pipeline.md index 727bc63cf..766d5b4b7 100644 --- a/docs/design/h264-encoder-pipeline.md +++ b/docs/design/h264-encoder-pipeline.md @@ -25,7 +25,7 @@ Each P macroblock passes a cost ladder: a cheap exact test settles most skips wi Each step is one pull request with one outcome, in this order. Before a step is built, its ceiling is counted in a throwaway build that deletes the work it removes; a step whose ceiling is negligible is dropped. Byte-identical steps must reproduce the current main stream and reconstruction in every case of both identity sets (`scripts/h264_g1_simd_identity.py` and the 57-case set), in dispatched and scalar builds. Output-changing steps report PSNR BD-rate against current main over QP 22, 28, 34 and 40 on A, B, C and D (PCHIP on log rate, common interval, no extrapolation), regenerate any byte fixture they change in the same pull request, and decode with the pinned FFmpeg to the encoder reconstruction. Throughput is an encoder-only reference figure from one GitHub-hosted job that runs current main and the candidate alternately, with instruction counts beside it. 1. **IDR cost ladder (output change).** For each I4x4 block, rank the legal modes by SATD plus mode bits and keep the exact trials of the best two that complete; a trial whose levels leave the coefficient range is not a finalist. Rank the I16x16 and chroma modes by SATD and complete only the best two of each, so an IDR macroblock normally builds at most six complete candidates. When none of them passes macroblock admission, the macroblock falls back to the complete search, which can build more, so the shortlist never refuses a macroblock the complete search accepts. P pictures keep the complete fallback. -2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. When coding the residual at the skip vector did not beat skip, the macroblock is already well predicted, so the bound doubles. Raising the bound for every macroblock instead cost one real-motion input more than +1.5% PSNR BD-rate per plane even at 1.25 times the bound, while the doubled bound for these macroblocks alone kept every input and plane within +0.5% of main. +2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. When coding the residual at the skip vector did not beat skip, the macroblock is already well predicted, so the bound doubles. Raising the bound for every macroblock instead cost one real-motion input up to +1.7% PSNR BD-rate on a plane even at 1.25 times the bound, while the doubled bound for these macroblocks alone kept every input and plane within +0.5% of main. 3. **P intra gating (output change, margin gate).** Comparing the best I16x16 SATD with the inter SATD cost directly proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The shipped gate keeps the cheap I16x16 proposal and skips only the cheap I4x4 proposal, and only when four times the best inter plan's SATD rank (16x16 or 8x8, motion cost included) is at most the best I16x16 SATD. With a factor of one or two, an I16x16-based gate still lost I4x4 decisions on one input; with four, every input and plane stayed within +0.5% PSNR BD-rate of main. The gate applies only once an inter candidate has been admitted; without one, the I4x4 proposal runs as before, and a rejected cheap proposal still falls back to the complete search, so admission never depends on SATD. 4. **Integer search early termination (dropped).** Stopping the integer refinement once the best integer cost falls below a QP-scaled bound saved little time, because integer probes are SAD-only and cheap next to fractional probes and exact evaluations, and bounds high enough to matter cost rate on real motion. Fractional-probe cost is addressed by step 8 instead. 5. **Narrow transform path (byte-identical).** Forward transform, quantization, scaling and inverse transform on 32-bit fixed arrays, with the residual taken in the same pass and range checks at the block boundary instead of per call.