diff --git a/crates/coreav-h264-encoder/src/codec.rs b/crates/coreav-h264-encoder/src/codec.rs index bd4c80b3..d1efef02 100644 --- a/crates/coreav-h264-encoder/src/codec.rs +++ b/crates/coreav-h264-encoder/src/codec.rs @@ -23,6 +23,7 @@ const STRIDE: usize = 21; const ORIGIN: usize = STRIDE + 1; const P_INTRA_TYPE_BASE: u32 = 5; const FINALISTS: usize = 2; +const INTRA4_INTER_MARGIN: u64 = 4; const UNRANKED: (u64, u8) = (u64::MAX, u8::MAX); type Ranked = [(u64, u8); 4]; @@ -901,7 +902,7 @@ impl Decision<'_> { base: &Tile, cbases: &[Tile; 2], a: IntraAvail, - four: bool, + scope: IntraScope, ) -> Result { let (lumas, chromas) = self.ranked_modes(base, cbases, a)?; let luma = *lumas @@ -914,7 +915,13 @@ impl Decision<'_> { .filter(|r| **r != UNRANKED) .ok_or(Error::Internal)? .1; - let four = if four { + let trial = match scope { + IntraScope::Quick(rank) => { + rank.is_none_or(|rank| rank.saturating_mul(INTRA4_INTER_MARGIN) > luma.0) + } + _ => false, + }; + let four = if trial { Some(intra4( base, self.source, @@ -969,7 +976,7 @@ impl Decision<'_> { } } if !exhaustive { - match self.quick_intra(base, cbases, a, scope == IntraScope::Quick) { + match self.quick_intra(base, cbases, a, scope) { Ok(candidate) => { consider(best, candidate); return Ok(()); @@ -1085,7 +1092,7 @@ impl Decision<'_> { pub(crate) enum IntraScope { None, Sixteen, - Quick, + Quick(Option), Exhaustive, } @@ -1481,10 +1488,14 @@ mod tests { pending_skip: 0, }; INTRA_PAIRS.with(|count| count.set(0)); - let candidate = decision.quick_intra(&base, &csources, a, true).unwrap(); + let candidate = decision + .quick_intra(&base, &csources, a, IntraScope::Quick(None)) + .unwrap(); selected_four |= candidate.luma.mode.is_none(); assert_eq!(INTRA_PAIRS.with(|count| count.get()), 1); - let sixteen = decision.quick_intra(&base, &csources, a, false).unwrap(); + let sixteen = decision + .quick_intra(&base, &csources, a, IntraScope::Sixteen) + .unwrap(); assert!(sixteen.luma.mode.is_some_and(|mode| legal(mode, 16, a))); if candidate.luma.mode.is_some() { assert!(sixteen.key == candidate.key); @@ -1493,6 +1504,50 @@ mod tests { assert!(selected_four); } + #[test] + fn p_intra_four_trial_needs_the_inter_rank_margin_over_intra16_satd() { + let base = Tile::empty(); + let csources = [Tile::empty(), Tile::empty()]; + let a = intra_availability(Border::default()); + let mut gated_four = false; + for pattern in 0..8 { + let mut source = Tile::empty(); + for y in 0..16 { + for x in 0..16 { + source.0[Tile::at(x, y)] = + ((x * (17 + pattern) + y * (7 + pattern * 3)) % 256) as u8; + } + } + let decision = Decision { + source: &source, + csources: &csources, + context: MbContext { + top: None, + left: None, + p_slice: true, + }, + qp: 28, + pending_skip: 0, + }; + let satd = decision.ranked_modes(&base, &csources, a).unwrap().0[0].0; + let quick = |scope| { + decision + .quick_intra(&base, &csources, a, scope) + .unwrap() + .key + }; + let open = decision + .quick_intra(&base, &csources, a, IntraScope::Quick(None)) + .unwrap(); + gated_four |= open.luma.mode.is_none(); + let sixteen = quick(IntraScope::Sixteen); + assert!(quick(IntraScope::Quick(Some(satd / INTRA4_INTER_MARGIN))) == sixteen); + assert!(quick(IntraScope::Quick(Some(satd / INTRA4_INTER_MARGIN + 1))) == open.key); + assert!(quick(IntraScope::Quick(Some(u64::MAX))) == open.key); + } + assert!(gated_four); + } + #[test] fn cheap_intra_completes_one_legal_pair_for_every_availability_and_qp_boundary() { let base = Tile::empty(); @@ -1523,7 +1578,7 @@ mod tests { &base, &csources, a, - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut Error::Coefficient, ) @@ -1568,7 +1623,12 @@ mod tests { }; let a = intra_availability(Border::default()); assert!(matches!( - decision.quick_intra(&base, &[base.clone(), base.clone()], a, true), + decision.quick_intra( + &base, + &[base.clone(), base.clone()], + a, + IntraScope::Quick(None) + ), Err(Error::MacroblockLimit) )); INTRA_PAIRS.with(|count| count.set(0)); @@ -1579,7 +1639,7 @@ mod tests { &base, &[base.clone(), base.clone()], a, - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut refusal, ) diff --git a/crates/coreav-h264-encoder/src/inter.rs b/crates/coreav-h264-encoder/src/inter.rs index f9818c95..b4fcf0ac 100644 --- a/crates/coreav-h264-encoder/src/inter.rs +++ b/crates/coreav-h264-encoder/src/inter.rs @@ -877,6 +877,11 @@ impl Search<'_> { } else { None }; + let rank = [whole.as_ref(), split.as_ref()] + .into_iter() + .flatten() + .map(|plan| plan.2) + .min(); let mut plans = [whole, split]; plans.sort_by_key(|plan| plan.as_ref().map_or(usize::MAX, |plan| plan.0)); let [first, second] = &mut plans; @@ -912,7 +917,7 @@ impl Search<'_> { Ok(if settled { IntraScope::Sixteen } else { - IntraScope::Quick + IntraScope::Quick(rank.filter(|_| best.is_some())) }) } } @@ -1271,7 +1276,12 @@ mod tests { consider( &mut expected, decision - .quick_intra(&base, &csources, intra_availability(search.border), true) + .quick_intra( + &base, + &csources, + intra_availability(search.border), + IntraScope::Quick(None), + ) .unwrap(), ); let mut best = Some(search.evaluate(skip).unwrap()); @@ -1281,7 +1291,7 @@ mod tests { &base, &csources, intra_availability(search.border), - IntraScope::Quick, + IntraScope::Quick(None), &mut best, &mut Error::Coefficient, ) diff --git a/docs/design/h264-encoder-pipeline.md b/docs/design/h264-encoder-pipeline.md index 38c87191..b1cadfd3 100644 --- a/docs/design/h264-encoder-pipeline.md +++ b/docs/design/h264-encoder-pipeline.md @@ -26,7 +26,7 @@ Each step is one pull request with one outcome, in this order. Before a step is 1. **IDR cost ladder (output change).** For each I4x4 block, rank the legal modes by SATD plus mode bits and keep the exact trials of the best two that complete; a trial whose levels leave the coefficient range is not a finalist. Rank the I16x16 and chroma modes by SATD and complete only the best two of each, so an IDR macroblock normally builds at most six complete candidates. When none of them passes macroblock admission, the macroblock falls back to the complete search, which can build more, so the shortlist never refuses a macroblock the complete search accepts. P pictures keep the complete fallback. 2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. -3. **P intra gating (output change, revised).** Comparing the best I16x16 SATD with the inter SATD cost proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The revised step gates on an I4x4-aware estimate (for example SATD from neighbouring source samples, without reconstruction), or keeps the cheap proposal and skips only the exact build that follows it. The gate applies only once an inter candidate has been admitted; without one, the complete intra search runs as now, so admission never depends on SATD. It is measured again before it is built. +3. **P intra gating (output change, margin gate).** Comparing the best I16x16 SATD with the inter SATD cost directly proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The shipped gate keeps the cheap I16x16 proposal and skips only the cheap I4x4 proposal, and only when four times the best inter plan's SATD rank (16x16 or 8x8, motion cost included) is at most the best I16x16 SATD. With a factor of one or two, an I16x16-based gate still lost I4x4 decisions on one input; with four, every input and plane stayed within +0.5% PSNR BD-rate of main. The gate applies only once an inter candidate has been admitted; without one, the I4x4 proposal runs as before, and a rejected cheap proposal still falls back to the complete search, so admission never depends on SATD. 4. **Integer search early termination (dropped).** Stopping the integer refinement once the best integer cost falls below a QP-scaled bound saved little time, because integer probes are SAD-only and cheap next to fractional probes and exact evaluations, and bounds high enough to matter cost rate on real motion. Fractional-probe cost is addressed by step 8 instead. 5. **Narrow transform path (byte-identical).** Forward transform, quantization, scaling and inverse transform on 32-bit fixed arrays, with the residual taken in the same pass and range checks at the block boundary instead of per call. 6. **Edge-extended reference planes (byte-identical).** Pad each reference picture once so motion compensation and SAD read rows without clamping, and keep candidate samples in reusable per-macroblock scratch instead of moved structures. @@ -48,7 +48,7 @@ The stage attribution suggests, without establishing a majority for any one stag |---|---|---|---|---|---| | 1 | Vector six-tap filter and quarter-sample averages over the half-sample planes | Identical | Step 8 | Largest on real motion: fractional refinement is the biggest part of motion search, and the plane pass becomes one streaming filter per picture | Obvious, under the [encoder kernel policy](encoder-policy.md#0044-simd-kernels-for-the-opt-in-h264-encoder) | | 2 | Step 10, P finalist shortlist | Changes | Steps 2 and 6 | Large on every input: complete P inter evaluation is the second-largest P stage | Obvious | -| 3 | Revised step 3, I4x4-aware P intra gate | Changes | None | Large on low motion, where the P intra proposal costs about as much as motion search | Obvious; measured before it is built | +| 3 | An I4x4-aware P intra gate that also covers the macroblocks step 3's margin gate leaves open | Changes | Step 3 | Medium on low motion, where the remaining P intra proposals cost about as much as motion search | Obvious; measured before it is built | | 4 | Vector SATD for 4x4 and 8x8 Hadamard sums | Identical | None | Medium, and growing as steps 1, 2, 3 and 10 move decisions to SATD ranking | Obvious, under the kernel policy | | 5 | Vector forward transform, quantization, inverse scaling and inverse transform on the 32-bit path | Identical | Step 5 | Medium on every input: these leaves run in every exact evaluation | Obvious, under the kernel policy | | 6 | Step 7, exact early quiet test | Identical | Step 5 | Medium on low motion, small on real motion | Obvious |