perf(h264): raise the split bound for settled macroblocks

crates/coreav-h264-encoder/src/inter.rs

@@ -5,6 +5,7 @@ use crate::motion::{Layout, Part};
 use coreav_kernels_h264::Block;
 
 const SPLIT_SATD: u64 = 64;
+const SETTLED_SPLIT_SATD: u64 = 2 * SPLIT_SATD;
 
 #[derive(Clone, PartialEq, Eq)]
 pub(super) struct Inter {
@@ -869,9 +870,14 @@ impl Search<'_> {
         }
         let ceiling = best.as_ref().map(|candidate| candidate.key.cost);
         let whole = self.propose(Layout::Whole, ceiling)?;
+        let bound = if settled {
+            SETTLED_SPLIT_SATD
+        } else {
+            SPLIT_SATD
+        };
         let split = if whole
             .as_ref()
-            .is_none_or(|plan| plan.2 > SPLIT_SATD << (self.decision.qp / 6))
+            .is_none_or(|plan| plan.2 > bound << (self.decision.qp / 6))
         {
             self.propose(Layout::Sub([0; 4]), ceiling)?
         } else {
@@ -1148,6 +1154,75 @@ mod tests {
         }
     }
 
+    #[test]
+    fn settled_macroblocks_need_twice_the_satd_before_the_split_search() {
+        let mut reference = [
+            FramePlane::new(16, 16).unwrap(),
+            FramePlane::new(8, 8).unwrap(),
+            FramePlane::new(8, 8).unwrap(),
+        ];
+        for plane in &mut reference {
+            plane.data.fill(128);
+        }
+        let csources = [Tile::empty(), Tile::empty()];
+        let mut settled_between_bounds = false;
+        for qp in [22, 28] {
+            for (noise, spike) in (0..8).flat_map(|noise| [0, 4, 8, 12].map(|spike| (noise, spike)))
+            {
+                let mut state = 99u32;
+                let mut source = Tile::empty();
+                for y in 0..16 {
+                    for x in 0..16 {
+                        state ^= state << 13;
+                        state ^= state >> 17;
+                        state ^= state << 5;
+                        let sample = (state % (2 * noise + 1)) as i32 - noise as i32;
+                        let peak = if x < 4 && y < 4 && (x + y) % 2 == 0 {
+                            spike
+                        } else {
+                            0
+                        };
+                        source.0[Tile::at(x, y)] = (128 + sample + peak) as u8;
+                    }
+                }
+                let decision = Decision {
+                    source: &source,
+                    csources: &csources,
+                    context: MbContext {
+                        top: None,
+                        left: None,
+                        p_slice: true,
+                    },
+                    qp,
+                    pending_skip: 0,
+                };
+                let search = Search {
+                    reference: &reference,
+                    decision: &decision,
+                    border: Border::default(),
+                    mx: 0,
+                    my: 0,
+                    previous: 0,
+                };
+                let whole = search.propose(Layout::Whole, None).unwrap().unwrap();
+                SEARCHES.with(|count| count.set(0));
+                let scope = search
+                    .candidates(&mut None, &mut Error::Coefficient)
+                    .unwrap();
+                let settled = scope == IntraScope::Sixteen;
+                let bound = if settled { 2 * SPLIT_SATD } else { SPLIT_SATD } << (qp / 6);
+                let expected = match scope {
+                    IntraScope::None => 0,
+                    _ => 1 + usize::from(whole.2 > bound),
+                };
+                assert_eq!(SEARCHES.with(|count| count.get()), expected);
+                settled_between_bounds |=
+                    settled && whole.2 > SPLIT_SATD << (qp / 6) && whole.2 <= bound;
+            }
+        }
+        assert!(settled_between_bounds);
+    }
+
     #[test]
     fn inter_syntax_floors_preserve_equal_bounds_and_skip_noncompetitive_trials() {
         let mut reference = [

docs/design/h264-encoder-pipeline.md

@@ -25,7 +25,7 @@ Each P macroblock passes a cost ladder: a cheap exact test settles most skips wi
 Each step is one pull request with one outcome, in this order. Before a step is built, its ceiling is counted in a throwaway build that deletes the work it removes; a step whose ceiling is negligible is dropped. Byte-identical steps must reproduce the current main stream and reconstruction in every case of both identity sets (`scripts/h264_g1_simd_identity.py` and the 57-case set), in dispatched and scalar builds. Output-changing steps report PSNR BD-rate against current main over QP 22, 28, 34 and 40 on A, B, C and D (PCHIP on log rate, common interval, no extrapolation), regenerate any byte fixture they change in the same pull request, and decode with the pinned FFmpeg to the encoder reconstruction. Throughput is an encoder-only reference figure from one GitHub-hosted job that runs current main and the candidate alternately, with instruction counts beside it.
 
 1. **IDR cost ladder (output change).** For each I4x4 block, rank the legal modes by SATD plus mode bits and keep the exact trials of the best two that complete; a trial whose levels leave the coefficient range is not a finalist. Rank the I16x16 and chroma modes by SATD and complete only the best two of each, so an IDR macroblock normally builds at most six complete candidates. When none of them passes macroblock admission, the macroblock falls back to the complete search, which can build more, so the shortlist never refuses a macroblock the complete search accepts. P pictures keep the complete fallback.
-2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound.
+2. **P partition gating (output change).** Search the four 8x8 partitions only when the 16x16 result leaves a SATD above a QP-scaled bound. When coding the residual at the skip vector did not beat skip, the macroblock is already well predicted, so the bound doubles. Raising the bound for every macroblock instead cost one real-motion input up to +1.7% PSNR BD-rate on a plane even at 1.25 times the bound, while the doubled bound for these macroblocks alone kept every input and plane within +0.5% of main.
 3. **P intra gating (output change, margin gate).** Comparing the best I16x16 SATD with the inter SATD cost directly proved unsafe: the few P macroblocks that intra wins are mostly I4x4 decisions, which an I16x16-only estimate cannot see, so the gate dropped them and cost rate. The shipped gate keeps the cheap I16x16 proposal and skips only the cheap I4x4 proposal, and only when four times the best inter plan's SATD rank (16x16 or 8x8, motion cost included) is at most the best I16x16 SATD. With a factor of one or two, an I16x16-based gate still lost I4x4 decisions on one input; with four, every input and plane stayed within +0.5% PSNR BD-rate of main. The gate applies only once an inter candidate has been admitted; without one, the I4x4 proposal runs as before, and a rejected cheap proposal still falls back to the complete search, so admission never depends on SATD.
 4. **Integer search early termination (dropped).** Stopping the integer refinement once the best integer cost falls below a QP-scaled bound saved little time, because integer probes are SAD-only and cheap next to fractional probes and exact evaluations, and bounds high enough to matter cost rate on real motion. Fractional-probe cost is addressed by step 8 instead.
 5. **Narrow transform path (byte-identical).** Forward transform, quantization, scaling and inverse transform on 32-bit fixed arrays, with the residual taken in the same pass and range checks at the block boundary instead of per call.