New solver opts for diesel (part 3)

compiler/rustc_infer/src/infer/at.rs

@@ -88,7 +88,6 @@ impl<'tcx> InferCtxt<'tcx> {
             next_trait_solver: self.next_trait_solver,
             enable_next_solver_overflow_fcw: self.enable_next_solver_overflow_fcw.clone(),
             obligation_inspector: self.obligation_inspector.clone(),
-            canonicalizer_state: Default::default(),
         }
     }
 
@@ -119,7 +118,6 @@ impl<'tcx> InferCtxt<'tcx> {
             next_trait_solver: self.next_trait_solver,
             enable_next_solver_overflow_fcw: self.enable_next_solver_overflow_fcw.clone(),
             obligation_inspector: self.obligation_inspector.clone(),
-            canonicalizer_state: Default::default(),
         };
         forked.inner.borrow_mut().projection_cache().clear();
         forked

compiler/rustc_infer/src/infer/canonical/mod.rs

@@ -67,10 +67,13 @@ impl<'tcx> InferCtxt<'tcx> {
             .chain((1..=canonical.max_universe.as_u32()).map(|_| self.create_next_universe()))
             .collect();
 
-        let var_values =
-            CanonicalVarValues::instantiate(self.tcx, &canonical.var_kinds, |var_values, info| {
-                self.instantiate_canonical_var(span, info, &var_values, |ui| universes[ui])
-            });
+        let var_values = CanonicalVarValues::instantiate(
+            self.tcx,
+            &canonical.var_kinds,
+            |prev_var_values, info| {
+                self.instantiate_canonical_var(span, info, &prev_var_values, |ui| universes[ui])
+            },
+        );
         let result = canonical.instantiate(self.tcx, &var_values);
         (result, var_values)
     }
@@ -87,15 +90,15 @@ impl<'tcx> InferCtxt<'tcx> {
         &self,
         span: Span,
         kind: CanonicalVarKind<'tcx>,
-        previous_var_values: &[GenericArg<'tcx>],
+        prev_var_values: &[GenericArg<'tcx>],
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> GenericArg<'tcx> {
         match kind {
             CanonicalVarKind::Ty { ui, sub_root } => {
                 let vid = self.next_ty_vid_in_universe(span, universe_map(ui));
                 // If this inference variable is related to an earlier variable
                 // via subtyping, we need to add that info to the inference context.
-                if let Some(prev) = previous_var_values.get(sub_root.as_usize()) {
+                if let Some(prev) = prev_var_values.get(sub_root.as_usize()) {
                     if let &ty::Infer(ty::TyVar(sub_root)) = prev.expect_ty().kind() {
                         self.sub_unify_ty_vids_raw(vid, sub_root);
                     } else {

compiler/rustc_infer/src/infer/canonical/query_response.rs

@@ -488,27 +488,31 @@ impl<'tcx> InferCtxt<'tcx> {
         // a fresh inference variable.
         let tcx = self.tcx;
         let var_kinds = query_response.var_kinds;
-        let var_values = CanonicalVarValues::instantiate(tcx, var_kinds, |var_values, kind| {
-            if kind.universe() != ty::UniverseIndex::ROOT {
-                // A variable from inside a binder of the query. While ideally these shouldn't
-                // exist at all, we have to deal with them for now.
-                self.instantiate_canonical_var(cause.span, kind, &var_values, |u| {
-                    universe_map[u.as_usize()]
-                })
-            } else if kind.is_existential() {
-                match opt_values[BoundVar::new(var_values.len())] {
-                    Some(k) => k,
-                    None => self.instantiate_canonical_var(cause.span, kind, &var_values, |u| {
+        let var_values =
+            CanonicalVarValues::instantiate(tcx, var_kinds, |prev_var_values, kind| {
+                if kind.universe() != ty::UniverseIndex::ROOT {
+                    // A variable from inside a binder of the query. While ideally these shouldn't
+                    // exist at all, we have to deal with them for now.
+                    self.instantiate_canonical_var(cause.span, kind, &prev_var_values, |u| {
                         universe_map[u.as_usize()]
-                    }),
+                    })
+                } else if kind.is_existential() {
+                    match opt_values[BoundVar::new(prev_var_values.len())] {
+                        Some(k) => k,
+                        None => self.instantiate_canonical_var(
+                            cause.span,
+                            kind,
+                            &prev_var_values,
+                            |u| universe_map[u.as_usize()],
+                        ),
+                    }
+                } else {
+                    // For placeholders which were already part of the input, we simply map this
+                    // universal bound variable back the placeholder of the input.
+                    opt_values[BoundVar::new(prev_var_values.len())]
+                        .expect("expected placeholder to be unified with itself during response")
                 }
-            } else {
-                // For placeholders which were already part of the input, we simply map this
-                // universal bound variable back the placeholder of the input.
-                opt_values[BoundVar::new(var_values.len())]
-                    .expect("expected placeholder to be unified with itself during response")
-            }
-        });
+            });
 
         let mut obligations = PredicateObligations::new();
 

compiler/rustc_infer/src/infer/mod.rs

@@ -34,7 +34,7 @@ use rustc_middle::ty::{
     TypeVisitableExt, TypingEnv, TypingMode, fold_regions,
 };
 use rustc_span::{DUMMY_SP, Span, Symbol, bug};
-use rustc_type_ir::{CanonicalizerState, MayBeErased};
+use rustc_type_ir::MayBeErased;
 use snapshot::undo_log::InferCtxtUndoLogs;
 use tracing::{debug, instrument};
 use ty::solve::TyOrConstInferVar;
@@ -348,11 +348,6 @@ pub struct InferCtxt<'tcx> {
     enable_next_solver_overflow_fcw: Cell<bool>,
 
     pub obligation_inspector: Cell<Option<ObligationInspector<'tcx>>>,
-
-    /// State reused by each new canonicalizer, and then cleared (but not deallocated) once the
-    /// canonicalizer is finished. A performance win, because it avoids reallocating new
-    /// vecs/hashmaps for every canonicalizer.
-    pub canonicalizer_state: RefCell<CanonicalizerState<TyCtxt<'tcx>>>,
 }
 
 impl<'tcx> Drop for InferCtxt<'tcx> {
@@ -696,7 +691,6 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
             next_trait_solver,
             enable_next_solver_overflow_fcw: Cell::new(enable_next_solver_overflow_fcw),
             obligation_inspector: Cell::new(None),
-            canonicalizer_state: Default::default(),
         }
     }
 }

compiler/rustc_middle/src/ty/context.rs

@@ -73,11 +73,11 @@ use crate::traits::solve::{
 };
 use crate::ty::predicate::ExistentialPredicateStableCmpExt as _;
 use crate::ty::{
-    self, AdtDef, AdtDefData, AdtKind, Binder, Clause, ClausePolarity, Clauses, Const, ConstKind,
-    FnSigKind, GenericArg, GenericArgsUnsized, GenericParamDefKind, List, ListWithCachedTypeInfo,
-    ParamConst, Pattern, PatternKind, PolyExistentialPredicate, PolyFnSig, Predicate,
-    PredicateKind, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid, ValTree,
-    ValTreeKind, Visibility,
+    self, AdtDef, AdtDefData, AdtKind, Binder, CanonicalizerState, Clause, ClausePolarity, Clauses,
+    Const, ConstKind, FnSigKind, GenericArg, GenericArgsUnsized, GenericParamDefKind, List,
+    ListWithCachedTypeInfo, ParamConst, Pattern, PatternKind, PolyExistentialPredicate, PolyFnSig,
+    Predicate, PredicateKind, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind,
+    TyVid, ValTree, ValTreeKind, Visibility,
 };
 
 impl<'tcx> rustc_type_ir::inherent::DefId<TyCtxt<'tcx>> for DefId {
@@ -703,6 +703,13 @@ pub struct GlobalCaches<'tcx> {
     new_solver_evaluation_cache: Lock<search_graph::GlobalCache<TyCtxt<'tcx>>>,
     new_solver_canonical_param_env_cache: Lock<ty::CanonicalParamEnvCache<TyCtxt<'tcx>>>,
 
+    /// State reused by each new canonicalizer, and then cleared (but not deallocated) once the
+    /// canonicalizer is finished. A performance win, because it avoids reallocating new
+    /// vecs/hashmaps for every canonicalizer. The `Vec` is needed because there can be multiple
+    /// canonicalizers in use at once if the number of threads > 1. The `Box` avoids memcpy because
+    /// `CanonicalizerState` is moderately big.
+    pub canonicalizer_states: Lock<Vec<Box<CanonicalizerState<TyCtxt<'tcx>>>>>,
+
     pub canonical_param_env_cache: CanonicalParamEnvCache<'tcx>,
 
     /// Caches the index of the highest bound var in clauses in a canonical binder.

compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs

@@ -1,5 +1,4 @@
 use std::collections::hash_map::Entry;
-use std::mem;
 
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::solve::{Goal, QueryInput};
@@ -8,7 +7,6 @@ use rustc_type_ir::{
     Const, Flags, InferCtxtLike, Interner, PlaceholderConst, PlaceholderType, PredicateProxy,
     Region, TypeFlags, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
 };
-use thin_vec::ThinVec;
 
 use crate::delegate::SolverDelegate;
 
@@ -73,7 +71,7 @@ pub(super) struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner
     canonicalize_mode: CanonicalizeMode,
 
     // Mutable fields.
-    state: CanonicalizerState<I>,
+    state: Box<CanonicalizerState<I>>,
 }
 
 impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
@@ -95,7 +93,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         };
         debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
         debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
-        let (max_universe, _variables, var_kinds) = canonicalizer.finalize();
+        let (max_universe, var_kinds) = canonicalizer.finalize();
 
         Canonical { max_universe, var_kinds, value }
     }
@@ -204,7 +202,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
     pub(super) fn canonicalize_input<P: TypeFoldable<I>>(
         delegate: &'a D,
         input: QueryInput<I, P>,
-    ) -> (ThinVec<I::GenericArg>, ty::Canonical<I, QueryInput<I, P>>) {
+    ) -> (Vec<I::GenericArg>, ty::Canonical<I, QueryInput<I, P>>) {
         // First canonicalize the `param_env` while keeping `'static`. This produces a
         // canonicalizer that can canonicalize the rest of the input without keeping `'static`.
         let (param_env, mut rest_canonicalizer) =
@@ -226,7 +224,8 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
 
         debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
         debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
-        let (max_universe, variables, var_kinds) = rest_canonicalizer.finalize();
+        let variables = rest_canonicalizer.state.variables.clone();
+        let (max_universe, var_kinds) = rest_canonicalizer.finalize();
         (variables, Canonical { max_universe, var_kinds, value })
     }
 
@@ -274,7 +273,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         ty::BoundVar::from(idx)
     }
 
-    fn finalize(mut self) -> (ty::UniverseIndex, ThinVec<I::GenericArg>, I::CanonicalVarKinds) {
+    fn finalize(mut self) -> (ty::UniverseIndex, I::CanonicalVarKinds) {
         // See the rustc-dev-guide section about how we deal with universes
         // during canonicalization in the new solver.
         let max_universe = match self.canonicalize_mode {
@@ -310,14 +309,13 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                     .unwrap_or(ty::UniverseIndex::ROOT)
             }
         };
-        let variables = mem::take(&mut self.state.variables);
         let var_kinds = self.delegate.cx().mk_canonical_var_kinds(&self.state.var_kinds);
 
         // We have finished with this canonicalizer and can return its state to the delegate for
         // later reuse.
         self.delegate.release_canonicalizer_state(self.state);
 
-        (max_universe, variables, var_kinds)
+        (max_universe, var_kinds)
     }
 
     fn inner_fold_ty(&mut self, t: I::Ty) -> I::Ty {

compiler/rustc_next_trait_solver/src/canonical/mod.rs

@@ -21,7 +21,6 @@ use rustc_type_ir::{
     self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, Canonicalized, Const,
     InferCtxtLike, Interner, Region, TypeFoldable, TypingMode, TypingModeEqWrapper,
 };
-use thin_vec::ThinVec;
 use tracing::instrument;
 
 use crate::delegate::SolverDelegate;
@@ -58,7 +57,7 @@ pub(super) fn canonicalize_goal<D, I>(
     goal: Goal<I, I::Predicate>,
     opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
     typing_mode: TypingMode<I>,
-) -> (ThinVec<I::GenericArg>, I::CanonicalInput)
+) -> (Vec<I::GenericArg>, I::CanonicalInput)
 where
     D: SolverDelegate<Interner = I>,
     I: Interner,
@@ -223,51 +222,63 @@ where
     let var_values = response.value.var_values();
     assert_eq!(original_values.len(), var_values.len());
 
-    // If the query did not make progress with constraining inference variables,
-    // we would normally create a new inference variables for bound existential variables
-    // only then unify this new inference variable with the inference variable from
-    // the input.
-    //
-    // We therefore instantiate the existential variable in the canonical response with the
-    // inference variable of the input right away, which is more performant.
-    let mut opt_values = IndexVec::from_elem_n(None, response.var_kinds.len());
-    for (original_value, result_value) in iter::zip(original_values, var_values.iter()) {
-        match result_value.kind() {
-            ty::GenericArgKind::Type(t) => {
-                // We disable the instantiation guess for inference variables
-                // and only use it for placeholders. We need to handle the
-                // `sub_root` of type inference variables which would make this
-                // more involved. They are also a lot rarer than region variables.
-                if let ty::Bound(index_kind, b) = t.kind()
-                    && !matches!(
-                        response.var_kinds.get(b.var().as_usize()).unwrap(),
-                        CanonicalVarKind::Ty { .. }
-                    )
-                {
-                    assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
-                    opt_values[b.var()] = Some(*original_value);
+    // `opt_values` below is expensive to build and rarely needed. Do a quick scan to see if it's
+    // needed. (This duplicates the test in the `instantiate` closure below.)
+    let has_root_existentials = response
+        .var_kinds
+        .iter()
+        .any(|kind| kind.universe() == ty::UniverseIndex::ROOT && kind.is_existential());
+
+    // Build it only if necessary.
+    let opt_values = has_root_existentials.then(|| {
+        // If the query did not make progress with constraining inference variables,
+        // we would normally create a new inference variables for bound existential variables
+        // only then unify this new inference variable with the inference variable from
+        // the input.
+        //
+        // We therefore instantiate the existential variable in the canonical response with the
+        // inference variable of the input right away, which is more performant.
+        let mut opt_values = IndexVec::from_elem_n(None, response.var_kinds.len());
+        for (original_value, result_value) in iter::zip(original_values, var_values.iter()) {
+            match result_value.kind() {
+                ty::GenericArgKind::Type(t) => {
+                    // We disable the instantiation guess for inference variables
+                    // and only use it for placeholders. We need to handle the
+                    // `sub_root` of type inference variables which would make this
+                    // more involved. They are also a lot rarer than region variables.
+                    if let ty::Bound(index_kind, b) = t.kind()
+                        && !matches!(
+                            response.var_kinds.get(b.var().as_usize()).unwrap(),
+                            CanonicalVarKind::Ty { .. }
+                        )
+                    {
+                        assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
+                        opt_values[b.var()] = Some(*original_value);
+                    }
                 }
-            }
-            ty::GenericArgKind::Lifetime(r) => {
-                if let ty::ReBound(index_kind, br) = r.kind() {
-                    assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
-                    opt_values[br.var()] = Some(*original_value);
+                ty::GenericArgKind::Lifetime(r) => {
+                    if let ty::ReBound(index_kind, br) = r.kind() {
+                        assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
+                        opt_values[br.var()] = Some(*original_value);
+                    }
                 }
-            }
-            ty::GenericArgKind::Const(c) => {
-                if let ty::ConstKind::Bound(index_kind, bc) = c.kind() {
-                    assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
-                    opt_values[bc.var()] = Some(*original_value);
+                ty::GenericArgKind::Const(c) => {
+                    if let ty::ConstKind::Bound(index_kind, bc) = c.kind() {
+                        assert!(matches!(index_kind, ty::BoundVarIndexKind::Canonical));
+                        opt_values[bc.var()] = Some(*original_value);
+                    }
                 }
             }
         }
-    }
-    CanonicalVarValues::instantiate(delegate.cx(), response.var_kinds, |var_values, kind| {
+        opt_values
+    });
+
+    CanonicalVarValues::instantiate(delegate.cx(), response.var_kinds, |prev_var_values, kind| {
         if kind.universe() != ty::UniverseIndex::ROOT {
             // A variable from inside a binder of the query. While ideally these shouldn't
             // exist at all (see the FIXME at the start of this method), we have to deal with
             // them for now.
-            delegate.instantiate_canonical_var(kind, span, &var_values, |idx| {
+            delegate.instantiate_canonical_var(kind, span, &prev_var_values, |idx| {
                 prev_universe + idx.index()
             })
         } else if kind.is_existential() {
@@ -278,10 +289,12 @@ where
             // more placeholders then they should be able to. However the inference variables have
             // to "come from somewhere", so by equating them with the original values of the caller
             // later on, we pull them down into their correct universe again.
-            if let Some(v) = opt_values[ty::BoundVar::from_usize(var_values.len())] {
+            if let Some(v) =
+                opt_values.as_ref().unwrap()[ty::BoundVar::from_usize(prev_var_values.len())]
+            {
                 v
             } else {
-                delegate.instantiate_canonical_var(kind, span, &var_values, |_| prev_universe)
+                delegate.instantiate_canonical_var(kind, span, &prev_var_values, |_| prev_universe)
             }
         } else {
             // For placeholders which were already part of the input, we simply map this
@@ -639,7 +652,7 @@ pub fn instantiate_canonical_state<D, I, T>(
     delegate: &D,
     span: I::Span,
     prev_universe: ty::UniverseIndex,
-    orig_values: &mut ThinVec<I::GenericArg>,
+    orig_values: &mut Vec<I::GenericArg>,
     state: inspect::CanonicalState<I, T>,
 ) -> T
 where