Encode HashMap and HashSet in deterministic order when an encoder requires determinism

compiler/rustc_hir_id/src/lib.rs

@@ -16,6 +16,7 @@ use rustc_data_structures::stable_hash::{
     StableHash, StableHashCtxt, StableHasher, StableOrd, ToStableHashKey,
 };
 use rustc_macros::{Decodable, Encodable, StableHash};
+use rustc_serialize::ContextOrd;
 use rustc_span::def_id::{CRATE_DEF_ID, DefId, DefIndex, DefPathHash, LocalDefId, LocalModId};
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable)]
@@ -176,6 +177,12 @@ rustc_index::newtype_index! {
     pub struct ItemLocalId {}
 }
 
+impl<E> ContextOrd<E> for ItemLocalId {
+    fn stable_cmp_with_context(_context: E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        <Self as Ord>::cmp
+    }
+}
+
 impl ItemLocalId {
     /// Signal local id which should never be used.
     pub const INVALID: ItemLocalId = ItemLocalId::MAX;

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -102,6 +102,8 @@ macro_rules! encoder_methods {
 }
 
 impl<'a, 'tcx> Encoder for EncodeContext<'a, 'tcx> {
+    const DETERMINISTIC_HASH_MAP_ENCODE: bool = true;
+
     encoder_methods! {
         emit_usize(usize);
         emit_u128(u128);
@@ -150,6 +152,10 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for ExpnIndex {
 }
 
 impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> {
+    fn def_path_hasher(&self) -> impl Fn(DefId) -> DefPathHash {
+        self.tcx.prelocked_def_path_hasher()
+    }
+
     fn encode_crate_num(&mut self, crate_num: CrateNum) {
         if crate_num != LOCAL_CRATE && self.is_proc_macro {
             panic!("Attempted to encode non-local CrateNum {crate_num:?} for proc-macro crate");

compiler/rustc_middle/src/query/on_disk_cache.rs

@@ -14,7 +14,7 @@ use rustc_hir::definitions::DefPathHash;
 use rustc_index::IndexVec;
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder};
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
+use rustc_serialize::{ContextOrd, Decodable, Decoder, Encodable, Encoder};
 use rustc_session::Session;
 use rustc_span::hygiene::{
     ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextKey,
@@ -116,6 +116,9 @@ struct Footer {
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)]
 struct SourceFileIndex(u32);
 
+// this is never used by the on disk cache, since it is not required to be deterministic
+impl<E> ContextOrd<E> for SourceFileIndex {}
+
 #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)]
 struct AbsoluteBytePos(u64);
 

compiler/rustc_middle/src/ty/context.rs

@@ -1144,6 +1144,22 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Useful when one needs to access a lot of `DefPathHash`-es in quick succession. Stable
+    /// sorting a collection for example.
+    ///
+    /// Read locks the structures containing the `DefPathHash` mappings, allowing us to skip lots of
+    /// locking and releasing using `[Self::def_path_hash]` would do
+    pub fn prelocked_def_path_hasher(
+        self,
+    ) -> impl Fn(DefId) -> rustc_hir::definitions::DefPathHash {
+        let definitions = self.definitions_untracked();
+        let cstore = self.cstore_untracked();
+        move |def_id| match def_id.as_local() {
+            Some(def_id) => definitions.def_path_hash(def_id),
+            None => cstore.def_path_hash(def_id),
+        }
+    }
+
     #[inline]
     pub fn crate_types(self) -> &'tcx [CrateType] {
         &self.crate_types

compiler/rustc_monomorphize/src/offload/manifest.rs

@@ -8,7 +8,7 @@ use std::fs;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lock;
-use rustc_hir::def_id::{DefId, DefIndex, LOCAL_CRATE, StableCrateId};
+use rustc_hir::def_id::{DefId, DefIndex, DefPathHash, LOCAL_CRATE, StableCrateId};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mono::MonoItem;
 use rustc_middle::ty::codec::{TyDecoder, TyEncoder};
@@ -44,6 +44,8 @@ impl<'a, 'tcx> OffloadManifestEncoder<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> Encoder for OffloadManifestEncoder<'a, 'tcx> {
+    const DETERMINISTIC_HASH_MAP_ENCODE: bool = true;
+
     fn emit_usize(&mut self, v: usize) {
         self.encoder.emit_usize(v);
     }
@@ -86,6 +88,10 @@ impl<'a, 'tcx> Encoder for OffloadManifestEncoder<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> SpanEncoder for OffloadManifestEncoder<'a, 'tcx> {
+    fn def_path_hasher(&self) -> impl Fn(DefId) -> DefPathHash {
+        self.tcx.prelocked_def_path_hasher()
+    }
+
     fn encode_span(&mut self, _span: Span) {
         // Spans are not needed in the manifest, encode a dummy span.
         self.emit_usize(0);

compiler/rustc_serialize/src/lib.rs

@@ -14,7 +14,7 @@
 #[cfg(test)]
 extern crate self as rustc_serialize;
 
-pub use self::serialize::{Decodable, Decoder, Encodable, Encoder};
+pub use self::serialize::{ContextOrd, Decodable, Decoder, Encodable, Encoder};
 
 mod serialize;
 

compiler/rustc_serialize/src/serialize.rs

@@ -85,6 +85,67 @@ pub trait Encoder {
     }
 
     fn emit_raw_bytes(&mut self, s: &[u8]);
+
+    /// Controls whether `HashMap`-s are stable sorted by their key before encoding. Some encoders
+    /// don't need determinism, improving performance.
+    const DETERMINISTIC_HASH_MAP_ENCODE: bool = false;
+
+    fn encode_hash_map<K, V, S>(&mut self, map: &HashMap<K, V, S>)
+    where
+        Self: Sized,
+        K: Encodable<Self> + for<'a> ContextOrd<&'a Self>,
+        V: Encodable<Self>,
+        S: BuildHasher,
+    {
+        self.emit_usize(map.len());
+        if Self::DETERMINISTIC_HASH_MAP_ENCODE {
+            // we are sorting right after
+            #[expect(rustc::potential_query_instability)]
+            let mut values: Vec<_> = map.iter().collect();
+            let mut cmp = <K as ContextOrd<&Self>>::stable_cmp_with_context(self);
+            values.sort_by(|(k1, _), (k2, _)| cmp(*k1, *k2));
+            drop(cmp);
+
+            for (key, val) in values {
+                key.encode(self);
+                val.encode(self);
+            }
+        } else {
+            // this is explicitly allowed when Self::DETERMINISTIC_HASH_MAP_ENCODE is false
+            #[expect(rustc::potential_query_instability)]
+            for (key, val) in map.iter() {
+                key.encode(self);
+                val.encode(self);
+            }
+        }
+    }
+
+    fn encode_hash_set<K, S>(&mut self, map: &HashSet<K, S>)
+    where
+        Self: Sized,
+        K: Encodable<Self> + for<'a> ContextOrd<&'a Self>,
+        S: BuildHasher,
+    {
+        self.emit_usize(map.len());
+        if Self::DETERMINISTIC_HASH_MAP_ENCODE {
+            // we are sorting right after
+            #[expect(rustc::potential_query_instability)]
+            let mut values: Vec<_> = map.iter().collect();
+            let mut cmp = <K as ContextOrd<&Self>>::stable_cmp_with_context(self);
+            values.sort_by(|k1, k2| cmp(*k1, *k2));
+            drop(cmp);
+
+            for key in values {
+                key.encode(self);
+            }
+        } else {
+            // this is explicitly allowed when Self::DETERMINISTIC_HASH_MAP_ENCODE is false
+            #[expect(rustc::potential_query_instability)]
+            for key in map.iter() {
+                key.encode(self);
+            }
+        }
+    }
 }
 
 // Note: all the methods in this trait are infallible, which may be surprising.
@@ -608,16 +669,13 @@ where
 
 impl<E: Encoder, K, V, S> Encodable<E> for HashMap<K, V, S>
 where
-    K: Encodable<E> + Eq,
+    E: Sized,
+    K: Encodable<E> + Eq + for<'a> ContextOrd<&'a E>,
     V: Encodable<E>,
     S: BuildHasher,
 {
     fn encode(&self, e: &mut E) {
-        e.emit_usize(self.len());
-        for (key, val) in self {
-            key.encode(e);
-            val.encode(e);
-        }
+        e.encode_hash_map(self)
     }
 }
 
@@ -635,14 +693,11 @@ where
 
 impl<E: Encoder, T, S> Encodable<E> for HashSet<T, S>
 where
-    T: Encodable<E> + Eq,
+    T: Encodable<E> + for<'a> ContextOrd<&'a E>,
     S: BuildHasher,
 {
-    fn encode(&self, s: &mut E) {
-        s.emit_usize(self.len());
-        for e in self {
-            e.encode(s);
-        }
+    fn encode(&self, e: &mut E) {
+        e.encode_hash_set(self)
     }
 }
 
@@ -763,3 +818,26 @@ impl<D: Decoder> Decodable<D> for Hash128 {
         Self::new(u128::from_le_bytes(d.read_raw_bytes(16).try_into().unwrap()))
     }
 }
+
+pub trait ContextOrd<Ctx> {
+    /// The returned comparator must be stable.
+    /// E. g. compares `DefPathHash` instead of `DefIndex`
+    fn stable_cmp_with_context(_context: Ctx) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        unimplemented!();
+        #[allow(unused)]
+        |_i, _i2| unreachable!()
+    }
+}
+
+impl<Ctx> ContextOrd<Ctx> for String {
+    fn stable_cmp_with_context(_context: Ctx) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        <Self as Ord>::cmp
+    }
+}
+
+// FIXME remove this. This is only here because CacheEncoder, which does not need to be
+// deterministic, encodes a HashMap<u32,V>. If a real implementation is ever needed, that use should
+// be switched to a new type instead of u32
+impl<Ctx> ContextOrd<Ctx> for u32 {}
+
+impl<Ctx> ContextOrd<Ctx> for i64 {}

compiler/rustc_span/src/def_id.rs

@@ -10,7 +10,7 @@ use rustc_data_structures::unhash::Unhasher;
 use rustc_hashes::Hash64;
 use rustc_index::Idx;
 use rustc_macros::{BlobDecodable, Decodable, Encodable, StableHash};
-use rustc_serialize::{Decodable, Encodable};
+use rustc_serialize::{ContextOrd, Decodable, Encodable};
 
 use crate::{BlobDecoder, SpanEncoder, Symbol};
 
@@ -23,6 +23,13 @@ rustc_index::newtype_index! {
     pub struct CrateNum {}
 }
 
+impl<E: SpanEncoder> ContextOrd<&E> for CrateNum {
+    fn stable_cmp_with_context(context: &E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        let hasher = context.def_path_hasher();
+        move |id1, id2| hasher(id1.as_def_id()).cmp(&hasher(id2.as_def_id()))
+    }
+}
+
 /// Item definitions in the currently-compiled crate would have the `CrateNum`
 /// `LOCAL_CRATE` in their `DefId`.
 pub const LOCAL_CRATE: CrateNum = CrateNum::ZERO;
@@ -410,13 +417,27 @@ impl<D: BlobDecoder> Decodable<D> for LocalDefId {
     }
 }
 
+impl<E: SpanEncoder> ContextOrd<&E> for LocalDefId {
+    fn stable_cmp_with_context(context: &E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        let hasher = context.def_path_hasher();
+        move |id1, id2| hasher(id1.to_def_id()).cmp(&hasher(id2.to_def_id()))
+    }
+}
+
 rustc_data_structures::define_id_collections!(
     LocalDefIdMap,
     LocalDefIdSet,
     LocalDefIdMapEntry,
     LocalDefId
 );
 
+impl<E: SpanEncoder> ContextOrd<&E> for DefId {
+    fn stable_cmp_with_context(context: &E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        let hasher = context.def_path_hasher();
+        move |id1, id2| hasher(*id1).cmp(&hasher(*id2))
+    }
+}
+
 impl StableHash for DefId {
     #[inline]
     fn stable_hash<Hcx: StableHashCtxt>(&self, hcx: &mut Hcx, hasher: &mut StableHasher) {

compiler/rustc_span/src/hygiene.rs

@@ -39,7 +39,7 @@ use rustc_data_structures::unhash::UnhashMap;
 use rustc_hashes::Hash64;
 use rustc_index::IndexVec;
 use rustc_macros::{Decodable, Encodable, StableHash};
-use rustc_serialize::{Decodable, Decoder, Encodable};
+use rustc_serialize::{ContextOrd, Decodable, Decoder, Encodable};
 use tracing::{debug, trace};
 
 use crate::def_id::{CRATE_DEF_ID, CrateNum, DefId, LOCAL_CRATE, ModId, StableCrateId};
@@ -133,6 +133,12 @@ impl !PartialOrd for LocalExpnId {}
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Encodable, Decodable, StableHash)]
 pub struct ExpnHash(Fingerprint);
 
+impl<E> ContextOrd<E> for ExpnHash {
+    fn stable_cmp_with_context(_context: E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        |h1, h2| h1.0.cmp(&h2.0)
+    }
+}
+
 impl ExpnHash {
     /// Returns the [StableCrateId] identifying the crate this [ExpnHash]
     /// originates from.

compiler/rustc_span/src/lib.rs

@@ -46,6 +46,7 @@ pub mod source_map;
 use source_map::{SourceMap, SourceMapInputs};
 
 pub use self::caching_source_map_view::CachingSourceMapView;
+use crate::def_id::DefPathHash;
 use crate::fatal_error::FatalError;
 
 pub mod edition;
@@ -1344,6 +1345,16 @@ rustc_index::newtype_index! {
 /// This trait is used to allow encoder specific encodings of certain types.
 /// It is similar to rustc_type_ir's TyEncoder.
 pub trait SpanEncoder: Encoder {
+    /// Only needed if the encoder wants deterministic encoding for `HashMap` and `HashSet` with
+    /// `CrateNum`, `DefId` or `LocalDefId` keys. Can be unimplemented otherwise
+    fn def_path_hasher(&self) -> impl Fn(DefId) -> DefPathHash {
+        panic!(
+            "Deterministic encoding of HashMap<DefId, V> requires overriding `SpanEncoder::def_path_hasher`"
+        );
+        #[allow(unreachable_code)]
+        |_def_id| unimplemented!()
+    }
+
     fn encode_span(&mut self, span: Span);
     fn encode_symbol(&mut self, sym: Symbol);
     fn encode_byte_symbol(&mut self, byte_sym: ByteSymbol);

compiler/rustc_span/src/symbol.rs

@@ -11,6 +11,7 @@ use rustc_data_structures::hash_table::{Entry, HashTable};
 use rustc_data_structures::stable_hash::{StableCompare, StableHash, StableHashCtxt, StableHasher};
 use rustc_data_structures::sync::Lock;
 use rustc_macros::{Decodable, Encodable, StableHash, symbols};
+use rustc_serialize::ContextOrd;
 
 use crate::edit_distance::find_best_match_for_name;
 use crate::{DUMMY_SP, Edition, Span, with_session_globals};
@@ -2735,6 +2736,12 @@ impl fmt::Display for MacroRulesNormalizedIdent {
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Symbol(SymbolIndex);
 
+impl<E> ContextOrd<E> for Symbol {
+    fn stable_cmp_with_context(_context: E) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        StableCompare::stable_cmp
+    }
+}
+
 // Used within both `Symbol` and `ByteSymbol`.
 rustc_index::newtype_index! {
     #[orderable]

compiler/rustc_structures/src/crate_type.rs

@@ -1,3 +1,4 @@
+use rustc_serialize::ContextOrd;
 #[cfg(feature = "nightly")]
 use {
     rustc_macros::{Decodable_NoContext, Encodable_NoContext, StableHash},
@@ -25,6 +26,13 @@ pub enum CrateType {
     // Unstable; feature(export_stable)
     Sdylib,
 }
+
+impl<Ctx> ContextOrd<Ctx> for CrateType {
+    fn stable_cmp_with_context(_context: Ctx) -> impl FnMut(&Self, &Self) -> std::cmp::Ordering {
+        <Self as Ord>::cmp
+    }
+}
+
 #[cfg(feature = "nightly")]
 impl CrateType {
     /// Pairs of each `#[crate_type] = "..."` value and the crate type it resolves to