diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 92d1a15177f79eb..c7d79622eec5804 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -38,6 +38,40 @@ PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key); PyObject * _PySet_Freeze(PyObject *set); +/* Bit mixer applied to each element hash by frozenset_hash() (and to each + (key, value) pair hash by frozendict_hash()) before the results are + combined with xor. + + Because xor does no mixing of its own, all of the avalanche has to come + from this function: a change in *any* input bit must be able to affect + *every* output bit. In particular, high-order input bits must reach the + low-order output bits, which are the ones a hash table looks at first + (gh-150134). + + On 64-bit builds this is the SplitMix64 output function, including its + additive "golden gamma" step, which also keeps 0 from being a fixed + point. On 32-bit builds it is the MurmurHash3 fmix32 finalizer. */ +static inline Py_uhash_t +_PySet_ShuffleHashBits(Py_uhash_t h) +{ +#if SIZEOF_PY_UHASH_T > 4 + h += (Py_uhash_t)0x9e3779b97f4a7c15ULL; + h ^= h >> 30; + h *= (Py_uhash_t)0xbf58476d1ce4e5b9ULL; + h ^= h >> 27; + h *= (Py_uhash_t)0x94d049bb133111ebULL; + h ^= h >> 31; +#else + h += (Py_uhash_t)0x9e3779b9UL; + h ^= h >> 16; + h *= (Py_uhash_t)0x85ebca6bUL; + h ^= h >> 13; + h *= (Py_uhash_t)0xc2b2ae35UL; + h ^= h >> 16; +#endif + return h; +} + #ifdef __cplusplus } #endif diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 24d08605429d340..a5c1950fd0e9e62 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -665,13 +665,29 @@ def _hash(self): """ MAX = sys.maxsize MASK = 2 * MAX + 1 + # Element hash mixer; see _PySet_ShuffleHashBits() in + # Include/internal/pycore_setobject.h. + if MAX > 0x7FFFFFFF: + # SplitMix64 finalizer + G, S1, M1, S2, M2, S3 = (0x9e3779b97f4a7c15, + 30, 0xbf58476d1ce4e5b9, + 27, 0x94d049bb133111eb, 31) + else: + # MurmurHash3 fmix32 finalizer + G, S1, M1, S2, M2, S3 = (0x9e3779b9, + 16, 0x85ebca6b, + 13, 0xc2b2ae35, 16) n = len(self) h = 1927868237 * (n + 1) h &= MASK for x in self: - hx = hash(x) - h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 - h &= MASK + hx = (hash(x) + G) & MASK + hx ^= hx >> S1 + hx = (hx * M1) & MASK + hx ^= hx >> S2 + hx = (hx * M2) & MASK + hx ^= hx >> S3 + h ^= hx h ^= (h >> 11) ^ (h >> 25) h = h * 69069 + 907133923 h &= MASK diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 9bfd4bc7d636699..4bdb494239b9e84 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -821,6 +821,17 @@ def powerset(s): u = len({h & mask for h in map(hash, powerset(nums(n)))}) self.assertGreater(4*u, t) + # gh-150134: element hashes which differ only in their high-order + # bits, e.g. hash(i / 256) == i << 53 on 64-bit builds. + floats = [i / 256 for i in range(256)] + hashes = [hash(self.thetype(c)) + for c in itertools.combinations(floats, 2)] + self.assertGreater(len(set(hashes)), 0.99 * len(hashes)) + t = 2 ** 12 + mask = t - 1 + u = len({h & mask for h in hashes}) + self.assertGreater(u, 0.9 * t) + class FrozenSetSubclass(frozenset): pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-20-56-19.gh-issue-150134.JnbjSG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-20-56-19.gh-issue-150134.JnbjSG.rst new file mode 100644 index 000000000000000..6c14ba2acd9c97f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-20-56-19.gh-issue-150134.JnbjSG.rst @@ -0,0 +1,6 @@ +Improve the hash of :class:`frozenset` (and :class:`frozendict`, which +shares the algorithm) for elements whose hashes differ only in their +high-order bits, such as ``i / 256``. Previously, many distinct frozensets +of such elements collapsed onto very few hash values. The hash of each +element is now mixed with the SplitMix64 finalizer (MurmurHash3 ``fmix32`` +on 32-bit builds), so that every input bit can affect every output bit. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ce6f4a61ad3c262..bf6a2f82baee19d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8534,12 +8534,6 @@ frozendict_repr(PyObject *self) return res; } -static Py_uhash_t -_shuffle_bits(Py_uhash_t h) -{ - return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; -} - // Compute hash((key, value)). // Code copied from tuple_hash(). static Py_hash_t @@ -8603,7 +8597,7 @@ frozendict_hash(PyObject *op) _Py_LeaveRecursiveCall(); return -1; } - hash ^= _shuffle_bits(pair_hash); + hash ^= _PySet_ShuffleHashBits(pair_hash); } _Py_LeaveRecursiveCall(); diff --git a/Objects/setobject.c b/Objects/setobject.c index 8c5f244d4894d8a..5e2ce42850060c6 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -945,13 +945,8 @@ set_traverse(PyObject *self, visitproc visit, void *arg) /* Work to increase the bit dispersion for closely spaced hash values. This is important because some use cases have many combinations of a small number of elements with nearby hashes so that many distinct - combinations collapse to only a handful of distinct hash values. */ - -static Py_uhash_t -_shuffle_bits(Py_uhash_t h) -{ - return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; -} + combinations collapse to only a handful of distinct hash values. + See _PySet_ShuffleHashBits() in pycore_setobject.h. */ /* Most of the constants in this hash algorithm are randomly chosen large primes with "interesting bit patterns" and that passed tests @@ -984,15 +979,15 @@ frozenset_hash_impl(PyObject *self) entries on every iteration. */ for (entry = so->table; entry <= &so->table[so->mask]; entry++) - hash ^= _shuffle_bits(entry->hash); + hash ^= _PySet_ShuffleHashBits(entry->hash); /* Remove the effect of an odd number of NULL entries */ if ((so->mask + 1 - so->fill) & 1) - hash ^= _shuffle_bits(0); + hash ^= _PySet_ShuffleHashBits(0); /* Remove the effect of an odd number of dummy entries */ if ((so->fill - so->used) & 1) - hash ^= _shuffle_bits(-1); + hash ^= _PySet_ShuffleHashBits(-1); /* Factor in the number of active entries */ hash ^= ((Py_uhash_t)PySet_GET_SIZE(self) + 1) * 1927868237UL;