Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions Include/internal/pycore_setobject.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,40 @@ PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key);
PyObject *
_PySet_Freeze(PyObject *set);

/* Bit mixer applied to each element hash by frozenset_hash() (and to each
(key, value) pair hash by frozendict_hash()) before the results are
combined with xor.

Because xor does no mixing of its own, all of the avalanche has to come
from this function: a change in *any* input bit must be able to affect
*every* output bit. In particular, high-order input bits must reach the
low-order output bits, which are the ones a hash table looks at first
(gh-150134).

On 64-bit builds this is the SplitMix64 output function, including its
additive "golden gamma" step, which also keeps 0 from being a fixed
point. On 32-bit builds it is the MurmurHash3 fmix32 finalizer. */
static inline Py_uhash_t
_PySet_ShuffleHashBits(Py_uhash_t h)
{
#if SIZEOF_PY_UHASH_T > 4
h += (Py_uhash_t)0x9e3779b97f4a7c15ULL;
h ^= h >> 30;
h *= (Py_uhash_t)0xbf58476d1ce4e5b9ULL;
h ^= h >> 27;
h *= (Py_uhash_t)0x94d049bb133111ebULL;
h ^= h >> 31;
#else
h += (Py_uhash_t)0x9e3779b9UL;
h ^= h >> 16;
h *= (Py_uhash_t)0x85ebca6bUL;
h ^= h >> 13;
h *= (Py_uhash_t)0xc2b2ae35UL;
h ^= h >> 16;
#endif
return h;
}

#ifdef __cplusplus
}
#endif
Expand Down
22 changes: 19 additions & 3 deletions Lib/_collections_abc.py
Original file line number Diff line number Diff line change
Expand Up @@ -665,13 +665,29 @@ def _hash(self):
"""
MAX = sys.maxsize
MASK = 2 * MAX + 1
# Element hash mixer; see _PySet_ShuffleHashBits() in
# Include/internal/pycore_setobject.h.
if MAX > 0x7FFFFFFF:
# SplitMix64 finalizer
G, S1, M1, S2, M2, S3 = (0x9e3779b97f4a7c15,
30, 0xbf58476d1ce4e5b9,
27, 0x94d049bb133111eb, 31)
else:
# MurmurHash3 fmix32 finalizer
G, S1, M1, S2, M2, S3 = (0x9e3779b9,
16, 0x85ebca6b,
13, 0xc2b2ae35, 16)
n = len(self)
h = 1927868237 * (n + 1)
h &= MASK
for x in self:
hx = hash(x)
h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167
h &= MASK
hx = (hash(x) + G) & MASK
hx ^= hx >> S1
hx = (hx * M1) & MASK
hx ^= hx >> S2
hx = (hx * M2) & MASK
hx ^= hx >> S3
h ^= hx
h ^= (h >> 11) ^ (h >> 25)
h = h * 69069 + 907133923
h &= MASK
Expand Down
11 changes: 11 additions & 0 deletions Lib/test/test_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,17 @@ def powerset(s):
u = len({h & mask for h in map(hash, powerset(nums(n)))})
self.assertGreater(4*u, t)

# gh-150134: element hashes which differ only in their high-order
# bits, e.g. hash(i / 256) == i << 53 on 64-bit builds.
floats = [i / 256 for i in range(256)]
hashes = [hash(self.thetype(c))
for c in itertools.combinations(floats, 2)]
self.assertGreater(len(set(hashes)), 0.99 * len(hashes))
t = 2 ** 12
mask = t - 1
u = len({h & mask for h in hashes})
self.assertGreater(u, 0.9 * t)

class FrozenSetSubclass(frozenset):
pass

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Improve the hash of :class:`frozenset` (and :class:`frozendict`, which
shares the algorithm) for elements whose hashes differ only in their
high-order bits, such as ``i / 256``. Previously, many distinct frozensets
of such elements collapsed onto very few hash values. The hash of each
element is now mixed with the SplitMix64 finalizer (MurmurHash3 ``fmix32``
on 32-bit builds), so that every input bit can affect every output bit.
8 changes: 1 addition & 7 deletions Objects/dictobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -8534,12 +8534,6 @@ frozendict_repr(PyObject *self)
return res;
}

static Py_uhash_t
_shuffle_bits(Py_uhash_t h)
{
return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL;
}

// Compute hash((key, value)).
// Code copied from tuple_hash().
static Py_hash_t
Expand Down Expand Up @@ -8603,7 +8597,7 @@ frozendict_hash(PyObject *op)
_Py_LeaveRecursiveCall();
return -1;
}
hash ^= _shuffle_bits(pair_hash);
hash ^= _PySet_ShuffleHashBits(pair_hash);
}
_Py_LeaveRecursiveCall();

Expand Down
15 changes: 5 additions & 10 deletions Objects/setobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -945,13 +945,8 @@ set_traverse(PyObject *self, visitproc visit, void *arg)
/* Work to increase the bit dispersion for closely spaced hash values.
This is important because some use cases have many combinations of a
small number of elements with nearby hashes so that many distinct
combinations collapse to only a handful of distinct hash values. */

static Py_uhash_t
_shuffle_bits(Py_uhash_t h)
{
return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL;
}
combinations collapse to only a handful of distinct hash values.
See _PySet_ShuffleHashBits() in pycore_setobject.h. */

/* Most of the constants in this hash algorithm are randomly chosen
large primes with "interesting bit patterns" and that passed tests
Expand Down Expand Up @@ -984,15 +979,15 @@ frozenset_hash_impl(PyObject *self)
entries on every iteration. */

for (entry = so->table; entry <= &so->table[so->mask]; entry++)
hash ^= _shuffle_bits(entry->hash);
hash ^= _PySet_ShuffleHashBits(entry->hash);

/* Remove the effect of an odd number of NULL entries */
if ((so->mask + 1 - so->fill) & 1)
hash ^= _shuffle_bits(0);
hash ^= _PySet_ShuffleHashBits(0);

/* Remove the effect of an odd number of dummy entries */
if ((so->fill - so->used) & 1)
hash ^= _shuffle_bits(-1);
hash ^= _PySet_ShuffleHashBits(-1);

/* Factor in the number of active entries */
hash ^= ((Py_uhash_t)PySet_GET_SIZE(self) + 1) * 1927868237UL;
Expand Down
Loading