diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 490ae7b23e6279..0f9798a62f692d 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -56,12 +56,16 @@ def test_set_existing_item(self): self.assert_events(["mod:foo:baz"]) def test_clone(self): - d = {} - d2 = {"foo": "bar"} - with self.watcher() as wid: - self.watch(wid, d) - d.update(d2) - self.assert_events(["clone"]) + sparse = {str(i): i for i in range(100)} + for i in range(99): + del sparse[str(i)] + for d2 in ({"foo": "bar"}, sparse): + d = {} + with self.watcher() as wid: + self.watch(wid, d) + d.update(d2) + self.assert_events(["clone"]) + self.assertEqual(d, d2) def test_no_event_if_not_watched(self): d = {} diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index e8b2aeb1d53125..008a3e7162e078 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -465,17 +465,32 @@ class A: self.assertEqual(gc.is_tracked(d), gc.is_tracked(d2)) def test_copy_noncompact(self): - # Dicts don't compact themselves on del/pop operations. - # Copy will use a slow merging strategy that produces - # a compacted copy when roughly 33% of dict is a non-used - # keys-space (to optimize memory footprint). - # In this test we want to hit the slow/compacting - # branch of dict.copy() and make sure it works OK. - d = {k: k for k in range(1000)} - for k in range(950): - del d[k] - d2 = d.copy() - self.assertEqual(d2, d) + # Dicts don't compact themselves on del/pop operations. Check + # copying a sparse table with both unicode and generic keys. + class Key(CustomHash): + hash_calls = 0 + + def __hash__(self): + self.hash_calls += 1 + return super().__hash__() + + for keys in ([str(i) for i in range(100)], + [Key(i % 5) for i in range(100)]): + with self.subTest(key_type=type(keys[0])): + d = dict(zip(keys, range(100))) + for i, key in enumerate(keys): + if i not in (1, 50, 98): + del d[key] + expected = list(d.items()) + if isinstance(keys[0], Key): + calls = [key.hash_calls for key in d] + copies = (d.copy(), dict(d), {**d}, frozendict(d)) + if isinstance(keys[0], Key): + self.assertEqual([key.hash_calls for key in d], calls) + for copy in copies: + self.assertEqual(list(copy.items()), expected) + for key, value in expected: + self.assertEqual(copy[key], value) @support.nomemtest def test_copy_empty_no_memory(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-08-18-55-00.gh-issue-161524.KV7s2N.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-08-18-55-00.gh-issue-161524.KV7s2N.rst new file mode 100644 index 00000000000000..4b6e0942c37380 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-08-18-55-00.gh-issue-161524.KV7s2N.rst @@ -0,0 +1,2 @@ +Speed up copying dictionaries with many deleted entries by copying live +entries directly into a compact table instead of inserting them one by one. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b8ca74aedc2adf..3409f7755d758d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4296,6 +4296,66 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) return dict_merge_from_seq2(d, seq2, override); } + +/* Clone only live entries from a combined table, preserving insertion order. + The source is locked while its keys and values are referenced. */ +static PyDictKeysObject * +clone_compact_dict_keys(PyDictObject *orig) +{ + assert(PyAnyDict_Check(orig)); + assert(Py_TYPE(orig)->tp_iter == dict_iter); + assert(orig->ma_values == NULL); + assert(orig->ma_used > 0); + if (!PyFrozenDict_Check(orig)) { + ASSERT_DICT_LOCKED(orig); + } + + PyDictKeysObject *oldkeys = orig->ma_keys; + Py_ssize_t used = orig->ma_used; + int unicode = DK_IS_UNICODE(oldkeys); + PyDictKeysObject *keys = new_keys_object(estimate_log2_keysize(used), + unicode); + if (keys == NULL) { + return NULL; + } + + Py_ssize_t n = 0; + if (unicode) { + PyDictUnicodeEntry *src = DK_UNICODE_ENTRIES(oldkeys); + PyDictUnicodeEntry *dst = DK_UNICODE_ENTRIES(keys); + for (Py_ssize_t i = 0; n < used; i++) { + while (src[i].me_value == NULL) { + i++; + } + assert(i < oldkeys->dk_nentries); + dst[n].me_key = Py_NewRef(src[i].me_key); + dst[n].me_value = Py_NewRef(src[i].me_value); + n++; + } + assert(n == used); + build_indices_unicode(keys, dst, n); + } + else { + PyDictKeyEntry *src = DK_ENTRIES(oldkeys); + PyDictKeyEntry *dst = DK_ENTRIES(keys); + for (Py_ssize_t i = 0; n < used; i++) { + while (src[i].me_value == NULL) { + i++; + } + assert(i < oldkeys->dk_nentries); + dst[n].me_key = Py_NewRef(src[i].me_key); + dst[n].me_hash = src[i].me_hash; + dst[n].me_value = Py_NewRef(src[i].me_value); + n++; + } + assert(n == used); + build_indices_generic(keys, dst, n); + } + keys->dk_nentries = n; + keys->dk_usable -= n; + return keys; +} + static int dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject **dupkey) { @@ -4315,15 +4375,18 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject ** override = 1; PyDictKeysObject *okeys = other->ma_keys; - // If other is clean, combined, and just allocated, just clone it. - if (mp->ma_values == NULL && - other->ma_values == NULL && - other->ma_used == okeys->dk_nentries && - (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || - USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used) - ) { + // Clone a combined table directly, compacting deleted entries. + if (mp->ma_values == NULL && other->ma_values == NULL) { _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL); - PyDictKeysObject *keys = clone_combined_dict_keys(other); + PyDictKeysObject *keys; + if (other->ma_used == okeys->dk_nentries && + (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || + USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { + keys = clone_combined_dict_keys(other); + } + else { + keys = clone_compact_dict_keys(other); + } if (keys == NULL) return -1; @@ -4619,25 +4682,30 @@ copy_lock_held_untracked(PyObject *o, int as_frozendict) return (PyObject *)split_copy; } - if (Py_TYPE(mp)->tp_iter == dict_iter && - mp->ma_values == NULL && - (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3)) + if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL) { /* Use fast-copy if: (1) type(mp) doesn't override tp_iter; and - (2) 'mp' is not a split-dict; and + (2) 'mp' is not a split-dict. - (3) if 'mp' is non-compact ('del' operation does not resize dicts), - do fast-copy only if it has at most 1/3 non-used keys. + Dicts don't compact themselves on del/pop operations, so a large + dict can have very few live entries. Previously, fast-copy was + restricted to tables with at most 1/3 non-used entries; otherwise + PyDict_Merge produced a compacted copy. - The last condition (3) is important to guard against a pathological - case when a large dict is almost emptied with multiple del/pop - operations and copied after that. In cases like this, we defer to - PyDict_Merge, which produces a compacted copy. + Clone tables with few deleted entries as-is. For sparse tables, + copy only live entries into a compact table instead of retaining + deleted slots or reinserting entries one by one. */ - PyDictKeysObject *keys = clone_combined_dict_keys(mp); + PyDictKeysObject *keys; + if (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3) { + keys = clone_combined_dict_keys(mp); + } + else { + keys = clone_compact_dict_keys(mp); + } if (keys == NULL) { return NULL; }