Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions Lib/test/test_capi/test_watchers.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,16 @@ def test_set_existing_item(self):
self.assert_events(["mod:foo:baz"])

def test_clone(self):
d = {}
d2 = {"foo": "bar"}
with self.watcher() as wid:
self.watch(wid, d)
d.update(d2)
self.assert_events(["clone"])
sparse = {str(i): i for i in range(100)}
for i in range(99):
del sparse[str(i)]
for d2 in ({"foo": "bar"}, sparse):
d = {}
with self.watcher() as wid:
self.watch(wid, d)
d.update(d2)
self.assert_events(["clone"])
self.assertEqual(d, d2)

def test_no_event_if_not_watched(self):
d = {}
Expand Down
37 changes: 26 additions & 11 deletions Lib/test/test_dict.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,17 +465,32 @@ class A:
self.assertEqual(gc.is_tracked(d), gc.is_tracked(d2))

def test_copy_noncompact(self):
# Dicts don't compact themselves on del/pop operations.
# Copy will use a slow merging strategy that produces
# a compacted copy when roughly 33% of dict is a non-used
# keys-space (to optimize memory footprint).
# In this test we want to hit the slow/compacting
# branch of dict.copy() and make sure it works OK.
d = {k: k for k in range(1000)}
for k in range(950):
del d[k]
d2 = d.copy()
self.assertEqual(d2, d)
# Dicts don't compact themselves on del/pop operations. Check
# copying a sparse table with both unicode and generic keys.
class Key(CustomHash):
hash_calls = 0

def __hash__(self):
self.hash_calls += 1
return super().__hash__()

for keys in ([str(i) for i in range(100)],
[Key(i % 5) for i in range(100)]):
with self.subTest(key_type=type(keys[0])):
d = dict(zip(keys, range(100)))
for i, key in enumerate(keys):
if i not in (1, 50, 98):
del d[key]
expected = list(d.items())
if isinstance(keys[0], Key):
calls = [key.hash_calls for key in d]
copies = (d.copy(), dict(d), {**d}, frozendict(d))
if isinstance(keys[0], Key):
self.assertEqual([key.hash_calls for key in d], calls)
for copy in copies:
self.assertEqual(list(copy.items()), expected)
for key, value in expected:
self.assertEqual(copy[key], value)

@support.nomemtest
def test_copy_empty_no_memory(self):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Speed up copying dictionaries with many deleted entries by copying live
entries directly into a compact table instead of inserting them one by one.
106 changes: 87 additions & 19 deletions Objects/dictobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -4296,6 +4296,66 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override)
return dict_merge_from_seq2(d, seq2, override);
}


/* Clone only live entries from a combined table, preserving insertion order.
The source is locked while its keys and values are referenced. */
static PyDictKeysObject *
clone_compact_dict_keys(PyDictObject *orig)
{
assert(PyAnyDict_Check(orig));
assert(Py_TYPE(orig)->tp_iter == dict_iter);
assert(orig->ma_values == NULL);
assert(orig->ma_used > 0);
if (!PyFrozenDict_Check(orig)) {
ASSERT_DICT_LOCKED(orig);
}

PyDictKeysObject *oldkeys = orig->ma_keys;
Py_ssize_t used = orig->ma_used;
int unicode = DK_IS_UNICODE(oldkeys);
PyDictKeysObject *keys = new_keys_object(estimate_log2_keysize(used),
unicode);
if (keys == NULL) {
return NULL;
}

Py_ssize_t n = 0;
if (unicode) {
PyDictUnicodeEntry *src = DK_UNICODE_ENTRIES(oldkeys);
PyDictUnicodeEntry *dst = DK_UNICODE_ENTRIES(keys);
for (Py_ssize_t i = 0; n < used; i++) {
while (src[i].me_value == NULL) {
i++;
}
assert(i < oldkeys->dk_nentries);
dst[n].me_key = Py_NewRef(src[i].me_key);
dst[n].me_value = Py_NewRef(src[i].me_value);
n++;
}
assert(n == used);
build_indices_unicode(keys, dst, n);
}
else {
PyDictKeyEntry *src = DK_ENTRIES(oldkeys);
PyDictKeyEntry *dst = DK_ENTRIES(keys);
for (Py_ssize_t i = 0; n < used; i++) {
while (src[i].me_value == NULL) {
i++;
}
assert(i < oldkeys->dk_nentries);
dst[n].me_key = Py_NewRef(src[i].me_key);
dst[n].me_hash = src[i].me_hash;
dst[n].me_value = Py_NewRef(src[i].me_value);
n++;
}
assert(n == used);
build_indices_generic(keys, dst, n);
}
keys->dk_nentries = n;
keys->dk_usable -= n;
return keys;
}

static int
dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject **dupkey)
{
Expand All @@ -4315,15 +4375,18 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject **
override = 1;
PyDictKeysObject *okeys = other->ma_keys;

// If other is clean, combined, and just allocated, just clone it.
if (mp->ma_values == NULL &&
other->ma_values == NULL &&
other->ma_used == okeys->dk_nentries &&
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
) {
// Clone a combined table directly, compacting deleted entries.
if (mp->ma_values == NULL && other->ma_values == NULL) {
_PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
PyDictKeysObject *keys = clone_combined_dict_keys(other);
PyDictKeysObject *keys;
if (other->ma_used == okeys->dk_nentries &&
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) {
keys = clone_combined_dict_keys(other);
}
else {
keys = clone_compact_dict_keys(other);
}
if (keys == NULL)
return -1;

Expand Down Expand Up @@ -4619,25 +4682,30 @@ copy_lock_held_untracked(PyObject *o, int as_frozendict)
return (PyObject *)split_copy;
}

if (Py_TYPE(mp)->tp_iter == dict_iter &&
mp->ma_values == NULL &&
(mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3))
if (Py_TYPE(mp)->tp_iter == dict_iter && mp->ma_values == NULL)
{
/* Use fast-copy if:

(1) type(mp) doesn't override tp_iter; and

(2) 'mp' is not a split-dict; and
(2) 'mp' is not a split-dict.

(3) if 'mp' is non-compact ('del' operation does not resize dicts),
do fast-copy only if it has at most 1/3 non-used keys.
Dicts don't compact themselves on del/pop operations, so a large
dict can have very few live entries. Previously, fast-copy was
restricted to tables with at most 1/3 non-used entries; otherwise
PyDict_Merge produced a compacted copy.

The last condition (3) is important to guard against a pathological
case when a large dict is almost emptied with multiple del/pop
operations and copied after that. In cases like this, we defer to
PyDict_Merge, which produces a compacted copy.
Clone tables with few deleted entries as-is. For sparse tables,
copy only live entries into a compact table instead of retaining
deleted slots or reinserting entries one by one.
*/
PyDictKeysObject *keys = clone_combined_dict_keys(mp);
PyDictKeysObject *keys;
if (mp->ma_used >= (mp->ma_keys->dk_nentries * 2) / 3) {
keys = clone_combined_dict_keys(mp);
}
else {
keys = clone_compact_dict_keys(mp);
}
if (keys == NULL) {
return NULL;
}
Expand Down
Loading