diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9ad61f458944ab..41cc647fc447302 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -825,6 +825,8 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +struct _PyList_DequeHooks; + #define _PyInterpreterGuard_GUARDS_NOT_ALLOWED UINTPTR_MAX /* PyInterpreterState holds the global state for one of the runtime's @@ -996,6 +998,8 @@ struct _is { struct _py_code_state code_state; struct _Py_dict_state dict_state; + // Installed by _collections in this interpreter; NULL until then. + const struct _PyList_DequeHooks *list_deque_hooks; struct _Py_exc_state exc_state; struct _Py_mem_interp_free_queue mem_free_queue; diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index df0d00f752573b5..e3bdc9531ebd247 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -77,6 +77,31 @@ union _PyStackRef; PyAPI_FUNC(PyObject *)_PyList_FromStackRefStealOnSuccess(const union _PyStackRef *src, Py_ssize_t n); PyAPI_FUNC(PyObject *)_PyList_AsTupleAndClear(PyListObject *v); +// Bulk extension fast path for collections.deque, which lives in the +// _collections module and is therefore unknown to listobject.c. +// +// A deque (or a subclass that keeps deque's tp_iter and sq_length) is +// recognised by deque_iter/deque_len; a forward deque iterator by +// dequeiter_next. extend() is called without any lock held and returns 0 on +// success, -1 with an exception set, or 1 to request the generic path. +typedef struct _PyList_DequeHooks { + getiterfunc deque_iter; + lenfunc deque_len; + iternextfunc dequeiter_next; + int (*extend)(PyListObject *self, PyObject *iterable); +} _PyList_DequeHooks; + +PyAPI_FUNC(void) _PyList_SetDequeHooks(const _PyList_DequeHooks *hooks); + +// Make room for 'n' (> 0) more items at the end of 'self' without changing +// its size; return a pointer to the first free slot, or NULL with an +// exception set. The caller must hold self's critical section, store new +// references with FT_ATOMIC_STORE_PTR_RELEASE() and then publish them with +// Py_SET_SIZE(). Matches the allocation behaviour of the generic +// list.extend(iterable) path when the length hint is exact. +PyAPI_FUNC(PyObject **) _PyList_ReserveForExtendLockHeld(PyListObject *self, + Py_ssize_t n); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 0102f203340bd9c..3a72b321c293b0b 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -8,6 +8,7 @@ import pickle import random import struct +import sys BIG = 100000 @@ -798,6 +799,128 @@ def test_runtime_error_on_empty_deque(self): d.append(10) self.assertRaises(RuntimeError, next, it) +class TestBulkMaterialization(unittest.TestCase): + # list(), list.extend(), [*d] and str.join() copy a deque (or a forward + # deque iterator) into a list in one step. The result must be identical + # to element-by-element iteration, which a comprehension still uses. + + def check(self, d): + expected = [x for x in d] + self.assertEqual(list(d), expected) + self.assertEqual([*d], expected) + lst = ['head'] + lst.extend(d) + self.assertEqual(lst, ['head'] + expected) + self.assertEqual(list(iter(d)), expected) + + def test_block_boundaries(self): + for n in (0, 1, 2, 31, 32, 33, 63, 64, 65, 127, 128, 129, 1000): + for shift in (0, 1, 31, 32, 63, 64, 65, 100): + d = deque(range(n + shift)) + for _ in range(shift): + d.popleft() + self.check(d) + d.rotate(shift) + self.check(d) + d.appendleft(-1) + self.check(d) + + def test_partially_consumed_iterator(self): + for n in (0, 1, 63, 64, 65, 200): + for k in sorted(k for k in {0, 1, n // 2, n - 1, n} if 0 <= k <= n): + d = deque(range(n)) + it = iter(d) + for _ in range(k): + next(it) + self.assertEqual(list(it), list(range(k, n))) + self.assertEqual(it.__length_hint__(), 0) + self.assertEqual(list(it), []) + self.assertRaises(StopIteration, next, it) + # __reduce__ reports the iterator as exhausted + self.assertEqual(it.__reduce__()[1][1], n) + + def test_reverse_iterator(self): + d = deque(range(300)) + it = reversed(d) + next(it) + self.assertEqual(list(it), list(range(298, -1, -1))) + + def test_join(self): + items = [str(i) for i in range(500)] + d = deque(items) + self.assertEqual(''.join(d), ''.join(items)) + self.assertEqual(', '.join(d), ', '.join(items)) + self.assertRaises(TypeError, ''.join, deque(['a', 1])) + + def test_mutation_during_iteration(self): + d = deque(range(10)) + it = iter(d) + next(it) + d.append(10) + self.assertRaises(RuntimeError, list, it) + # Like next(), the error exhausts the iterator. + self.assertEqual(it.__length_hint__(), 0) + self.assertRaises(RuntimeError, next, it) + + d = deque() + it = iter(d) + d.append(1) + self.assertRaises(RuntimeError, list, it) + + # The state is checked before exhaustion, as in next(). + d = deque([1]) + it = iter(d) + self.assertEqual(list(it), [1]) + d.append(2) + self.assertRaises(RuntimeError, list, it) + + lst = [0] + d = deque(range(5)) + it = iter(d) + d.pop() + self.assertRaises(RuntimeError, lst.extend, it) + self.assertEqual(lst, [0]) + + def test_subclasses(self): + class D(deque): + pass + self.check(D(range(200))) + + calls = [] + class WithLen(deque): + def __len__(self): + calls.append(1) + return deque.__len__(self) + self.assertEqual(list(WithLen(range(5))), list(range(5))) + self.assertTrue(calls) + + class WithIter(deque): + def __iter__(self): + return iter(['x']) + self.assertEqual(list(WithIter(range(5))), ['x']) + self.assertEqual(''.join(WithIter('abc')), 'x') + + def test_allocation_matches_generic_path(self): + for n in (1, 2, 5, 64, 65, 129): + self.assertEqual(sys.getsizeof(list(deque(range(n)))), + sys.getsizeof(list(iter(range(n))))) + a = list(range(3)) + a.extend(deque(range(n))) + b = list(range(3)) + b.extend(iter(range(n))) + self.assertEqual(a, b) + self.assertEqual(sys.getsizeof(a), sys.getsizeof(b)) + + def test_refcounts(self): + obj = object() + before = sys.getrefcount(obj) + d = deque([obj] * 200) + lst = list(d) + lst2 = list(iter(d)) + self.assertEqual(sys.getrefcount(obj), before + 600) + del d, lst, lst2 + self.assertEqual(sys.getrefcount(obj), before) + class Deque(deque): pass diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py index d6efde55e698d0a..65539e398011217 100644 --- a/Lib/test/test_free_threading/test_collections.py +++ b/Lib/test/test_free_threading/test_collections.py @@ -87,5 +87,92 @@ def iterate(): ) + def test_shared_iterator(self): + # Deque iterators are advanced under the deque's lock, so threads + # sharing one iterator must together see every item exactly once. + n = 20000 + for make_iter in (iter, reversed): + with self.subTest(make_iter=make_iter.__name__): + d = deque(range(n)) + it = make_iter(d) + results = [] + + def consume(): + results.append(list(x for x in it)) + + threading_helper.run_concurrently([consume] * 4) + self.assertEqual(sorted(x for r in results for x in r), + list(range(n))) + + def test_iterate_while_mutating(self): + d = deque(range(100)) + + def mutate(): + for i in range(5000): + d.append(i) + d.popleft() + + def iterate(): + for _ in range(500): + it = iter(d) + try: + for _ in it: + pass + except RuntimeError: + pass + it.__length_hint__() + it.__reduce__() + + threading_helper.run_concurrently([mutate, iterate, iterate]) + + def test_bulk_materialize_race(self): + # list(d), list(iter(d)) and str.join(d) copy the deque in chunks + # under its critical section while another thread mutates it. + d = deque(str(i) for i in range(200)) + + def mutate(): + for i in range(5000): + d.append(str(i)) + d.popleft() + if i % 64 == 0: + d.rotate(7) + + def materialize(): + # The copy holds the deque's lock for at most one chunk at a + # time, so, as with next(), a mutation between chunks raises + # RuntimeError; a completed copy is consistent. + for _ in range(2000): + for func in (list, ''.join): + for arg in (d, iter(d)): + try: + res = func(arg) + except RuntimeError: + continue + if func is list: + self.assertIn(len(res), (200, 201)) + + threading_helper.run_concurrently([mutate, materialize, materialize]) + + def test_shared_iterator_bulk_and_next(self): + # list.extend(it) and next(it) both advance a shared iterator while + # holding the deque's lock: every item is produced exactly once. + n = 20000 + for _ in range(5): + d = deque(range(n)) + it = iter(d) + results = [] + + def bulk(): + got = [] + got.extend(it) + results.append(got) + + def step(): + results.append([x for x in it]) + + threading_helper.run_concurrently([step, bulk, step, step]) + self.assertEqual(sorted(x for r in results for x in r), + list(range(n))) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-11-12-15-30-00.gh-issue-161850.Xq3dLk.rst b/Misc/NEWS.d/next/Library/2026-11-12-15-30-00.gh-issue-161850.Xq3dLk.rst new file mode 100644 index 000000000000000..a785be5f1448860 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-12-15-30-00.gh-issue-161850.Xq3dLk.rst @@ -0,0 +1,2 @@ +Improve performance of iterating over a :class:`collections.deque` in the +:term:`free-threaded build` by locking only the deque, not the iterator. diff --git a/Misc/NEWS.d/next/Library/2026-11-12-15-31-00.gh-issue-161850.Rm8pVa.rst b/Misc/NEWS.d/next/Library/2026-11-12-15-31-00.gh-issue-161850.Rm8pVa.rst new file mode 100644 index 000000000000000..ac026e235c9bb0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-12-15-31-00.gh-issue-161850.Rm8pVa.rst @@ -0,0 +1,4 @@ +Speed up :func:`list`, :meth:`list.extend`, :meth:`str.join` and unpacking of +a :class:`collections.deque` by copying up to 64 items per critical section +in the :term:`free-threaded build`. Release the locks between chunks and +check for iterator-invalidating mutations before each chunk. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c63dc0661bd69e7..285313683a086c3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2,6 +2,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDict_GetItemRef_KnownHash_LockHeld() +#include "pycore_list.h" // _PyList_SetDequeHooks() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" @@ -1966,7 +1967,7 @@ dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) PyObject *item; if (it->deque->state != it->state) { - it->counter = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, 0); PyErr_SetString(PyExc_RuntimeError, "deque mutated during iteration"); return NULL; @@ -1978,7 +1979,7 @@ dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) item = it->b->data[it->index]; it->index++; - it->counter--; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, it->counter - 1); if (it->index == BLOCKLEN && it->counter > 0) { CHECK_NOT_END(it->b->rightlink); it->b = it->b->rightlink; @@ -1987,6 +1988,14 @@ dequeiter_next_lock_held(dequeiterobject *it, dequeobject *deque) return Py_NewRef(item); } +/* The mutable fields of a deque iterator (b, index and counter) are only + * written while holding the critical section of the deque it iterates over, + * never the iterator's own lock. Concurrent next() calls on a shared + * iterator are therefore serialized by the deque's lock, and locking the + * iterator as well is unnecessary. This halves the number of atomic + * operations per item in the free-threaded build. __length_hint__ reads + * counter without locking, so writes to it use relaxed atomic stores. + */ static PyObject * dequeiter_next(PyObject *op) { @@ -1995,9 +2004,9 @@ dequeiter_next(PyObject *op) // It's safe to access it->deque without holding the per-object lock for it // here; it->deque is only assigned during construction of it. dequeobject *deque = it->deque; - Py_BEGIN_CRITICAL_SECTION2(it, deque); + Py_BEGIN_CRITICAL_SECTION(deque); result = dequeiter_next_lock_held(it, deque); - Py_END_CRITICAL_SECTION2(); + Py_END_CRITICAL_SECTION(); return result; } @@ -2056,10 +2065,10 @@ dequeiter_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) // here; it->deque is only assigned during construction of it. dequeobject *deque = it->deque; Py_ssize_t size, counter; - Py_BEGIN_CRITICAL_SECTION2(it, deque); + Py_BEGIN_CRITICAL_SECTION(deque); size = Py_SIZE(deque); counter = it->counter; - Py_END_CRITICAL_SECTION2(); + Py_END_CRITICAL_SECTION(); return Py_BuildValue("O(On)", ty, deque, size - counter); } @@ -2089,6 +2098,148 @@ static PyType_Spec dequeiter_spec = { .slots = dequeiter_slots, }; +/******************** Bulk materialization into a list ********************/ + +/* list(d), [*d], lst.extend(d) and PySequence_Fast(d) -- and therefore + * str.join(d), which hands the deque's *iterator* to list.extend() -- used to + * fetch one item per tp_iternext call, each taking a critical section. The + * functions below instead copy up to DEQUE_EXTEND_CHUNK items per critical + * section on (list, deque). + * + * Bounded chunks amortize the per-element locking cost while releasing the + * critical section between groups of items. This bounds element transfer, + * not total lock duration: initial list capacity reservation can still do + * work proportional to the remaining size while holding both locks. + * Between chunks, iterator-invalidating changes to deque->state raise + * RuntimeError using the same check as next(). State-preserving mutations + * are not rejected, and no atomic snapshot or starvation guarantee is made. + * The element-copy loop does not call Python iteration methods. + */ +#define DEQUE_EXTEND_CHUNK BLOCKLEN + +/* An iteration position. For a deque iterator the pointers alias the + * iterator's own fields, which are only accessed while holding the deque's + * critical section (see dequeiter_next()). */ +typedef struct { + block **b; + Py_ssize_t *index; + Py_ssize_t *counter; /* number of items remaining */ + size_t state; /* deque->state the position is valid for */ +} deque_cursor; + +/* Append the next chunk of items at the cursor to 'list'. On the first + * call, room for all remaining items is reserved, mirroring the generic + * list.extend() path, which sizes the list from the length hint up front. + * Returns 1 if items remain, 0 when the cursor is exhausted, or -1 with an + * exception set. */ +static int +deque_extend_list_chunk_lock_held(dequeobject *deque, PyListObject *list, + deque_cursor *cur, int first) +{ + // Same checks, in the same order, as dequeiter_next_lock_held(). + if (deque->state != cur->state) { + FT_ATOMIC_STORE_SSIZE_RELAXED(*cur->counter, 0); + PyErr_SetString(PyExc_RuntimeError, + "deque mutated during iteration"); + return -1; + } + Py_ssize_t remaining = *cur->counter; + if (remaining == 0) { + return 0; + } + Py_ssize_t n = Py_MIN(remaining, DEQUE_EXTEND_CHUNK); + Py_ssize_t m = Py_SIZE(list); + PyObject **dest; + if (!first && list->ob_item != NULL && list->allocated - m >= n) { + dest = list->ob_item + m; + } + else { + dest = _PyList_ReserveForExtendLockHeld(list, remaining); + if (dest == NULL) { + return -1; + } + } + + block *b = *cur->b; + Py_ssize_t index = *cur->index; + Py_ssize_t left = n; + while (left > 0) { + Py_ssize_t k = Py_MIN(left, BLOCKLEN - index); + PyObject **src = &b->data[index]; + for (Py_ssize_t i = 0; i < k; i++) { + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(src[i])); + } + dest += k; + index += k; + left -= k; + // Advance like dequeiter_next_lock_held(): only while items remain. + if (index == BLOCKLEN && remaining - (n - left) > 0) { + CHECK_NOT_END(b->rightlink); + b = b->rightlink; + index = 0; + } + } + *cur->b = b; + *cur->index = index; + FT_ATOMIC_STORE_SSIZE_RELAXED(*cur->counter, remaining - n); + Py_SET_SIZE(list, m + n); + return remaining - n > 0; +} + +/* Installed in listobject.c through _PyList_SetDequeHooks(). 'iterable' is + * either a deque (or a subclass that keeps deque's __iter__ and __len__) or + * a forward deque iterator. Returns 0 on success, -1 with an exception set, + * or 1 if the generic iteration path should be used instead. */ +static int +deque_list_extend_hook(PyListObject *list, PyObject *iterable) +{ + int res; + int first = 1; + if (Py_TYPE(iterable)->tp_iternext == dequeiter_next) { + dequeiterobject *it = dequeiterobject_CAST(iterable); + // it->deque and it->state are only assigned during construction + // (see dequeiter_next()). + dequeobject *deque = it->deque; + if (deque == NULL) { + return 1; + } + deque_cursor cur = {&it->b, &it->index, &it->counter, it->state}; + do { + Py_BEGIN_CRITICAL_SECTION2(list, deque); + res = deque_extend_list_chunk_lock_held(deque, list, &cur, first); + Py_END_CRITICAL_SECTION2(); + first = 0; + } while (res > 0); + } + else { + dequeobject *deque = dequeobject_CAST(iterable); + block *b = NULL; + Py_ssize_t index = 0, counter = 0; + deque_cursor cur = {&b, &index, &counter, 0}; + do { + Py_BEGIN_CRITICAL_SECTION2(list, deque); + if (first) { + // The position deque_iter() would record for a new iterator. + b = deque->leftblock; + index = deque->leftindex; + counter = Py_SIZE(deque); + cur.state = deque->state; + } + res = deque_extend_list_chunk_lock_held(deque, list, &cur, first); + Py_END_CRITICAL_SECTION2(); + first = 0; + } while (res > 0); + } + return res; +} + +static const _PyList_DequeHooks deque_list_hooks = { + .deque_iter = deque_iter, + .deque_len = deque_len, + .dequeiter_next = dequeiter_next, + .extend = deque_list_extend_hook, +}; + /*********************** Deque Reverse Iterator **************************/ static PyObject * @@ -2119,7 +2270,7 @@ dequereviter_next_lock_held(dequeiterobject *it, dequeobject *deque) return NULL; if (it->deque->state != it->state) { - it->counter = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, 0); PyErr_SetString(PyExc_RuntimeError, "deque mutated during iteration"); return NULL; @@ -2129,7 +2280,7 @@ dequereviter_next_lock_held(dequeiterobject *it, dequeobject *deque) item = it->b->data[it->index]; it->index--; - it->counter--; + FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, it->counter - 1); if (it->index < 0 && it->counter > 0) { CHECK_NOT_END(it->b->leftlink); it->b = it->b->leftlink; @@ -2146,9 +2297,10 @@ dequereviter_next(PyObject *self) // It's safe to access it->deque without holding the per-object lock for it // here; it->deque is only assigned during construction of it. dequeobject *deque = it->deque; - Py_BEGIN_CRITICAL_SECTION2(it, deque); + // See the comment above dequeiter_next(). + Py_BEGIN_CRITICAL_SECTION(deque); item = dequereviter_next_lock_held(it, deque); - Py_END_CRITICAL_SECTION2(); + Py_END_CRITICAL_SECTION(); return item; } @@ -2893,6 +3045,10 @@ collections_exec(PyObject *module) { return -1; } + // Every module instance installs the same statically linked functions, + // so repeated or concurrent registration is harmless. + _PyList_SetDequeHooks(&deque_list_hooks); + return 0; } diff --git a/Objects/listobject.c b/Objects/listobject.c index c5fe704df0e23de..ec5dced076128e8 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -6,7 +6,7 @@ #include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDictViewObject #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() -#include "pycore_interp.h" // PyInterpreterState.list +#include "pycore_interp.h" // PyInterpreterState.list_deque_hooks #include "pycore_list.h" // struct _Py_list_freelist, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() @@ -1508,6 +1508,64 @@ list_extend_dictitems(PyListObject *self, PyDictObject *dict) return 0; } +PyObject ** +_PyList_ReserveForExtendLockHeld(PyListObject *self, Py_ssize_t n) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(n > 0); + Py_ssize_t m = Py_SIZE(self); + if (m > PY_SSIZE_T_MAX - n) { + PyErr_NoMemory(); + return NULL; + } + // Same steps as list_extend_iter_lock_held() with an exact length hint. + if (self->ob_item == NULL) { + if (list_preallocate_exact(self, n) < 0) { + return NULL; + } + } + else { + if (list_resize(self, m + n) < 0) { + return NULL; + } + /* Make the list sane again. */ + Py_SET_SIZE(self, m); + } + return self->ob_item + m; +} + +void +_PyList_SetDequeHooks(const _PyList_DequeHooks *hooks) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_atomic_store_ptr_release(&interp->list_deque_hooks, (void *)hooks); +} + +static inline const _PyList_DequeHooks * +list_get_deque_hooks(PyObject *iterable) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + const _PyList_DequeHooks *hooks = + _Py_atomic_load_ptr_acquire(&interp->list_deque_hooks); + if (hooks == NULL) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(iterable); + if (tp->tp_iternext == hooks->dequeiter_next) { + // The deque iterator type cannot be subclassed. + return hooks; + } + if (tp->tp_iter == hooks->deque_iter + && tp->tp_as_sequence != NULL + && tp->tp_as_sequence->sq_length == hooks->deque_len) + { + // The generic path would only call deque_len() and deque_iter(), + // neither of which runs user code. + return hooks; + } + return NULL; +} + static int _list_extend(PyListObject *self, PyObject *iterable) { @@ -1558,6 +1616,13 @@ _list_extend(PyListObject *self, PyObject *iterable) Py_END_CRITICAL_SECTION2(); } else { + const _PyList_DequeHooks *hooks = list_get_deque_hooks(iterable); + if (hooks != NULL) { + res = hooks->extend(self, iterable); + if (res != 1) { + return res; + } + } Py_BEGIN_CRITICAL_SECTION(self); res = list_extend_iter_lock_held(self, iterable); Py_END_CRITICAL_SECTION();