Skip to content

Commit da66db3

Browse files
113xiaojiclaude
andcommitted
Copy deques into lists in bounded chunks
Copying a whole deque under one critical section holds the deque's lock for O(n). When several threads keep copying a shared deque, a writer then rarely gets the lock: on a Kunpeng 950, 7 threads calling list(d) on a 1000-item deque cut a rotate() writer's throughput by ~99% and pushed its p99 latency to ~1 ms (the PyMutex fairness timeout). Copy at most BLOCKLEN items per critical section instead. Between chunks the deque's state is checked again and a mutation raises RuntimeError, exactly as next() does, so the observable behavior under concurrent mutation is the same as iterating item by item. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent b83e424 commit da66db3

2 files changed

Lines changed: 104 additions & 81 deletions

File tree

‎Lib/test/test_free_threading/test_collections.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,8 @@ def iterate():
126126
threading_helper.run_concurrently([mutate, iterate, iterate])
127127

128128
def test_bulk_materialize_race(self):
129-
# list(d) copies the deque under its critical section and must see
130-
# a consistent snapshot; list(iter(d)) and str.join() first create
131-
# an iterator, so a mutation in between is reported as by next().
129+
# list(d), list(iter(d)) and str.join(d) copy the deque in chunks
130+
# under its critical section while another thread mutates it.
132131
d = deque(str(i) for i in range(200))
133132

134133
def mutate():
@@ -139,15 +138,18 @@ def mutate():
139138
d.rotate(7)
140139

141140
def materialize():
141+
# The copy holds the deque's lock for at most one chunk at a
142+
# time, so, as with next(), a mutation between chunks raises
143+
# RuntimeError; a completed copy is consistent.
142144
for _ in range(2000):
143-
self.assertIn(len(list(d)), (200, 201))
144145
for func in (list, ''.join):
145-
try:
146-
res = func(iter(d))
147-
except RuntimeError:
148-
continue
149-
if func is list:
150-
self.assertIn(len(res), (200, 201))
146+
for arg in (d, iter(d)):
147+
try:
148+
res = func(arg)
149+
except RuntimeError:
150+
continue
151+
if func is list:
152+
self.assertIn(len(res), (200, 201))
151153

152154
threading_helper.run_concurrently([mutate, materialize, materialize])
153155

‎Modules/_collectionsmodule.c‎

Lines changed: 92 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -2102,84 +2102,86 @@ static PyType_Spec dequeiter_spec = {
21022102

21032103
/* list(d), [*d], lst.extend(d) and PySequence_Fast(d) -- and therefore
21042104
* str.join(d), which hands the deque's *iterator* to list.extend() -- used to
2105-
* fetch one element per tp_iternext call, each taking a critical section.
2106-
* The functions below copy all remaining elements while holding the list's
2107-
* and the deque's critical sections once. The result is identical to
2108-
* running the iterator to exhaustion: no user code can run during either
2109-
* form of iteration, and holding the deque's lock gives an atomic snapshot.
2105+
* fetch one item per tp_iternext call, each taking a critical section. The
2106+
* functions below instead copy up to DEQUE_EXTEND_CHUNK items per critical
2107+
* section on (list, deque).
2108+
*
2109+
* Copying in bounded chunks, rather than all at once, keeps the time the
2110+
* deque's lock is held independent of the deque's length, so threads waiting
2111+
* for it (for example a writer while other threads keep copying the deque)
2112+
* are not starved. Between chunks the copy behaves exactly like iteration:
2113+
* the deque's state is checked again and a mutation raises RuntimeError, as
2114+
* next() would. No user code runs during either form of iteration.
21102115
*/
2116+
#define DEQUE_EXTEND_CHUNK BLOCKLEN
21112117

2112-
/* Store new references to 'n' items, starting at (b, index), into dest.
2113-
* Leaves (*pb, *pindex) at the position dequeiter_next_lock_held() would
2114-
* have reached after returning the same 'n' items. */
2115-
static void
2116-
deque_copy_forward_lock_held(block *b, Py_ssize_t index, Py_ssize_t n,
2117-
PyObject **dest, block **pb, Py_ssize_t *pindex)
2118-
{
2119-
while (n > 0) {
2120-
Py_ssize_t chunk = Py_MIN(n, BLOCKLEN - index);
2121-
PyObject **src = &b->data[index];
2122-
for (Py_ssize_t i = 0; i < chunk; i++) {
2123-
FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(src[i]));
2124-
}
2125-
dest += chunk;
2126-
index += chunk;
2127-
n -= chunk;
2128-
if (index == BLOCKLEN && n > 0) {
2129-
CHECK_NOT_END(b->rightlink);
2130-
b = b->rightlink;
2131-
index = 0;
2132-
}
2133-
}
2134-
*pb = b;
2135-
*pindex = index;
2136-
}
2137-
2138-
static int
2139-
deque_extend_list_lock_held(dequeobject *deque, PyListObject *list)
2140-
{
2141-
Py_ssize_t n = Py_SIZE(deque);
2142-
if (n == 0) {
2143-
return 0;
2144-
}
2145-
Py_ssize_t m = Py_SIZE(list);
2146-
PyObject **dest = _PyList_ReserveForExtendLockHeld(list, n);
2147-
if (dest == NULL) {
2148-
return -1;
2149-
}
2150-
block *b;
2151-
Py_ssize_t index;
2152-
deque_copy_forward_lock_held(deque->leftblock, deque->leftindex, n,
2153-
dest, &b, &index);
2154-
Py_SET_SIZE(list, m + n);
2155-
return 0;
2156-
}
2157-
2118+
/* An iteration position. For a deque iterator the pointers alias the
2119+
* iterator's own fields, which are only accessed while holding the deque's
2120+
* critical section (see dequeiter_next()). */
2121+
typedef struct {
2122+
block **b;
2123+
Py_ssize_t *index;
2124+
Py_ssize_t *counter; /* number of items remaining */
2125+
size_t state; /* deque->state the position is valid for */
2126+
} deque_cursor;
2127+
2128+
/* Append the next chunk of items at the cursor to 'list'. On the first
2129+
* call, room for all remaining items is reserved, mirroring the generic
2130+
* list.extend() path, which sizes the list from the length hint up front.
2131+
* Returns 1 if items remain, 0 when the cursor is exhausted, or -1 with an
2132+
* exception set. */
21582133
static int
2159-
dequeiter_extend_list_lock_held(dequeiterobject *it, dequeobject *deque,
2160-
PyListObject *list)
2134+
deque_extend_list_chunk_lock_held(dequeobject *deque, PyListObject *list,
2135+
deque_cursor *cur, int first)
21612136
{
21622137
// Same checks, in the same order, as dequeiter_next_lock_held().
2163-
if (deque->state != it->state) {
2164-
FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, 0);
2138+
if (deque->state != cur->state) {
2139+
FT_ATOMIC_STORE_SSIZE_RELAXED(*cur->counter, 0);
21652140
PyErr_SetString(PyExc_RuntimeError,
21662141
"deque mutated during iteration");
21672142
return -1;
21682143
}
2169-
Py_ssize_t n = it->counter;
2170-
if (n == 0) {
2144+
Py_ssize_t remaining = *cur->counter;
2145+
if (remaining == 0) {
21712146
return 0;
21722147
}
2148+
Py_ssize_t n = Py_MIN(remaining, DEQUE_EXTEND_CHUNK);
21732149
Py_ssize_t m = Py_SIZE(list);
2174-
PyObject **dest = _PyList_ReserveForExtendLockHeld(list, n);
2175-
if (dest == NULL) {
2176-
return -1;
2150+
PyObject **dest;
2151+
if (!first && list->ob_item != NULL && list->allocated - m >= n) {
2152+
dest = list->ob_item + m;
2153+
}
2154+
else {
2155+
dest = _PyList_ReserveForExtendLockHeld(list, remaining);
2156+
if (dest == NULL) {
2157+
return -1;
2158+
}
21772159
}
2178-
deque_copy_forward_lock_held(it->b, it->index, n, dest,
2179-
&it->b, &it->index);
2180-
FT_ATOMIC_STORE_SSIZE_RELAXED(it->counter, 0);
2160+
2161+
block *b = *cur->b;
2162+
Py_ssize_t index = *cur->index;
2163+
Py_ssize_t left = n;
2164+
while (left > 0) {
2165+
Py_ssize_t k = Py_MIN(left, BLOCKLEN - index);
2166+
PyObject **src = &b->data[index];
2167+
for (Py_ssize_t i = 0; i < k; i++) {
2168+
FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(src[i]));
2169+
}
2170+
dest += k;
2171+
index += k;
2172+
left -= k;
2173+
// Advance like dequeiter_next_lock_held(): only while items remain.
2174+
if (index == BLOCKLEN && remaining - (n - left) > 0) {
2175+
CHECK_NOT_END(b->rightlink);
2176+
b = b->rightlink;
2177+
index = 0;
2178+
}
2179+
}
2180+
*cur->b = b;
2181+
*cur->index = index;
2182+
FT_ATOMIC_STORE_SSIZE_RELAXED(*cur->counter, remaining - n);
21812183
Py_SET_SIZE(list, m + n);
2182-
return 0;
2184+
return remaining - n > 0;
21832185
}
21842186

21852187
/* Installed in listobject.c through _PyList_SetDequeHooks(). 'iterable' is
@@ -2190,22 +2192,41 @@ static int
21902192
deque_list_extend_hook(PyListObject *list, PyObject *iterable)
21912193
{
21922194
int res;
2195+
int first = 1;
21932196
if (Py_TYPE(iterable)->tp_iternext == dequeiter_next) {
21942197
dequeiterobject *it = dequeiterobject_CAST(iterable);
2195-
// it->deque is only assigned during construction (see dequeiter_next).
2198+
// it->deque and it->state are only assigned during construction
2199+
// (see dequeiter_next()).
21962200
dequeobject *deque = it->deque;
21972201
if (deque == NULL) {
21982202
return 1;
21992203
}
2200-
Py_BEGIN_CRITICAL_SECTION2(list, deque);
2201-
res = dequeiter_extend_list_lock_held(it, deque, list);
2202-
Py_END_CRITICAL_SECTION2();
2204+
deque_cursor cur = {&it->b, &it->index, &it->counter, it->state};
2205+
do {
2206+
Py_BEGIN_CRITICAL_SECTION2(list, deque);
2207+
res = deque_extend_list_chunk_lock_held(deque, list, &cur, first);
2208+
Py_END_CRITICAL_SECTION2();
2209+
first = 0;
2210+
} while (res > 0);
22032211
}
22042212
else {
22052213
dequeobject *deque = dequeobject_CAST(iterable);
2206-
Py_BEGIN_CRITICAL_SECTION2(list, deque);
2207-
res = deque_extend_list_lock_held(deque, list);
2208-
Py_END_CRITICAL_SECTION2();
2214+
block *b = NULL;
2215+
Py_ssize_t index = 0, counter = 0;
2216+
deque_cursor cur = {&b, &index, &counter, 0};
2217+
do {
2218+
Py_BEGIN_CRITICAL_SECTION2(list, deque);
2219+
if (first) {
2220+
// The position deque_iter() would record for a new iterator.
2221+
b = deque->leftblock;
2222+
index = deque->leftindex;
2223+
counter = Py_SIZE(deque);
2224+
cur.state = deque->state;
2225+
}
2226+
res = deque_extend_list_chunk_lock_held(deque, list, &cur, first);
2227+
Py_END_CRITICAL_SECTION2();
2228+
first = 0;
2229+
} while (res > 0);
22092230
}
22102231
return res;
22112232
}

0 commit comments

Comments
 (0)