@@ -2102,84 +2102,86 @@ static PyType_Spec dequeiter_spec = {
21022102
21032103/* list(d), [*d], lst.extend(d) and PySequence_Fast(d) -- and therefore
21042104 * str.join(d), which hands the deque's *iterator* to list.extend() -- used to
2105- * fetch one element per tp_iternext call, each taking a critical section.
2106- * The functions below copy all remaining elements while holding the list's
2107- * and the deque's critical sections once. The result is identical to
2108- * running the iterator to exhaustion: no user code can run during either
2109- * form of iteration, and holding the deque's lock gives an atomic snapshot.
2105+ * fetch one item per tp_iternext call, each taking a critical section. The
2106+ * functions below instead copy up to DEQUE_EXTEND_CHUNK items per critical
2107+ * section on (list, deque).
2108+ *
2109+ * Copying in bounded chunks, rather than all at once, keeps the time the
2110+ * deque's lock is held independent of the deque's length, so threads waiting
2111+ * for it (for example a writer while other threads keep copying the deque)
2112+ * are not starved. Between chunks the copy behaves exactly like iteration:
2113+ * the deque's state is checked again and a mutation raises RuntimeError, as
2114+ * next() would. No user code runs during either form of iteration.
21102115 */
2116+ #define DEQUE_EXTEND_CHUNK BLOCKLEN
21112117
2112- /* Store new references to 'n' items, starting at (b, index), into dest.
2113- * Leaves (*pb, *pindex) at the position dequeiter_next_lock_held() would
2114- * have reached after returning the same 'n' items. */
2115- static void
2116- deque_copy_forward_lock_held (block * b , Py_ssize_t index , Py_ssize_t n ,
2117- PyObject * * dest , block * * pb , Py_ssize_t * pindex )
2118- {
2119- while (n > 0 ) {
2120- Py_ssize_t chunk = Py_MIN (n , BLOCKLEN - index );
2121- PyObject * * src = & b -> data [index ];
2122- for (Py_ssize_t i = 0 ; i < chunk ; i ++ ) {
2123- FT_ATOMIC_STORE_PTR_RELEASE (dest [i ], Py_NewRef (src [i ]));
2124- }
2125- dest += chunk ;
2126- index += chunk ;
2127- n -= chunk ;
2128- if (index == BLOCKLEN && n > 0 ) {
2129- CHECK_NOT_END (b -> rightlink );
2130- b = b -> rightlink ;
2131- index = 0 ;
2132- }
2133- }
2134- * pb = b ;
2135- * pindex = index ;
2136- }
2137-
2138- static int
2139- deque_extend_list_lock_held (dequeobject * deque , PyListObject * list )
2140- {
2141- Py_ssize_t n = Py_SIZE (deque );
2142- if (n == 0 ) {
2143- return 0 ;
2144- }
2145- Py_ssize_t m = Py_SIZE (list );
2146- PyObject * * dest = _PyList_ReserveForExtendLockHeld (list , n );
2147- if (dest == NULL ) {
2148- return -1 ;
2149- }
2150- block * b ;
2151- Py_ssize_t index ;
2152- deque_copy_forward_lock_held (deque -> leftblock , deque -> leftindex , n ,
2153- dest , & b , & index );
2154- Py_SET_SIZE (list , m + n );
2155- return 0 ;
2156- }
2157-
2118+ /* An iteration position. For a deque iterator the pointers alias the
2119+ * iterator's own fields, which are only accessed while holding the deque's
2120+ * critical section (see dequeiter_next()). */
2121+ typedef struct {
2122+ block * * b ;
2123+ Py_ssize_t * index ;
2124+ Py_ssize_t * counter ; /* number of items remaining */
2125+ size_t state ; /* deque->state the position is valid for */
2126+ } deque_cursor ;
2127+
2128+ /* Append the next chunk of items at the cursor to 'list'. On the first
2129+ * call, room for all remaining items is reserved, mirroring the generic
2130+ * list.extend() path, which sizes the list from the length hint up front.
2131+ * Returns 1 if items remain, 0 when the cursor is exhausted, or -1 with an
2132+ * exception set. */
21582133static int
2159- dequeiter_extend_list_lock_held ( dequeiterobject * it , dequeobject * deque ,
2160- PyListObject * list )
2134+ deque_extend_list_chunk_lock_held ( dequeobject * deque , PyListObject * list ,
2135+ deque_cursor * cur , int first )
21612136{
21622137 // Same checks, in the same order, as dequeiter_next_lock_held().
2163- if (deque -> state != it -> state ) {
2164- FT_ATOMIC_STORE_SSIZE_RELAXED (it -> counter , 0 );
2138+ if (deque -> state != cur -> state ) {
2139+ FT_ATOMIC_STORE_SSIZE_RELAXED (* cur -> counter , 0 );
21652140 PyErr_SetString (PyExc_RuntimeError ,
21662141 "deque mutated during iteration" );
21672142 return -1 ;
21682143 }
2169- Py_ssize_t n = it -> counter ;
2170- if (n == 0 ) {
2144+ Py_ssize_t remaining = * cur -> counter ;
2145+ if (remaining == 0 ) {
21712146 return 0 ;
21722147 }
2148+ Py_ssize_t n = Py_MIN (remaining , DEQUE_EXTEND_CHUNK );
21732149 Py_ssize_t m = Py_SIZE (list );
2174- PyObject * * dest = _PyList_ReserveForExtendLockHeld (list , n );
2175- if (dest == NULL ) {
2176- return -1 ;
2150+ PyObject * * dest ;
2151+ if (!first && list -> ob_item != NULL && list -> allocated - m >= n ) {
2152+ dest = list -> ob_item + m ;
2153+ }
2154+ else {
2155+ dest = _PyList_ReserveForExtendLockHeld (list , remaining );
2156+ if (dest == NULL ) {
2157+ return -1 ;
2158+ }
21772159 }
2178- deque_copy_forward_lock_held (it -> b , it -> index , n , dest ,
2179- & it -> b , & it -> index );
2180- FT_ATOMIC_STORE_SSIZE_RELAXED (it -> counter , 0 );
2160+
2161+ block * b = * cur -> b ;
2162+ Py_ssize_t index = * cur -> index ;
2163+ Py_ssize_t left = n ;
2164+ while (left > 0 ) {
2165+ Py_ssize_t k = Py_MIN (left , BLOCKLEN - index );
2166+ PyObject * * src = & b -> data [index ];
2167+ for (Py_ssize_t i = 0 ; i < k ; i ++ ) {
2168+ FT_ATOMIC_STORE_PTR_RELEASE (dest [i ], Py_NewRef (src [i ]));
2169+ }
2170+ dest += k ;
2171+ index += k ;
2172+ left -= k ;
2173+ // Advance like dequeiter_next_lock_held(): only while items remain.
2174+ if (index == BLOCKLEN && remaining - (n - left ) > 0 ) {
2175+ CHECK_NOT_END (b -> rightlink );
2176+ b = b -> rightlink ;
2177+ index = 0 ;
2178+ }
2179+ }
2180+ * cur -> b = b ;
2181+ * cur -> index = index ;
2182+ FT_ATOMIC_STORE_SSIZE_RELAXED (* cur -> counter , remaining - n );
21812183 Py_SET_SIZE (list , m + n );
2182- return 0 ;
2184+ return remaining - n > 0 ;
21832185}
21842186
21852187/* Installed in listobject.c through _PyList_SetDequeHooks(). 'iterable' is
@@ -2190,22 +2192,41 @@ static int
21902192deque_list_extend_hook (PyListObject * list , PyObject * iterable )
21912193{
21922194 int res ;
2195+ int first = 1 ;
21932196 if (Py_TYPE (iterable )-> tp_iternext == dequeiter_next ) {
21942197 dequeiterobject * it = dequeiterobject_CAST (iterable );
2195- // it->deque is only assigned during construction (see dequeiter_next).
2198+ // it->deque and it->state are only assigned during construction
2199+ // (see dequeiter_next()).
21962200 dequeobject * deque = it -> deque ;
21972201 if (deque == NULL ) {
21982202 return 1 ;
21992203 }
2200- Py_BEGIN_CRITICAL_SECTION2 (list , deque );
2201- res = dequeiter_extend_list_lock_held (it , deque , list );
2202- Py_END_CRITICAL_SECTION2 ();
2204+ deque_cursor cur = {& it -> b , & it -> index , & it -> counter , it -> state };
2205+ do {
2206+ Py_BEGIN_CRITICAL_SECTION2 (list , deque );
2207+ res = deque_extend_list_chunk_lock_held (deque , list , & cur , first );
2208+ Py_END_CRITICAL_SECTION2 ();
2209+ first = 0 ;
2210+ } while (res > 0 );
22032211 }
22042212 else {
22052213 dequeobject * deque = dequeobject_CAST (iterable );
2206- Py_BEGIN_CRITICAL_SECTION2 (list , deque );
2207- res = deque_extend_list_lock_held (deque , list );
2208- Py_END_CRITICAL_SECTION2 ();
2214+ block * b = NULL ;
2215+ Py_ssize_t index = 0 , counter = 0 ;
2216+ deque_cursor cur = {& b , & index , & counter , 0 };
2217+ do {
2218+ Py_BEGIN_CRITICAL_SECTION2 (list , deque );
2219+ if (first ) {
2220+ // The position deque_iter() would record for a new iterator.
2221+ b = deque -> leftblock ;
2222+ index = deque -> leftindex ;
2223+ counter = Py_SIZE (deque );
2224+ cur .state = deque -> state ;
2225+ }
2226+ res = deque_extend_list_chunk_lock_held (deque , list , & cur , first );
2227+ Py_END_CRITICAL_SECTION2 ();
2228+ first = 0 ;
2229+ } while (res > 0 );
22092230 }
22102231 return res ;
22112232}
0 commit comments