Skip to content

Commit fa427cb

Browse files
authored
[3.15] gh-158195: Do not hold the type lock during stop-the-world (gh-159972) (gh-160987)
Fixes the free-threaded deadlock described in gh-158195 by removing type-lock pinning. The type lock and class dict lock can now be released normally while a thread waits to stop the world, rather than remaining pinned across that wait. For 3.15 backport, avoid changing the default build behavior. The following changes are now only for the free-threaded build: - fresh __bases__ tuple copying - type_new_copy_dict() and namespace-key normalization - addition of the new exact-Unicode dictionary helpers In the default build, you can still create a type with string-subclass keys.
1 parent 6b0001a commit fa427cb

8 files changed

Lines changed: 893 additions & 440 deletions

File tree

‎Include/internal/pycore_dict.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,12 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *k
169169
PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result);
170170
PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result);
171171
extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result);
172+
#ifdef Py_GIL_DISABLED
173+
extern PyObject *_PyDict_GetItem_ExactUnicode(PyDictObject *mp, PyObject *key);
174+
extern int _PyDict_GetItemStackRef_ExactUnicode(PyDictObject *mp,
175+
PyObject *key,
176+
_PyStackRef *value_addr);
177+
#endif
172178
PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value);
173179

174180
extern int _PyDict_Pop_KnownHash(

‎Lib/test/test_descr.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4371,6 +4371,22 @@ class B_tuple(tuple):
43714371
B_tuple.__bases__ = (tuple,)
43724372
self.assertEqual(repr(b), '(1, 2)')
43734373

4374+
@support.cpython_only
4375+
def test_assign_bases_tuple_identity(self):
4376+
class A:
4377+
pass
4378+
4379+
class B(A):
4380+
pass
4381+
4382+
bases = (A,)
4383+
B.__bases__ = bases
4384+
self.assertEqual(B.__bases__, bases)
4385+
if support.Py_GIL_DISABLED:
4386+
self.assertIsNot(B.__bases__, bases)
4387+
else:
4388+
self.assertIs(B.__bases__, bases)
4389+
43744390
def test_assign_bases_many_subclasses(self):
43754391
# This is intended to check that typeobject.c:queue_slot_update() can
43764392
# handle updating many subclasses when a slot method is re-assigned.
@@ -5098,6 +5114,47 @@ def __new__(mcls, name, bases, ns):
50985114
with self.assertWarnsRegex(RuntimeWarning, 'MyClass'):
50995115
MyClass = meta('MyClass', (), {})
51005116

5117+
@support.cpython_only
5118+
def test_str_subclass_namespace_key_identity(self):
5119+
class Key(str):
5120+
pass
5121+
5122+
key = Key('other')
5123+
C = type('C', (), {key: 1})
5124+
stored_key = next(iter(C.__dict__))
5125+
self.assertEqual(stored_key, key)
5126+
self.assertEqual(C.other, 1)
5127+
if support.Py_GIL_DISABLED:
5128+
self.assertIs(type(stored_key), str)
5129+
else:
5130+
self.assertIs(stored_key, key)
5131+
5132+
@unittest.skipUnless(support.Py_GIL_DISABLED, 'needs a free-threaded build')
5133+
def test_str_subclass_namespace_keys(self):
5134+
# Keys of the class namespace that are str subclasses are copied
5135+
# to exact str in the type's __dict__, like type.__setattr__ does.
5136+
class Key(str):
5137+
def __hash__(self):
5138+
# Collide with the slot name assigned below.
5139+
return hash('__hash__')
5140+
def __eq__(self, other):
5141+
# Must not be called by type dict lookups.
5142+
raise AssertionError('Key.__eq__ called')
5143+
5144+
ns = {Key('other'): 1, '__repr__': lambda self: 'r', 'c': 2}
5145+
C = type('C', (), ns)
5146+
for key in C.__dict__:
5147+
self.assertIs(type(key), str)
5148+
self.assertEqual([str(k) for k in ns], list(C.__dict__)[:3])
5149+
self.assertEqual(repr(C()), 'r')
5150+
self.assertEqual(C.other, 1)
5151+
# Lookups and slot updates that hash-collide with 'other' do not
5152+
# call Key.__eq__.
5153+
C.__hash__ = None
5154+
with self.assertRaises(TypeError):
5155+
hash(C())
5156+
del C.__hash__
5157+
51015158
def test_proxy_call(self):
51025159
class FakeStr:
51035160
__class__ = str

‎Lib/test/test_free_threading/test_threading.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,30 @@ def mutate_thread():
2626

2727

2828
class TestThreadState(unittest.TestCase):
29+
def test_stop_the_world(self):
30+
import _testinternalcapi
31+
32+
_testinternalcapi.stop_the_world()
33+
_testinternalcapi.stop_the_world(0)
34+
35+
def pause():
36+
for _ in range(10):
37+
_testinternalcapi.stop_the_world(0.0001)
38+
39+
threading_helper.run_concurrently(pause, nthreads=2)
40+
for duration in (-1, -0.1, float("nan")):
41+
with self.subTest(duration=duration):
42+
with self.assertRaises(ValueError):
43+
_testinternalcapi.stop_the_world(duration)
44+
for duration in (float("inf"), 10**100):
45+
with self.subTest(duration=duration):
46+
with self.assertRaises(OverflowError):
47+
_testinternalcapi.stop_the_world(duration)
48+
for duration in (None, "0"):
49+
with self.subTest(duration=duration):
50+
with self.assertRaises(TypeError):
51+
_testinternalcapi.stop_the_world(duration)
52+
2953
@support.requires_subprocess()
3054
def test_tight_stw_loop_does_not_starve_attach(self):
3155
script = textwrap.dedent(f"""

‎Lib/test/test_free_threading/test_type.py‎

Lines changed: 173 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1+
import gc
12
import threading
23
import unittest
34

45
from concurrent.futures import ThreadPoolExecutor
56
from threading import Thread
67
from unittest import TestCase
78

8-
from test.support import threading_helper
9+
from test.support import import_helper, threading_helper
10+
11+
_testinternalcapi = import_helper.import_module("_testinternalcapi")
912

1013

1114

@@ -17,6 +20,24 @@
1720
class A:
1821
attr = 1
1922

23+
def world_stoppers():
24+
"""Finite workloads that contend for the stop-the-world mutex.
25+
26+
Waiting for the mutex releases the type lock, so that other threads
27+
can change types in the meantime. Use a short pause rather than full collections
28+
to prolong the contention without depending on the size of the heap.
29+
"""
30+
def stopper():
31+
for _ in range(100):
32+
_testinternalcapi.stop_the_world()
33+
34+
def pauser():
35+
for _ in range(100):
36+
_testinternalcapi.stop_the_world(0.0001)
37+
38+
return [stopper, pauser]
39+
40+
2041
@threading_helper.requires_working_threading()
2142
class TestType(TestCase):
2243
def test_attr_cache(self):
@@ -182,9 +203,8 @@ def wrapper():
182203
reader.join()
183204

184205
def test_setattr_many_subclasses(self):
185-
# gh-155978: Updating a special method queues a slot update for every
186-
# affected subclass. Keep enough subclasses alive to require
187-
# heap-allocated queue chunks in addition to the stack chunk.
206+
# gh-155978: Updating a special method updates the slot of every
207+
# affected subclass.
188208
class Base:
189209
pass
190210

@@ -201,6 +221,10 @@ def custom_repr(self):
201221
self.assertTrue(all(repr(cls()) != "custom repr"
202222
for cls in subclasses))
203223

224+
# The slot update must not keep references to the types.
225+
del subclasses
226+
gc.collect()
227+
self.assertEqual(Base.__subclasses__(), [])
204228

205229
def test_concurrent_setattr_deadlock(self):
206230
# gh-155400: two threads assigning to a special method of the same
@@ -259,6 +283,151 @@ class B(A):
259283
with threading_helper.start_threads(threads):
260284
pass
261285

286+
def check_stop_the_world_under_type_lock(self, setter):
287+
# gh-158195: Python code that runs with the type lock held and stops
288+
# the world could deadlock with a thread that held the type lock
289+
# pinned while it waited for the stop-the-world mutex (__bases__
290+
# assignment). The first thread released the type lock while it was
291+
# blocked on the stop-the-world mutex, was handed the mutex, and then
292+
# waited for the type lock while owning the mutex.
293+
def basesetter():
294+
class A:
295+
pass
296+
class B:
297+
pass
298+
class C(A):
299+
pass
300+
for _ in range(100):
301+
C.__bases__ = (B,)
302+
C.__bases__ = (A,)
303+
304+
def stopper():
305+
for _ in range(100):
306+
_testinternalcapi.stop_the_world()
307+
308+
# The stopper keeps the stop-the-world mutex contended, which makes
309+
# the setters block on it. All workers have finite workloads so
310+
# that repeated invalidation cannot indefinitely delay a setter.
311+
threading_helper.run_concurrently(
312+
[setter, setter, basesetter, stopper])
313+
314+
def test_stop_the_world_in_mro_deadlock(self):
315+
# type_ready() calls a metaclass mro() with the type lock held.
316+
N = 50
317+
318+
class Meta(type):
319+
def mro(cls):
320+
_testinternalcapi.stop_the_world()
321+
return type.mro(cls)
322+
323+
def setter():
324+
for _ in range(N):
325+
Meta("K", (), {})
326+
327+
self.check_stop_the_world_under_type_lock(setter)
328+
329+
def test_reentrant_mro_always_publishes(self):
330+
# A custom mro() that updates a slot on every call must not make
331+
# the __bases__ assignment retry forever.
332+
class Other:
333+
pass
334+
335+
class Meta(type):
336+
def mro(cls):
337+
Other.__repr__ = lambda self: "x"
338+
return type.mro(cls)
339+
340+
class A:
341+
pass
342+
343+
class B:
344+
pass
345+
346+
class C(A, metaclass=Meta):
347+
pass
348+
349+
C.__bases__ = (B,)
350+
self.assertEqual(C.__mro__, (C, B, object))
351+
352+
def test_slot_update_skips_non_str_keys(self):
353+
# Slots are computed (with the world stopped, for an update) without
354+
# comparing the slot name with non-str keys of the type dicts, which
355+
# could run user code. Here, that code would also update a slot
356+
# every time.
357+
calls = 0
358+
359+
class Other:
360+
pass
361+
362+
class Key:
363+
def __hash__(self):
364+
return hash("__hash__")
365+
366+
def __eq__(self, other):
367+
nonlocal calls
368+
calls += 1
369+
Other.__repr__ = lambda self: "x"
370+
return False
371+
372+
class Base:
373+
pass
374+
375+
with self.assertWarns(RuntimeWarning):
376+
Sub = type("Sub", (Base,), {Key(): None})
377+
self.assertEqual(calls, 0)
378+
379+
Base.__hash__ = None
380+
self.assertEqual(calls, 0)
381+
with self.assertRaises(TypeError):
382+
hash(Sub())
383+
del Base.__hash__
384+
hash(Sub())
385+
386+
def test_concurrent_bases_assignment(self):
387+
# Several threads reassign __bases__ of the same class, and of its
388+
# base, while others stop the world. The type lock is released while
389+
# waiting for the stop-the-world. At the end, the MRO must agree
390+
# with __bases__.
391+
N = 50
392+
393+
class A:
394+
def meth(self):
395+
return "A"
396+
def __repr__(self):
397+
return "A"
398+
class B:
399+
def meth(self):
400+
return "B"
401+
def __repr__(self):
402+
return "B"
403+
class C(A):
404+
pass
405+
class D(C):
406+
pass
407+
408+
def setter():
409+
for _ in range(N):
410+
C.__bases__ = (B,)
411+
C.__bases__ = (A,)
412+
413+
def subsetter():
414+
for _ in range(N):
415+
D.__bases__ = (C, B)
416+
D.__bases__ = (C,)
417+
418+
threading_helper.run_concurrently(
419+
[setter, setter, subsetter] + world_stoppers())
420+
421+
self.assertEqual(C.__bases__, (A,))
422+
self.assertEqual(C.__mro__, (C, A, object))
423+
self.assertEqual(D.__bases__, (C,))
424+
self.assertEqual(D.__mro__, (D, C, A, object))
425+
self.assertEqual(D().meth(), "A")
426+
self.assertEqual(repr(C()), "A")
427+
self.assertEqual(repr(D()), "A")
428+
self.assertIn(C, A.__subclasses__())
429+
self.assertNotIn(C, B.__subclasses__())
430+
262431

263432
if __name__ == "__main__":
264433
unittest.main()
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Fix a deadlock in the free-threaded build between a thread that stops the
2+
world while running Python code under the type lock (for example a custom
3+
metaclass ``mro()`` or the ``__eq__`` of a :class:`str` subclass used as a
4+
class ``__dict__`` key) and a thread that assigns to a special method or to
5+
``__bases__`` of a class. The type lock is no longer held while waiting
6+
for a stop-the-world; the prepared slot, MRO and bases updates are
7+
validated with the world stopped and recomputed if they went stale.

‎Modules/_testinternalcapi.c‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
#include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromDict()
3939
#include "pycore_pystate.h" // _PyThreadState_GET()
4040
#include "pycore_runtime_structs.h" // _PY_NSMALLPOSINTS
41+
#include "pycore_time.h" // _PyTime_FromSecondsObject()
4142
#include "pycore_unicodeobject.h" // _PyUnicode_TransformDecimalAndSpaceToASCII()
4243

4344
#include "clinic/_testinternalcapi.c.h"
@@ -203,6 +204,42 @@ get_stack_margin(PyObject *self, PyObject *Py_UNUSED(args))
203204
return PyLong_FromSize_t(_PyOS_STACK_MARGIN_BYTES);
204205
}
205206

207+
static PyObject *
208+
stop_the_world(PyObject *self, PyObject *args)
209+
{
210+
PyObject *duration_obj = NULL;
211+
PyTime_t duration = 0;
212+
if (!PyArg_ParseTuple(args, "|O:stop_the_world", &duration_obj)) {
213+
return NULL;
214+
}
215+
if (duration_obj != NULL &&
216+
_PyTime_FromSecondsObject(&duration, duration_obj,
217+
_PyTime_ROUND_TIMEOUT) < 0) {
218+
return NULL;
219+
}
220+
if (duration < 0) {
221+
PyErr_SetString(PyExc_ValueError, "duration must be non-negative");
222+
return NULL;
223+
}
224+
#ifdef Py_GIL_DISABLED
225+
PyInterpreterState *interp = _PyInterpreterState_GET();
226+
_PyEval_StopTheWorld(interp);
227+
if (duration != 0) {
228+
PyEvent event = {0};
229+
PyEvent_WaitTimed(&event, duration, /*detach=*/0);
230+
}
231+
_PyEval_StartTheWorld(interp);
232+
#endif
233+
Py_RETURN_NONE;
234+
}
235+
236+
PyDoc_STRVAR(stop_the_world_doc,
237+
"stop_the_world($module, duration=0, /)\n"
238+
"--\n"
239+
"\n"
240+
"Stop the world, wait for duration seconds, then restart it.\n"
241+
"Does nothing in GIL-enabled builds.");
242+
206243
static PyObject *
207244
test_stop_the_world(PyObject *self, PyObject *Py_UNUSED(args))
208245
{
@@ -3299,6 +3336,7 @@ static PyMethodDef module_functions[] = {
32993336
{"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS},
33003337
{"get_stack_pointer", get_stack_pointer, METH_NOARGS},
33013338
{"get_stack_margin", get_stack_margin, METH_NOARGS},
3339+
{"stop_the_world", stop_the_world, METH_VARARGS, stop_the_world_doc},
33023340
{"test_stop_the_world", test_stop_the_world, METH_NOARGS},
33033341
{"classify_stack_addresses", classify_stack_addresses, METH_VARARGS},
33043342
{"get_jit_code_ranges", get_jit_code_ranges, METH_NOARGS},

0 commit comments

Comments
 (0)