diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9ad61f458944a..91cb4d86b9465b 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -861,6 +861,11 @@ struct _is { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + /* The linked list of thread states that have been created but are + not bound to an OS thread yet. They are moved to `head` once + they are bound, so that `head` only lists fully initialized + thread states. Protected by HEAD_LOCK. */ + PyThreadState *pending; _PyThreadStateImpl *preallocated; /* The thread currently executing in the __main__ module, if any. */ PyThreadState *main; diff --git a/Python/pystate.c b/Python/pystate.c index 81360f4999849d..21aefc8ab70217 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -159,11 +159,29 @@ static void bind_gilstate_tstate(PyThreadState *); static void unbind_gilstate_tstate(PyThreadState *); static void tstate_mimalloc_bind(PyThreadState *); +static void add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, + PyThreadState *next); +static void unlink_threadstate(PyThreadState **head, PyThreadState *tstate); -static void +// Bind the thread state to the current OS thread and publish it to the +// interpreter's thread list. Returns 0 without binding if the interpreter +// started finalizing before the thread state could be bound. +static int bind_tstate(PyThreadState *tstate) { assert(tstate != NULL); + // Finalization marks the pending thread states as shutting down with + // the lock held, see _PyThreadState_RemoveExcept(). Hold the lock until + // the thread state is published so that the interpreter cannot be + // finalized (and freed) while it is still being used here. + // Don't use tstate->interp before the check: it may be already freed. + HEAD_LOCK(&_PyRuntime); + int state = _Py_atomic_load_int_relaxed(&tstate->state); + if (state == _Py_THREAD_SHUTTING_DOWN) { + HEAD_UNLOCK(&_PyRuntime); + return 0; + } + assert(tstate_is_alive(tstate) && !tstate->_status.bound); assert(!tstate->_status.unbound); // just in case assert(!tstate->_status.bound_gilstate); @@ -190,6 +208,19 @@ bind_tstate(PyThreadState *tstate) tstate_mimalloc_bind(tstate); tstate->_status.bound = 1; + + // Publish the thread state to the interpreter's thread list only now + // that it is fully initialized, so that readers of the list never see + // a partially initialized thread state. + PyInterpreterState *interp = tstate->interp; + if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { + // Start in the suspended state if there is an ongoing stop-the-world. + tstate->state = _Py_THREAD_SUSPENDED; + } + unlink_threadstate(&interp->threads.pending, tstate); + add_threadstate(interp, tstate, interp->threads.head); + HEAD_UNLOCK(&_PyRuntime); + return 1; } static void @@ -1059,7 +1090,7 @@ PyInterpreterState_Delete(PyInterpreterState *interp) break; } } - if (interp->threads.head != NULL) { + if (interp->threads.head != NULL || interp->threads.pending != NULL) { Py_FatalError("remaining threads"); } *p = interp->next; @@ -1576,8 +1607,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, assert(interp != NULL); tstate->interp = interp; - tstate->eval_breaker = - _Py_atomic_load_uintptr_relaxed(&interp->ceval.instrumentation_version); + // eval_breaker is set in add_threadstate(). // next/prev are set in add_threadstate(). assert(tstate->next == NULL); @@ -1654,10 +1684,6 @@ init_threadstate(_PyThreadStateImpl *_tstate, llist_init(&_tstate->mem_free_queue); llist_init(&_tstate->asyncio_tasks_head); - if (interp->stoptheworld.requested || _PyRuntime.stoptheworld.requested) { - // Start in the suspended state if there is an ongoing stop-the-world. - tstate->state = _Py_THREAD_SUSPENDED; - } tstate->_status.initialized = 1; } @@ -1667,6 +1693,11 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState *next) { assert(interp->threads.head != tstate); + // Set the instrumentation version, which in the free-threaded build is + // broadcast to the thread states on the interpreter's thread list. + uintptr_t version = _Py_atomic_load_uintptr_relaxed( + &interp->ceval.instrumentation_version); + _Py_atomic_or_uintptr(&tstate->eval_breaker, version); if (next != NULL) { assert(next->prev == NULL || next->prev == tstate); next->prev = tstate; @@ -1676,6 +1707,40 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, interp->threads.head = tstate; } +// Add a new, not yet bound, thread state to the interpreter's list of +// pending thread states. bind_tstate() moves it to the thread list. +static void + +add_pending_threadstate(PyInterpreterState *interp, PyThreadState *tstate) +{ + assert(tstate->prev == NULL && tstate->next == NULL); + PyThreadState *next = interp->threads.pending; + if (next != NULL) { + assert(next->prev == NULL); + next->prev = tstate; + } + tstate->next = next; + interp->threads.pending = tstate; +} + +// Remove the thread state from the list starting at `*head`, which is +// either the interpreter's thread list or its list of pending thread states. +static void +unlink_threadstate(PyThreadState **head, PyThreadState *tstate) +{ + if (tstate->prev) { + tstate->prev->next = tstate->next; + } + else { + assert(*head == tstate); + *head = tstate->next; + } + if (tstate->next) { + tstate->next->prev = tstate->prev; + } + tstate->prev = tstate->next = NULL; +} + static PyThreadState * new_threadstate(PyInterpreterState *interp, int whence) { @@ -1715,9 +1780,9 @@ new_threadstate(PyInterpreterState *interp, int whence) uint64_t id = interp->threads.next_unique_id; init_threadstate(tstate, interp, id, whence); - // Add the new thread state to the interpreter. - PyThreadState *old_head = interp->threads.head; - add_threadstate(interp, (PyThreadState *)tstate, old_head); + // Track the new thread state so that finalization can find it. It is + // only added to the interpreter's thread list once it is bound. + add_pending_threadstate(interp, (PyThreadState *)tstate); HEAD_UNLOCK(interp->runtime); @@ -1741,10 +1806,9 @@ _PyThreadState_NewBound(PyInterpreterState *interp, int whence) { PyThreadState *tstate = new_threadstate(interp, whence); if (tstate) { - bind_tstate(tstate); // This makes sure there's a gilstate tstate bound // as soon as possible. - if (gilstate_get() == NULL) { + if (bind_tstate(tstate) && gilstate_get() == NULL) { bind_gilstate_tstate(tstate); } } @@ -1961,27 +2025,27 @@ tstate_delete_common(PyThreadState *tstate, int release_gil, int is_unlinked) _PyRuntimeState *runtime = interp->runtime; HEAD_LOCK(runtime); - if (tstate->prev) { - tstate->prev->next = tstate->next; + if (tstate->_status.bound) { + unlink_threadstate(&interp->threads.head, tstate); + int state = _Py_atomic_load_int_relaxed(&tstate->state); + if (state != _Py_THREAD_SUSPENDED && + state != _Py_THREAD_SUSPENDED_WAITING) + { + // Any ongoing stop-the-world request should not wait for us + // because our thread is getting deleted. + if (interp->stoptheworld.requested) { + decrement_stoptheworld_countdown(&interp->stoptheworld); + } + if (runtime->stoptheworld.requested) { + decrement_stoptheworld_countdown(&runtime->stoptheworld); + } + } } else { - interp->threads.head = tstate->next; - } - if (tstate->next) { - tstate->next->prev = tstate->prev; - } - int state = _Py_atomic_load_int_relaxed(&tstate->state); - if (state != _Py_THREAD_SUSPENDED && - state != _Py_THREAD_SUSPENDED_WAITING) - { - // Any ongoing stop-the-world request should not wait for us because - // our thread is getting deleted. - if (interp->stoptheworld.requested) { - decrement_stoptheworld_countdown(&interp->stoptheworld); - } - if (runtime->stoptheworld.requested) { - decrement_stoptheworld_countdown(&runtime->stoptheworld); - } + // The thread state was never bound, so it is still on the list of + // pending thread states and no stop-the-world request is waiting + // on it. + unlink_threadstate(&interp->threads.pending, tstate); } #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) @@ -2030,7 +2094,9 @@ zapthreads(PyInterpreterState *interp) Cannot use _Py_FOR_EACH_TSTATE_UNLOCKED because we are freeing the thread states here. */ - while ((tstate = interp->threads.head) != NULL) { + while ((tstate = interp->threads.head) != NULL || + (tstate = interp->threads.pending) != NULL) + { tstate_verify_not_active(tstate); tstate_delete_common(tstate, /*release_gil=*/0, /*is_unlinked=*/0); free_threadstate((_PyThreadStateImpl *)tstate); @@ -2106,6 +2172,26 @@ _PyThreadState_RemoveExcept(PyThreadState *tstate) } tstate->prev = tstate->next = NULL; interp->threads.head = tstate; + + /* Also remove the thread states which are not bound yet. Mark them as + shutting down while holding the lock, so that bind_tstate() does not + bind them to an interpreter which is being finalized. */ + PyThreadState *pending = interp->threads.pending; + interp->threads.pending = NULL; + for (PyThreadState *p = pending; p != NULL; p = p->next) { + _Py_atomic_store_int(&p->state, _Py_THREAD_SHUTTING_DOWN); + } + if (list == NULL) { + list = pending; + } + else if (pending != NULL) { + PyThreadState *tail = list; + while (tail->next != NULL) { + tail = tail->next; + } + tail->next = pending; + pending->prev = tail; + } HEAD_UNLOCK(runtime); return list; @@ -2755,7 +2841,11 @@ _PyThreadState_Bind(PyThreadState *tstate) // was called, tstate becomes a dangling pointer. assert(_PyThreadState_CheckConsistency(tstate)); - bind_tstate(tstate); + // If the interpreter is already finalizing, the thread state is left + // unbound and the thread hangs once it tries to attach. + if (!bind_tstate(tstate)) { + return; + } // This makes sure there's a gilstate tstate bound // as soon as possible. if (gilstate_get() == NULL) {