From e89873fbc95027f04e6e4c03259b87738412d34f Mon Sep 17 00:00:00 2001 From: Tomek Paczkowski Date: Thu, 12 Nov 2026 15:27:43 +0000 Subject: [PATCH 1/2] Keep the GIL switch interval deadline across early wake-ups take_gil() set the drop request only when a single COND_TIMED_WAIT call timed out. Every GIL release signals gil->cond, so each release by the holder woke the waiter early and restarted the full switch interval. On a multi-core machine, a holder that releases the GIL briefly and often (for example zlib.crc32 or hashlib on large inputs between short stretches of Python code) re-takes it before the waiter runs. The waiter's interval then never expires, and it can wait for seconds. Measure the interval from the start of the wait, or from the last switch to another thread, so that early wake-ups no longer restart it. Wait only for the remaining time, and decide from the monotonic clock whether the interval has passed. On macOS, the condition variable uses the wall clock, so the result of the wait alone is not sufficient. --- Python/ceval_gil.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 91c75cdd2ed413..7a0c52ee5c2b9b 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -7,6 +7,7 @@ #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_pystats.h" // _Py_PrintSpecializationStats() #include "pycore_runtime.h" // _PyRuntime +#include "pycore_time.h" // _PyTime_FromMicrosecondsClamp() /* @@ -20,8 +21,9 @@ - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be able to release the GIL on demand by another thread. A volatile boolean variable (gil_drop_request) is used for that purpose, which is checked - at every turn of the eval loop. That variable is set after a wait of - `interval` microseconds on `gil_cond` has timed out. + at every turn of the eval loop. That variable is set after a thread has + waited on `gil_cond` for `interval` microseconds in total, with no + switch to another thread in the meantime. [Actually, another volatile boolean variable (eval_breaker) is used which ORs several conditions into one. Volatile booleans are @@ -324,22 +326,47 @@ take_gil(PyThreadState *tstate) tstate->gil_requested = 1; int drop_requested = 0; + /* The switch interval runs from the start of the wait, from the last + switch to another thread, or from the last drop request. A wake-up + caused by the holder releasing the GIL and taking it back before this + thread runs does not restart it. */ + int restart_interval = 1; + unsigned long saved_switchnum = 0; + PyTime_t interval_start = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { - unsigned long saved_switchnum = gil->switch_number; + PyTime_t now; + (void)PyTime_MonotonicRaw(&now); + if (restart_interval || gil->switch_number != saved_switchnum) { + restart_interval = 0; + saved_switchnum = gil->switch_number; + interval_start = now; + } unsigned long interval = _Py_atomic_load_ulong_relaxed(&gil->interval); if (interval < 1) { interval = 1; } + PyTime_t deadline = _PyTime_Add( + interval_start, _PyTime_FromMicrosecondsClamp(interval)); + long long remaining_us = _PyTime_AsMicroseconds( + deadline - now, _PyTime_ROUND_CEILING); + if (remaining_us < 1) { + remaining_us = 1; + } int timed_out = 0; - COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out); - - /* If we timed out and no switch occurred in the meantime, it is time - to ask the GIL-holding thread to drop it. */ + COND_TIMED_WAIT(gil->cond, gil->mutex, remaining_us, timed_out); + /* Use the clock to decide: the wait can end early, and a timeout + reported by the wait does not prove that the deadline passed. */ + (void)PyTime_MonotonicRaw(&now); + timed_out = (now >= deadline); + + /* If the interval has passed and no switch occurred in the meantime, + it is time to ask the GIL-holding thread to drop it. */ if (timed_out && _Py_atomic_load_int_relaxed(&gil->locked) && gil->switch_number == saved_switchnum) { + restart_interval = 1; PyThreadState *holder_tstate = (PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder); if (_PyThreadState_MustExit(tstate)) { From 50403744f3bb673b6c28ac1cdb766b081c013f52 Mon Sep 17 00:00:00 2001 From: Tomek Paczkowski Date: Thu, 12 Nov 2026 16:30:00 +0000 Subject: [PATCH 2/2] Add a test and a NEWS entry for the GIL switch interval fix --- Lib/test/test_threading.py | 43 +++++++++++++++++++ ...-11-12-15-41-30.gh-issue-161868.zRIwaU.rst | 4 ++ 2 files changed, 47 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-15-41-30.gh-issue-161868.zRIwaU.rst diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 96b43936be92cd..267c1359180776 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -592,6 +592,49 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) + @requires_gil_enabled() + def test_switch_interval_with_frequent_gil_releases(self): + # gh-161868: A thread that releases the GIL briefly and often must + # not keep a waiting thread off the GIL for much longer than the + # switch interval. + zlib = import_module("zlib") + # zlib.crc32() releases the GIL for inputs over 5 KiB. + data = b"x" * 6000 + self.addCleanup(sys.setswitchinterval, sys.getswitchinterval()) + support.setswitchinterval(0.005) + + stop = False + timed_out = False + releases = 0 + + def worker(): + nonlocal timed_out, releases + # Stop if the main thread cannot take the GIL back. + deadline = time.monotonic() + support.SHORT_TIMEOUT + while not stop: + if time.monotonic() > deadline: + timed_out = True + break + end = time.perf_counter() + 0.001 + while time.perf_counter() < end: + pass + zlib.crc32(data) + releases += 1 + + thread = threading.Thread(target=worker) + thread.start() + try: + # Each round takes the GIL back from the busy worker. This should + # take about one switch interval. + for _ in range(20): + previous = releases + while releases == previous and thread.is_alive(): + time.sleep(0.001) + finally: + stop = True + thread.join() + self.assertFalse(timed_out, "the main thread could not take the GIL") + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) def test_join_from_multiple_threads(self, size): # Thread.join() should be thread-safe diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-15-41-30.gh-issue-161868.zRIwaU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-15-41-30.gh-issue-161868.zRIwaU.rst new file mode 100644 index 00000000000000..f434a0f3404b18 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-15-41-30.gh-issue-161868.zRIwaU.rst @@ -0,0 +1,4 @@ +Fix a thread that waits for the :term:`GIL` being kept waiting much longer +than the :func:`switch interval ` when the thread +that holds the GIL releases it briefly and often, for example in +:func:`zlib.crc32` on large inputs. Patch by Tomek Paczkowski.