Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions Lib/test/test_threading.py
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,49 @@ def test_enumerate_after_join(self):
finally:
sys.setswitchinterval(old_interval)

@requires_gil_enabled()
def test_switch_interval_with_frequent_gil_releases(self):
# gh-161868: A thread that releases the GIL briefly and often must
# not keep a waiting thread off the GIL for much longer than the
# switch interval.
zlib = import_module("zlib")
# zlib.crc32() releases the GIL for inputs over 5 KiB.
data = b"x" * 6000
self.addCleanup(sys.setswitchinterval, sys.getswitchinterval())
support.setswitchinterval(0.005)

stop = False
timed_out = False
releases = 0

def worker():
nonlocal timed_out, releases
# Stop if the main thread cannot take the GIL back.
deadline = time.monotonic() + support.SHORT_TIMEOUT
while not stop:
if time.monotonic() > deadline:
timed_out = True
break
end = time.perf_counter() + 0.001
while time.perf_counter() < end:
pass
zlib.crc32(data)
releases += 1

thread = threading.Thread(target=worker)
thread.start()
try:
# Each round takes the GIL back from the busy worker. This should
# take about one switch interval.
for _ in range(20):
previous = releases
while releases == previous and thread.is_alive():
time.sleep(0.001)
finally:
stop = True
thread.join()
self.assertFalse(timed_out, "the main thread could not take the GIL")

@support.bigmemtest(size=20, memuse=72*2**20, dry_run=False)
def test_join_from_multiple_threads(self, size):
# Thread.join() should be thread-safe
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix a thread that waits for the :term:`GIL` being kept waiting much longer
than the :func:`switch interval <sys.setswitchinterval>` when the thread
that holds the GIL releases it briefly and often, for example in
:func:`zlib.crc32` on large inputs. Patch by Tomek Paczkowski.
41 changes: 34 additions & 7 deletions Python/ceval_gil.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_pystats.h" // _Py_PrintSpecializationStats()
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_time.h" // _PyTime_FromMicrosecondsClamp()


/*
Expand All @@ -20,8 +21,9 @@
- In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be
able to release the GIL on demand by another thread. A volatile boolean
variable (gil_drop_request) is used for that purpose, which is checked
at every turn of the eval loop. That variable is set after a wait of
`interval` microseconds on `gil_cond` has timed out.
at every turn of the eval loop. That variable is set after a thread has
waited on `gil_cond` for `interval` microseconds in total, with no
switch to another thread in the meantime.

[Actually, another volatile boolean variable (eval_breaker) is used
which ORs several conditions into one. Volatile booleans are
Expand Down Expand Up @@ -324,22 +326,47 @@ take_gil(PyThreadState *tstate)
tstate->gil_requested = 1;

int drop_requested = 0;
/* The switch interval runs from the start of the wait, from the last
switch to another thread, or from the last drop request. A wake-up
caused by the holder releasing the GIL and taking it back before this
thread runs does not restart it. */
int restart_interval = 1;
unsigned long saved_switchnum = 0;
PyTime_t interval_start = 0;
while (_Py_atomic_load_int_relaxed(&gil->locked)) {
unsigned long saved_switchnum = gil->switch_number;
PyTime_t now;
(void)PyTime_MonotonicRaw(&now);
if (restart_interval || gil->switch_number != saved_switchnum) {
restart_interval = 0;
saved_switchnum = gil->switch_number;
interval_start = now;
}

unsigned long interval = _Py_atomic_load_ulong_relaxed(&gil->interval);
if (interval < 1) {
interval = 1;
}
PyTime_t deadline = _PyTime_Add(
interval_start, _PyTime_FromMicrosecondsClamp(interval));
long long remaining_us = _PyTime_AsMicroseconds(
deadline - now, _PyTime_ROUND_CEILING);
if (remaining_us < 1) {
remaining_us = 1;
}
int timed_out = 0;
COND_TIMED_WAIT(gil->cond, gil->mutex, interval, timed_out);

/* If we timed out and no switch occurred in the meantime, it is time
to ask the GIL-holding thread to drop it. */
COND_TIMED_WAIT(gil->cond, gil->mutex, remaining_us, timed_out);
/* Use the clock to decide: the wait can end early, and a timeout
reported by the wait does not prove that the deadline passed. */
(void)PyTime_MonotonicRaw(&now);
timed_out = (now >= deadline);

/* If the interval has passed and no switch occurred in the meantime,
it is time to ask the GIL-holding thread to drop it. */
if (timed_out &&
_Py_atomic_load_int_relaxed(&gil->locked) &&
gil->switch_number == saved_switchnum)
{
restart_interval = 1;
PyThreadState *holder_tstate =
(PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder);
if (_PyThreadState_MustExit(tstate)) {
Expand Down
Loading