From 303ef7dbea490bc18de0fc84c09dca4208dc4d83 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Wed, 11 Nov 2026 13:50:02 -0800 Subject: [PATCH 1/2] Don't wait in Thread.join(timeout) when draining supervised output On CPython 3.13 to 3.15, Thread.join(timeout) can wait without a timeout (python/cpython#161712): ThreadHandle_join reads the clock twice through Py_MAX, and a negative second read becomes an INFINITE wait. Windows hits the window far more often, because its waits end a fraction of a timer tick early and loop back around. _OutputPump.drain() polled the reader with join(0.05) while the reader sat in os.read on the child's output pipe. When the join lost its timeout, the drain waited until every writer closed the pipe: - test_supervise_output_does_not_wait_for_lingering_descendants took 30 s, the lifetime of the grandchild that inherited the pipe; - test_output_pump_drain_grace_ends_with_the_write deadlocked, since the test closes the pipe only after drain() returns. The reader now sets an Event when run() ends, and drain() waits on that. Event.wait goes through _PyMutex_LockTimed, which reads the clock once and clamps a negative remainder to zero. Every test in test_supervisor.py also gets a faulthandler watchdog that aborts the run after 120 s and dumps all thread stacks to the real stderr, so a regression fails in minutes instead of using up the job. Fixes #7819 Co-Authored-By: Claude Opus 5.5 --- ...sor-windows-lingering-descendant.bugfix.md | 1 + .../src/reflex_base/utils/supervisor.py | 41 ++++++++------ .../reflex_base/utils/test_supervisor.py | 54 +++++++++++++++++++ 3 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 packages/reflex-base/news/+json-supervisor-windows-lingering-descendant.bugfix.md diff --git a/packages/reflex-base/news/+json-supervisor-windows-lingering-descendant.bugfix.md b/packages/reflex-base/news/+json-supervisor-windows-lingering-descendant.bugfix.md new file mode 100644 index 00000000000..0ff68043540 --- /dev/null +++ b/packages/reflex-base/news/+json-supervisor-windows-lingering-descendant.bugfix.md @@ -0,0 +1 @@ +`reflex run --json` no longer waits on exit for a descendant process that outlives the app and keeps its output open, which could happen on Python 3.13 to 3.15, most often on Windows. diff --git a/packages/reflex-base/src/reflex_base/utils/supervisor.py b/packages/reflex-base/src/reflex_base/utils/supervisor.py index f2573b24d34..9b47cd4e1ee 100644 --- a/packages/reflex-base/src/reflex_base/utils/supervisor.py +++ b/packages/reflex-base/src/reflex_base/utils/supervisor.py @@ -201,26 +201,34 @@ def __init__(self, read_fd: int, write_fd: int, level: str, name: str): # Clear while a write is in progress. self._write_done = threading.Event() self._write_done.set() + # Set when run() ends. drain() waits on this rather than in + # Thread.join(timeout), which CPython 3.13 to 3.15 can turn into an + # untimed wait (python/cpython#161712): that would hold the drain until + # a lingering descendant closes the pipe. + self._finished = threading.Event() def run(self): """Forward the pipe until every writer closed it.""" pending = b"" traceback: list[str] = [] - while True: - self.idle_since = time.monotonic() - chunk = os.read(self.read_fd, 1 << 16) - self.idle_since = None - if not chunk: - break - data = pending + chunk - # A trailing \r may be the first half of a \r\n. - cut = len(data) - data.endswith(b"\r") - *lines, pending = _LINE_END.split(data[:cut]) - pending += data[cut:] - self._write(lines, traceback, final=False) - os.close(self.read_fd) - pending = pending.removesuffix(b"\r") - self._write([pending] if pending else [], traceback, final=True) + try: + while True: + self.idle_since = time.monotonic() + chunk = os.read(self.read_fd, 1 << 16) + self.idle_since = None + if not chunk: + break + data = pending + chunk + # A trailing \r may be the first half of a \r\n. + cut = len(data) - data.endswith(b"\r") + *lines, pending = _LINE_END.split(data[:cut]) + pending += data[cut:] + self._write(lines, traceback, final=False) + os.close(self.read_fd) + pending = pending.removesuffix(b"\r") + self._write([pending] if pending else [], traceback, final=True) + finally: + self._finished.set() def _write(self, lines: list[bytes], traceback: list[str], final: bool): """Write converted lines, dropping them if the consumer is gone. @@ -274,8 +282,7 @@ def drain(self, exited_at: float, writing_at_exit: float, wall_deadline: float): writing_at_exit: The cumulative write time at that snapshot. wall_deadline: The shared monotonic deadline for shutdown. """ - while self.is_alive(): - self.join(0.05) + while not self._finished.wait(0.05): now, writing_elapsed = self.writing_snapshot() idle_since = self.idle_since # Seconds charged against the five-second active drain budget. diff --git a/tests/units/reflex_base/utils/test_supervisor.py b/tests/units/reflex_base/utils/test_supervisor.py index c86447a17e7..4d00e70a543 100644 --- a/tests/units/reflex_base/utils/test_supervisor.py +++ b/tests/units/reflex_base/utils/test_supervisor.py @@ -1,6 +1,7 @@ """Tests for the ``reflex run --json`` output supervisor.""" import contextlib +import faulthandler import json import os import signal @@ -14,6 +15,34 @@ import pytest from reflex_base.utils import supervisor +# Seconds any test in this module may take before the run is aborted. +_TEST_TIMEOUT_SECONDS = 120 + + +@pytest.fixture(autouse=True) +def _abort_on_hang(capfd: pytest.CaptureFixture[str]): + """Abort the run, dumping every thread's stack, when a test hangs. + + Several waits here cannot be bounded from Python: a pipe read held open by + a stray process, the untimed wait in ``Popen.__exit__``, or a blocked + interpreter call. faulthandler's watchdog runs outside the GIL, so a + regression fails in minutes with a traceback instead of using up the job. + + Args: + capfd: Used to reach the real stderr; during a test, file descriptor 2 + is pytest's capture file, which is lost when the watchdog exits. + + Yields: + None. + """ + with capfd.disabled(): + stderr_fd = os.dup(2) + faulthandler.dump_traceback_later(_TEST_TIMEOUT_SECONDS, exit=True, file=stderr_fd) + yield + faulthandler.cancel_dump_traceback_later() + os.close(stderr_fd) + + _SUPERVISED_SCRIPT = """ import logging import multiprocessing @@ -639,3 +668,28 @@ def slow_write(_fd, _data): finally: os.close(write_fd) pump.join(5) + + +def test_output_pump_drain_does_not_join_the_reader(monkeypatch): + """The drain never waits in Thread.join, whose timeout CPython can ignore. + + On CPython 3.13 to 3.15 a timed join can wait without a timeout + (python/cpython#161712), so a reader blocked on a pipe that a lingering + descendant holds would keep the drain waiting until that descendant exits. + """ + # The write end stays open, like a descendant that outlives the command. + read_fd, write_fd = os.pipe() + pump = supervisor._OutputPump(read_fd, 1, "info", "stdout") + pump.start() + joins = [] + monkeypatch.setattr(pump, "join", joins.append) + try: + begin = time.monotonic() + pump.drain(begin, 0.0, begin + supervisor._DRAIN_WALL_SECONDS) + elapsed = time.monotonic() - begin + assert joins == [] + assert supervisor._DRAIN_IDLE_SECONDS <= elapsed < 2 + finally: + os.close(write_fd) + threading.Thread.join(pump, 5) + assert not pump.is_alive() From 67d13d52c9e3dd0eefe3a5c1286f87827f49b7b1 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Wed, 11 Nov 2026 15:16:08 -0800 Subject: [PATCH 2/2] Close the output pump's pipe even when forwarding raises The reader closed its end of the pipe only after the read loop ended normally, so an exception while forwarding leaked the descriptor. Close it in the finally block, before signalling that the reader finished. Co-Authored-By: Claude Opus 5.5 --- .../src/reflex_base/utils/supervisor.py | 3 ++- .../reflex_base/utils/test_supervisor.py | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/reflex-base/src/reflex_base/utils/supervisor.py b/packages/reflex-base/src/reflex_base/utils/supervisor.py index 9b47cd4e1ee..fbc9c3e0731 100644 --- a/packages/reflex-base/src/reflex_base/utils/supervisor.py +++ b/packages/reflex-base/src/reflex_base/utils/supervisor.py @@ -224,10 +224,11 @@ def run(self): *lines, pending = _LINE_END.split(data[:cut]) pending += data[cut:] self._write(lines, traceback, final=False) - os.close(self.read_fd) pending = pending.removesuffix(b"\r") self._write([pending] if pending else [], traceback, final=True) finally: + with contextlib.suppress(OSError): + os.close(self.read_fd) self._finished.set() def _write(self, lines: list[bytes], traceback: list[str], final: bool): diff --git a/tests/units/reflex_base/utils/test_supervisor.py b/tests/units/reflex_base/utils/test_supervisor.py index 4d00e70a543..3b0f34859cf 100644 --- a/tests/units/reflex_base/utils/test_supervisor.py +++ b/tests/units/reflex_base/utils/test_supervisor.py @@ -693,3 +693,29 @@ def test_output_pump_drain_does_not_join_the_reader(monkeypatch): os.close(write_fd) threading.Thread.join(pump, 5) assert not pump.is_alive() + + +def test_output_pump_closes_the_pipe_when_forwarding_fails(monkeypatch): + """The reader closes its end of the pipe even when forwarding raises.""" + read_fd, write_fd = os.pipe() + pump = supervisor._OutputPump(read_fd, 1, "info", "stdout") + + def fail(*_args, **_kwargs): + msg = "forwarding failed" + raise RuntimeError(msg) + + monkeypatch.setattr(pump, "_write", fail) + errors = [] + monkeypatch.setattr(threading, "excepthook", errors.append) + pump.start() + try: + os.write(write_fd, b"line\n") + assert pump._finished.wait(5) + # The write end fails only once the pipe has no reader left. + with pytest.raises(OSError): + os.write(write_fd, b"line\n") + finally: + os.close(write_fd) + threading.Thread.join(pump, 5) + assert not pump.is_alive() + assert [str(error.exc_value) for error in errors] == ["forwarding failed"]