From 530ea8a2bb5fb7056864b44d485808c36a2421b2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 13 Nov 2026 03:21:13 +0000 Subject: [PATCH 1/2] gh-159986: Avoid manager proxy finalizer deadlock in local server (cherry picked from commit c32cb4a5f4bec1a717bc908ffbd8eef1c2b93bea) --- Lib/multiprocessing/managers.py | 25 +++++++- Lib/test/_test_multiprocessing.py | 63 +++++++++++++++++++ ...-11-12-19-20-00.gh-issue-159986.gpt6k4.rst | 2 + 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-12-19-20-00.gh-issue-159986.gpt6k4.rst diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 11c6e21de90e693..7959825958ff7ba 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -164,6 +164,10 @@ def __init__(self, registry, address, authkey, serializer): self.id_to_refcount = {} self.id_to_local_proxy_obj = {} self.mutex = threading.Lock() + # Decrements from in-process proxy finalizers cannot block a thread + # that this server needs to accept incoming requests. + self._local_decrefs = queue.SimpleQueue() + self._server_pid = os.getpid() def serve_forever(self): ''' @@ -177,7 +181,14 @@ def serve_forever(self): accepter.start() try: while not self.stop_event.is_set(): - self.stop_event.wait(1) + try: + ident = self._local_decrefs.get(timeout=1) + except queue.Empty: + continue + try: + self.decref(None, ident) + except Exception as e: + util.debug('... local decref failed %s', e) except (KeyboardInterrupt, SystemExit): pass finally: @@ -884,8 +895,16 @@ def _decref(token, serial, authkey, state, tls, idset, _Client): # tell manager this process no longer cares about referent try: util.debug('DECREF %r', token.id) - conn = _Client(token.address, authkey=authkey) - dispatch(conn, None, 'decref', (token.id,)) + server = getattr(process.current_process(), + '_manager_server', None) + if (server is not None and server.address == token.address + and server._server_pid == os.getpid()): + # A finalizer may run on this server's accepter thread. + # Do not wait for that same thread to serve an RPC. + server._local_decrefs.put(token.id) + else: + conn = _Client(token.address, authkey=authkey) + dispatch(conn, None, 'decref', (token.id,)) except Exception as e: util.debug('... decref failed %s', e) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 540189147cb4d7b..0f0d34b29767bb2 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3751,6 +3751,69 @@ class _TestMyManager(BaseTestCase): ALLOWED_TYPES = ('manager',) + def test_proxy_finalizer_does_not_block_local_server(self): + # gh-159986: the cyclic GC may run a proxy finalizer on the + # manager's own accepter thread, which must stay free to accept RPCs. + code = r''' +import gc +import os +import threading +import time +from multiprocessing.managers import BaseManager + +class Manager(BaseManager): + pass + +class Value: + def get(self): + return 123 + +Manager.register('Value', Value) +server = Manager(address=('127.0.0.1', 0), authkey=b'secret').get_server() +threading.Thread(target=server.serve_forever, daemon=True).start() +manager = Manager(address=server.address, authkey=b'secret') +manager.connect() + +# Ensure only the accepter's explicit collection can run the finalizer. +gc.disable() +proxy = manager.Value() +cycle = [proxy] +cycle.append(cycle) +del proxy, cycle + +triggered = threading.Event() +original_init = threading.Thread.__init__ + +def collect_during_accept(self, *args, **kwargs): + target = kwargs.get('target') + if (getattr(target, '__self__', None) is server and + getattr(target, '__name__', '') == 'handle_request' and + not triggered.is_set()): + triggered.set() + gc.collect() + return original_init(self, *args, **kwargs) + +threading.Thread.__init__ = collect_during_accept +result = [] + +def request(): + result.append(manager.Value().get()) + +worker = threading.Thread(target=request, daemon=True) +worker.start() +worker.join(timeout=10) +if worker.is_alive(): + os._exit(2) +assert triggered.is_set() and result == [123], result + +# The local decrement is asynchronous, but it must actually be processed. +deadline = time.monotonic() + 5 +while server.id_to_refcount and time.monotonic() < deadline: + time.sleep(0.01) +assert not server.id_to_refcount, server.id_to_refcount +''' + test.support.script_helper.assert_python_ok('-c', code) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.skip_if_sanitizer('TSan: leaks threads', thread=True) def test_mymanager(self): diff --git a/Misc/NEWS.d/next/Library/2026-11-12-19-20-00.gh-issue-159986.gpt6k4.rst b/Misc/NEWS.d/next/Library/2026-11-12-19-20-00.gh-issue-159986.gpt6k4.rst new file mode 100644 index 000000000000000..ecb0880ca8947b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-12-19-20-00.gh-issue-159986.gpt6k4.rst @@ -0,0 +1,2 @@ +Fix a deadlock in :mod:`multiprocessing.managers` when cyclic garbage +collection finalizes a proxy on its own manager server's accepting thread. From 4a95ae946a8995ea18e9d56dd70a44ff32d24d91 Mon Sep 17 00:00:00 2001 From: HAMON GPT-6 worker Date: Mon, 16 Nov 2026 03:54:06 +0000 Subject: [PATCH 2/2] gh-159986: Handle finalizers for all in-process manager servers --- Lib/multiprocessing/managers.py | 12 ++++++++---- Lib/test/_test_multiprocessing.py | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 7959825958ff7ba..9fa97a2d5452a51 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -20,6 +20,7 @@ import array import collections.abc import queue +import weakref import time import types import os @@ -144,6 +145,8 @@ class Server(object): ''' Server class which runs in a process controlled by a manager object ''' + _local_servers = weakref.WeakValueDictionary() + public = ['shutdown', 'create', 'accept_connection', 'get_methods', 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref'] @@ -175,6 +178,7 @@ def serve_forever(self): ''' self.stop_event = threading.Event() process.current_process()._manager_server = self + self._local_servers[self.address] = self try: accepter = threading.Thread(target=self.accepter) accepter.daemon = True @@ -192,6 +196,8 @@ def serve_forever(self): except (KeyboardInterrupt, SystemExit): pass finally: + if self._local_servers.get(self.address) is self: + self._local_servers.pop(self.address, None) if sys.stdout != sys.__stdout__: # what about stderr? util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ @@ -895,10 +901,8 @@ def _decref(token, serial, authkey, state, tls, idset, _Client): # tell manager this process no longer cares about referent try: util.debug('DECREF %r', token.id) - server = getattr(process.current_process(), - '_manager_server', None) - if (server is not None and server.address == token.address - and server._server_pid == os.getpid()): + server = Server._local_servers.get(token.address) + if (server is not None and server._server_pid == os.getpid()): # A finalizer may run on this server's accepter thread. # Do not wait for that same thread to serve an RPC. server._local_decrefs.put(token.id) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 0f0d34b29767bb2..46afd968d6a5b78 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3757,8 +3757,10 @@ def test_proxy_finalizer_does_not_block_local_server(self): code = r''' import gc import os +import sys import threading import time +from multiprocessing import process from multiprocessing.managers import BaseManager class Manager(BaseManager): @@ -3771,6 +3773,17 @@ def get(self): Manager.register('Value', Value) server = Manager(address=('127.0.0.1', 0), authkey=b'secret').get_server() threading.Thread(target=server.serve_forever, daemon=True).start() +if sys.argv[1] == 'multiple': + deadline = time.monotonic() + 3 + while getattr(process.current_process(), '_manager_server', None) is not server: + assert time.monotonic() < deadline + time.sleep(.001) + second_server = Manager(address=('127.0.0.1', 0), authkey=b'secret').get_server() + threading.Thread(target=second_server.serve_forever, daemon=True).start() + deadline = time.monotonic() + 3 + while getattr(process.current_process(), '_manager_server', None) is not second_server: + assert time.monotonic() < deadline + time.sleep(.001) manager = Manager(address=server.address, authkey=b'secret') manager.connect() @@ -3812,7 +3825,8 @@ def request(): time.sleep(0.01) assert not server.id_to_refcount, server.id_to_refcount ''' - test.support.script_helper.assert_python_ok('-c', code) + for variant in ('single', 'multiple'): + test.support.script_helper.assert_python_ok('-c', code, variant) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.skip_if_sanitizer('TSan: leaks threads', thread=True)