Skip to content

Commit aebb8b2

Browse files
committed
gh-161962: Fix LazyLoader deadlock on circular imports between threads
1 parent 797087c commit aebb8b2

3 files changed

Lines changed: 94 additions & 5 deletions

File tree

‎Lib/importlib/util.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""Utility code for constructing importers, etc."""
22

3+
from . import _bootstrap
34
from ._bootstrap import _resolve_name
45
from ._bootstrap import _find_spec
56

@@ -168,6 +169,33 @@ def override(self):
168169
return -1 if self.disable_check else 1
169170

170171

172+
class _LazyModuleLock:
173+
174+
"""Hold the import system's lock for a module while it loads lazily.
175+
176+
Unlike a plain lock, the module lock detects a deadlock between threads
177+
that are loading each other's modules. On a deadlock, proceed without the
178+
lock as _bootstrap._lock_unlock_module() does for eager imports, so that
179+
the partially initialized module is returned.
180+
"""
181+
182+
def __init__(self, name):
183+
self._lock = _bootstrap._get_module_lock(name)
184+
self._acquired = False
185+
186+
def __enter__(self):
187+
try:
188+
self._lock.acquire()
189+
except _bootstrap._DeadlockError:
190+
pass
191+
else:
192+
self._acquired = True
193+
194+
def __exit__(self, *args):
195+
if self._acquired:
196+
self._lock.release()
197+
198+
171199
class _LazyModule(types.ModuleType):
172200

173201
"""A subclass of the module type which triggers loading upon attribute access."""
@@ -176,7 +204,7 @@ def __getattribute__(self, attr):
176204
"""Trigger the load of the module and return the attribute."""
177205
__spec__ = object.__getattribute__(self, '__spec__')
178206
loader_state = __spec__.loader_state
179-
with loader_state['lock']:
207+
with _LazyModuleLock(__spec__.name):
180208
# Only the first thread to get the lock should trigger the load
181209
# and reset the module's class. The rest can now getattr().
182210
if object.__getattribute__(self, '__class__') is _LazyModule:
@@ -259,9 +287,6 @@ def create_module(self, spec):
259287

260288
def exec_module(self, module):
261289
"""Make the module load lazily."""
262-
# Threading is only needed for lazy loading, and importlib.util can
263-
# be pulled in at interpreter startup, so defer until needed.
264-
import threading
265290
module.__spec__.loader = self.loader
266291
module.__loader__ = self.loader
267292
# Don't need to worry about deep-copying as trying to set an attribute
@@ -271,7 +296,6 @@ def exec_module(self, module):
271296
loader_state = {}
272297
loader_state['__dict__'] = module.__dict__.copy()
273298
loader_state['__class__'] = module.__class__
274-
loader_state['lock'] = threading.RLock()
275299
loader_state['is_loading'] = False
276300
module.__spec__.loader_state = loader_state
277301
module.__class__ = _LazyModule

‎Lib/test/test_importlib/test_lazy.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
import importlib
22
from importlib import abc
33
from importlib import util
4+
import pathlib
45
import sys
6+
import tempfile
57
import time
68
import threading
79
import types
@@ -178,6 +180,68 @@ def access_module():
178180
# Or multiple load attempts
179181
self.assertEqual(loader.load_count, 1)
180182

183+
def test_lazy_circular_import_no_deadlock(self):
184+
# gh-161962: two threads first-touching two lazily loaded modules
185+
# that import each other must not deadlock. The import system's
186+
# module lock detects the deadlock and one thread proceeds with the
187+
# partially initialized module, as with eager circular imports.
188+
# The barrier makes the timing deterministic: both threads are
189+
# inside a module load before either touches the other module.
190+
sources = {
191+
"sync": "import threading\nbarrier = threading.Barrier(2)\n",
192+
"mod_a": (
193+
"import sys, sync\n"
194+
"A = 1\n"
195+
"sync.barrier.wait()\n"
196+
"FROM_B = sys.modules['mod_b'].B\n"
197+
),
198+
"mod_b": (
199+
"import sys, sync\n"
200+
"B = 2\n"
201+
"sync.barrier.wait()\n"
202+
"FROM_A = sys.modules['mod_a'].A\n"
203+
),
204+
}
205+
with tempfile.TemporaryDirectory() as directory:
206+
for name, source in sources.items():
207+
pathlib.Path(directory, f"{name}.py").write_text(source)
208+
sys.path.insert(0, directory)
209+
try:
210+
with test_util.uncache("sync", "mod_a", "mod_b"):
211+
mod_a = self._lazy_import("mod_a")
212+
mod_b = self._lazy_import("mod_b")
213+
results = {}
214+
threads = [
215+
threading.Thread(
216+
target=lambda i=i, f=f: results.__setitem__(i, f()),
217+
daemon=True,
218+
)
219+
for i, f in enumerate(
220+
[lambda: mod_a.FROM_B, lambda: mod_b.FROM_A]
221+
)
222+
]
223+
for thread in threads:
224+
thread.start()
225+
for thread in threads:
226+
thread.join(timeout=10)
227+
self.assertFalse(
228+
any(thread.is_alive() for thread in threads),
229+
"threads deadlocked",
230+
)
231+
self.assertEqual(results, {0: 2, 1: 1})
232+
finally:
233+
sys.path.remove(directory)
234+
235+
def _lazy_import(self, name):
236+
# The recipe from the importlib docs.
237+
spec = util.find_spec(name)
238+
loader = util.LazyLoader(spec.loader)
239+
spec.loader = loader
240+
module = util.module_from_spec(spec)
241+
sys.modules[name] = module
242+
loader.exec_module(module)
243+
return module
244+
181245
def test_lazy_self_referential_modules(self):
182246
# Directory modules with submodules that reference the parent can attempt to access
183247
# the parent module during a load. Verify that this common pattern works with lazy loading.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix a deadlock in :class:`importlib.util.LazyLoader` when two threads first access two lazily loaded modules that import each other.

0 commit comments

Comments
 (0)