From 473a44984f6c5f07916ea7ec981fcfc4979166b1 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 18 Nov 2026 13:13:50 -0800 Subject: [PATCH 1/2] gh-136267: Use the target configuration in build-details.json (GH-161529) With --with-build-python, generate-build-details.py runs on a different interpreter than the one being built. abi.flags, suffixes.extensions, abi.stable_abi_suffix and supports_isolated_interpreters came from sys and importlib, so they described the build interpreter. Read them from sysconfig instead. The extension suffix list is rebuilt the same way Python/dynload_shlib.c builds it; a test checks that the result matches importlib.machinery.EXTENSION_SUFFIXES on native builds. Also set abi.extension_suffix and abi.stable_abi_suffix for static libpython builds. They were only set with a shared libpython, but extension modules work either way and PEP 739 requires them. (cherry picked from commit fb9330bbacb39c847bd3a2d5437836a04e10ad14) Co-authored-by: Itamar Oren Co-authored-by: Claude Fable 5.1 --- Lib/test/test_build_details.py | 88 ++++++++++++++++--- ...-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst | 8 ++ Tools/build/generate-build-details.py | 59 +++++++++++-- 3 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 6bbca7f472bed43..9988e4883918ab4 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -142,6 +142,17 @@ def contents(self): def test_location(self): self.assertTrue(os.path.isfile(self.location)) + def test_abi(self): + self.assertEqual(self.key('abi.flags'), list(sys.abiflags)) + self.assertEqual(self.key('abi.extension_suffix'), + sysconfig.get_config_var('EXT_SUFFIX')) + self.assertIn(self.key('abi.stable_abi_suffix'), + importlib.machinery.EXTENSION_SUFFIXES) + + def test_extension_suffixes(self): + self.assertEqual(self.key('suffixes.extensions'), + importlib.machinery.EXTENSION_SUFFIXES) + # Override generic format tests with tests for our specific implemenation. @needs_installed_python @@ -289,6 +300,62 @@ def check_relative_paths(self, data_abs_path, data_rel_path): self.assertEqual(data_abs_path[part][key], abs_rel_path) +def generate_data(**config_vars): + """Generate build-details data with overridden config vars.""" + get_config_var = sysconfig.get_config_var + def patched_get_config_var(name): + if name in config_vars: + return config_vars[name] + return get_config_var(name) + with mock.patch('sysconfig.get_config_var', patched_get_config_var): + data = generate_build_details.generate_data(schema_version='1.0') + return json.loads(json.dumps(data)) + + +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX') +class BuildDetailsTargetTests(unittest.TestCase): + """Values that must come from the target configuration (gh-136267).""" + + def test_native_extension_suffixes(self): + self.assertEqual(generate_build_details.get_extension_suffixes(), + importlib.machinery.EXTENSION_SUFFIXES) + + def test_free_threaded_debug(self): + # A GIL-enabled release build Python generating for this target + # used to report its own flags and suffixes. + data = generate_data( + ABIFLAGS='td', MACHDEP='linux', DYNLOADFILE='dynload_shlib.o', + SOABI_PLATFORM='aarch64-linux-gnu', + SOABI='cpython-3XXtd-aarch64-linux-gnu', + ALT_SOABI='cpython-3XXt-aarch64-linux-gnu', + EXT_SUFFIX='.cpython-3XXtd-aarch64-linux-gnu.so', + Py_GIL_DISABLED=1) + self.assertEqual(data['abi']['flags'], ['t', 'd']) + self.assertEqual(data['abi']['extension_suffix'], + '.cpython-3XXtd-aarch64-linux-gnu.so') + self.assertEqual(data['abi']['stable_abi_suffix'], + '.abi3t-aarch64-linux-gnu.so') + self.assertEqual(data['suffixes']['extensions'], [ + '.cpython-3XXtd-aarch64-linux-gnu.so', + '.cpython-3XXt-aarch64-linux-gnu.so', + '.abi3t-aarch64-linux-gnu.so', + '.abi3t.so', + '.so', + ]) + + def test_no_dynamic_loading(self): + data = generate_data(MACHDEP='wasi', DYNLOADFILE='dynload_stub.o') + self.assertEqual(data['suffixes']['extensions'], []) + self.assertNotIn('extension_suffix', data['abi']) + self.assertNotIn('stable_abi_suffix', data['abi']) + self.assertFalse( + data['implementation']['supports_isolated_interpreters']) + + @unittest.skipIf( generate_build_details is None, "Failed to import generate-build-details" @@ -299,26 +366,25 @@ class BuildDetailsLibpythonTests(unittest.TestCase): LIBDIR = '/prefix/lib' LIBPL = '/prefix/lib/python3.X/config-3.X' + def generate(self, **config_vars): + return generate_data(LIBDIR=self.LIBDIR, LIBPL=self.LIBPL, + LIBRARY='libpython3.X.a', **config_vars) + def generate_libpython(self, **config_vars): """Generate the libpython section with overridden config vars.""" - config_vars = dict(LIBDIR=self.LIBDIR, LIBPL=self.LIBPL, - LIBRARY='libpython3.X.a', **config_vars) - get_config_var = sysconfig.get_config_var - def patched_get_config_var(name): - if name in config_vars: - return config_vars[name] - return get_config_var(name) - with mock.patch('sysconfig.get_config_var', patched_get_config_var): - data = generate_build_details.generate_data(schema_version='1.0') - return json.loads(json.dumps(data)).get('libpython', {}) + return self.generate(**config_vars).get('libpython', {}) def test_static_only(self): - libpython = self.generate_libpython( + data = self.generate( LDLIBRARY='libpython3.X.a', STATIC_LIBPYTHON=1, PYTHONFRAMEWORK='') + libpython = data['libpython'] self.assertNotIn('dynamic', libpython) self.assertEqual(libpython['static'], os.path.join(self.LIBDIR, 'libpython3.X.a')) + # Extension modules are supported without a shared libpython. + self.assertIn('extension_suffix', data['abi']) + self.assertIn('stable_abi_suffix', data['abi']) def test_shared_and_static(self): # The static library is only installed into LIBPL, not LIBDIR. diff --git a/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst b/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst new file mode 100644 index 000000000000000..43cc054a04ac7a2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst @@ -0,0 +1,8 @@ +``build-details.json`` now describes the interpreter being built rather than +the interpreter running :program:`generate-build-details.py`: ``abi.flags``, +``suffixes.extensions``, ``abi.stable_abi_suffix`` and +``implementation.supports_isolated_interpreters`` are derived from the +target's configuration, which fixes cross builds and builds that use +:option:`--with-build-python`. ``abi.extension_suffix`` and +``abi.stable_abi_suffix`` are now also present when libpython is only built +as a static library. diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 5370ad462eaea0c..485f59919b00456 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -29,6 +29,46 @@ def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: return container +def get_extension_suffixes() -> list[str]: + """Return the extension module suffixes of the target interpreter. + + importlib.machinery.EXTENSION_SUFFIXES describes the running interpreter, + which is not the target under --with-build-python. On POSIX, rebuild the + list from the target configuration instead, following + _PyImport_DynLoadFiletab in Python/dynload_*.c. The two are kept in sync + by test_build_details.BuildDetailsTargetTests.test_native_extension_suffixes. + """ + if os.name != 'posix': + return importlib.machinery.EXTENSION_SUFFIXES + + dynloadfile = sysconfig.get_config_var('DYNLOADFILE') + if dynloadfile == 'dynload_stub.o': + return [] + if dynloadfile == 'dynload_hpux.o': + return [sysconfig.get_config_var('EXT_SUFFIX'), '.sl'] + if dynloadfile != 'dynload_shlib.o': + raise ValueError(f'Unsupported DYNLOADFILE: {dynloadfile!r}') + + soabi = sysconfig.get_config_var('SOABI') + if sysconfig.get_config_var('MACHDEP') == 'cygwin': + return [f'.{soabi}.dll', '.dll'] + + suffixes = [f'.{soabi}.so'] + alt_soabi = sysconfig.get_config_var('ALT_SOABI') + if alt_soabi: + suffixes.append(f'.{alt_soabi}.so') + platform = sysconfig.get_config_var('SOABI_PLATFORM') + if not sysconfig.get_config_var('Py_GIL_DISABLED'): + if platform: + suffixes.append(f'.abi3-{platform}.so') + suffixes.append('.abi3.so') + if platform: + suffixes.append(f'.abi3t-{platform}.so') + suffixes.append('.abi3t.so') + suffixes.append('.so') + return suffixes + + def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: """Generate the build-details.json data (PEP 739). @@ -60,14 +100,20 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: # Fix cross-compilation if '_multiarch' in data['implementation']: data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') + if 'supports_isolated_interpreters' in data['implementation']: + # Mirrors Python/sysmodule.c. + data['implementation']['supports_isolated_interpreters'] = ( + sysconfig.get_config_var('MACHDEP') not in ('wasi', 'emscripten') + ) - data['abi']['flags'] = list(sys.abiflags) + data['abi']['flags'] = list(sysconfig.get_config_var('ABIFLAGS')) data['suffixes']['source'] = importlib.machinery.SOURCE_SUFFIXES data['suffixes']['bytecode'] = importlib.machinery.BYTECODE_SUFFIXES #data['suffixes']['optimized_bytecode'] = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES #data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES - data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES + extension_suffixes = get_extension_suffixes() + data['suffixes']['extensions'] = extension_suffixes LIBDIR = sysconfig.get_config_var('LIBDIR') LIBPL = sysconfig.get_config_var('LIBPL') @@ -116,17 +162,16 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: # regardless. # # We only want to set abi.extension_suffix and stable_abi_suffix if - # extension modules are supported. - if has_dynamic_library: + # extension modules are supported, regardless of how libpython is built. + if extension_suffixes: data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') - # EXTENSION_SUFFIXES has been constant for a long time, and currently we - # don't have a better information source to find the stable ABI suffix. - for suffix in importlib.machinery.EXTENSION_SUFFIXES: + for suffix in extension_suffixes: if suffix.startswith('.abi'): data['abi']['stable_abi_suffix'] = suffix break + if has_dynamic_library: data['libpython']['dynamic'] = os.path.join(dynamic_library_dir, LDLIBRARY) # FIXME: Not sure if windows has a different dll for the stable ABI, and # even if it does, currently we don't have a way to get its name. From a5a7e839c0d1055431ac9812788d929505204a57 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 18 Nov 2026 14:08:01 -0800 Subject: [PATCH 2/2] Follow 3.14's extension suffix table 3.14's `_PyImport_DynLoadFiletab` has no `abi3t` or platform-tagged `abi3` suffixes and no SOABI-tagged `.dll` on Cygwin, and its `sysconfig` keeps the quotes around `ALT_SOABI`. Co-Authored-By: Claude Fable 5.2 --- Lib/test/test_build_details.py | 13 +++++-------- Tools/build/generate-build-details.py | 15 +++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 9988e4883918ab4..5cbf9e6dc81680b 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -329,21 +329,18 @@ def test_free_threaded_debug(self): # used to report its own flags and suffixes. data = generate_data( ABIFLAGS='td', MACHDEP='linux', DYNLOADFILE='dynload_shlib.o', - SOABI_PLATFORM='aarch64-linux-gnu', SOABI='cpython-3XXtd-aarch64-linux-gnu', - ALT_SOABI='cpython-3XXt-aarch64-linux-gnu', - EXT_SUFFIX='.cpython-3XXtd-aarch64-linux-gnu.so', - Py_GIL_DISABLED=1) + # pyconfig.h string values keep their quotes in sysconfig. + ALT_SOABI='"cpython-3XXt-aarch64-linux-gnu"', + EXT_SUFFIX='.cpython-3XXtd-aarch64-linux-gnu.so') self.assertEqual(data['abi']['flags'], ['t', 'd']) self.assertEqual(data['abi']['extension_suffix'], '.cpython-3XXtd-aarch64-linux-gnu.so') - self.assertEqual(data['abi']['stable_abi_suffix'], - '.abi3t-aarch64-linux-gnu.so') + self.assertEqual(data['abi']['stable_abi_suffix'], '.abi3.so') self.assertEqual(data['suffixes']['extensions'], [ '.cpython-3XXtd-aarch64-linux-gnu.so', '.cpython-3XXt-aarch64-linux-gnu.so', - '.abi3t-aarch64-linux-gnu.so', - '.abi3t.so', + '.abi3.so', '.so', ]) diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 485f59919b00456..371d722c1c77bb8 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -49,22 +49,17 @@ def get_extension_suffixes() -> list[str]: if dynloadfile != 'dynload_shlib.o': raise ValueError(f'Unsupported DYNLOADFILE: {dynloadfile!r}') - soabi = sysconfig.get_config_var('SOABI') if sysconfig.get_config_var('MACHDEP') == 'cygwin': - return [f'.{soabi}.dll', '.dll'] + return ['.dll'] + soabi = sysconfig.get_config_var('SOABI') suffixes = [f'.{soabi}.so'] alt_soabi = sysconfig.get_config_var('ALT_SOABI') if alt_soabi: + # sysconfig keeps the quotes of string values from pyconfig.h. + alt_soabi = alt_soabi.strip('"') suffixes.append(f'.{alt_soabi}.so') - platform = sysconfig.get_config_var('SOABI_PLATFORM') - if not sysconfig.get_config_var('Py_GIL_DISABLED'): - if platform: - suffixes.append(f'.abi3-{platform}.so') - suffixes.append('.abi3.so') - if platform: - suffixes.append(f'.abi3t-{platform}.so') - suffixes.append('.abi3t.so') + suffixes.append('.abi3.so') suffixes.append('.so') return suffixes