diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index f86a6ec69172d8..f8be815f03dd04 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1106,7 +1106,7 @@ Optimizations Startup performance is tracked on the `Python Speed Center `__. (Contributed by Pablo Galindo Salgado, Cody Maloney, and Pieter Eendebak in - :gh:`159137`, :gh:`160858`, :gh:`160913`, and :gh:`87613`.) + :gh:`159137`, :gh:`160858`, :gh:`160913`, :gh:`87613`, and :gh:`161941`.) * Speed up iteration over asynchronous generators: values yielded to ``async for`` no longer create and discard a :exc:`StopIteration` exception diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9b2a77e883acf1..d759c842bf4fff 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -969,6 +969,7 @@ def _fix_up_module(cls, module): __path__ = spec.submodule_search_locations if ispkg: assert __path__ == [], __path__ + pkgdir = cls._package_dir(spec.name, pkgdir) if pkgdir: spec.submodule_search_locations.insert(0, pkgdir) else: @@ -997,10 +998,6 @@ def _fix_up_module(cls, module): (__file__, pkgdir, ) = cls._resolve_filename(state.origname, spec.name, ispkg) assert state.filename == __file__, (state.filename, __file__) - if pkgdir: - assert __path__ == [pkgdir], (__path__, pkgdir) - else: - assert __path__ == ([] if ispkg else None), __path__ else: __file__ = None assert state.filename is None, state.filename @@ -1043,6 +1040,25 @@ def _resolve_filename(cls, fullname, alias=None, ispkg=False): filename = f'{sys._stdlib_dir}{sep}{relfile}.py' return filename, pkgdir + @classmethod + def _package_dir(cls, fullname, pkgdir): + """Return the directory of the frozen package 'fullname'. + + Normally this is 'pkgdir', the package's directory in the standard + library, so that submodules which were not frozen are found there. + When that directory does not exist (e.g. the standard library is + zipped), a standard library package is looked up on sys.path + instead. + """ + if (_bootstrap_external is None + or fullname not in sys.stdlib_module_names + or (pkgdir and _bootstrap_external._path_isdir(pkgdir))): + return pkgdir + spec = _bootstrap_external.PathFinder.find_spec(fullname) + if spec is not None and spec.submodule_search_locations: + return spec.submodule_search_locations[0] + return pkgdir + @classmethod def find_spec(cls, fullname, path=None, target=None): info = _call_with_frames_removed(_imp.find_frozen, fullname) @@ -1069,6 +1085,8 @@ def find_spec(cls, fullname, path=None, target=None): filename=filename, origname=origname, ) + if ispkg: + pkgdir = cls._package_dir(fullname, pkgdir) if pkgdir: spec.submodule_search_locations.insert(0, pkgdir) return spec diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 971cc28b6d3e26..e2e9bf4201e9b6 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -3,9 +3,16 @@ machinery = util.import_importlib('importlib.machinery') import os.path +import sys +import tempfile +import textwrap import unittest +import zipfile +from unittest import mock -from test.support import import_helper, REPO_ROOT, STDLIB_DIR +from test import support +from test.support import (import_helper, os_helper, script_helper, + REPO_ROOT, STDLIB_DIR) def resolve_stdlib_file(name, ispkg=False): @@ -179,5 +186,64 @@ def test_not_using_frozen(self): ) = util.test_both(FindSpecTests, machinery=machinery) +class PackagePathTests(unittest.TestCase): + + """Test __path__ of frozen packages.""" + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, self.tmpdir) + self.missing_dir = os.path.join(self.tmpdir, 'does-not-exist') + + def import_phello(self): + with (mock.patch.dict(sys.modules), + import_helper.frozen_modules(), + support.captured_stdout()): + sys.modules.pop('__phello__', None) + import __phello__ + self.assertEqual(__phello__.__spec__.origin, 'frozen') + return __phello__ + + def test_not_in_stdlib(self): + # A frozen package which is not a standard library package + # is not looked up on sys.path. + os.mkdir(os.path.join(self.tmpdir, '__phello__')) + with open(os.path.join(self.tmpdir, '__phello__', '__init__.py'), 'wb'): + pass + for stdlib_dir, expected in [ + (self.missing_dir, [os.path.join(self.missing_dir, '__phello__')]), + (None, []), + ]: + with (self.subTest(stdlib_dir=stdlib_dir), + support.swap_attr(sys, 'path', [self.tmpdir, *sys.path]), + support.swap_attr(sys, '_stdlib_dir', stdlib_dir)): + module = self.import_phello() + self.assertEqual(module.__path__, expected) + + @support.requires_subprocess() + def test_zipped_stdlib(self): + # gh-148750: The codecs which are not frozen can be imported + # if the standard library is not available as a directory. + zipname = os.path.join(self.tmpdir, 'stdlib.zip') + with zipfile.ZipFile(zipname, 'w') as zf: + for name in ('__init__', 'cp437'): + zf.write(os.path.join(STDLIB_DIR, 'encodings', f'{name}.py'), + f'encodings/{name}.py') + code = textwrap.dedent(""" + import encodings + print(encodings.__spec__.origin) + print(encodings.__path__[0]) + import encodings.cp437 + print(encodings.cp437.__spec__.origin) + """) + # PYTHONHOME is a directory without a standard library. + proc = script_helper.assert_python_ok( + '-S', '-X', 'frozen_modules=on', '-c', code, + PYTHONHOME=self.tmpdir, PYTHONPATH=zipname) + pkgdir = os.path.join(zipname, 'encodings') + self.assertEqual(proc.out.decode().splitlines(), + ['frozen', pkgdir, os.path.join(pkgdir, 'cp437.py')]) + + if __name__ == '__main__': unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 1d109dac77fa90..a185dd6b4e0961 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1729,6 +1729,11 @@ FROZEN_FILES_IN = \ Lib/_pybuiltins.py \ Lib/abc.py \ Lib/codecs.py \ + Lib/encodings/__init__.py \ + Lib/encodings/aliases.py \ + Lib/encodings/utf_8.py \ + Lib/encodings/_win_cp_codecs.py \ + Lib/encodings/_iconv_codecs.py \ Lib/io.py \ Lib/_collections_abc.py \ Lib/_sitebuiltins.py \ @@ -1756,6 +1761,11 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/builtins.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ + Python/frozen_modules/encodings.h \ + Python/frozen_modules/encodings.aliases.h \ + Python/frozen_modules/encodings.utf_8.h \ + Python/frozen_modules/encodings._win_cp_codecs.h \ + Python/frozen_modules/encodings._iconv_codecs.h \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ @@ -1808,6 +1818,21 @@ Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/encodings.h: Lib/encodings/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings $(srcdir)/Lib/encodings/__init__.py Python/frozen_modules/encodings.h + +Python/frozen_modules/encodings.aliases.h: Lib/encodings/aliases.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.aliases $(srcdir)/Lib/encodings/aliases.py Python/frozen_modules/encodings.aliases.h + +Python/frozen_modules/encodings.utf_8.h: Lib/encodings/utf_8.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.utf_8 $(srcdir)/Lib/encodings/utf_8.py Python/frozen_modules/encodings.utf_8.h + +Python/frozen_modules/encodings._win_cp_codecs.h: Lib/encodings/_win_cp_codecs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings._win_cp_codecs $(srcdir)/Lib/encodings/_win_cp_codecs.py Python/frozen_modules/encodings._win_cp_codecs.h + +Python/frozen_modules/encodings._iconv_codecs.h: Lib/encodings/_iconv_codecs.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings._iconv_codecs $(srcdir)/Lib/encodings/_iconv_codecs.py Python/frozen_modules/encodings._iconv_codecs.h + Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-10-12-00.gh-issue-161941.pQ7vKe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-10-12-00.gh-issue-161941.pQ7vKe.rst new file mode 100644 index 00000000000000..e25f078e7369bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-10-12-00.gh-issue-161941.pQ7vKe.rst @@ -0,0 +1 @@ +The ``encodings`` package and the codecs that every interpreter imports at startup (``encodings.aliases``, ``encodings.utf_8`` and the platform codecs) are frozen again, which makes interpreter startup faster. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index bf430135e21ed5..63f9ed1e56f167 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -324,6 +324,31 @@ $(IntDir)codecs.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\codecs.h + + encodings + $(IntDir)encodings.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.h + + + encodings.aliases + $(IntDir)encodings.aliases.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.aliases.h + + + encodings.utf_8 + $(IntDir)encodings.utf_8.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings.utf_8.h + + + encodings._win_cp_codecs + $(IntDir)encodings._win_cp_codecs.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings._win_cp_codecs.h + + + encodings._iconv_codecs + $(IntDir)encodings._iconv_codecs.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\encodings._iconv_codecs.h + io $(IntDir)io.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 6ff943f89ade00..381e49bfdad838 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -567,6 +567,21 @@ Python Files + + Python Files + + + Python Files + + + Python Files + + + Python Files + + + Python Files + Python Files diff --git a/Python/frozen.c b/Python/frozen.c index 1f92ea01dc38bb..0049180c6e04b9 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -47,6 +47,11 @@ #include "frozen_modules/builtins.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" +#include "frozen_modules/encodings.h" +#include "frozen_modules/encodings.aliases.h" +#include "frozen_modules/encodings.utf_8.h" +#include "frozen_modules/encodings._win_cp_codecs.h" +#include "frozen_modules/encodings._iconv_codecs.h" #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" @@ -79,6 +84,11 @@ static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), false}, {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), false}, + {"encodings", _Py_M__encodings, (int)sizeof(_Py_M__encodings), true}, + {"encodings.aliases", _Py_M__encodings_aliases, (int)sizeof(_Py_M__encodings_aliases), false}, + {"encodings.utf_8", _Py_M__encodings_utf_8, (int)sizeof(_Py_M__encodings_utf_8), false}, + {"encodings._win_cp_codecs", _Py_M__encodings__win_cp_codecs, (int)sizeof(_Py_M__encodings__win_cp_codecs), false}, + {"encodings._iconv_codecs", _Py_M__encodings__iconv_codecs, (int)sizeof(_Py_M__encodings__iconv_codecs), false}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), false}, /* stdlib - startup, with site */ diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 8c817daec0f0b3..5cddc199aa6ddb 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -55,13 +55,16 @@ ('stdlib - startup, without site (python -S)', [ 'abc', 'codecs', - # gh-148750: Partially freezing encodings causes import errors of - # submodules when using a zipped standard library. - #'', - #'encodings.aliases', - #'encodings.utf_8', - #'encodings._win_cp_codecs', - #'encodings._iconv_codecs', + # Only the parts of the encodings package that every interpreter + # imports at startup are frozen. The other codecs are found via + # encodings.__path__, which FrozenImporter resolves through the + # path-based finders when the stdlib directory is not available + # (e.g. a zipped stdlib). + '', + 'encodings.aliases', + 'encodings.utf_8', + 'encodings._win_cp_codecs', + 'encodings._iconv_codecs', 'io', ]), ('stdlib - startup, with site', [