[Python-checkins] bpo-45020: Freeze os, site, and codecs. (gh-28398)
ericsnowcurrently
webhook-mailer at python.org
Fri Sep 17 18:31:39 EDT 2021
https://github.com/python/cpython/commit/090591636c4f03ce06a039079bd7716a5b23631e
commit: 090591636c4f03ce06a039079bd7716a5b23631e
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2021-09-17T16:31:31-06:00
summary:
bpo-45020: Freeze os, site, and codecs. (gh-28398)
https://bugs.python.org/issue45020
files:
M Lib/test/support/__init__.py
M Lib/test/test_embed.py
M Lib/test/test_import/__init__.py
M Lib/test/test_site.py
M Makefile.pre.in
M PCbuild/_freeze_module.vcxproj
M PCbuild/_freeze_module.vcxproj.filters
M Python/frozen.c
M Tools/scripts/freeze_modules.py
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 89f5e5a35c8f4..04516a5f9d719 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -99,6 +99,13 @@
# option.
LONG_TIMEOUT = 5 * 60.0
+# TEST_HOME_DIR refers to the top level directory of the "test" package
+# that contains Python's regression test suite
+TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
+STDLIB_DIR = os.path.dirname(TEST_HOME_DIR)
+REPO_ROOT = os.path.dirname(STDLIB_DIR)
+
class Error(Exception):
"""Base class for regression test exceptions."""
@@ -148,9 +155,7 @@ def load_tests(*args):
"""
if pattern is None:
pattern = "test*"
- top_dir = os.path.dirname( # Lib
- os.path.dirname( # test
- os.path.dirname(__file__))) # support
+ top_dir = STDLIB_DIR
package_tests = loader.discover(start_dir=pkg_dir,
top_level_dir=top_dir,
pattern=pattern)
@@ -459,11 +464,6 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
# PGO task. If this is True, PGO is also True.
PGO_EXTENDED = False
-# TEST_HOME_DIR refers to the top level directory of the "test" package
-# that contains Python's regression test suite
-TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
-
# TEST_DATA_DIR is used as a target download location for remote resources
TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data")
@@ -1390,7 +1390,7 @@ def _platform_specific(self):
self._env = {k.upper(): os.getenv(k) for k in os.environ}
self._env["PYTHONHOME"] = os.path.dirname(self.real)
if sysconfig.is_python_build(True):
- self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
+ self._env["PYTHONPATH"] = STDLIB_DIR
def __enter__(self):
os.symlink(self.real, self.link)
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e5e7c833e10d3..941e307ecbc9a 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -8,6 +8,7 @@
import contextlib
import json
import os
+import os.path
import re
import shutil
import subprocess
@@ -52,15 +53,13 @@ def remove_python_envvars():
class EmbeddingTestsMixin:
def setUp(self):
- here = os.path.abspath(__file__)
- basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
exename = "_testembed"
if MS_WINDOWS:
ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
exename += ext
exepath = os.path.dirname(sys.executable)
else:
- exepath = os.path.join(basepath, "Programs")
+ exepath = os.path.join(support.REPO_ROOT, "Programs")
self.test_exe = exe = os.path.join(exepath, exename)
if not os.path.exists(exe):
self.skipTest("%r doesn't exist" % exe)
@@ -68,7 +67,7 @@ def setUp(self):
# "Py_Initialize: Unable to get the locale encoding
# LookupError: no codec search functions registered: can't find encoding"
self.oldcwd = os.getcwd()
- os.chdir(basepath)
+ os.chdir(support.REPO_ROOT)
def tearDown(self):
os.chdir(self.oldcwd)
@@ -1304,7 +1303,10 @@ def test_init_pyvenv_cfg(self):
lib_dynload = os.path.join(pyvenv_home, 'lib')
os.makedirs(lib_dynload)
# getpathp.c uses Lib\os.py as the LANDMARK
- shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+ shutil.copyfile(
+ os.path.join(support.STDLIB_DIR, 'os.py'),
+ os.path.join(lib_dynload, 'os.py'),
+ )
filename = os.path.join(tmpdir, 'pyvenv.cfg')
with open(filename, "w", encoding="utf8") as fp:
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index afbc12cd7e1b8..8857fd5000bb1 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -19,7 +19,8 @@
from unittest import mock
from test.support import os_helper
-from test.support import (is_jython, swap_attr, swap_item, cpython_only)
+from test.support import (
+ STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only)
from test.support.import_helper import (
forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
from test.support.os_helper import (
@@ -495,7 +496,7 @@ def test_dll_dependency_import(self):
env = None
env = {k.upper(): os.environ[k] for k in os.environ}
- env["PYTHONPATH"] = tmp2 + ";" + os.path.dirname(os.__file__)
+ env["PYTHONPATH"] = tmp2 + ";" + STDLIB_DIR
# Test 1: import with added DLL directory
subprocess.check_call([
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index fa74e7770ede2..5f06a0d4b0372 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -591,7 +591,7 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines):
return sys_path
def test_underpth_nosite_file(self):
- libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+ libpath = test.support.STDLIB_DIR
exe_prefix = os.path.dirname(sys.executable)
pth_lines = [
'fake-path-name',
@@ -619,7 +619,7 @@ def test_underpth_nosite_file(self):
)
def test_underpth_file(self):
- libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+ libpath = test.support.STDLIB_DIR
exe_prefix = os.path.dirname(sys.executable)
exe_file = self._create_underpth_exe([
'fake-path-name',
@@ -644,7 +644,7 @@ def test_underpth_file(self):
def test_underpth_dll_file(self):
- libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+ libpath = test.support.STDLIB_DIR
exe_prefix = os.path.dirname(sys.executable)
exe_file = self._create_underpth_exe([
'fake-path-name',
diff --git a/Makefile.pre.in b/Makefile.pre.in
index d6b2179c022e5..9ea359b70a22c 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -762,6 +762,9 @@ Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
+Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py
+ Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h
+
Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
@@ -780,6 +783,12 @@ Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
+Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py
+ Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h
+
+Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py
+ Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h
+
Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
@@ -1023,12 +1032,15 @@ FROZEN_FILES = \
Python/frozen_modules/importlib__bootstrap_external.h \
Python/frozen_modules/zipimport.h \
Python/frozen_modules/abc.h \
+ Python/frozen_modules/codecs.h \
Python/frozen_modules/io.h \
Python/frozen_modules/_collections_abc.h \
Python/frozen_modules/_sitebuiltins.h \
Python/frozen_modules/genericpath.h \
Python/frozen_modules/ntpath.h \
Python/frozen_modules/posixpath.h \
+ Python/frozen_modules/os.h \
+ Python/frozen_modules/site.h \
Python/frozen_modules/stat.h \
Python/frozen_modules/__hello__.h
# End FROZEN_FILES
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index ef0069c83fb7f..382351ce23338 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -250,6 +250,11 @@
<IntFile>$(IntDir)abc.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
</None>
+ <None Include="..\Lib\codecs.py">
+ <ModName>codecs</ModName>
+ <IntFile>$(IntDir)codecs.g.h</IntFile>
+ <OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
+ </None>
<None Include="..\Lib\io.py">
<ModName>io</ModName>
<IntFile>$(IntDir)io.g.h</IntFile>
@@ -280,6 +285,16 @@
<IntFile>$(IntDir)posixpath.g.h</IntFile>
<OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
</None>
+ <None Include="..\Lib\os.py">
+ <ModName>os</ModName>
+ <IntFile>$(IntDir)os.g.h</IntFile>
+ <OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
+ </None>
+ <None Include="..\Lib\site.py">
+ <ModName>site</ModName>
+ <IntFile>$(IntDir)site.g.h</IntFile>
+ <OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
+ </None>
<None Include="..\Lib\stat.py">
<ModName>stat</ModName>
<IntFile>$(IntDir)stat.g.h</IntFile>
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 71fb41cb42ad3..4a1c90f668e64 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -28,6 +28,9 @@
<None Include="..\Lib\abc.py">
<Filter>Python Files</Filter>
</None>
+ <None Include="..\Lib\codecs.py">
+ <Filter>Python Files</Filter>
+ </None>
<None Include="..\Lib\io.py">
<Filter>Python Files</Filter>
</None>
@@ -46,6 +49,12 @@
<None Include="..\Lib\posixpath.py">
<Filter>Python Files</Filter>
</None>
+ <None Include="..\Lib\os.py">
+ <Filter>Python Files</Filter>
+ </None>
+ <None Include="..\Lib\site.py">
+ <Filter>Python Files</Filter>
+ </None>
<None Include="..\Lib\stat.py">
<Filter>Python Files</Filter>
</None>
diff --git a/Python/frozen.c b/Python/frozen.c
index cf384d31768eb..05b5281cce0c0 100644
--- a/Python/frozen.c
+++ b/Python/frozen.c
@@ -42,12 +42,15 @@
#include "frozen_modules/importlib__bootstrap_external.h"
#include "frozen_modules/zipimport.h"
#include "frozen_modules/abc.h"
+#include "frozen_modules/codecs.h"
#include "frozen_modules/io.h"
#include "frozen_modules/_collections_abc.h"
#include "frozen_modules/_sitebuiltins.h"
#include "frozen_modules/genericpath.h"
#include "frozen_modules/ntpath.h"
#include "frozen_modules/posixpath.h"
+#include "frozen_modules/os.h"
+#include "frozen_modules/site.h"
#include "frozen_modules/stat.h"
#include "frozen_modules/__hello__.h"
/* End includes */
@@ -62,15 +65,21 @@ static const struct _frozen _PyImport_FrozenModules[] = {
(int)sizeof(_Py_M__importlib__bootstrap_external)},
{"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
- /* stdlib */
+ /* stdlib - startup, without site (python -S) */
{"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
+ {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs)},
{"io", _Py_M__io, (int)sizeof(_Py_M__io)},
+
+ /* stdlib - startup, with site */
{"_collections_abc", _Py_M___collections_abc,
(int)sizeof(_Py_M___collections_abc)},
{"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
{"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
{"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
{"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
+ {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
+ {"os", _Py_M__os, (int)sizeof(_Py_M__os)},
+ {"site", _Py_M__site, (int)sizeof(_Py_M__site)},
{"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
/* Test module */
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index bfbea5d27f878..b3ae5c72a9a66 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -66,16 +66,16 @@ def find_tool():
# on a builtin zip file instead of a filesystem.
'zipimport',
]),
- ('stdlib', [
- # For the moment we skip codecs, encodings.*, os, and site.
- # These modules have different generated files depending on
- # if a debug or non-debug build. (See bpo-45186 and bpo-45188.)
- # without site (python -S)
+ ('stdlib - startup, without site (python -S)', [
'abc',
- #'codecs',
- # '<encodings.*>',
+ 'codecs',
+ # For now we do not freeze the encodings, due # to the noise all
+ # those extra modules add to the text printed during the build.
+ # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
+ #'<encodings.*>',
'io',
- # with site
+ ]),
+ ('stdlib - startup, with site', [
'_collections_abc',
'_sitebuiltins',
'genericpath',
@@ -83,9 +83,9 @@ def find_tool():
'posixpath',
# We must explicitly mark os.path as a frozen module
# even though it will never be imported.
- #f'{OS_PATH} : os.path',
- #'os',
- #'site',
+ f'{OS_PATH} : os.path',
+ 'os',
+ 'site',
'stat',
]),
('Test module', [
More information about the Python-checkins
mailing list