[Python-checkins] bpo-45019: Cleanup module freezing and deepfreeze (#29772)
gvanrossum
webhook-mailer at python.org
Fri Nov 26 11:51:23 EST 2021
https://github.com/python/cpython/commit/b0b10e146b1cbf9c5dfa44af116a2eeb0f210e8b
commit: b0b10e146b1cbf9c5dfa44af116a2eeb0f210e8b
branch: main
author: Kumar Aditya <rahuladitya303 at gmail.com>
committer: gvanrossum <gvanrossum at gmail.com>
date: 2021-11-26T08:50:54-08:00
summary:
bpo-45019: Cleanup module freezing and deepfreeze (#29772)
files:
M Tools/scripts/freeze_modules.py
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index 74b2c4857d088..7496b6c868407 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -8,12 +8,9 @@
import os
import ntpath
import posixpath
-import platform
-import subprocess
import sys
-import time
-from update_file import updating_file_with_tmpfile, update_file_with_tmpfile
+from update_file import updating_file_with_tmpfile
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
@@ -21,9 +18,10 @@
FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
-# If MODULES_DIR is changed then the .gitattributes and .gitignore files
-# need to be updated.
-MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
+# If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the
+# .gitattributes and .gitignore files needs to be updated.
+FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
+DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
@@ -111,16 +109,16 @@ def relpath_for_posix_display(path, base):
#######################################
# specs
-def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
+def parse_frozen_specs():
seen = {}
- for section, specs in sectionalspecs:
+ for section, specs in FROZEN:
parsed = _parse_specs(specs, section, seen)
for item in parsed:
frozenid, pyfile, modname, ispkg, section = item
try:
source = seen[frozenid]
except KeyError:
- source = FrozenSource.from_id(frozenid, pyfile, destdir)
+ source = FrozenSource.from_id(frozenid, pyfile)
seen[frozenid] = source
else:
assert not pyfile or pyfile == source.pyfile, item
@@ -225,15 +223,16 @@ def iter_subs():
#######################################
# frozen source files
-class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
+class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):
@classmethod
- def from_id(cls, frozenid, pyfile=None, destdir=MODULES_DIR):
+ def from_id(cls, frozenid, pyfile=None):
if not pyfile:
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
#assert os.path.exists(pyfile), (frozenid, pyfile)
- frozenfile = resolve_frozen_file(frozenid, destdir)
- return cls(frozenid, pyfile, frozenfile)
+ frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
+ deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR)
+ return cls(frozenid, pyfile, frozenfile, deepfreezefile)
@property
def frozenid(self):
@@ -261,7 +260,7 @@ def ispkg(self):
return os.path.basename(self.pyfile) == '__init__.py'
-def resolve_frozen_file(frozenid, destdir=MODULES_DIR):
+def resolve_frozen_file(frozenid, destdir):
"""Return the filename corresponding to the given frozen ID.
For stdlib modules the ID will always be the full name
@@ -570,41 +569,30 @@ def regen_makefile(modules):
deepfreezefiles = []
rules = ['']
deepfreezerules = ['']
-
- # TODO: Merge the two loops
for src in _iter_sources(modules):
- header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
- relfile = header.replace('\\', '/')
- _pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
-
- # TODO: This is a bit hackish
- xfile = relfile.replace("/frozen_modules/", "/deepfreeze/")
- cfile = xfile[:-2] + ".c"
- ofile = xfile[:-2] + ".o"
+ frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
+ deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR)
+ frozenfiles.append(f'\t\t{frozen_header} \\')
+ cfile = deepfreeze_header[:-2] + ".c"
+ ofile = deepfreeze_header[:-2] + ".o"
deepfreezefiles.append(f"\t\t{ofile} \\")
- # Also add a deepfreeze rule.
- deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)')
- deepfreezerules.append(
- f"\t$(PYTHON_FOR_REGEN) "
- f"$(srcdir)/Tools/scripts/deepfreeze.py "
- f"{header} -m {src.frozenid} -o {cfile}")
- deepfreezerules.append('')
-
- for src in _iter_sources(modules):
- header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
- frozenfiles.append(f'\t\t{header} \\')
-
pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
pyfiles.append(f'\t\t{pyfile} \\')
freeze = (f'$(FREEZE_MODULE) {src.frozenid} '
- f'$(srcdir)/{pyfile} {header}')
+ f'$(srcdir)/{pyfile} {frozen_header}')
rules.extend([
- f'{header}: $(FREEZE_MODULE) {pyfile}',
+ f'{frozen_header}: $(FREEZE_MODULE) {pyfile}',
f'\t{freeze}',
'',
])
+ deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
+ deepfreezerules.append(
+ f"\t$(PYTHON_FOR_REGEN) "
+ f"$(srcdir)/Tools/scripts/deepfreeze.py "
+ f"{frozen_header} -m {src.frozenid} -o {cfile}")
+ deepfreezerules.append('')
pyfiles[-1] = pyfiles[-1].rstrip(" \\")
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\")
@@ -715,7 +703,7 @@ def regen_pcbuild(modules):
def main():
# Expand the raw specs, preserving order.
- modules = list(parse_frozen_specs(destdir=MODULES_DIR))
+ modules = list(parse_frozen_specs())
# Regen build-related files.
regen_makefile(modules)
More information about the Python-checkins
mailing list