[Python-checkins] bpo-45629: Add a test for the "freeze" tool. (gh-29222)
ericsnowcurrently
webhook-mailer at python.org
Thu Oct 28 12:14:45 EDT 2021
https://github.com/python/cpython/commit/13d9205f4057eeeef80a25d410ad123876dc60cd
commit: 13d9205f4057eeeef80a25d410ad123876dc60cd
branch: main
author: Eric Snow <ericsnowcurrently at gmail.com>
committer: ericsnowcurrently <ericsnowcurrently at gmail.com>
date: 2021-10-28T10:14:37-06:00
summary:
bpo-45629: Add a test for the "freeze" tool. (gh-29222)
The "freeze" tool has been part of the repo for a long time. However, it hasn't had any tests in the test suite to guard against regressions. We add such a test here. This is especially important as there has been a lot of change recently related to frozen modules, with more to come.
Note that as part of the test we build Python out-of-tree and install it in a temp dir.
https://bugs.python.org/issue45629
files:
A Lib/test/test_tools/test_freeze.py
A Tools/freeze/test/freeze.py
M .gitignore
M Lib/test/support/__init__.py
diff --git a/.gitignore b/.gitignore
index c3fc748ea97c3..b2ad76689f12b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -120,6 +120,7 @@ Tools/unicode/data/
Tools/msi/obj
Tools/ssl/amd64
Tools/ssl/win32
+Tools/freeze/test/outdir
# The frozen modules are always generated by the build so we don't
# keep them in the repo. Also see Tools/scripts/freeze_modules.py.
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 85fd74126b5f4..fc3c99ec0ecc3 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -372,6 +372,17 @@ def wrapper(*args, **kw):
return decorator
+def skip_if_buildbot(reason=None):
+ """Decorator raising SkipTest if running on a buildbot."""
+ if not reason:
+ reason = 'not suitable for buildbots'
+ if sys.platform == 'win32':
+ isbuildbot = os.environ.get('USERNAME') == 'Buildbot'
+ else:
+ isbuildbot = os.environ.get('USER') == 'buildbot'
+ return unittest.skipIf(isbuildbot, reason)
+
+
def system_must_validate_cert(f):
"""Skip the test on TLS certificate validation failures."""
@functools.wraps(f)
diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py
new file mode 100644
index 0000000000000..392a776f042e4
--- /dev/null
+++ b/Lib/test/test_tools/test_freeze.py
@@ -0,0 +1,29 @@
+"""Sanity-check tests for the "freeze" tool."""
+
+import sys
+import textwrap
+import unittest
+
+from test import support
+
+from . import imports_under_tool, skip_if_missing
+skip_if_missing('freeze')
+with imports_under_tool('freeze', 'test'):
+ import freeze as helper
+
+
+ at unittest.skipIf(sys.platform.startswith('win'), 'not supported on Windows')
+ at support.skip_if_buildbot('not all buildbots have enough space')
+class TestFreeze(unittest.TestCase):
+
+ def test_freeze_simple_script(self):
+ script = textwrap.dedent("""
+ import sys
+ print('running...')
+ sys.exit(0)
+ """)
+ outdir, scriptfile, python = helper.prepare(script)
+
+ executable = helper.freeze(python, scriptfile, outdir)
+ text = helper.run(executable)
+ self.assertEqual(text, 'running...')
diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py
new file mode 100644
index 0000000000000..18a5d27cebf2e
--- /dev/null
+++ b/Tools/freeze/test/freeze.py
@@ -0,0 +1,194 @@
+import os
+import os.path
+import re
+import shlex
+import shutil
+import subprocess
+
+
+TESTS_DIR = os.path.dirname(__file__)
+TOOL_ROOT = os.path.dirname(TESTS_DIR)
+SRCDIR = os.path.dirname(os.path.dirname(TOOL_ROOT))
+
+MAKE = shutil.which('make')
+GIT = shutil.which('git')
+FREEZE = os.path.join(TOOL_ROOT, 'freeze.py')
+OUTDIR = os.path.join(TESTS_DIR, 'outdir')
+
+
+class UnsupportedError(Exception):
+ """The operation isn't supported."""
+
+
+def _run_quiet(cmd, cwd=None):
+ #print(f'# {" ".join(shlex.quote(a) for a in cmd)}')
+ return subprocess.run(
+ cmd,
+ cwd=cwd,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+
+
+def _run_stdout(cmd, cwd=None):
+ proc = _run_quiet(cmd, cwd)
+ return proc.stdout.strip()
+
+
+def find_opt(args, name):
+ opt = f'--{name}'
+ optstart = f'{opt}='
+ for i, arg in enumerate(args):
+ if arg == opt or arg.startswith(optstart):
+ return i
+ return -1
+
+
+def ensure_opt(args, name, value):
+ opt = f'--{name}'
+ pos = find_opt(args, name)
+ if value is None:
+ if pos < 0:
+ args.append(opt)
+ else:
+ args[pos] = opt
+ elif pos < 0:
+ args.extend([opt, value])
+ else:
+ arg = args[pos]
+ if arg == opt:
+ if pos == len(args) - 1:
+ raise NotImplementedError((args, opt))
+ args[pos + 1] = value
+ else:
+ args[pos] = f'{opt}={value}'
+
+
+def git_copy_repo(newroot, oldroot):
+ if not GIT:
+ raise UnsupportedError('git')
+
+ if os.path.exists(newroot):
+ print(f'updating copied repo {newroot}...')
+ if newroot == SRCDIR:
+ raise Exception('this probably isn\'t what you wanted')
+ _run_quiet([GIT, 'clean', '-d', '-f'], newroot)
+ _run_quiet([GIT, 'reset'], newroot)
+ _run_quiet([GIT, 'checkout', '.'], newroot)
+ _run_quiet([GIT, 'pull', '-f', oldroot], newroot)
+ else:
+ print(f'copying repo into {newroot}...')
+ _run_quiet([GIT, 'clone', oldroot, newroot])
+
+ # Copy over any uncommited files.
+ text = _run_stdout([GIT, 'status', '-s'], oldroot)
+ for line in text.splitlines():
+ _, _, relfile = line.strip().partition(' ')
+ relfile = relfile.strip()
+ isdir = relfile.endswith(os.path.sep)
+ relfile = relfile.rstrip(os.path.sep)
+ srcfile = os.path.join(oldroot, relfile)
+ dstfile = os.path.join(newroot, relfile)
+ os.makedirs(os.path.dirname(dstfile), exist_ok=True)
+ if isdir:
+ shutil.copytree(srcfile, dstfile, dirs_exist_ok=True)
+ else:
+ shutil.copy2(srcfile, dstfile)
+
+
+def get_makefile_var(builddir, name):
+ regex = re.compile(rf'^{name} *=\s*(.*?)\s*$')
+ filename = os.path.join(builddir, 'Makefile')
+ try:
+ infile = open(filename)
+ except FileNotFoundError:
+ return None
+ with infile:
+ for line in infile:
+ m = regex.match(line)
+ if m:
+ value, = m.groups()
+ return value or ''
+ return None
+
+
+def get_config_var(builddir, name):
+ python = os.path.join(builddir, 'python')
+ if os.path.isfile(python):
+ cmd = [python, '-c',
+ f'import sysconfig; print(sysconfig.get_config_var("{name}"))']
+ try:
+ return _run_stdout(cmd)
+ except subprocess.CalledProcessError:
+ pass
+ return get_makefile_var(builddir, name)
+
+
+##################################
+# freezing
+
+def prepare(script=None, outdir=None):
+ if not outdir:
+ outdir = OUTDIR
+ os.makedirs(outdir, exist_ok=True)
+
+ # Write the script to disk.
+ if script:
+ scriptfile = os.path.join(outdir, 'app.py')
+ with open(scriptfile, 'w') as outfile:
+ outfile.write(script)
+
+ # Make a copy of the repo to avoid affecting the current build.
+ srcdir = os.path.join(outdir, 'cpython')
+ git_copy_repo(srcdir, SRCDIR)
+
+ # We use an out-of-tree build (instead of srcdir).
+ builddir = os.path.join(outdir, 'python-build')
+ os.makedirs(builddir, exist_ok=True)
+
+ # Run configure.
+ print(f'configuring python in {builddir}...')
+ cmd = [
+ os.path.join(srcdir, 'configure'),
+ *shlex.split(get_config_var(builddir, 'CONFIG_ARGS') or ''),
+ ]
+ ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache'))
+ prefix = os.path.join(outdir, 'python-installation')
+ ensure_opt(cmd, 'prefix', prefix)
+ _run_quiet(cmd, builddir)
+
+ if not MAKE:
+ raise UnsupportedError('make')
+
+ # Build python.
+ print('building python...')
+ if os.path.exists(os.path.join(srcdir, 'Makefile')):
+ # Out-of-tree builds require a clean srcdir.
+ _run_quiet([MAKE, '-C', srcdir, 'clean'])
+ _run_quiet([MAKE, '-C', builddir, '-j8'])
+
+ # Install the build.
+ print(f'installing python into {prefix}...')
+ _run_quiet([MAKE, '-C', builddir, '-j8', 'install'])
+ python = os.path.join(prefix, 'bin', 'python3')
+
+ return outdir, scriptfile, python
+
+
+def freeze(python, scriptfile, outdir):
+ if not MAKE:
+ raise UnsupportedError('make')
+
+ print(f'freezing {scriptfile}...')
+ os.makedirs(outdir, exist_ok=True)
+ _run_quiet([python, FREEZE, '-o', outdir, scriptfile], outdir)
+ _run_quiet([MAKE, '-C', os.path.dirname(scriptfile)])
+
+ name = os.path.basename(scriptfile).rpartition('.')[0]
+ executable = os.path.join(outdir, name)
+ return executable
+
+
+def run(executable):
+ return _run_stdout([executable])
More information about the Python-checkins
mailing list