[Python-checkins] bpo-38470: Fix test_compileall.test_compile_dir_maxlevels() (GH-16789)
Petr Viktorin
webhook-mailer at python.org
Tue Oct 15 05:26:23 EDT 2019
https://github.com/python/cpython/commit/eb1dda2b56f67f09352c303588c28880c471ae87
commit: eb1dda2b56f67f09352c303588c28880c471ae87
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: Petr Viktorin <encukou at gmail.com>
date: 2019-10-15T11:26:13+02:00
summary:
bpo-38470: Fix test_compileall.test_compile_dir_maxlevels() (GH-16789)
Fix test_compile_dir_maxlevels() on Windows without long path
support: only create 3 subdirectories instead of between 20 and 100
subdirectories.
Fix also compile_dir() to use the current sys.getrecursionlimit()
value as the default maxlevels value, rather than using
sys.getrecursionlimit() value read at startup.
files:
A Misc/NEWS.d/next/Tests/2019-10-14-22-46-35.bpo-38470.NHtzpy.rst
M Lib/compileall.py
M Lib/test/test_compileall.py
diff --git a/Lib/compileall.py b/Lib/compileall.py
index 26caf34baec0a..8cfde5b7b3e0a 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -19,11 +19,9 @@
from functools import partial
from pathlib import Path
-RECURSION_LIMIT = sys.getrecursionlimit()
-
__all__ = ["compile_dir","compile_file","compile_path"]
-def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
+def _walk_dir(dir, maxlevels, quiet=0):
if quiet < 2 and isinstance(dir, os.PathLike):
dir = os.fspath(dir)
if not quiet:
@@ -46,7 +44,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0):
yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
quiet=quiet)
-def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
+def compile_dir(dir, maxlevels=None, ddir=None, force=False,
rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
invalidation_mode=None, stripdir=None,
prependdir=None, limit_sl_dest=None):
@@ -83,6 +81,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False,
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
+ if maxlevels is None:
+ maxlevels = sys.getrecursionlimit()
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
success = True
if workers != 1 and ProcessPoolExecutor is not None:
@@ -285,7 +285,7 @@ def main():
parser = argparse.ArgumentParser(
description='Utilities to support installing Python libraries.')
parser.add_argument('-l', action='store_const', const=0,
- default=RECURSION_LIMIT, dest='maxlevels',
+ default=None, dest='maxlevels',
help="don't recurse into subdirectories")
parser.add_argument('-r', type=int, dest='recursion',
help=('control the maximum recursion level. '
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 3bb03b3dd3846..2ebcb424095d8 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -46,57 +46,6 @@ def setUp(self):
def tearDown(self):
shutil.rmtree(self.directory)
- def create_long_path(self):
- long_path = os.path.join(self.directory, "long")
-
- # Create a long path, 10 directories at a time.
- # It will be 100 directories deep, or shorter if the OS limits it.
- for i in range(10):
- longer_path = os.path.join(
- long_path, *(f"dir_{i}_{j}" for j in range(10))
- )
-
- # Check if we can open __pycache__/*.pyc.
- # Also, put in the source file that we want to compile
- longer_source = os.path.join(longer_path, '_test_long.py')
- longer_cache = importlib.util.cache_from_source(longer_source)
- try:
- os.makedirs(longer_path)
- shutil.copyfile(self.source_path, longer_source)
- os.makedirs(os.path.dirname(longer_cache))
- # Make sure we can write to the cache
- with open(longer_cache, 'w'):
- pass
- except FileNotFoundError:
- # On Windows, a FileNotFoundError("The filename or extension
- # is too long") is raised for long paths
- if sys.platform == "win32":
- break
- else:
- raise
- except OSError as exc:
- if exc.errno == errno.ENAMETOOLONG:
- break
- else:
- raise
-
- # Remove the __pycache__
- shutil.rmtree(os.path.dirname(longer_cache))
-
- long_path = longer_path
- long_source = longer_source
- long_cache = longer_cache
-
- # On Windows, MAX_PATH is 260 characters, our path with the 20
- # directories is 160 characters long, leaving something for the
- # root (self.directory) as well.
- # Tests assume long_path contains at least 10 directories.
- if i < 2:
- raise ValueError(f'"Long path" is too short: {long_path}')
-
- self.source_path_long = long_source
- self.bc_path_long = long_cache
-
def add_bad_source_file(self):
self.bad_source_path = os.path.join(self.directory, '_test_bad.py')
with open(self.bad_source_path, 'w') as file:
@@ -247,14 +196,21 @@ def test_compile_missing_multiprocessing(self, compile_file_mock):
self.assertTrue(compile_file_mock.called)
def test_compile_dir_maxlevels(self):
- # Test the actual impact of maxlevels attr
- self.create_long_path()
- compileall.compile_dir(os.path.join(self.directory, "long"),
- maxlevels=10, quiet=True)
- self.assertFalse(os.path.isfile(self.bc_path_long))
- compileall.compile_dir(os.path.join(self.directory, "long"),
- quiet=True)
- self.assertTrue(os.path.isfile(self.bc_path_long))
+ # Test the actual impact of maxlevels parameter
+ depth = 3
+ path = self.directory
+ for i in range(1, depth + 1):
+ path = os.path.join(path, f"dir_{i}")
+ source = os.path.join(path, 'script.py')
+ os.mkdir(path)
+ shutil.copyfile(self.source_path, source)
+ pyc_filename = importlib.util.cache_from_source(source)
+
+ compileall.compile_dir(self.directory, quiet=True, maxlevels=depth - 1)
+ self.assertFalse(os.path.isfile(pyc_filename))
+
+ compileall.compile_dir(self.directory, quiet=True, maxlevels=depth)
+ self.assertTrue(os.path.isfile(pyc_filename))
def test_strip_only(self):
fullpath = ["test", "build", "real", "path"]
diff --git a/Misc/NEWS.d/next/Tests/2019-10-14-22-46-35.bpo-38470.NHtzpy.rst b/Misc/NEWS.d/next/Tests/2019-10-14-22-46-35.bpo-38470.NHtzpy.rst
new file mode 100644
index 0000000000000..bb4710f86d4c8
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2019-10-14-22-46-35.bpo-38470.NHtzpy.rst
@@ -0,0 +1,3 @@
+Fix ``test_compileall.test_compile_dir_maxlevels()`` on Windows without long
+path support: only create 3 subdirectories instead of between 20 and 100
+subdirectories.
More information about the Python-checkins
mailing list