[Python-checkins] distutils2: Fix bugs with MANIFEST.in parsing on Windows (#6884).
eric.araujo
python-checkins at python.org
Mon Feb 27 12:29:44 CET 2012
http://hg.python.org/distutils2/rev/4d6a9f287543
changeset: 1289:4d6a9f287543
user: Éric Araujo <merwok at netwok.org>
date: Mon Feb 27 11:52:34 2012 +0100
summary:
Fix bugs with MANIFEST.in parsing on Windows (#6884).
These regex changes fix a number of issues on Windows:
- #6884: impossible to include a file starting with 'build'
- #9691 and #14004: sdist includes too many files
- #13193: test_manifest failures
Thanks to Nadeem Vawda for his help.
files:
CHANGES.txt | 1 +
CONTRIBUTORS.txt | 1 +
distutils2/manifest.py | 18 +-
distutils2/tests/fixer/__init__.py | 1 +
distutils2/tests/test_manifest.py | 138 ++++++++++++----
5 files changed, 119 insertions(+), 40 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -167,6 +167,7 @@
- #1326113: build_ext now correctly parses multiple values given to the
--libraries option [éric]
- #13974: add test for util.set_platform [tshepang]
+- #6884: Fix MANIFEST.in parsing bugs on Windows [éric, nadeem]
1.0a3 - 2010-10-08
diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -69,3 +69,4 @@
- Vinay Sajip
- Victor Stinner
- Alexandre Vassalotti
+- Nadeem Vawda
diff --git a/distutils2/manifest.py b/distutils2/manifest.py
--- a/distutils2/manifest.py
+++ b/distutils2/manifest.py
@@ -278,6 +278,7 @@
Return True if files are found.
"""
+ # XXX docstring lying about what the special chars are?
files_found = False
pattern_re = _translate_pattern(pattern, anchor, prefix, is_regex)
@@ -341,10 +342,14 @@
# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
# and by extension they shouldn't match such "special characters" under
# any OS. So change all non-escaped dots in the RE to match any
- # character except the special characters.
- # XXX currently the "special characters" are just slash -- i.e. this is
- # Unix-only.
- pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re)
+ # character except the special characters (currently: just os.sep).
+ sep = os.sep
+ if os.sep == '\\':
+ # we're using a regex to manipulate a regex, so we need
+ # to escape the backslash twice
+ sep = r'\\\\'
+ escaped = r'\1[^%s]' % sep
+ pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
if pattern_re.endswith('$'):
# make it look like fnmatch in Python 2.6 and newer
@@ -376,7 +381,10 @@
# ditch end of pattern character
empty_pattern = _glob_to_re('')
prefix_re = _glob_to_re(prefix)[:-len(empty_pattern)]
- pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re)
+ sep = os.sep
+ if os.sep == '\\':
+ sep = r'\\'
+ pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
else: # no prefix -- respect anchor flag
if anchor:
pattern_re = "^" + pattern_re
diff --git a/distutils2/tests/fixer/__init__.py b/distutils2/tests/fixer/__init__.py
--- a/distutils2/tests/fixer/__init__.py
+++ b/distutils2/tests/fixer/__init__.py
@@ -0,0 +1,1 @@
+"""Example fixers used in tests."""
diff --git a/distutils2/tests/test_manifest.py b/distutils2/tests/test_manifest.py
--- a/distutils2/tests/test_manifest.py
+++ b/distutils2/tests/test_manifest.py
@@ -7,7 +7,22 @@
from distutils2.tests import unittest, support
-_MANIFEST = """\
+MANIFEST_IN = """\
+include ok
+include xo
+exclude xo
+include foo.tmp
+include buildout.cfg
+global-include *.x
+global-include *.txt
+global-exclude *.tmp
+recursive-include f *.oo
+recursive-exclude global *.x
+graft dir
+prune dir3
+"""
+
+MANIFEST_IN_2 = """\
recursive-include foo *.py # ok
# nothing here
@@ -17,12 +32,17 @@
*.dat *.txt
"""
-_MANIFEST2 = """\
+MANIFEST_IN_3 = """\
README
file1
"""
+def make_local_path(s):
+ """Converts '/' in a string to os.sep"""
+ return s.replace('/', os.sep)
+
+
class ManifestTestCase(support.TempdirManager,
support.LoggingCatcher,
unittest.TestCase):
@@ -38,7 +58,7 @@
MANIFEST = os.path.join(tmpdir, 'MANIFEST.in')
f = open(MANIFEST, 'w')
try:
- f.write(_MANIFEST)
+ f.write(MANIFEST_IN_2)
finally:
f.close()
@@ -69,27 +89,74 @@
os.chdir(tmpdir)
self.write_file('README', 'xxx')
self.write_file('file1', 'xxx')
- content = StringIO(_MANIFEST2)
+ content = StringIO(MANIFEST_IN_3)
manifest = Manifest()
manifest.read_template(content)
self.assertEqual(['README', 'file1'], manifest.files)
def test_glob_to_re(self):
- # simple cases
- self.assertEqual(_glob_to_re('foo*'), 'foo[^/]*\\Z(?ms)')
- self.assertEqual(_glob_to_re('foo?'), 'foo[^/]\\Z(?ms)')
- self.assertEqual(_glob_to_re('foo??'), 'foo[^/][^/]\\Z(?ms)')
+ sep = os.sep
+ if os.sep == '\\':
+ sep = r'\\'
- # special cases
- self.assertEqual(_glob_to_re(r'foo\\*'), r'foo\\\\[^/]*\Z(?ms)')
- self.assertEqual(_glob_to_re(r'foo\\\*'), r'foo\\\\\\[^/]*\Z(?ms)')
- self.assertEqual(_glob_to_re('foo????'), r'foo[^/][^/][^/][^/]\Z(?ms)')
- self.assertEqual(_glob_to_re(r'foo\\??'), r'foo\\\\[^/][^/]\Z(?ms)')
+ for glob, regex in (
+ # simple cases
+ ('foo*', r'foo[^%(sep)s]*\Z(?ms)'),
+ ('foo?', r'foo[^%(sep)s]\Z(?ms)'),
+ ('foo??', r'foo[^%(sep)s][^%(sep)s]\Z(?ms)'),
+ # special cases
+ (r'foo\\*', r'foo\\\\[^%(sep)s]*\Z(?ms)'),
+ (r'foo\\\*', r'foo\\\\\\[^%(sep)s]*\Z(?ms)'),
+ ('foo????', r'foo[^%(sep)s][^%(sep)s][^%(sep)s][^%(sep)s]\Z(?ms)'),
+ (r'foo\\??', r'foo\\\\[^%(sep)s][^%(sep)s]\Z(?ms)'),
+ ):
+ regex = regex % {'sep': sep}
+ self.assertEqual(_glob_to_re(glob), regex)
+
+ def test_process_template_line(self):
+ # testing all MANIFEST.in template patterns
+ manifest = Manifest()
+ l = make_local_path
+
+ # simulated file list
+ manifest.allfiles = ['foo.tmp', 'ok', 'xo', 'four.txt',
+ 'buildout.cfg',
+ # filelist does not filter out VCS directories,
+ # it's sdist that does
+ l('.hg/last-message.txt'),
+ l('global/one.txt'),
+ l('global/two.txt'),
+ l('global/files.x'),
+ l('global/here.tmp'),
+ l('f/o/f.oo'),
+ l('dir/graft-one'),
+ l('dir/dir2/graft2'),
+ l('dir3/ok'),
+ l('dir3/sub/ok.txt'),
+ ]
+
+ for line in MANIFEST_IN.split('\n'):
+ if line.strip() == '':
+ continue
+ manifest._process_template_line(line)
+
+ wanted = ['ok',
+ 'buildout.cfg',
+ 'four.txt',
+ l('.hg/last-message.txt'),
+ l('global/one.txt'),
+ l('global/two.txt'),
+ l('f/o/f.oo'),
+ l('dir/graft-one'),
+ l('dir/dir2/graft2'),
+ ]
+
+ self.assertEqual(manifest.files, wanted)
def test_remove_duplicates(self):
manifest = Manifest()
manifest.files = ['a', 'b', 'a', 'g', 'c', 'g']
- # files must be sorted beforehand
+ # files must be sorted beforehand (like sdist does)
manifest.sort()
manifest.remove_duplicates()
self.assertEqual(manifest.files, ['a', 'b', 'c', 'g'])
@@ -149,6 +216,7 @@
self.assertEqual(manifest.allfiles, ['a.py', 'b.txt'])
def test_process_template(self):
+ l = make_local_path
# invalid lines
manifest = Manifest()
for action in ('include', 'exclude', 'global-include',
@@ -159,7 +227,7 @@
# implicit include
manifest = Manifest()
- manifest.allfiles = ['a.py', 'b.txt', 'd/c.py']
+ manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')]
manifest._process_template_line('*.py')
self.assertEqual(manifest.files, ['a.py'])
@@ -167,7 +235,7 @@
# include
manifest = Manifest()
- manifest.allfiles = ['a.py', 'b.txt', 'd/c.py']
+ manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')]
manifest._process_template_line('include *.py')
self.assertEqual(manifest.files, ['a.py'])
@@ -179,31 +247,31 @@
# exclude
manifest = Manifest()
- manifest.files = ['a.py', 'b.txt', 'd/c.py']
+ manifest.files = ['a.py', 'b.txt', l('d/c.py')]
manifest._process_template_line('exclude *.py')
- self.assertEqual(manifest.files, ['b.txt', 'd/c.py'])
+ self.assertEqual(manifest.files, ['b.txt', l('d/c.py')])
self.assertNoWarnings()
manifest._process_template_line('exclude *.rb')
- self.assertEqual(manifest.files, ['b.txt', 'd/c.py'])
+ self.assertEqual(manifest.files, ['b.txt', l('d/c.py')])
self.assertWarnings()
# global-include
manifest = Manifest()
- manifest.allfiles = ['a.py', 'b.txt', 'd/c.py']
+ manifest.allfiles = ['a.py', 'b.txt', l('d/c.py')]
manifest._process_template_line('global-include *.py')
- self.assertEqual(manifest.files, ['a.py', 'd/c.py'])
+ self.assertEqual(manifest.files, ['a.py', l('d/c.py')])
self.assertNoWarnings()
manifest._process_template_line('global-include *.rb')
- self.assertEqual(manifest.files, ['a.py', 'd/c.py'])
+ self.assertEqual(manifest.files, ['a.py', l('d/c.py')])
self.assertWarnings()
# global-exclude
manifest = Manifest()
- manifest.files = ['a.py', 'b.txt', 'd/c.py']
+ manifest.files = ['a.py', 'b.txt', l('d/c.py')]
manifest._process_template_line('global-exclude *.py')
self.assertEqual(manifest.files, ['b.txt'])
@@ -215,50 +283,50 @@
# recursive-include
manifest = Manifest()
- manifest.allfiles = ['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py']
+ manifest.allfiles = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')]
manifest._process_template_line('recursive-include d *.py')
- self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py'])
+ self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')])
self.assertNoWarnings()
manifest._process_template_line('recursive-include e *.py')
- self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py'])
+ self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')])
self.assertWarnings()
# recursive-exclude
manifest = Manifest()
- manifest.files = ['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py']
+ manifest.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')]
manifest._process_template_line('recursive-exclude d *.py')
- self.assertEqual(manifest.files, ['a.py', 'd/c.txt'])
+ self.assertEqual(manifest.files, ['a.py', l('d/c.txt')])
self.assertNoWarnings()
manifest._process_template_line('recursive-exclude e *.py')
- self.assertEqual(manifest.files, ['a.py', 'd/c.txt'])
+ self.assertEqual(manifest.files, ['a.py', l('d/c.txt')])
self.assertWarnings()
# graft
manifest = Manifest()
- manifest.allfiles = ['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py']
+ manifest.allfiles = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')]
manifest._process_template_line('graft d')
- self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py'])
+ self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')])
self.assertNoWarnings()
manifest._process_template_line('graft e')
- self.assertEqual(manifest.files, ['d/b.py', 'd/d/e.py'])
+ self.assertEqual(manifest.files, [l('d/b.py'), l('d/d/e.py')])
self.assertWarnings()
# prune
manifest = Manifest()
- manifest.files = ['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py']
+ manifest.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')]
manifest._process_template_line('prune d')
- self.assertEqual(manifest.files, ['a.py', 'f/f.py'])
+ self.assertEqual(manifest.files, ['a.py', l('f/f.py')])
self.assertNoWarnings()
manifest._process_template_line('prune e')
- self.assertEqual(manifest.files, ['a.py', 'f/f.py'])
+ self.assertEqual(manifest.files, ['a.py', l('f/f.py')])
self.assertWarnings()
--
Repository URL: http://hg.python.org/distutils2
More information about the Python-checkins
mailing list