[Jython-checkins] jython: Resolve CRLF line-endings in updated lib2to3 test_parser (fixes #2808)
jeff.allen
jython-checkins at python.org
Sun Feb 2 02:27:38 EST 2020
https://hg.python.org/jython/rev/e81878891a4b
changeset: 8325:e81878891a4b
user: Adam Burke <adamburkemail at gmail.com>
date: Sat Feb 01 14:40:32 2020 +0000
summary:
Resolve CRLF line-endings in updated lib2to3 test_parser (fixes #2808)
This change updates lib2to3 tests from the CPython stdlib, skipping
where necessary. Rather than call out to the shell for diff (fails on
Windows) we use a comparison function in Python, being careful to choose
the mode rbU to deal with CRLF line endings.
files:
Lib/lib2to3/tests/test_parser.py | 185 ++++++++++++++++++-
NEWS | 1 +
2 files changed, 178 insertions(+), 8 deletions(-)
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -11,12 +11,23 @@
# Testing imports
from . import support
from .support import driver, test_dir
+from test import test_support
+
# Python imports
+import binascii
+import operator
import os
+import pickle
+import shutil
+import subprocess
import sys
+import tempfile
+import types
+import unittest
# Local imports
+from lib2to3.pgen2 import driver as pgen2_driver
from lib2to3.pgen2 import tokenize
from ..pgen2.parse import ParseError
from lib2to3.pygram import python_symbols as syms
@@ -31,6 +42,86 @@
self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
+class TestPgen2Caching(support.TestCase):
+ def test_load_grammar_from_txt_file(self):
+ pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
+
+ @unittest.skipIf(test_support.is_jython, "Not a valid test for Jython")
+ def test_load_grammar_from_pickle(self):
+ # Make a copy of the grammar file in a temp directory we are
+ # guaranteed to be able to write to.
+ tmpdir = tempfile.mkdtemp()
+ try:
+ grammar_copy = os.path.join(
+ tmpdir, os.path.basename(support.grammar_path))
+ shutil.copy(support.grammar_path, grammar_copy)
+ pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
+
+ pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
+ self.assertTrue(os.path.exists(pickle_name))
+
+ os.unlink(grammar_copy) # Only the pickle remains...
+ pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
+ finally:
+ shutil.rmtree(tmpdir)
+
+ @unittest.skipIf(test_support.is_jython, "Not a valid test for Jython")
+ @unittest.skipIf(sys.executable is None, 'sys.executable required')
+ def test_load_grammar_from_subprocess(self):
+ tmpdir = tempfile.mkdtemp()
+ tmpsubdir = os.path.join(tmpdir, 'subdir')
+ try:
+ os.mkdir(tmpsubdir)
+ grammar_base = os.path.basename(support.grammar_path)
+ grammar_copy = os.path.join(tmpdir, grammar_base)
+ grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
+ shutil.copy(support.grammar_path, grammar_copy)
+ shutil.copy(support.grammar_path, grammar_sub_copy)
+ pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
+ pickle_sub_name = pgen2_driver._generate_pickle_name(
+ grammar_sub_copy)
+ self.assertNotEqual(pickle_name, pickle_sub_name)
+
+ # Generate a pickle file from this process.
+ pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
+ self.assertTrue(os.path.exists(pickle_name))
+
+ # Generate a new pickle file in a subprocess with a most likely
+ # different hash randomization seed.
+ sub_env = dict(os.environ)
+ sub_env['PYTHONHASHSEED'] = 'random'
+ subprocess.check_call(
+ [sys.executable, '-c', """
+from lib2to3.pgen2 import driver as pgen2_driver
+pgen2_driver.load_grammar(%r, save=True, force=True)
+ """ % (grammar_sub_copy,)],
+ env=sub_env)
+ self.assertTrue(os.path.exists(pickle_sub_name))
+
+ with open(pickle_name, 'rb') as pickle_f_1, \
+ open(pickle_sub_name, 'rb') as pickle_f_2:
+ self.assertEqual(
+ pickle_f_1.read(), pickle_f_2.read(),
+ msg='Grammar caches generated using different hash seeds'
+ ' were not identical.')
+ finally:
+ shutil.rmtree(tmpdir)
+
+ @unittest.skipIf(test_support.is_jython, "Not a valid test for Jython")
+ def test_load_packaged_grammar(self):
+ modname = __name__ + '.load_test'
+ class MyLoader:
+ def get_data(self, where):
+ return pickle.dumps({'elephant': 19})
+ class MyModule(types.ModuleType):
+ __file__ = 'parsertestmodule'
+ __loader__ = MyLoader()
+ sys.modules[modname] = MyModule(modname)
+ self.addCleanup(operator.delitem, sys.modules, modname)
+ g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
+ self.assertEqual(g.elephant, 19)
+
+
class GrammarTest(support.TestCase):
def validate(self, code):
support.parse_string(code)
@@ -44,6 +135,21 @@
raise AssertionError("Syntax shouldn't have been valid")
+class TestMatrixMultiplication(GrammarTest):
+ @unittest.skipIf(test_support.is_jython, "Not supported yet")
+ def test_matrix_multiplication_operator(self):
+ self.validate("a @ b")
+ self.validate("a @= b")
+
+
+class TestYieldFrom(GrammarTest):
+ @unittest.skipIf(test_support.is_jython, "Not supported yet")
+ def test_matrix_multiplication_operator(self):
+ self.validate("yield from x")
+ self.validate("(yield from x) + y")
+ self.invalid_syntax("yield from")
+
+
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):
self.validate("raise")
@@ -72,8 +178,48 @@
def test_3x_style_invalid_4(self):
self.invalid_syntax("raise E from")
+# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
+# and Lib/test/text_parser.py test_list_displays, test_set_displays,
+# test_dict_displays, test_argument_unpacking, ... changes.
+class TestUnpackingGeneralizations(GrammarTest):
+ def test_mid_positional_star(self):
+ self.validate("""func(1, *(2, 3), 4)""")
-# Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
+ def test_double_star_dict_literal(self):
+ self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
+
+ def test_double_star_dict_literal_after_keywords(self):
+ self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
+
+ def test_list_display(self):
+ self.validate("""[*{2}, 3, *[4]]""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_set_display(self):
+ self.validate("""{*{2}, 3, *[4]}""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_dict_display_1(self):
+ self.validate("""{**{}}""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_dict_display_2(self):
+ self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_argument_unpacking_1(self):
+ self.validate("""f(a, *b, *c, d)""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_argument_unpacking_2(self):
+ self.validate("""f(**a, **b)""")
+
+ @unittest.skipIf(test_support.is_jython, "No Jython support yet")
+ def test_argument_unpacking_3(self):
+ self.validate("""f(2, *a, *b, **b, **c, **d)""")
+
+
+# Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
class TestFunctionAnnotations(GrammarTest):
def test_1(self):
self.validate("""def f(x) -> list: pass""")
@@ -162,15 +308,17 @@
for filepath in support.all_project_files():
with open(filepath, "rb") as fp:
encoding = tokenize.detect_encoding(fp.readline)[0]
- self.assertTrue(encoding is not None,
- "can't detect encoding for %s" % filepath)
+ self.assertIsNotNone(encoding,
+ "can't detect encoding for %s" % filepath)
with open(filepath, "r") as fp:
source = fp.read()
source = source.decode(encoding)
tree = driver.parse_string(source)
new = unicode(tree)
- if diff(filepath, new, encoding):
- self.fail("Idempotency failed: %s" % filepath)
+ diffResult = diff(filepath, new, encoding)
+ if diffResult:
+ self.fail("Idempotency failed: {} using {} encoding\n{}".
+ format(filepath,encoding,diffResult) )
def test_extended_unpacking(self):
driver.parse_string("a, *b, c = x\n")
@@ -212,12 +360,33 @@
self.validate(s)
+
def diff(fn, result, encoding):
- "A diff the result and original file content independent of OS."
+ """Diff the result and original file content independent of OS. This
+ implementation is in Jython only. CPython 2.x calls out to the shell diff
+ command. CPython 3.x str equality behaviour is simpler and should allow
+ this test file to be replaced wholesale in Jython 3.x."""
r = iter(result.encode(encoding).splitlines(True))
- with open(fn, "rb") as f:
+ lineNumber = 0
+ with open(fn, "rbU") as f:
for line in f:
+ lineNumber += 1
rline = next(r)
if rline != line:
- return True
+ return "original line {}:\n{}\n differs from:\n{}\n{}". \
+ format(lineNumber, line, rline, diffLine(line, rline) )
return False
+
+def diffLine(orig,result):
+ charNumber = 0
+ for c in orig:
+ if c != result[charNumber]:
+ return "Lines differ at char {}: {} vs {} (of {} vs {})".format(
+ charNumber,
+ binascii.hexlify(c),
+ binascii.hexlify(result[charNumber]),
+ len(orig),
+ len(result) )
+ charNumber += 1
+
+
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,7 @@
Jython 2.7.2b3
Bugs fixed
+ - [ 2808 ] lib2to3 test failures on Windows JDK 11
- [ 2846 ] Main module __name __ is not "__main__" under Java Scripting API
- [ 2828 ] Update netty JARs to 4.1.45
- [ 2044 ] CVE-2013-2027 Current umask sets privileges of class files and cache
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list