[Python-checkins] cpython: Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
guido.van.rossum
python-checkins at python.org
Fri Sep 9 12:06:47 EDT 2016
https://hg.python.org/cpython/rev/35447332ab19
changeset: 103402:35447332ab19
user: Guido van Rossum <guido at dropbox.com>
date: Fri Sep 09 09:06:11 2016 -0700
summary:
Issue #28038: Remove Tools/parser/com2ann.py and its unit test.
Development is moving to https://github.com/ilevkivskyi/com2ann
files:
Lib/test/test_tools/test_com2ann.py | 260 -------------
Tools/parser/com2ann.py | 308 ----------------
2 files changed, 0 insertions(+), 568 deletions(-)
diff --git a/Lib/test/test_tools/test_com2ann.py b/Lib/test/test_tools/test_com2ann.py
deleted file mode 100644
--- a/Lib/test/test_tools/test_com2ann.py
+++ /dev/null
@@ -1,260 +0,0 @@
-"""Tests for the com2ann.py script in the Tools/parser directory."""
-
-import unittest
-import test.support
-import os
-import re
-
-from test.test_tools import basepath, toolsdir, skip_if_missing
-
-skip_if_missing()
-
-parser_path = os.path.join(toolsdir, "parser")
-
-with test.support.DirsOnSysPath(parser_path):
- from com2ann import *
-
-class BaseTestCase(unittest.TestCase):
-
- def check(self, code, expected, n=False, e=False):
- self.assertEqual(com2ann(code,
- drop_None=n, drop_Ellipsis=e, silent=True),
- expected)
-
-class SimpleTestCase(BaseTestCase):
- # Tests for basic conversions
-
- def test_basics(self):
- self.check("z = 5", "z = 5")
- self.check("z: int = 5", "z: int = 5")
- self.check("z = 5 # type: int", "z: int = 5")
- self.check("z = 5 # type: int # comment",
- "z: int = 5 # comment")
-
- def test_type_ignore(self):
- self.check("foobar = foobaz() #type: ignore",
- "foobar = foobaz() #type: ignore")
- self.check("a = 42 #type: ignore #comment",
- "a = 42 #type: ignore #comment")
-
- def test_complete_tuple(self):
- self.check("t = 1, 2, 3 # type: Tuple[int, ...]",
- "t: Tuple[int, ...] = (1, 2, 3)")
- self.check("t = 1, # type: Tuple[int]",
- "t: Tuple[int] = (1,)")
- self.check("t = (1, 2, 3) # type: Tuple[int, ...]",
- "t: Tuple[int, ...] = (1, 2, 3)")
-
- def test_drop_None(self):
- self.check("x = None # type: int",
- "x: int", True)
- self.check("x = None # type: int # another",
- "x: int # another", True)
- self.check("x = None # type: int # None",
- "x: int # None", True)
-
- def test_drop_Ellipsis(self):
- self.check("x = ... # type: int",
- "x: int", False, True)
- self.check("x = ... # type: int # another",
- "x: int # another", False, True)
- self.check("x = ... # type: int # ...",
- "x: int # ...", False, True)
-
- def test_newline(self):
- self.check("z = 5 # type: int\r\n", "z: int = 5\r\n")
- self.check("z = 5 # type: int # comment\x85",
- "z: int = 5 # comment\x85")
-
- def test_wrong(self):
- self.check("#type : str", "#type : str")
- self.check("x==y #type: bool", "x==y #type: bool")
-
- def test_pattern(self):
- for line in ["#type: int", " # type: str[:] # com"]:
- self.assertTrue(re.search(TYPE_COM, line))
- for line in ["", "#", "# comment", "#type", "type int:"]:
- self.assertFalse(re.search(TYPE_COM, line))
-
-class BigTestCase(BaseTestCase):
- # Tests for really crazy formatting, to be sure
- # that script works reasonably in extreme situations
-
- def test_crazy(self):
- self.maxDiff = None
- self.check(crazy_code, big_result, False, False)
- self.check(crazy_code, big_result_ne, True, True)
-
-crazy_code = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x = 5 #type : int # this one is OK
-ttt \\
- = \\
- 1.0, \\
- 2.0, \\
- 3.0, #type: Tuple[float, float, float]
-with foo(x==1) as f: #type: str
- print(f)
-
-for i, j in my_inter(x=1): # type: ignore
- i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], [] # type: (List[int], List[int], List[str])
-class C:
-
-
- l[f(x
- =1)] = [
-
- 1,
- 2,
- ] # type: List[int]
-
-
- (C.x[1]) = \\
- 42 == 5# type: bool
-lst[...] = \\
- ((\\
-...)) # type: int # comment ..
-
-y = ... # type: int # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1] \\
- = 0 == 5# type: bool
-
-TD.y[1] =5 == 5# type: bool # one more here
-F[G(x == y,
-
-# hm...
-
- z)]\\
- = None # type: OMG[int] # comment: None
-x = None#type:int #comment : None"""
-
-big_result = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x: int = 5 # this one is OK
-ttt: Tuple[float, float, float] \\
- = \\
- (1.0, \\
- 2.0, \\
- 3.0,)
-with foo(x==1) as f: #type: str
- print(f)
-
-for i, j in my_inter(x=1): # type: ignore
- i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], [] # type: (List[int], List[int], List[str])
-class C:
-
-
- l[f(x
- =1)]: List[int] = [
-
- 1,
- 2,
- ]
-
-
- (C.x[1]): bool = \\
- 42 == 5
-lst[...]: int = \\
- ((\\
-...)) # comment ..
-
-y: int = ... # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1]: bool \\
- = 0 == 5
-
-TD.y[1]: bool =5 == 5 # one more here
-F[G(x == y,
-
-# hm...
-
- z)]: OMG[int]\\
- = None # comment: None
-x: int = None #comment : None"""
-
-big_result_ne = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x: int = 5 # this one is OK
-ttt: Tuple[float, float, float] \\
- = \\
- (1.0, \\
- 2.0, \\
- 3.0,)
-with foo(x==1) as f: #type: str
- print(f)
-
-for i, j in my_inter(x=1): # type: ignore
- i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], [] # type: (List[int], List[int], List[str])
-class C:
-
-
- l[f(x
- =1)]: List[int] = [
-
- 1,
- 2,
- ]
-
-
- (C.x[1]): bool = \\
- 42 == 5
-lst[...]: int \\
- \\
- # comment ..
-
-y: int # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1]: bool \\
- = 0 == 5
-
-TD.y[1]: bool =5 == 5 # one more here
-F[G(x == y,
-
-# hm...
-
- z)]: OMG[int]\\
- # comment: None
-x: int #comment : None"""
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/Tools/parser/com2ann.py b/Tools/parser/com2ann.py
deleted file mode 100644
--- a/Tools/parser/com2ann.py
+++ /dev/null
@@ -1,308 +0,0 @@
-"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations."""
-import re
-import os
-import ast
-import argparse
-import tokenize
-from collections import defaultdict
-from textwrap import dedent
-from io import BytesIO
-
-__all__ = ['com2ann', 'TYPE_COM']
-
-TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL)
-TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL)
-
-
-class _Data:
- """Internal class describing global data on file."""
- def __init__(self, lines, tokens):
- self.lines = lines
- self.tokens = tokens
- ttab = defaultdict(list) # maps line number to token numbers
- for i, tok in enumerate(tokens):
- ttab[tok.start[0]].append(i)
- self.ttab = ttab
- self.success = [] # list of lines where type comments where processed
- self.fail = [] # list of lines where type comments where rejected
-
-
-def skip_blank(d, lno):
- while d.lines[lno].strip() == '':
- lno += 1
- return lno
-
-
-def find_start(d, lcom):
- """Find first char of the assignment target."""
- i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list
- while ((d.tokens[i].exact_type != tokenize.NEWLINE) and
- (d.tokens[i].exact_type != tokenize.ENCODING)):
- i -= 1
- lno = d.tokens[i].start[0]
- return skip_blank(d, lno)
-
-
-def check_target(stmt):
- if len(stmt.body):
- assign = stmt.body[0]
- else:
- return False
- if isinstance(assign, ast.Assign) and len(assign.targets) == 1:
- targ = assign.targets[0]
- else:
- return False
- if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute)
- or isinstance(targ, ast.Subscript)):
- return True
- return False
-
-
-def find_eq(d, lstart):
- """Find equal sign starting from lstart taking care about d[f(x=1)] = 5."""
- col = pars = 0
- lno = lstart
- while d.lines[lno][col] != '=' or pars != 0:
- ch = d.lines[lno][col]
- if ch in '([{':
- pars += 1
- elif ch in ')]}':
- pars -= 1
- if ch == '#' or col == len(d.lines[lno])-1:
- lno = skip_blank(d, lno+1)
- col = 0
- else:
- col += 1
- return lno, col
-
-
-def find_val(d, poseq):
- """Find position of first char of assignment value starting from poseq."""
- lno, col = poseq
- while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
- if col == len(d.lines[lno])-1:
- lno += 1
- col = 0
- else:
- col += 1
- return lno, col
-
-
-def find_targ(d, poseq):
- """Find position of last char of target (annotation goes here)."""
- lno, col = poseq
- while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
- if col == 0:
- lno -= 1
- col = len(d.lines[lno])-1
- else:
- col -= 1
- return lno, col+1
-
-
-def trim(new_lines, string, ltarg, poseq, lcom, ccom):
- """Remove None or Ellipsis from assignment value.
-
- Also remove parens if one has (None), (...) etc.
- string -- 'None' or '...'
- ltarg -- line where last char of target is located
- poseq -- position of equal sign
- lcom, ccom -- position of type comment
- """
- nopars = lambda s: s.replace('(', '').replace(')', '')
- leq, ceq = poseq
- end = ccom if leq == lcom else len(new_lines[leq])
- subline = new_lines[leq][:ceq]
- if leq == ltarg:
- subline = subline.rstrip()
- new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom
- else new_lines[leq][ceq+1:end])
-
- for lno in range(leq+1,lcom):
- new_lines[lno] = nopars(new_lines[lno])
-
- if lcom != leq:
- subline = nopars(new_lines[lcom][:ccom]).replace(string, '')
- if (not subline.isspace()):
- subline = subline.rstrip()
- new_lines[lcom] = subline + new_lines[lcom][ccom:]
-
-
-def _com2ann(d, drop_None, drop_Ellipsis):
- new_lines = d.lines[:]
- for lcom, line in enumerate(d.lines):
- match = re.search(TYPE_COM, line)
- if match:
- # strip " # type : annotation \n" -> "annotation \n"
- tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip()
- submatch = re.search(TRAIL_OR_COM, tp)
- subcom = ''
- if submatch and submatch.group():
- subcom = submatch.group()
- tp = tp[:submatch.start()]
- if tp == 'ignore':
- continue
- ccom = match.start()
- if not any(d.tokens[i].exact_type == tokenize.COMMENT
- for i in d.ttab[lcom + 1]):
- d.fail.append(lcom)
- continue # type comment inside string
- lstart = find_start(d, lcom)
- stmt_str = dedent(''.join(d.lines[lstart:lcom+1]))
- try:
- stmt = ast.parse(stmt_str)
- except SyntaxError:
- d.fail.append(lcom)
- continue # for or with statements
- if not check_target(stmt):
- d.fail.append(lcom)
- continue
-
- d.success.append(lcom)
- val = stmt.body[0].value
-
- # writing output now
- poseq = find_eq(d, lstart)
- lval, cval = find_val(d, poseq)
- ltarg, ctarg = find_targ(d, poseq)
-
- op_par = ''
- cl_par = ''
- if isinstance(val, ast.Tuple):
- if d.lines[lval][cval] != '(':
- op_par = '('
- cl_par = ')'
- # write the comment first
- new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom
- ccom = len(d.lines[lcom][:ccom].rstrip())
-
- string = False
- if isinstance(val, ast.Tuple):
- # t = 1, 2 -> t = (1, 2); only latter is allowed with annotation
- free_place = int(new_lines[lval][cval-2:cval] == ' ')
- new_lines[lval] = (new_lines[lval][:cval-free_place] +
- op_par + new_lines[lval][cval:])
- elif isinstance(val, ast.Ellipsis) and drop_Ellipsis:
- string = '...'
- elif (isinstance(val, ast.NameConstant) and
- val.value is None and drop_None):
- string = 'None'
- if string:
- trim(new_lines, string, ltarg, poseq, lcom, ccom)
-
- # finally write an annotation
- new_lines[ltarg] = (new_lines[ltarg][:ctarg] +
- ': ' + tp + new_lines[ltarg][ctarg:])
- return ''.join(new_lines)
-
-
-def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False):
- """Translate type comments to type annotations in code.
-
- Take code as string and return this string where::
-
- variable = value # type: annotation # real comment
-
- is translated to::
-
- variable: annotation = value # real comment
-
- For unsupported syntax cases, the type comments are
- left intact. If drop_None is True or if drop_Ellipsis
- is True translate correcpondingly::
-
- variable = None # type: annotation
- variable = ... # type: annotation
-
- into::
-
- variable: annotation
-
- The tool tries to preserve code formatting as much as
- possible, but an exact translation is not guarateed.
- A summary of translated comments id printed by default.
- """
- try:
- ast.parse(code) # we want to work only with file without syntax errors
- except SyntaxError:
- return None
- lines = code.splitlines(keepends=True)
- rl = BytesIO(code.encode('utf-8')).readline
- tokens = list(tokenize.tokenize(rl))
-
- data = _Data(lines, tokens)
- new_code = _com2ann(data, drop_None, drop_Ellipsis)
-
- if not silent:
- if data.success:
- print('Comments translated on lines:',
- ', '.join(str(lno+1) for lno in data.success))
- if data.fail:
- print('Comments rejected on lines:',
- ', '.join(str(lno+1) for lno in data.fail))
- if not data.success and not data.fail:
- print('No type comments found')
-
- return new_code
-
-
-def translate_file(infile, outfile, dnone, dell, silent):
- try:
- descr = tokenize.open(infile)
- except SyntaxError:
- print("Cannot open", infile)
- return
- with descr as f:
- code = f.read()
- enc = f.encoding
- if not silent:
- print('File:', infile)
- new_code = com2ann(code, drop_None=dnone,
- drop_Ellipsis=dell,
- silent=silent)
- if new_code is None:
- print("SyntaxError in", infile)
- return
- with open(outfile, 'wb') as f:
- f.write((new_code).encode(enc))
-
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("-o", "--outfile",
- help="output file, will be overwritten if exists,\n"
- "defaults to input file")
- parser.add_argument("infile",
- help="input file or directory for translation, must\n"
- "contain no syntax errors, for directory\n"
- "the outfile is ignored and translation is\n"
- "made in place")
- parser.add_argument("-s", "--silent",
- help="Do not print summary for line numbers of\n"
- "translated and rejected comments",
- action="store_true")
- parser.add_argument("-n", "--drop-none",
- help="drop any None as assignment value during\n"
- "translation if it is annotated by a type coment",
- action="store_true")
- parser.add_argument("-e", "--drop-ellipsis",
- help="drop any Ellipsis (...) as assignment value during\n"
- "translation if it is annotated by a type coment",
- action="store_true")
- args = parser.parse_args()
- if args.outfile is None:
- args.outfile = args.infile
-
- if os.path.isfile(args.infile):
- translate_file(args.infile, args.outfile,
- args.drop_none, args.drop_ellipsis, args.silent)
- else:
- for root, dirs, files in os.walk(args.infile):
- for afile in files:
- _, ext = os.path.splitext(afile)
- if ext == '.py' or ext == '.pyi':
- fname = os.path.join(root, afile)
- translate_file(fname, fname,
- args.drop_none, args.drop_ellipsis,
- args.silent)
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list