mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

Issue #28038: Remove Tools/parser/com2ann.py and its unit test.

Browse source 
Development is moving to https://github.com/ilevkivskyi/com2ann
This commit is contained in:
Guido van Rossum 2016-09-09 09:06:11 -07:00
parent 6230aaf561
commit deed5a18ca
2 changed files with 0 additions and 568 deletions
Download patch file Download diff file Expand all files Collapse all files

260
Lib/test/test_tools/test_com2ann.py
View file

@ -1,260 +0,0 @@
"""Tests for the com2ann.py script in the Tools/parser directory."""
import unittest
import test.support
import os
import re
from test.test_tools import basepath, toolsdir, skip_if_missing
skip_if_missing()
parser_path = os.path.join(toolsdir, "parser")
with test.support.DirsOnSysPath(parser_path):
from com2ann import *
class BaseTestCase(unittest.TestCase):
def check(self, code, expected, n=False, e=False):
self.assertEqual(com2ann(code,
drop_None=n, drop_Ellipsis=e, silent=True),
expected)
class SimpleTestCase(BaseTestCase):
# Tests for basic conversions
def test_basics(self):
self.check("z = 5", "z = 5")
self.check("z: int = 5", "z: int = 5")
self.check("z = 5 # type: int", "z: int = 5")
self.check("z = 5 # type: int # comment",
"z: int = 5 # comment")
def test_type_ignore(self):
self.check("foobar = foobaz() #type: ignore",
"foobar = foobaz() #type: ignore")
self.check("a = 42 #type: ignore #comment",
"a = 42 #type: ignore #comment")
def test_complete_tuple(self):
self.check("t = 1, 2, 3 # type: Tuple[int, ...]",
"t: Tuple[int, ...] = (1, 2, 3)")
self.check("t = 1, # type: Tuple[int]",
"t: Tuple[int] = (1,)")
self.check("t = (1, 2, 3) # type: Tuple[int, ...]",
"t: Tuple[int, ...] = (1, 2, 3)")
def test_drop_None(self):
self.check("x = None # type: int",
"x: int", True)
self.check("x = None # type: int # another",
"x: int # another", True)
self.check("x = None # type: int # None",
"x: int # None", True)
def test_drop_Ellipsis(self):
self.check("x = ... # type: int",
"x: int", False, True)
self.check("x = ... # type: int # another",
"x: int # another", False, True)
self.check("x = ... # type: int # ...",
"x: int # ...", False, True)
def test_newline(self):
self.check("z = 5 # type: int\r\n", "z: int = 5\r\n")
self.check("z = 5 # type: int # comment\x85",
"z: int = 5 # comment\x85")
def test_wrong(self):
self.check("#type : str", "#type : str")
self.check("x==y #type: bool", "x==y #type: bool")
def test_pattern(self):
for line in ["#type: int", " # type: str[:] # com"]:
self.assertTrue(re.search(TYPE_COM, line))
for line in ["", "#", "# comment", "#type", "type int:"]:
self.assertFalse(re.search(TYPE_COM, line))
class BigTestCase(BaseTestCase):
# Tests for really crazy formatting, to be sure
# that script works reasonably in extreme situations
def test_crazy(self):
self.maxDiff = None
self.check(crazy_code, big_result, False, False)
self.check(crazy_code, big_result_ne, True, True)
crazy_code = """\
# -*- coding: utf-8 -*- # this should not be spoiled
'''
Docstring here
'''
import testmod
x = 5 #type : int # this one is OK
ttt \\
= \\
1.0, \\
2.0, \\
3.0, #type: Tuple[float, float, float]
with foo(x==1) as f: #type: str
print(f)
for i, j in my_inter(x=1): # type: ignore
i + j # type: int # what about this
x = y = z = 1 # type: int
x, y, z = [], [], [] # type: (List[int], List[int], List[str])
class C:
l[f(x
=1)] = [
1,
2,
] # type: List[int]
(C.x[1]) = \\
42 == 5# type: bool
lst[...] = \\
((\\
...)) # type: int # comment ..
y = ... # type: int # comment ...
z = ...
#type: int
#DONE placement of annotation after target rather than before =
TD.x[1] \\
= 0 == 5# type: bool
TD.y[1] =5 == 5# type: bool # one more here
F[G(x == y,
# hm...
z)]\\
= None # type: OMG[int] # comment: None
x = None#type:int #comment : None"""
big_result = """\
# -*- coding: utf-8 -*- # this should not be spoiled
'''
Docstring here
'''
import testmod
x: int = 5 # this one is OK
ttt: Tuple[float, float, float] \\
= \\
(1.0, \\
2.0, \\
3.0,)
with foo(x==1) as f: #type: str
print(f)
for i, j in my_inter(x=1): # type: ignore
i + j # type: int # what about this
x = y = z = 1 # type: int
x, y, z = [], [], [] # type: (List[int], List[int], List[str])
class C:
l[f(x
=1)]: List[int] = [
1,
2,
]
(C.x[1]): bool = \\
42 == 5
lst[...]: int = \\
((\\
...)) # comment ..
y: int = ... # comment ...
z = ...
#type: int
#DONE placement of annotation after target rather than before =
TD.x[1]: bool \\
= 0 == 5
TD.y[1]: bool =5 == 5 # one more here
F[G(x == y,
# hm...
z)]: OMG[int]\\
= None # comment: None
x: int = None #comment : None"""
big_result_ne = """\
# -*- coding: utf-8 -*- # this should not be spoiled
'''
Docstring here
'''
import testmod
x: int = 5 # this one is OK
ttt: Tuple[float, float, float] \\
= \\
(1.0, \\
2.0, \\
3.0,)
with foo(x==1) as f: #type: str
print(f)
for i, j in my_inter(x=1): # type: ignore
i + j # type: int # what about this
x = y = z = 1 # type: int
x, y, z = [], [], [] # type: (List[int], List[int], List[str])
class C:
l[f(x
=1)]: List[int] = [
1,
2,
]
(C.x[1]): bool = \\
42 == 5
lst[...]: int \\
\\
# comment ..
y: int # comment ...
z = ...
#type: int
#DONE placement of annotation after target rather than before =
TD.x[1]: bool \\
= 0 == 5
TD.y[1]: bool =5 == 5 # one more here
F[G(x == y,
# hm...
z)]: OMG[int]\\
# comment: None
x: int #comment : None"""
if __name__ == '__main__':
unittest.main()

308
Tools/parser/com2ann.py
View file

@ -1,308 +0,0 @@
"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations."""
import re
import os
import ast
import argparse
import tokenize
from collections import defaultdict
from textwrap import dedent
from io import BytesIO
__all__ = ['com2ann', 'TYPE_COM']
TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL)
TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL)
class _Data:
"""Internal class describing global data on file."""
def __init__(self, lines, tokens):
self.lines = lines
self.tokens = tokens
ttab = defaultdict(list) # maps line number to token numbers
for i, tok in enumerate(tokens):
ttab[tok.start[0]].append(i)
self.ttab = ttab
self.success = [] # list of lines where type comments where processed
self.fail = [] # list of lines where type comments where rejected
def skip_blank(d, lno):
while d.lines[lno].strip() == '':
lno += 1
return lno
def find_start(d, lcom):
"""Find first char of the assignment target."""
i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list
while ((d.tokens[i].exact_type != tokenize.NEWLINE) and
(d.tokens[i].exact_type != tokenize.ENCODING)):
i -= 1
lno = d.tokens[i].start[0]
return skip_blank(d, lno)
def check_target(stmt):
if len(stmt.body):
assign = stmt.body[0]
else:
return False
if isinstance(assign, ast.Assign) and len(assign.targets) == 1:
targ = assign.targets[0]
else:
return False
if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute)
or isinstance(targ, ast.Subscript)):
return True
return False
def find_eq(d, lstart):
"""Find equal sign starting from lstart taking care about d[f(x=1)] = 5."""
col = pars = 0
lno = lstart
while d.lines[lno][col] != '=' or pars != 0:
ch = d.lines[lno][col]
if ch in '([{':
pars += 1
elif ch in ')]}':
pars -= 1
if ch == '#' or col == len(d.lines[lno])-1:
lno = skip_blank(d, lno+1)
col = 0
else:
col += 1
return lno, col
def find_val(d, poseq):
"""Find position of first char of assignment value starting from poseq."""
lno, col = poseq
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
if col == len(d.lines[lno])-1:
lno += 1
col = 0
else:
col += 1
return lno, col
def find_targ(d, poseq):
"""Find position of last char of target (annotation goes here)."""
lno, col = poseq
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
if col == 0:
lno -= 1
col = len(d.lines[lno])-1
else:
col -= 1
return lno, col+1
def trim(new_lines, string, ltarg, poseq, lcom, ccom):
"""Remove None or Ellipsis from assignment value.
Also remove parens if one has (None), (...) etc.
string -- 'None' or '...'
ltarg -- line where last char of target is located
poseq -- position of equal sign
lcom, ccom -- position of type comment
"""
nopars = lambda s: s.replace('(', '').replace(')', '')
leq, ceq = poseq
end = ccom if leq == lcom else len(new_lines[leq])
subline = new_lines[leq][:ceq]
if leq == ltarg:
subline = subline.rstrip()
new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom
else new_lines[leq][ceq+1:end])
for lno in range(leq+1,lcom):
new_lines[lno] = nopars(new_lines[lno])
if lcom != leq:
subline = nopars(new_lines[lcom][:ccom]).replace(string, '')
if (not subline.isspace()):
subline = subline.rstrip()
new_lines[lcom] = subline + new_lines[lcom][ccom:]
def _com2ann(d, drop_None, drop_Ellipsis):
new_lines = d.lines[:]
for lcom, line in enumerate(d.lines):
match = re.search(TYPE_COM, line)
if match:
# strip " # type : annotation \n" -> "annotation \n"
tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip()
submatch = re.search(TRAIL_OR_COM, tp)
subcom = ''
if submatch and submatch.group():
subcom = submatch.group()
tp = tp[:submatch.start()]
if tp == 'ignore':
continue
ccom = match.start()
if not any(d.tokens[i].exact_type == tokenize.COMMENT
for i in d.ttab[lcom + 1]):
d.fail.append(lcom)
continue # type comment inside string
lstart = find_start(d, lcom)
stmt_str = dedent(''.join(d.lines[lstart:lcom+1]))
try:
stmt = ast.parse(stmt_str)
except SyntaxError:
d.fail.append(lcom)
continue # for or with statements
if not check_target(stmt):
d.fail.append(lcom)
continue
d.success.append(lcom)
val = stmt.body[0].value
# writing output now
poseq = find_eq(d, lstart)
lval, cval = find_val(d, poseq)
ltarg, ctarg = find_targ(d, poseq)
op_par = ''
cl_par = ''
if isinstance(val, ast.Tuple):
if d.lines[lval][cval] != '(':
op_par = '('
cl_par = ')'
# write the comment first
new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom
ccom = len(d.lines[lcom][:ccom].rstrip())
string = False
if isinstance(val, ast.Tuple):
# t = 1, 2 -> t = (1, 2); only latter is allowed with annotation
free_place = int(new_lines[lval][cval-2:cval] == ' ')
new_lines[lval] = (new_lines[lval][:cval-free_place] +
op_par + new_lines[lval][cval:])
elif isinstance(val, ast.Ellipsis) and drop_Ellipsis:
string = '...'
elif (isinstance(val, ast.NameConstant) and
val.value is None and drop_None):
string = 'None'
if string:
trim(new_lines, string, ltarg, poseq, lcom, ccom)
# finally write an annotation
new_lines[ltarg] = (new_lines[ltarg][:ctarg] +
': ' + tp + new_lines[ltarg][ctarg:])
return ''.join(new_lines)
def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False):
"""Translate type comments to type annotations in code.
Take code as string and return this string where::
variable = value # type: annotation # real comment
is translated to::
variable: annotation = value # real comment
For unsupported syntax cases, the type comments are
left intact. If drop_None is True or if drop_Ellipsis
is True translate correcpondingly::
variable = None # type: annotation
variable = ... # type: annotation
into::
variable: annotation
The tool tries to preserve code formatting as much as
possible, but an exact translation is not guarateed.
A summary of translated comments id printed by default.
"""
try:
ast.parse(code) # we want to work only with file without syntax errors
except SyntaxError:
return None
lines = code.splitlines(keepends=True)
rl = BytesIO(code.encode('utf-8')).readline
tokens = list(tokenize.tokenize(rl))
data = _Data(lines, tokens)
new_code = _com2ann(data, drop_None, drop_Ellipsis)
if not silent:
if data.success:
print('Comments translated on lines:',
', '.join(str(lno+1) for lno in data.success))
if data.fail:
print('Comments rejected on lines:',
', '.join(str(lno+1) for lno in data.fail))
if not data.success and not data.fail:
print('No type comments found')
return new_code
def translate_file(infile, outfile, dnone, dell, silent):
try:
descr = tokenize.open(infile)
except SyntaxError:
print("Cannot open", infile)
return
with descr as f:
code = f.read()
enc = f.encoding
if not silent:
print('File:', infile)
new_code = com2ann(code, drop_None=dnone,
drop_Ellipsis=dell,
silent=silent)
if new_code is None:
print("SyntaxError in", infile)
return
with open(outfile, 'wb') as f:
f.write((new_code).encode(enc))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("-o", "--outfile",
help="output file, will be overwritten if exists,\n"
"defaults to input file")
parser.add_argument("infile",
help="input file or directory for translation, must\n"
"contain no syntax errors, for directory\n"
"the outfile is ignored and translation is\n"
"made in place")
parser.add_argument("-s", "--silent",
help="Do not print summary for line numbers of\n"
"translated and rejected comments",
action="store_true")
parser.add_argument("-n", "--drop-none",
help="drop any None as assignment value during\n"
"translation if it is annotated by a type coment",
action="store_true")
parser.add_argument("-e", "--drop-ellipsis",
help="drop any Ellipsis (...) as assignment value during\n"
"translation if it is annotated by a type coment",
action="store_true")
args = parser.parse_args()
if args.outfile is None:
args.outfile = args.infile
if os.path.isfile(args.infile):
translate_file(args.infile, args.outfile,
args.drop_none, args.drop_ellipsis, args.silent)
else:
for root, dirs, files in os.walk(args.infile):
for afile in files:
_, ext = os.path.splitext(afile)
if ext == '.py' or ext == '.pyi':
fname = os.path.join(root, afile)
translate_file(fname, fname,
args.drop_none, args.drop_ellipsis,
args.silent)