mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-02 16:13:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Merged revisions 72491-72493 via svnmerge from

Browse source 
svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r72491 | benjamin.peterson | 2009-05-08 19:33:27 -0500 (Fri, 08 May 2009) | 7 lines

  make 2to3 use unicode internally on 2.x

  This started out as a fix for #2660, but became this large refactoring
  when I realized the dire state this was in. 2to3 now uses
  tokenize.detect_encoding to decode the files correctly into unicode.
........
  r72492 | benjamin.peterson | 2009-05-08 19:35:38 -0500 (Fri, 08 May 2009) | 1 line

  remove compat code
........
  r72493 | benjamin.peterson | 2009-05-08 19:54:15 -0500 (Fri, 08 May 2009) | 1 line

  add a test for \r\n newlines
........
This commit is contained in:
Benjamin Peterson 2009-05-09 01:01:14 +00:00
parent 5edb1a1b0a
commit 84ad84e0bb
60 changed files with 386 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

64
Lib/lib2to3/fixer_util.py
View file

@ -14,13 +14,13 @@ from . import patcomp
def KeywordArg(keyword, value):
return Node(syms.argument,
[keyword, Leaf(token.EQUAL, '='), value])
[keyword, Leaf(token.EQUAL, u'='), value])
def LParen():
return Leaf(token.LPAR, "(")
return Leaf(token.LPAR, u"(")
def RParen():
return Leaf(token.RPAR, ")")
return Leaf(token.RPAR, u")")
def Assign(target, source):
"""Build an assignment statement"""
@ -43,11 +43,11 @@ def Attr(obj, attr):
def Comma():
"""A comma leaf"""
return Leaf(token.COMMA, ",")
return Leaf(token.COMMA, u",")
def Dot():
"""A period (.) leaf"""
return Leaf(token.DOT, ".")
return Leaf(token.DOT, u".")
def ArgList(args, lparen=LParen(), rparen=RParen()):
"""A parenthesised argument list, used by Call()"""
@ -65,20 +65,20 @@ def Call(func_name, args=None, prefix=None):
def Newline():
"""A newline literal"""
return Leaf(token.NEWLINE, "\n")
return Leaf(token.NEWLINE, u"\n")
def BlankLine():
"""A blank line"""
return Leaf(token.NEWLINE, "")
return Leaf(token.NEWLINE, u"")
def Number(n, prefix=None):
return Leaf(token.NUMBER, n, prefix=prefix)
def Subscript(index_node):
"""A numeric or string subscript"""
return Node(syms.trailer, [Leaf(token.LBRACE, '['),
return Node(syms.trailer, [Leaf(token.LBRACE, u'['),
index_node,
Leaf(token.RBRACE, ']')])
Leaf(token.RBRACE, u']')])
def String(string, prefix=None):
"""A string leaf"""
@ -89,24 +89,24 @@ def ListComp(xp, fp, it, test=None):
If test is None, the "if test" part is omitted.
"""
xp.set_prefix("")
fp.set_prefix(" ")
it.set_prefix(" ")
for_leaf = Leaf(token.NAME, "for")
for_leaf.set_prefix(" ")
in_leaf = Leaf(token.NAME, "in")
in_leaf.set_prefix(" ")
xp.set_prefix(u"")
fp.set_prefix(u" ")
it.set_prefix(u" ")
for_leaf = Leaf(token.NAME, u"for")
for_leaf.set_prefix(u" ")
in_leaf = Leaf(token.NAME, u"in")
in_leaf.set_prefix(u" ")
inner_args = [for_leaf, fp, in_leaf, it]
if test:
test.set_prefix(" ")
if_leaf = Leaf(token.NAME, "if")
if_leaf.set_prefix(" ")
test.set_prefix(u" ")
if_leaf = Leaf(token.NAME, u"if")
if_leaf.set_prefix(u" ")
inner_args.append(Node(syms.comp_if, [if_leaf, test]))
inner = Node(syms.listmaker, [xp, Node(syms.comp_for, inner_args)])
return Node(syms.atom,
[Leaf(token.LBRACE, "["),
[Leaf(token.LBRACE, u"["),
inner,
Leaf(token.RBRACE, "]")])
Leaf(token.RBRACE, u"]")])
def FromImport(package_name, name_leafs):
""" Return an import statement in the form:
@ -120,9 +120,9 @@ def FromImport(package_name, name_leafs):
# Pull the leaves out of their old tree
leaf.remove()
children = [Leaf(token.NAME, 'from'),
Leaf(token.NAME, package_name, prefix=" "),
Leaf(token.NAME, 'import', prefix=" "),
children = [Leaf(token.NAME, u'from'),
Leaf(token.NAME, package_name, prefix=u" "),
Leaf(token.NAME, u'import', prefix=u" "),
Node(syms.import_as_names, name_leafs)]
imp = Node(syms.import_from, children)
return imp
@ -141,8 +141,8 @@ def is_tuple(node):
and isinstance(node.children[0], Leaf)
and isinstance(node.children[1], Node)
and isinstance(node.children[2], Leaf)
and node.children[0].value == "("
and node.children[2].value == ")")
and node.children[0].value == u"("
and node.children[2].value == u")")
def is_list(node):
"""Does the node represent a list literal?"""
@ -150,8 +150,8 @@ def is_list(node):
and len(node.children) > 1
and isinstance(node.children[0], Leaf)
and isinstance(node.children[-1], Leaf)
and node.children[0].value == "["
and node.children[-1].value == "]")
and node.children[0].value == u"["
and node.children[-1].value == u"]")
###########################################################
@ -317,11 +317,11 @@ def touch_import(package, name, node):
if package is None:
import_ = Node(syms.import_name, [
Leaf(token.NAME, 'import'),
Leaf(token.NAME, name, prefix=' ')
Leaf(token.NAME, u'import'),
Leaf(token.NAME, name, prefix=u' ')
])
else:
import_ = FromImport(package, [Leaf(token.NAME, name, prefix=' ')])
import_ = FromImport(package, [Leaf(token.NAME, name, prefix=u' ')])
children = [import_, Newline()]
if add_newline_before:
@ -409,7 +409,7 @@ def _is_import_binding(node, name, package=None):
if package and unicode(node.children[1]).strip() != package:
return None
n = node.children[3]
if package and _find('as', n):
if package and _find(u'as', n):
# See test_from_import_as for explanation
return None
elif n.type == syms.import_as_names and _find(name, n):