Issue #27952: Get fixcid.py working with the re module

Lib/test/test_tools/test_fixcid.py

@@ -0,0 +1,91 @@
+'''Test Tools/scripts/fixcid.py.'''
+
+from io import StringIO
+import os, os.path
+import runpy
+import sys
+from test import support
+from test.test_tools import skip_if_missing, scriptsdir
+import unittest
+
+skip_if_missing()
+
+class Test(unittest.TestCase):
+    def test_parse_strings(self):
+        old1 = 'int xx = "xx\\"xx"[xx];\n'
+        old2 = "int xx = 'x\\'xx' + xx;\n"
+        output = self.run_script(old1 + old2)
+        new1 = 'int yy = "xx\\"xx"[yy];\n'
+        new2 = "int yy = 'x\\'xx' + yy;\n"
+        self.assertMultiLineEqual(output,
+            "1\n"
+            "< {old1}"
+            "> {new1}"
+            "{new1}"
+            "2\n"
+            "< {old2}"
+            "> {new2}"
+            "{new2}".format(old1=old1, old2=old2, new1=new1, new2=new2)
+        )
+
+    def test_alter_comments(self):
+        output = self.run_script(
+            substfile=
+                "xx yy\n"
+                "*aa bb\n",
+            args=("-c", "-",),
+            input=
+                "/* xx altered */\n"
+                "int xx;\n"
+                "/* aa unaltered */\n"
+                "int aa;\n",
+        )
+        self.assertMultiLineEqual(output,
+            "1\n"
+            "< /* xx altered */\n"
+            "> /* yy altered */\n"
+            "/* yy altered */\n"
+            "2\n"
+            "< int xx;\n"
+            "> int yy;\n"
+            "int yy;\n"
+            "/* aa unaltered */\n"
+            "4\n"
+            "< int aa;\n"
+            "> int bb;\n"
+            "int bb;\n"
+        )
+
+    def test_directory(self):
+        os.mkdir(support.TESTFN)
+        self.addCleanup(support.rmtree, support.TESTFN)
+        c_filename = os.path.join(support.TESTFN, "file.c")
+        with open(c_filename, "w") as file:
+            file.write("int xx;\n")
+        with open(os.path.join(support.TESTFN, "file.py"), "w") as file:
+            file.write("xx = 'unaltered'\n")
+        script = os.path.join(scriptsdir, "fixcid.py")
+        output = self.run_script(args=(support.TESTFN,))
+        self.assertMultiLineEqual(output,
+            "{}:\n"
+            "1\n"
+            '< int xx;\n'
+            '> int yy;\n'.format(c_filename)
+        )
+
+    def run_script(self, input="", *, args=("-",), substfile="xx yy\n"):
+        substfilename = support.TESTFN + ".subst"
+        with open(substfilename, "w") as file:
+            file.write(substfile)
+        self.addCleanup(support.unlink, substfilename)
+
+        argv = ["fixcid.py", "-s", substfilename] + list(args)
+        script = os.path.join(scriptsdir, "fixcid.py")
+        with support.swap_attr(sys, "argv", argv), \
+                support.swap_attr(sys, "stdin", StringIO(input)), \
+                support.captured_stdout() as output:
+            try:
+                runpy.run_path(script, run_name="__main__")
+            except SystemExit as exit:
+                self.assertEqual(exit.code, 0)
+        return output.getvalue()

Misc/NEWS

@@ -293,6 +293,10 @@ Tests
 Tools/Demos
 -----------
 
+- Issue #27952: Get Tools/scripts/fixcid.py working with Python 3 and the
+  current "re" module, avoid invalid Python backslash escapes, and fix a bug
+  parsing escaped C quote signs.
+
 - Issue #27332: Fixed the type of the first argument of module-level functions
   generated by Argument Clinic.  Patch by Petr Viktorin.
 

Tools/scripts/fixcid.py

@@ -88,9 +88,9 @@ def main():
     sys.exit(bad)
 
 # Change this regular expression to select a different set of files
-Wanted = '^[a-zA-Z0-9_]+\.[ch]$'
+Wanted = r'^[a-zA-Z0-9_]+\.[ch]$'
 def wanted(name):
-    return re.match(Wanted, name) >= 0
+    return re.match(Wanted, name)
 
 def recursedown(dirname):
     dbg('recursedown(%r)\n' % (dirname,))
@@ -168,6 +168,7 @@ def fix(filename):
     if filename == '-': return 0 # Done in filter mode
     f.close()
     if not g: return 0 # No changes
+    g.close()
 
     # Finishing touch -- move files
 
@@ -193,21 +194,21 @@ def fix(filename):
 
 # Tokenizing ANSI C (partly)
 
-Identifier = '\(struct \)?[a-zA-Z_][a-zA-Z0-9_]+'
+Identifier = '(struct )?[a-zA-Z_][a-zA-Z0-9_]+'
-String = '"\([^\n\\"]\|\\\\.\)*"'
+String = r'"([^\n\\"]|\\.)*"'
-Char = '\'\([^\n\\\']\|\\\\.\)*\''
+Char = r"'([^\n\\']|\\.)*'"
-CommentStart = '/\*'
+CommentStart = r'/\*'
-CommentEnd = '\*/'
+CommentEnd = r'\*/'
 
 Hexnumber = '0[xX][0-9a-fA-F]*[uUlL]*'
 Octnumber = '0[0-7]*[uUlL]*'
 Decnumber = '[1-9][0-9]*[uUlL]*'
-Intnumber = Hexnumber + '\|' + Octnumber + '\|' + Decnumber
+Intnumber = Hexnumber + '|' + Octnumber + '|' + Decnumber
 Exponent = '[eE][-+]?[0-9]+'
-Pointfloat = '\([0-9]+\.[0-9]*\|\.[0-9]+\)\(' + Exponent + '\)?'
+Pointfloat = r'([0-9]+\.[0-9]*|\.[0-9]+)(' + Exponent + r')?'
 Expfloat = '[0-9]+' + Exponent
-Floatnumber = Pointfloat + '\|' + Expfloat
+Floatnumber = Pointfloat + '|' + Expfloat
-Number = Floatnumber + '\|' + Intnumber
+Number = Floatnumber + '|' + Intnumber
 
 # Anything else is an operator -- don't list this explicitly because of '/*'
 
@@ -225,15 +226,16 @@ def initfixline():
 
 def fixline(line):
     global Program
-##  print '-->', repr(line)
+##  print('-->', repr(line))
     i = 0
     while i < len(line):
-        i = Program.search(line, i)
+        match = Program.search(line, i)
-        if i < 0: break
+        if match is None: break
-        found = Program.group(0)
+        i = match.start()
-##      if Program is InsideCommentProgram: print '...',
+        found = match.group(0)
-##      else: print '   ',
+##      if Program is InsideCommentProgram: print(end='... ')
-##      print found
+##      else: print(end='    ')
+##      print(found)
         if len(found) == 2:
             if found == '/*':
                 Program = InsideCommentProgram
@@ -247,15 +249,15 @@ def fixline(line):
                     print('Found in comment:', found)
                     i = i + n
                     continue
-                if NotInComment.has_key(found):
+                if found in NotInComment:
-##                  print 'Ignored in comment:',
+##                  print(end='Ignored in comment: ')
-##                  print found, '-->', subst
+##                  print(found, '-->', subst)
-##                  print 'Line:', line,
+##                  print('Line:', line, end='')
                     subst = found
 ##              else:
-##                  print 'Substituting in comment:',
+##                  print(end='Substituting in comment: ')
-##                  print found, '-->', subst
+##                  print(found, '-->', subst)
-##                  print 'Line:', line,
+##                  print('Line:', line, end='')
             line = line[:i] + subst + line[i+n:]
             n = len(subst)
         i = i + n