bpo-44010: IDLE: colorize pattern-matching soft keywords (GH-25851)

(cherry picked from commit 60d343a816)

Co-authored-by: Tal Einat <532281+taleinat@users.noreply.github.com>

Lib/idlelib/colorizer.py

@@ -16,6 +16,32 @@ def any(name, alternates):
 
 def make_pat():
     kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
+    match_softkw = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<MATCH_SOFTKW>match)\b" +
+        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
+            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
+                                 # pattern-matching statement
+            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
+        ]) +
+        r"))"
+    )
+    case_default = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<CASE_SOFTKW>case)" +
+        r"[ \t]+(?P<CASE_DEFAULT_UNDERSCORE>_\b)"
+    )
+    case_softkw_and_pattern = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<CASE_SOFTKW2>case)\b" +
+        r"(?![ \t]*(?:" + "|".join([  # not followed by ...
+            r"_\b",  # a lone underscore
+            r"[:,;=^&|@~)\]}]",  # a character which means it can't be a
+                                 # pattern-matching case
+            r"\b(?:" + r"|".join(keyword.kwlist) + r")\b",  # a keyword
+        ]) +
+        r"))"
+    )
     builtinlist = [str(name) for name in dir(builtins)
                    if not name.startswith('_') and
                    name not in keyword.kwlist]
@@ -27,12 +53,29 @@ def make_pat():
     sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
     dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
     string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
-    return (kw + "|" + builtin + "|" + comment + "|" + string +
-            "|" + any("SYNC", [r"\n"]))
+    prog = re.compile("|".join([
+                                builtin, comment, string, kw,
+                                match_softkw, case_default,
+                                case_softkw_and_pattern,
+                                any("SYNC", [r"\n"]),
+                               ]),
+                      re.DOTALL | re.MULTILINE)
+    return prog
 
 
-prog = re.compile(make_pat(), re.S)
-idprog = re.compile(r"\s+(\w+)", re.S)
+prog = make_pat()
+idprog = re.compile(r"\s+(\w+)")
+prog_group_name_to_tag = {
+    "MATCH_SOFTKW": "KEYWORD",
+    "CASE_SOFTKW": "KEYWORD",
+    "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
+    "CASE_SOFTKW2": "KEYWORD",
+}
+
+
+def matched_named_groups(re_match):
+    "Get only the non-empty named groups from an re.Match object."
+    return ((k, v) for (k, v) in re_match.groupdict().items() if v)
 
 
 def color_config(text):
@@ -231,14 +274,10 @@ class ColorDelegator(Delegator):
     def recolorize_main(self):
         "Evaluate text and apply colorizing tags."
         next = "1.0"
-        while True:
-            item = self.tag_nextrange("TODO", next)
-            if not item:
-                break
-            head, tail = item
-            self.tag_remove("SYNC", head, tail)
-            item = self.tag_prevrange("SYNC", head)
-            head = item[1] if item else "1.0"
+        while todo_tag_range := self.tag_nextrange("TODO", next):
+            self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1])
+            sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0])
+            head = sync_tag_range[1] if sync_tag_range else "1.0"
 
             chars = ""
             next = head
@@ -256,23 +295,8 @@ class ColorDelegator(Delegator):
                     return
                 for tag in self.tagdefs:
                     self.tag_remove(tag, mark, next)
-                chars = chars + line
-                m = self.prog.search(chars)
-                while m:
-                    for key, value in m.groupdict().items():
-                        if value:
-                            a, b = m.span(key)
-                            self.tag_add(key,
-                                         head + "+%dc" % a,
-                                         head + "+%dc" % b)
-                            if value in ("def", "class"):
-                                m1 = self.idprog.match(chars, b)
-                                if m1:
-                                    a, b = m1.span(1)
-                                    self.tag_add("DEFINITION",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
-                    m = self.prog.search(chars, m.end())
+                chars += line
+                self._add_tags_in_section(chars, head)
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
                     chars = ""
@@ -291,6 +315,40 @@ class ColorDelegator(Delegator):
                     if DEBUG: print("colorizing stopped")
                     return
 
+    def _add_tag(self, start, end, head, matched_group_name):
+        """Add a tag to a given range in the text widget.
+
+        This is a utility function, receiving the range as `start` and
+        `end` positions, each of which is a number of characters
+        relative to the given `head` index in the text widget.
+
+        The tag to add is determined by `matched_group_name`, which is
+        the name of a regular expression "named group" as matched by
+        by the relevant highlighting regexps.
+        """
+        tag = prog_group_name_to_tag.get(matched_group_name,
+                                         matched_group_name)
+        self.tag_add(tag,
+                     f"{head}+{start:d}c",
+                     f"{head}+{end:d}c")
+
+    def _add_tags_in_section(self, chars, head):
+        """Parse and add highlighting tags to a given part of the text.
+
+        `chars` is a string with the text to parse and to which
+        highlighting is to be applied.
+
+            `head` is the index in the text widget where the text is found.
+        """
+        for m in self.prog.finditer(chars):
+            for name, matched_text in matched_named_groups(m):
+                a, b = m.span(name)
+                self._add_tag(a, b, head, name)
+                if matched_text in ("def", "class"):
+                    if m1 := self.idprog.match(chars, b):
+                        a, b = m1.span(1)
+                        self._add_tag(a, b, head, "DEFINITION")
+
     def removecolors(self):
         "Remove all colorizing tags."
         for tag in self.tagdefs:
@@ -299,27 +357,14 @@ class ColorDelegator(Delegator):
 
 def _color_delegator(parent):  # htest #
     from tkinter import Toplevel, Text
+    from idlelib.idle_test.test_colorizer import source
     from idlelib.percolator import Percolator
 
     top = Toplevel(parent)
     top.title("Test ColorDelegator")
     x, y = map(int, parent.geometry().split('+')[1:])
-    top.geometry("700x250+%d+%d" % (x + 20, y + 175))
-    source = (
-        "if True: int ('1') # keyword, builtin, string, comment\n"
-        "elif False: print(0)\n"
-        "else: float(None)\n"
-        "if iF + If + IF: 'keyword matching must respect case'\n"
-        "if'': x or''  # valid keyword-string no-space combinations\n"
-        "async def f(): await g()\n"
-        "# All valid prefixes for unicode and byte strings should be colored.\n"
-        "'x', '''x''', \"x\", \"\"\"x\"\"\"\n"
-        "r'x', u'x', R'x', U'x', f'x', F'x'\n"
-        "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n"
-        "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n"
-        "# Invalid combinations of legal characters should be half colored.\n"
-        "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n"
-        )
+    top.geometry("700x550+%d+%d" % (x + 20, y + 175))
+
     text = Text(top, background="white")
     text.pack(expand=1, fill="both")
     text.insert("insert", source)