Simplified tuple returned by string._formatter_parser to only have

4 elements.  No need for old is_markup element, the same information
is indicated by literal_string being None.

Factored string.Formatter class to make subclasses easier to write.

Lib/string.py

@@ -208,15 +208,72 @@ class Formatter:
         return self.vformat(format_string, args, kwargs)
 
     def vformat(self, format_string, args, kwargs):
-        used_args = set()
+        used_args = self.get_empty_used_args()
         result = []
-        for (is_markup, literal, field_name, format_spec, conversion) in \
+        for literal_text, field_name, format_spec, conversion in \
-                format_string._formatter_parser():
+                self.parse(format_string):
-            if is_markup:
+            if literal_text is None:
-                # given the field_name, find the object it references
+                # this is some markup, find the object and do
+                #  the formatting
 
-                # split it into the first part, and and iterator that
+                # given the field_name, find the object it references
-                #  looks over the rest
+                obj = self.get_field(field_name, args, kwargs, used_args)
+
+                # do any conversion on the resulting object
+                obj = self.convert_field(obj, conversion)
+
+                # format the object and append to the result
+                result.append(self.format_field(obj, format_spec))
+            else:
+                # this is literal text, use it directly
+                result.append(literal_text)
+        self.check_unused_args(used_args, args, kwargs)
+        return ''.join(result)
+
+
+    def get_empty_used_args(self):
+        return set()
+
+
+    def get_value(self, key, args, kwargs):
+        if isinstance(key, int):
+            return args[key]
+        else:
+            return kwargs[key]
+
+
+    def check_unused_args(self, used_args, args, kwargs):
+        pass
+
+
+    def format_field(self, value, format_spec):
+        return format(value, format_spec)
+
+
+    def convert_field(self, value, conversion):
+        # do any conversion on the resulting object
+        if conversion == 'r':
+            return repr(value)
+        elif conversion == 's':
+            return str(value)
+        else:
+            assert conversion is None
+            return value
+
+
+    # returns an iterable that contains tuples of the form:
+    # (literal_text, field_name, format_spec, conversion)
+    def parse(self, format_string):
+        return format_string._formatter_parser()
+
+
+    # given a field_name, find the object it references.
+    #  field_name:   the field being looked up, e.g. "0.name"
+    #                 or "lookup[3]"
+    #  used_args:    a set of which args have been used
+    #  args, kwargs: as passed in to vformat
+    # also, mark it as used in 'used_args'
+    def get_field(self, field_name, args, kwargs, used_args):
         first, rest = field_name._formatter_field_name_split()
 
         used_args.add(first)
@@ -230,27 +287,4 @@ class Formatter:
             else:
                 obj = obj[i]
 
-                # do any conversion on the resulting object
+        return obj
-                if conversion == 'r':
-                    obj = repr(obj)
-                elif conversion == 's':
-                    obj = str(obj)
-
-                # format the object and append to the result
-                result.append(self.format_field(obj, format_spec))
-            else:
-                result.append(literal)
-        self.check_unused_args(used_args, args, kwargs)
-        return ''.join(result)
-
-    def get_value(self, key, args, kwargs):
-        if isinstance(key, int):
-            return args[key]
-        else:
-            return kwargs[key]
-
-    def check_unused_args(self, used_args, args, kwargs):
-        pass
-
-    def format_field(self, value, format_spec):
-        return format(value, format_spec)

Objects/stringlib/string_format.h

@@ -979,17 +979,12 @@ formatteriter_next(formatteriterobject *it)
         /* if 0, error has already been set, if 1, iterator is empty */
         return NULL;
     else {
-        PyObject *is_markup_bool = NULL;
         PyObject *literal_str = NULL;
         PyObject *field_name_str = NULL;
         PyObject *format_spec_str = NULL;
         PyObject *conversion_str = NULL;
         PyObject *tuple = NULL;
 
-        is_markup_bool = PyBool_FromLong(is_markup);
-        if (!is_markup_bool)
-            return NULL;
-
         if (is_markup) {
             /* field_name, format_spec, and conversion are returned */
             literal_str = Py_None;
@@ -1030,11 +1025,9 @@ formatteriter_next(formatteriterobject *it)
             Py_INCREF(format_spec_str);
             Py_INCREF(conversion_str);
         }
-        tuple = PyTuple_Pack(5, is_markup_bool, literal_str,
+        tuple = PyTuple_Pack(4, literal_str, field_name_str, format_spec_str,
-                             field_name_str, format_spec_str,
                              conversion_str);
     error:
-        Py_XDECREF(is_markup_bool);
         Py_XDECREF(literal_str);
         Py_XDECREF(field_name_str);
         Py_XDECREF(format_spec_str);