bpo-42901: [Enum] move member creation to __set_name__ (GH-24196)

`type.__new__` calls `__set_name__` and `__init_subclass__`, which means
that any work metaclasses do after calling `super().__new__()` will not
be available to those two methods.  In particular, `Enum` classes that
want to make use of `__init_subclass__` will not see any members.

Almost all customization is therefore moved to before the
`type.__new__()` call, including changing all members to a proto member
descriptor with a `__set_name__` that will do the final conversion of a
member to be an instance of the `Enum` class.

Lib/enum.py

@@ -1,11 +1,13 @@
 import sys
 from types import MappingProxyType, DynamicClassAttribute
+from builtins import property as _bltin_property
 
 
 __all__ = [
         'EnumMeta',
         'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag',
         'auto', 'unique',
+        'property',
         ]
 
 
@@ -54,14 +56,20 @@ def _is_private(cls_name, name):
     else:
         return False
 
-def _make_class_unpicklable(cls):
+def _make_class_unpicklable(obj):
     """
-    Make the given class un-picklable.
+    Make the given obj un-picklable.
+
+    obj should be either a dictionary, on an Enum
     """
     def _break_on_call_reduce(self, proto):
         raise TypeError('%r cannot be pickled' % self)
-    cls.__reduce_ex__ = _break_on_call_reduce
+    if isinstance(obj, dict):
-    cls.__module__ = '<unknown>'
+        obj['__reduce_ex__'] = _break_on_call_reduce
+        obj['__module__'] = '<unknown>'
+    else:
+        setattr(obj, '__reduce_ex__', _break_on_call_reduce)
+        setattr(obj, '__module__', '<unknown>')
 
 _auto_null = object()
 class auto:
@@ -70,6 +78,125 @@ class auto:
     """
     value = _auto_null
 
+class property(DynamicClassAttribute):
+    """
+    This is a descriptor, used to define attributes that act differently
+    when accessed through an enum member and through an enum class.
+    Instance access is the same as property(), but access to an attribute
+    through the enum class will instead look in the class' _member_map_ for
+    a corresponding enum member.
+    """
+
+    def __get__(self, instance, ownerclass=None):
+        if instance is None:
+            try:
+                return ownerclass._member_map_[self.name]
+            except KeyError:
+                raise AttributeError('%r not found in %r' % (self.name, ownerclass.__name__))
+        else:
+            if self.fget is None:
+                raise AttributeError('%s: cannot read attribute %r' % (ownerclass.__name__, self.name))
+            else:
+                return self.fget(instance)
+
+    def __set__(self, instance, value):
+        if self.fset is None:
+            raise AttributeError("%s: cannot set attribute %r" % (self.clsname, self.name))
+        else:
+            return self.fset(instance, value)
+
+    def __delete__(self, instance):
+        if self.fdel is None:
+            raise AttributeError("%s: cannot delete attribute %r" % (self.clsname, self.name))
+        else:
+            return self.fdel(instance)
+
+    def __set_name__(self, ownerclass, name):
+        self.name = name
+        self.clsname = ownerclass.__name__
+
+
+class _proto_member:
+    """
+    intermediate step for enum members between class execution and final creation
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+    def __set_name__(self, enum_class, member_name):
+        """
+        convert each quasi-member into an instance of the new enum class
+        """
+        # first step: remove ourself from enum_class
+        delattr(enum_class, member_name)
+        # second step: create member based on enum_class
+        value = self.value
+        if not isinstance(value, tuple):
+            args = (value, )
+        else:
+            args = value
+        if enum_class._member_type_ is tuple:   # special case for tuple enums
+            args = (args, )     # wrap it one more time
+        if not enum_class._use_args_:
+            enum_member = enum_class._new_member_(enum_class)
+            if not hasattr(enum_member, '_value_'):
+                enum_member._value_ = value
+        else:
+            enum_member = enum_class._new_member_(enum_class, *args)
+            if not hasattr(enum_member, '_value_'):
+                if enum_class._member_type_ is object:
+                    enum_member._value_ = value
+                else:
+                    enum_member._value_ = enum_class._member_type_(*args)
+        value = enum_member._value_
+        enum_member._name_ = member_name
+        enum_member.__objclass__ = enum_class
+        enum_member.__init__(*args)
+        # If another member with the same value was already defined, the
+        # new member becomes an alias to the existing one.
+        for name, canonical_member in enum_class._member_map_.items():
+            if canonical_member._value_ == enum_member._value_:
+                enum_member = canonical_member
+                break
+        else:
+            # no other instances found, record this member in _member_names_
+            enum_class._member_names_.append(member_name)
+        # get redirect in place before adding to _member_map_
+        # but check for other instances in parent classes first
+        need_override = False
+        descriptor = None
+        for base in enum_class.__mro__[1:]:
+            descriptor = base.__dict__.get(member_name)
+            if descriptor is not None:
+                if isinstance(descriptor, (property, DynamicClassAttribute)):
+                    break
+                else:
+                    need_override = True
+                    # keep looking for an enum.property
+        if descriptor and not need_override:
+            # previous enum.property found, no further action needed
+            pass
+        else:
+            redirect = property()
+            redirect.__set_name__(enum_class, member_name)
+            if descriptor and need_override:
+                # previous enum.property found, but some other inherited attribute
+                # is in the way; copy fget, fset, fdel to this one
+                redirect.fget = descriptor.fget
+                redirect.fset = descriptor.fset
+                redirect.fdel = descriptor.fdel
+            setattr(enum_class, member_name, redirect)
+        # now add to _member_map_ (even aliases)
+        enum_class._member_map_[member_name] = enum_member
+        try:
+            # This may fail if value is not hashable. We can't add the value
+            # to the map, and by-value lookups for this value will be
+            # linear.
+            enum_class._value2member_map_[value] = enum_member
+        except TypeError:
+            pass
+
 
 class _EnumDict(dict):
     """
@@ -195,46 +322,39 @@ class EnumMeta(type):
         ignore = classdict['_ignore_']
         for key in ignore:
             classdict.pop(key, None)
+        #
+        # grab member names
+        member_names = classdict._member_names
+        #
+        # check for illegal enum names (any others?)
+        invalid_names = set(member_names) & {'mro', ''}
+        if invalid_names:
+            raise ValueError('Invalid enum member name: {0}'.format(
+                ','.join(invalid_names)))
+        #
+        # adjust the sunders
+        _order_ = classdict.pop('_order_', None)
+        # convert to normal dict
+        classdict = dict(classdict.items())
+        #
+        # data type of member and the controlling Enum class
         member_type, first_enum = metacls._get_mixins_(cls, bases)
         __new__, save_new, use_args = metacls._find_new_(
                 classdict, member_type, first_enum,
                 )
-
+        classdict['_new_member_'] = __new__
-        # save enum items into separate mapping so they don't get baked into
+        classdict['_use_args_'] = use_args
-        # the new class
+        #
-        enum_members = {k: classdict[k] for k in classdict._member_names}
+        # convert future enum members into temporary _proto_members
-        for name in classdict._member_names:
+        for name in member_names:
-            del classdict[name]
+            classdict[name] = _proto_member(classdict[name])
-
+        #
-        # adjust the sunders
+        # house keeping structures
-        _order_ = classdict.pop('_order_', None)
+        classdict['_member_names_'] = []
-
+        classdict['_member_map_'] = {}
-        # check for illegal enum names (any others?)
+        classdict['_value2member_map_'] = {}
-        invalid_names = set(enum_members) & {'mro', ''}
+        classdict['_member_type_'] = member_type
-        if invalid_names:
+        #
-            raise ValueError('Invalid enum member name: {0}'.format(
-                ','.join(invalid_names)))
-
-        # create a default docstring if one has not been provided
-        if '__doc__' not in classdict:
-            classdict['__doc__'] = 'An enumeration.'
-
-        enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
-        enum_class._member_names_ = []               # names in definition order
-        enum_class._member_map_ = {}                 # name->value map
-        enum_class._member_type_ = member_type
-
-        # save DynamicClassAttribute attributes from super classes so we know
-        # if we can take the shortcut of storing members in the class dict
-        dynamic_attributes = {
-                k for c in enum_class.mro()
-                for k, v in c.__dict__.items()
-                if isinstance(v, DynamicClassAttribute)
-                }
-
-        # Reverse value->name map for hashable values.
-        enum_class._value2member_map_ = {}
-
         # If a custom type is mixed into the Enum, and it does not know how
         # to pickle itself, pickle.dumps will succeed but pickle.loads will
         # fail.  Rather than have the error show up later and possibly far
@@ -250,58 +370,21 @@ class EnumMeta(type):
                 methods = ('__getnewargs_ex__', '__getnewargs__',
                         '__reduce_ex__', '__reduce__')
                 if not any(m in member_type.__dict__ for m in methods):
-                    _make_class_unpicklable(enum_class)
+                    _make_class_unpicklable(classdict)
-
+        #
-        # instantiate them, checking for duplicates as we go
+        # create a default docstring if one has not been provided
-        # we instantiate first instead of checking for duplicates first in case
+        if '__doc__' not in classdict:
-        # a custom __new__ is doing something funky with the values -- such as
+            classdict['__doc__'] = 'An enumeration.'
-        # auto-numbering ;)
+        try:
-        for member_name in classdict._member_names:
+            exc = None
-            value = enum_members[member_name]
+            enum_class = super().__new__(metacls, cls, bases, classdict, **kwds)
-            if not isinstance(value, tuple):
+        except RuntimeError as e:
-                args = (value, )
+            # any exceptions raised by member.__new__ will get converted to a
-            else:
+            # RuntimeError, so get that original exception back and raise it instead
-                args = value
+            exc = e.__cause__ or e
-            if member_type is tuple:   # special case for tuple enums
+        if exc is not None:
-                args = (args, )     # wrap it one more time
+            raise exc
-            if not use_args:
+        #
-                enum_member = __new__(enum_class)
-                if not hasattr(enum_member, '_value_'):
-                    enum_member._value_ = value
-            else:
-                enum_member = __new__(enum_class, *args)
-                if not hasattr(enum_member, '_value_'):
-                    if member_type is object:
-                        enum_member._value_ = value
-                    else:
-                        enum_member._value_ = member_type(*args)
-            value = enum_member._value_
-            enum_member._name_ = member_name
-            enum_member.__objclass__ = enum_class
-            enum_member.__init__(*args)
-            # If another member with the same value was already defined, the
-            # new member becomes an alias to the existing one.
-            for name, canonical_member in enum_class._member_map_.items():
-                if canonical_member._value_ == enum_member._value_:
-                    enum_member = canonical_member
-                    break
-            else:
-                # Aliases don't appear in member names (only in __members__).
-                enum_class._member_names_.append(member_name)
-            # performance boost for any member that would not shadow
-            # a DynamicClassAttribute
-            if member_name not in dynamic_attributes:
-                setattr(enum_class, member_name, enum_member)
-            # now add to _member_map_
-            enum_class._member_map_[member_name] = enum_member
-            try:
-                # This may fail if value is not hashable. We can't add the value
-                # to the map, and by-value lookups for this value will be
-                # linear.
-                enum_class._value2member_map_[value] = enum_member
-            except TypeError:
-                pass
-
         # double check that repr and friends are not the mixin's or various
         # things break (such as pickle)
         # however, if the method is defined in the Enum itself, don't replace
@@ -314,7 +397,7 @@ class EnumMeta(type):
             enum_method = getattr(first_enum, name, None)
             if obj_method is not None and obj_method is class_method:
                 setattr(enum_class, name, enum_method)
-
+        #
         # replace any other __new__ with our own (as long as Enum is not None,
         # anyway) -- again, this is to support pickle
         if Enum is not None:
@@ -323,14 +406,14 @@ class EnumMeta(type):
             if save_new:
                 enum_class.__new_member__ = __new__
             enum_class.__new__ = Enum.__new__
-
+        #
         # py3 support for definition order (helps keep py2/py3 code in sync)
         if _order_ is not None:
             if isinstance(_order_, str):
                 _order_ = _order_.replace(',', ' ').split()
             if _order_ != enum_class._member_names_:
                 raise TypeError('member order does not match _order_')
-
+        #
         return enum_class
 
     def __bool__(self):
@@ -424,7 +507,7 @@ class EnumMeta(type):
     def __len__(cls):
         return len(cls._member_names_)
 
-    @property
+    @_bltin_property
     def __members__(cls):
         """
         Returns a mapping of member name->value.
@@ -491,7 +574,6 @@ class EnumMeta(type):
             else:
                 member_name, member_value = item
             classdict[member_name] = member_value
-        enum_class = metacls.__new__(metacls, class_name, bases, classdict)
 
         # TODO: replace the frame hack if a blessed way to know the calling
         # module is ever developed
@@ -501,13 +583,13 @@ class EnumMeta(type):
             except (AttributeError, ValueError, KeyError):
                 pass
         if module is None:
-            _make_class_unpicklable(enum_class)
+            _make_class_unpicklable(classdict)
         else:
-            enum_class.__module__ = module
+            classdict['__module__'] = module
         if qualname is not None:
-            enum_class.__qualname__ = qualname
+            classdict['__qualname__'] = qualname
 
-        return enum_class
+        return metacls.__new__(metacls, class_name, bases, classdict)
 
     def _convert_(cls, name, module, filter, source=None):
         """
@@ -756,19 +838,20 @@ class Enum(metaclass=EnumMeta):
     def __reduce_ex__(self, proto):
         return self.__class__, (self._value_, )
 
-    # DynamicClassAttribute is used to provide access to the `name` and
+    # enum.property is used to provide access to the `name` and
-    # `value` properties of enum members while keeping some measure of
+    # `value` attributes of enum members while keeping some measure of
     # protection from modification, while still allowing for an enumeration
     # to have members named `name` and `value`.  This works because enumeration
-    # members are not set directly on the enum class -- __getattr__ is
+    # members are not set directly on the enum class; they are kept in a
-    # used to look them up.
+    # separate structure, _member_map_, which is where enum.property looks for
+    # them
 
-    @DynamicClassAttribute
+    @property
     def name(self):
         """The name of the Enum member."""
         return self._name_
 
-    @DynamicClassAttribute
+    @property
     def value(self):
         """The value of the Enum member."""
         return self._value_

Lib/inspect.py

@@ -408,7 +408,7 @@ def classify_class_attrs(cls):
     # attribute with the same name as a DynamicClassAttribute exists.
     for base in mro:
         for k, v in base.__dict__.items():
-            if isinstance(v, types.DynamicClassAttribute):
+            if isinstance(v, types.DynamicClassAttribute) and v.fget is not None:
                 names.append(k)
     result = []
     processed = set()

Lib/test/test_enum.py

@@ -1677,6 +1677,13 @@ class TestEnum(unittest.TestCase):
         class Test(Base):
             test = 1
         self.assertEqual(Test.test.test, 'dynamic')
+        class Base2(Enum):
+            @enum.property
+            def flash(self):
+                return 'flashy dynamic'
+        class Test(Base2):
+            flash = 1
+        self.assertEqual(Test.flash.flash, 'flashy dynamic')
 
     def test_no_duplicates(self):
         class UniqueEnum(Enum):
@@ -2118,7 +2125,7 @@ class TestEnum(unittest.TestCase):
             class ThirdFailedStrEnum(StrEnum):
                 one = '1'
                 two = b'2', 'ascii', 9
-                
+
 
 
     @unittest.skipUnless(
@@ -3269,7 +3276,7 @@ class TestStdLib(unittest.TestCase):
                 ('value', Enum.__dict__['value']),
                 ))
         result = dict(inspect.getmembers(self.Color))
-        self.assertEqual(values.keys(), result.keys())
+        self.assertEqual(set(values.keys()), set(result.keys()))
         failed = False
         for k in values.keys():
             if result[k] != values[k]:
@@ -3306,6 +3313,10 @@ class TestStdLib(unittest.TestCase):
         values.sort(key=lambda item: item.name)
         result = list(inspect.classify_class_attrs(self.Color))
         result.sort(key=lambda item: item.name)
+        self.assertEqual(
+                len(values), len(result),
+                "%s != %s" % ([a.name for a in values], [a.name for a in result])
+                )
         failed = False
         for v, r in zip(values, result):
             if r != v:

Misc/NEWS.d/next/Library/2021-01-11-17-36-59.bpo-42901.gFd-ta.rst

@@ -0,0 +1,3 @@
+[Enum] move member creation from ``EnumMeta.__new__`` to
+``_proto_member.__set_name__``, allowing members to be created and visible
+in ``__init_subclass__``.