From 26d6c3831f237a511507c9cf5498e89f6f1d72d4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 15 Aug 2025 18:20:14 +0100 Subject: [PATCH] [ty] Represent `NamedTuple` as an opaque special form, not a class (#19915) --- .../resources/mdtest/named_tuple.md | 82 +++++++++++++++++++ crates/ty_python_semantic/src/types.rs | 21 ++++- crates/ty_python_semantic/src/types/class.rs | 32 ++++---- .../src/types/class_base.rs | 26 +++--- crates/ty_python_semantic/src/types/infer.rs | 34 +++++--- .../src/types/special_form.rs | 10 +++ .../ty_extensions/ty_extensions.pyi | 25 +++++- 7 files changed, 182 insertions(+), 48 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/named_tuple.md b/crates/ty_python_semantic/resources/mdtest/named_tuple.md index e642b25b2e..94070455c9 100644 --- a/crates/ty_python_semantic/resources/mdtest/named_tuple.md +++ b/crates/ty_python_semantic/resources/mdtest/named_tuple.md @@ -268,6 +268,88 @@ alice = Person(1, "Alice", 42) bob = Person(2, "Bob") ``` +## The symbol `NamedTuple` itself + +At runtime, `NamedTuple` is a function, and we understand this: + +```py +import types +import typing + +def expects_functiontype(x: types.FunctionType): ... + +expects_functiontype(typing.NamedTuple) +``` + +This means we also understand that all attributes on function objects are available on the symbol +`typing.NamedTuple`: + +```py +reveal_type(typing.NamedTuple.__name__) # revealed: str +reveal_type(typing.NamedTuple.__qualname__) # revealed: str +reveal_type(typing.NamedTuple.__kwdefaults__) # revealed: dict[str, Any] | None + +# TODO: this should cause us to emit a diagnostic and reveal `Unknown` (function objects don't have an `__mro__` attribute), +# but the fact that we don't isn't actually a `NamedTuple` bug (https://github.com/astral-sh/ty/issues/986) +reveal_type(typing.NamedTuple.__mro__) # revealed: tuple[, ] +``` + +By the normal rules, `NamedTuple` and `type[NamedTuple]` should not be valid in type expressions -- +there is no object at runtime that is an "instance of `NamedTuple`", nor is there any class at +runtime that is a "subclass of `NamedTuple`" -- these are both impossible, since `NamedTuple` is a +function and not a class. However, for compatibility with other type checkers, we allow `NamedTuple` +in type expressions and understand it as describing an interface that all `NamedTuple` classes would +satisfy: + +```py +def expects_named_tuple(x: typing.NamedTuple): + reveal_type(x) # revealed: tuple[object, ...] & NamedTupleLike + reveal_type(x._make) # revealed: bound method type[NamedTupleLike]._make(iterable: Iterable[Any]) -> Self@_make + reveal_type(x._replace) # revealed: bound method NamedTupleLike._replace(**kwargs) -> Self@_replace + # revealed: Overload[(value: tuple[object, ...], /) -> tuple[object, ...], (value: tuple[_T@__add__, ...], /) -> tuple[object, ...]] + reveal_type(x.__add__) + reveal_type(x.__iter__) # revealed: bound method tuple[object, ...].__iter__() -> Iterator[object] + +def _(y: type[typing.NamedTuple]): + reveal_type(y) # revealed: @Todo(unsupported type[X] special form) +``` + +Any instance of a `NamedTuple` class can therefore be passed for a function parameter that is +annotated with `NamedTuple`: + +```py +from typing import NamedTuple, Protocol, Iterable, Any +from ty_extensions import static_assert, is_assignable_to + +class Point(NamedTuple): + x: int + y: int + +reveal_type(Point._make) # revealed: bound method ._make(iterable: Iterable[Any]) -> Self@_make +reveal_type(Point._asdict) # revealed: def _asdict(self) -> dict[str, Any] +reveal_type(Point._replace) # revealed: def _replace(self, **kwargs: Any) -> Self@_replace + +static_assert(is_assignable_to(Point, NamedTuple)) + +expects_named_tuple(Point(x=42, y=56)) # fine + +# error: [invalid-argument-type] "Argument to function `expects_named_tuple` is incorrect: Expected `tuple[object, ...] & NamedTupleLike`, found `tuple[Literal[1], Literal[2]]`" +expects_named_tuple((1, 2)) +``` + +The type described by `NamedTuple` in type expressions is understood as being assignable to +`tuple[object, ...]` and `tuple[Any, ...]`: + +```py +static_assert(is_assignable_to(NamedTuple, tuple)) +static_assert(is_assignable_to(NamedTuple, tuple[object, ...])) +static_assert(is_assignable_to(NamedTuple, tuple[Any, ...])) + +def expects_tuple(x: tuple[object, ...]): ... +def _(x: NamedTuple): + expects_tuple(x) # fine +``` + ## NamedTuple with custom `__getattr__` This is a regression test for . Make sure that the diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 89b66a6b34..ad7a4c0ade 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -4264,10 +4264,6 @@ impl<'db> Type<'db> { .into() } - Some(KnownClass::NamedTuple) => { - Binding::single(self, Signature::todo("functional `NamedTuple` syntax")).into() - } - Some(KnownClass::Object) => { // ```py // class object: @@ -4583,6 +4579,10 @@ impl<'db> Type<'db> { .into() } + Type::SpecialForm(SpecialFormType::NamedTuple) => { + Binding::single(self, Signature::todo("functional `NamedTuple` syntax")).into() + } + Type::GenericAlias(_) => { // TODO annotated return type on `__new__` or metaclass `__call__` // TODO check call vs signatures of `__new__` and/or `__init__` @@ -5471,6 +5471,19 @@ impl<'db> Type<'db> { // TODO: Use an opt-in rule for a bare `Callable` SpecialFormType::Callable => Ok(CallableType::unknown(db)), + // Special case: `NamedTuple` in a type expression is understood to describe the type + // `tuple[object, ...] & `. + // This isn't very principled (since at runtime, `NamedTuple` is just a function), + // but it appears to be what users often expect, and it improves compatibility with + // other type checkers such as mypy. + // See conversation in https://github.com/astral-sh/ruff/pull/19915. + SpecialFormType::NamedTuple => Ok(IntersectionBuilder::new(db) + .positive_elements([ + Type::homogeneous_tuple(db, Type::object(db)), + KnownClass::NamedTupleLike.to_instance(db), + ]) + .build()), + SpecialFormType::TypingSelf => { let module = parsed_module(db, scope_id.file(db)).load(db); let index = semantic_index(db, scope_id.file(db)); diff --git a/crates/ty_python_semantic/src/types/class.rs b/crates/ty_python_semantic/src/types/class.rs index 60cf7c491e..cd749f8444 100644 --- a/crates/ty_python_semantic/src/types/class.rs +++ b/crates/ty_python_semantic/src/types/class.rs @@ -197,10 +197,7 @@ impl CodeGeneratorKind { Some(CodeGeneratorKind::DataclassLike) } else if class .explicit_bases(db) - .iter() - .copied() - .filter_map(Type::into_class_literal) - .any(|class| class.is_known(db, KnownClass::NamedTuple)) + .contains(&Type::SpecialForm(SpecialFormType::NamedTuple)) { Some(CodeGeneratorKind::NamedTuple) } else if class.is_typed_dict(db) { @@ -3137,7 +3134,6 @@ pub enum KnownClass { TypeVarTuple, TypeAliasType, NoDefaultType, - NamedTuple, NewType, SupportsIndex, Iterable, @@ -3160,6 +3156,7 @@ pub enum KnownClass { InitVar, // _typeshed._type_checker_internals NamedTupleFallback, + NamedTupleLike, TypedDictFallback, } @@ -3245,8 +3242,6 @@ impl KnownClass { | Self::ABCMeta | Self::Iterable | Self::Iterator - // Empty tuples are AlwaysFalse; non-empty tuples are AlwaysTrue - | Self::NamedTuple // Evaluating `NotImplementedType` in a boolean context was deprecated in Python 3.9 // and raises a `TypeError` in Python >=3.14 // (see https://docs.python.org/3/library/constants.html#NotImplemented) @@ -3260,6 +3255,7 @@ impl KnownClass { | Self::KwOnly | Self::InitVar | Self::NamedTupleFallback + | Self::NamedTupleLike | Self::TypedDictFallback => Some(Truthiness::Ambiguous), Self::Tuple => None, @@ -3342,8 +3338,8 @@ impl KnownClass { | Self::ExceptionGroup | Self::Field | Self::SupportsIndex - | Self::NamedTuple | Self::NamedTupleFallback + | Self::NamedTupleLike | Self::TypedDictFallback | Self::Counter | Self::DefaultDict @@ -3412,7 +3408,6 @@ impl KnownClass { | KnownClass::TypeVarTuple | KnownClass::TypeAliasType | KnownClass::NoDefaultType - | KnownClass::NamedTuple | KnownClass::NewType | KnownClass::SupportsIndex | KnownClass::Iterable @@ -3429,6 +3424,7 @@ impl KnownClass { | KnownClass::KwOnly | KnownClass::InitVar | KnownClass::NamedTupleFallback + | KnownClass::NamedTupleLike | KnownClass::TypedDictFallback => false, } } @@ -3489,7 +3485,6 @@ impl KnownClass { | KnownClass::TypeVarTuple | KnownClass::TypeAliasType | KnownClass::NoDefaultType - | KnownClass::NamedTuple | KnownClass::NewType | KnownClass::SupportsIndex | KnownClass::Iterable @@ -3506,6 +3501,7 @@ impl KnownClass { | KnownClass::KwOnly | KnownClass::InitVar | KnownClass::NamedTupleFallback + | KnownClass::NamedTupleLike | KnownClass::TypedDictFallback => false, } } @@ -3566,7 +3562,6 @@ impl KnownClass { | KnownClass::TypeVarTuple | KnownClass::TypeAliasType | KnownClass::NoDefaultType - | KnownClass::NamedTuple | KnownClass::NewType | KnownClass::SupportsIndex | KnownClass::Iterable @@ -3582,6 +3577,7 @@ impl KnownClass { | KnownClass::KwOnly | KnownClass::InitVar | KnownClass::TypedDictFallback + | KnownClass::NamedTupleLike | KnownClass::NamedTupleFallback => false, } } @@ -3604,6 +3600,7 @@ impl KnownClass { | Self::Iterable | Self::Iterator | Self::Awaitable + | Self::NamedTupleLike | Self::Generator => true, Self::Any @@ -3648,7 +3645,6 @@ impl KnownClass { | Self::TypeVarTuple | Self::TypeAliasType | Self::NoDefaultType - | Self::NamedTuple | Self::NewType | Self::ChainMap | Self::Counter @@ -3713,7 +3709,6 @@ impl KnownClass { Self::GeneratorType => "GeneratorType", Self::AsyncGeneratorType => "AsyncGeneratorType", Self::CoroutineType => "CoroutineType", - Self::NamedTuple => "NamedTuple", Self::NoneType => "NoneType", Self::SpecialForm => "_SpecialForm", Self::TypeVar => "TypeVar", @@ -3767,6 +3762,7 @@ impl KnownClass { Self::KwOnly => "KW_ONLY", Self::InitVar => "InitVar", Self::NamedTupleFallback => "NamedTupleFallback", + Self::NamedTupleLike => "NamedTupleLike", Self::TypedDictFallback => "TypedDictFallback", } } @@ -3984,7 +3980,6 @@ impl KnownClass { | Self::Generator | Self::SpecialForm | Self::TypeVar - | Self::NamedTuple | Self::StdlibAlias | Self::Iterable | Self::Iterator @@ -4025,6 +4020,7 @@ impl KnownClass { | Self::OrderedDict => KnownModule::Collections, Self::Field | Self::KwOnly | Self::InitVar => KnownModule::Dataclasses, Self::NamedTupleFallback | Self::TypedDictFallback => KnownModule::TypeCheckerInternals, + Self::NamedTupleLike => KnownModule::TyExtensions, } } @@ -4095,7 +4091,6 @@ impl KnownClass { | Self::Nonmember | Self::ABCMeta | Self::Super - | Self::NamedTuple | Self::NewType | Self::Field | Self::KwOnly @@ -4103,6 +4098,7 @@ impl KnownClass { | Self::Iterable | Self::Iterator | Self::NamedTupleFallback + | Self::NamedTupleLike | Self::TypedDictFallback => Some(false), Self::Tuple => None, @@ -4177,7 +4173,6 @@ impl KnownClass { | Self::ABCMeta | Self::Super | Self::UnionType - | Self::NamedTuple | Self::NewType | Self::Field | Self::KwOnly @@ -4185,6 +4180,7 @@ impl KnownClass { | Self::Iterable | Self::Iterator | Self::NamedTupleFallback + | Self::NamedTupleLike | Self::TypedDictFallback => false, } } @@ -4234,7 +4230,6 @@ impl KnownClass { "UnionType" => Self::UnionType, "MethodWrapperType" => Self::MethodWrapperType, "WrapperDescriptorType" => Self::WrapperDescriptorType, - "NamedTuple" => Self::NamedTuple, "NewType" => Self::NewType, "TypeAliasType" => Self::TypeAliasType, "TypeVar" => Self::TypeVar, @@ -4275,6 +4270,7 @@ impl KnownClass { "KW_ONLY" => Self::KwOnly, "InitVar" => Self::InitVar, "NamedTupleFallback" => Self::NamedTupleFallback, + "NamedTupleLike" => Self::NamedTupleLike, "TypedDictFallback" => Self::TypedDictFallback, _ => return None, }; @@ -4341,6 +4337,7 @@ impl KnownClass { | Self::InitVar | Self::NamedTupleFallback | Self::TypedDictFallback + | Self::NamedTupleLike | Self::Awaitable | Self::Generator => module == self.canonical_module(db), Self::NoneType => matches!(module, KnownModule::Typeshed | KnownModule::Types), @@ -4353,7 +4350,6 @@ impl KnownClass { | Self::ParamSpecArgs | Self::ParamSpecKwargs | Self::TypeVarTuple - | Self::NamedTuple | Self::Iterable | Self::Iterator | Self::NewType => matches!(module, KnownModule::Typing | KnownModule::TypingExtensions), diff --git a/crates/ty_python_semantic/src/types/class_base.rs b/crates/ty_python_semantic/src/types/class_base.rs index c74dc815f3..aff7e833ce 100644 --- a/crates/ty_python_semantic/src/types/class_base.rs +++ b/crates/ty_python_semantic/src/types/class_base.rs @@ -80,18 +80,6 @@ impl<'db> ClassBase<'db> { Type::ClassLiteral(literal) => { if literal.is_known(db, KnownClass::Any) { Some(Self::Dynamic(DynamicType::Any)) - } else if literal.is_known(db, KnownClass::NamedTuple) { - let fields = subclass.own_fields(db, None); - Self::try_from_type( - db, - TupleType::heterogeneous( - db, - fields.values().map(|field| field.declared_ty), - )? - .to_class_type(db) - .into(), - subclass, - ) } else { Some(Self::Class(literal.default_specialization(db))) } @@ -215,6 +203,20 @@ impl<'db> ClassBase<'db> { SpecialFormType::Protocol => Some(Self::Protocol), SpecialFormType::Generic => Some(Self::Generic), + SpecialFormType::NamedTuple => { + let fields = subclass.own_fields(db, None); + Self::try_from_type( + db, + TupleType::heterogeneous( + db, + fields.values().map(|field| field.declared_ty), + )? + .to_class_type(db) + .into(), + subclass, + ) + } + // TODO: Classes inheriting from `typing.Type` et al. also have `Generic` in their MRO SpecialFormType::Dict => { Self::try_from_type(db, KnownClass::Dict.to_class_literal(db), subclass) diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index 3f8ade1195..acd7fbb0ba 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -64,7 +64,7 @@ use super::string_annotation::{ use super::subclass_of::SubclassOfInner; use super::{ClassBase, add_inferred_python_version_hint_to_diagnostic}; use crate::module_name::{ModuleName, ModuleNameResolutionError}; -use crate::module_resolver::resolve_module; +use crate::module_resolver::{KnownModule, file_to_module, resolve_module}; use crate::node_key::NodeKey; use crate::place::{ Boundness, ConsideredDefinitions, LookupError, Place, PlaceAndQualifiers, @@ -3034,20 +3034,29 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let maybe_known_class = KnownClass::try_from_file_and_name(self.db(), self.file(), name); - let class_ty = Type::from(ClassLiteral::new( - self.db(), - name.id.clone(), - body_scope, - maybe_known_class, - deprecated, - dataclass_params, - dataclass_transformer_params, - )); + let ty = if maybe_known_class.is_none() + && &name.id == "NamedTuple" + && matches!( + file_to_module(self.db(), self.file()).and_then(|module| module.known(self.db())), + Some(KnownModule::Typing | KnownModule::TypingExtensions) + ) { + Type::SpecialForm(SpecialFormType::NamedTuple) + } else { + Type::from(ClassLiteral::new( + self.db(), + name.id.clone(), + body_scope, + maybe_known_class, + deprecated, + dataclass_params, + dataclass_transformer_params, + )) + }; self.add_declaration_with_binding( class_node.into(), definition, - &DeclaredAndInferredType::are_the_same_type(class_ty), + &DeclaredAndInferredType::are_the_same_type(ty), ); // if there are type parameters, then the keywords and bases are within that scope @@ -6206,7 +6215,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { | KnownClass::Property | KnownClass::Super | KnownClass::TypeVar - | KnownClass::NamedTuple | KnownClass::TypeAliasType | KnownClass::Deprecated ) @@ -10800,7 +10808,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { SpecialFormType::Tuple => { Type::tuple(self.infer_tuple_type_expression(arguments_slice)) } - SpecialFormType::Generic | SpecialFormType::Protocol => { + SpecialFormType::Generic | SpecialFormType::Protocol | SpecialFormType::NamedTuple => { self.infer_expression(arguments_slice); if let Some(builder) = self.context.report_lint(&INVALID_TYPE_FORM, subscript) { builder.into_diagnostic(format_args!( diff --git a/crates/ty_python_semantic/src/types/special_form.rs b/crates/ty_python_semantic/src/types/special_form.rs index a502a6864d..f0dd81b150 100644 --- a/crates/ty_python_semantic/src/types/special_form.rs +++ b/crates/ty_python_semantic/src/types/special_form.rs @@ -117,6 +117,11 @@ pub enum SpecialFormType { /// Note that instances of subscripted `typing.Generic` are not represented by this type; /// see also [`super::KnownInstanceType::SubscriptedGeneric`]. Generic, + + /// The symbol `typing.NamedTuple` (which can also be found as `typing_extensions.NamedTuple`). + /// Typeshed defines this symbol as a class, but this isn't accurate: it's actually a factory function + /// at runtime. We therefore represent it as a special form internally. + NamedTuple, } impl SpecialFormType { @@ -163,6 +168,8 @@ impl SpecialFormType { | Self::OrderedDict => KnownClass::StdlibAlias, Self::Unknown | Self::AlwaysTruthy | Self::AlwaysFalsy => KnownClass::Object, + + Self::NamedTuple => KnownClass::FunctionType, } } @@ -230,6 +237,7 @@ impl SpecialFormType { | Self::TypeIs | Self::TypingSelf | Self::Protocol + | Self::NamedTuple | Self::ReadOnly => { matches!(module, KnownModule::Typing | KnownModule::TypingExtensions) } @@ -261,6 +269,7 @@ impl SpecialFormType { | Self::Counter | Self::DefaultDict | Self::Deque + | Self::NamedTuple | Self::OrderedDict => true, // All other special forms are not callable @@ -344,6 +353,7 @@ impl SpecialFormType { SpecialFormType::CallableTypeOf => "ty_extensions.CallableTypeOf", SpecialFormType::Protocol => "typing.Protocol", SpecialFormType::Generic => "typing.Generic", + SpecialFormType::NamedTuple => "typing.NamedTuple", } } } diff --git a/crates/ty_vendored/ty_extensions/ty_extensions.pyi b/crates/ty_vendored/ty_extensions/ty_extensions.pyi index 4dd041762f..6968bcb75a 100644 --- a/crates/ty_vendored/ty_extensions/ty_extensions.pyi +++ b/crates/ty_vendored/ty_extensions/ty_extensions.pyi @@ -1,5 +1,15 @@ +import sys +from collections.abc import Iterable from enum import Enum -from typing import Any, LiteralString, _SpecialForm +from typing import ( + Any, + ClassVar, + LiteralString, + Protocol, + _SpecialForm, +) + +from typing_extensions import Self # noqa: UP035 # Special operations def static_assert(condition: object, msg: LiteralString | None = None) -> None: ... @@ -69,3 +79,16 @@ def has_member(obj: Any, name: str) -> bool: ... # diagnostic describing the protocol's interface. Passing a non-protocol type # will cause ty to emit an error diagnostic. def reveal_protocol_interface(protocol: type) -> None: ... + +# A protocol describing an interface that should be satisfied by all named tuples +# created using `typing.NamedTuple` or `collections.namedtuple`. +class NamedTupleLike(Protocol): + # from typing.NamedTuple stub + _field_defaults: ClassVar[dict[str, Any]] + _fields: ClassVar[tuple[str, ...]] + @classmethod + def _make(self: Self, iterable: Iterable[Any]) -> Self: ... + def _asdict(self, /) -> dict[str, Any]: ... + def _replace(self: Self, /, **kwargs) -> Self: ... + if sys.version_info >= (3, 13): + def __replace__(self: Self, **kwargs) -> Self: ...