[ty] implement typing.NewType by adding Type::NewTypeInstance

This commit is contained in:
Jack O'Connor 2025-10-23 10:10:10 -07:00
parent 039a69fa8c
commit 5f3e086ee4
25 changed files with 1343 additions and 191 deletions

View file

@ -1,7 +1,5 @@
# NewType
Currently, ty doesn't support `typing.NewType` in type annotations.
## Valid forms
```py
@ -12,13 +10,389 @@ X = GenericAlias(type, ())
A = NewType("A", int)
# TODO: typeshed for `typing.GenericAlias` uses `type` for the first argument. `NewType` should be special-cased
# to be compatible with `type`
# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found `NewType`"
# error: [invalid-argument-type] "Argument to function `__new__` is incorrect: Expected `type`, found `<NewType pseudo-class 'A'>`"
B = GenericAlias(A, ())
def _(
a: A,
b: B,
):
reveal_type(a) # revealed: @Todo(Support for `typing.NewType` instances in type expressions)
reveal_type(a) # revealed: A
reveal_type(b) # revealed: @Todo(Support for `typing.GenericAlias` instances in type expressions)
```
## Subtyping
The basic purpose of `NewType` is that it acts like a subtype of its base, but not the exact same
type (i.e. not an alias).
```py
from typing_extensions import NewType
from ty_extensions import static_assert, is_subtype_of, is_equivalent_to
Foo = NewType("Foo", int)
Bar = NewType("Bar", Foo)
static_assert(is_subtype_of(Foo, int))
static_assert(not is_equivalent_to(Foo, int))
static_assert(is_subtype_of(Bar, Foo))
static_assert(is_subtype_of(Bar, int))
static_assert(not is_equivalent_to(Bar, Foo))
Foo(42)
Foo(Foo(42)) # allowed: `Foo` is a subtype of `int`.
Foo(Bar(Foo(42))) # allowed: `Bar` is a subtype of `int`.
Foo(True) # allowed: `bool` is a subtype of `int`.
Foo("forty-two") # error: [invalid-argument-type] "Argument is incorrect: Expected `int`, found `Literal["forty-two"]`"
def f(_: int): ...
def g(_: Foo): ...
def h(_: Bar): ...
f(42)
f(Foo(42))
f(Bar(Foo(42)))
g(42) # error: [invalid-argument-type] "Argument to function `g` is incorrect: Expected `Foo`, found `Literal[42]`"
g(Foo(42))
g(Bar(Foo(42)))
h(42) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Literal[42]`"
h(Foo(42)) # error: [invalid-argument-type] "Argument to function `h` is incorrect: Expected `Bar`, found `Foo`"
h(Bar(Foo(42)))
```
## Member and method lookup work
```py
from typing_extensions import NewType
class Foo:
foo_member: str = "hello"
def foo_method(self) -> int:
return 42
Bar = NewType("Bar", Foo)
Baz = NewType("Baz", Bar)
baz = Baz(Bar(Foo()))
reveal_type(baz.foo_member) # revealed: str
reveal_type(baz.foo_method()) # revealed: int
```
We also infer member access on the `NewType` pseudo-type itself correctly:
```py
reveal_type(Bar.__supertype__) # revealed: type | NewType
reveal_type(Baz.__supertype__) # revealed: type | NewType
```
## `NewType` wrapper functions are `Callable`
```py
from collections.abc import Callable
from typing_extensions import NewType
from ty_extensions import CallableTypeOf
Foo = NewType("Foo", int)
def _(obj: CallableTypeOf[Foo]):
reveal_type(obj) # revealed: (int, /) -> Foo
def f(_: Callable[[int], Foo]): ...
f(Foo)
map(Foo, [1, 2, 3])
def g(_: Callable[[str], Foo]): ...
g(Foo) # error: [invalid-argument-type]
```
## `NewType` instances are `Callable` if the base type is
```py
from typing import NewType, Callable, Any
from ty_extensions import CallableTypeOf
N = NewType("N", int)
i = N(42)
y: Callable[..., Any] = i # error: [invalid-assignment] "Object of type `N` is not assignable to `(...) -> Any`"
# error: [invalid-type-form] "Expected the first argument to `ty_extensions.CallableTypeOf` to be a callable object, but got an object of type `N`"
def f(x: CallableTypeOf[i]):
reveal_type(x) # revealed: Unknown
class SomethingCallable:
def __call__(self, a: str) -> bytes:
raise NotImplementedError
N2 = NewType("N2", SomethingCallable)
j = N2(SomethingCallable())
z: Callable[[str], bytes] = j # fine
def g(x: CallableTypeOf[j]):
reveal_type(x) # revealed: (a: str) -> bytes
```
## The name must be a string literal
```py
from typing_extensions import NewType
def _(name: str) -> None:
_ = NewType(name, int) # error: [invalid-newtype] "The first argument to `NewType` must be a string literal"
```
However, the literal doesn't necessarily need to be inline, as long as we infer it:
```py
name = "Foo"
Foo = NewType(name, int)
reveal_type(Foo) # revealed: <NewType pseudo-class 'Foo'>
```
## The second argument must be a class type or another newtype
Other typing constructs like `Union` are not allowed.
```py
from typing_extensions import NewType
# error: [invalid-newtype] "invalid base for `typing.NewType`"
Foo = NewType("Foo", int | str)
```
We don't emit the "invalid base" diagnostic for `Unknown`, because that typically results from other
errors that already have a diagnostic, and there's no need to pile on. For example, this mistake
gives you an "Int literals are not allowed" error, and we'd rather not see an "invalid base" error
on top of that:
```py
# error: [invalid-type-form] "Int literals are not allowed in this context in a type expression"
Foo = NewType("Foo", 42)
```
## A `NewType` definition must be a simple variable assignment
```py
from typing import NewType
N: NewType = NewType("N", int) # error: [invalid-newtype] "A `NewType` definition must be a simple variable assignment"
```
## Newtypes can be cyclic in various ways
Cyclic newtypes are kind of silly, but it's possible for the user to express them, and it's
important that we don't go into infinite recursive loops and crash with a stack overflow. In fact,
this is *why* base type evaluation is deferred; otherwise Salsa itself would crash.
```py
from typing_extensions import NewType, reveal_type, cast
# Define a directly cyclic newtype.
A = NewType("A", "A")
reveal_type(A) # revealed: <NewType pseudo-class 'A'>
# Typechecking still works. We can't construct an `A` "honestly", but we can `cast` into one.
a: A
a = 42 # error: [invalid-assignment] "Object of type `Literal[42]` is not assignable to `A`"
a = A(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `A`, found `Literal[42]`"
a = cast(A, 42)
reveal_type(a) # revealed: A
# A newtype cycle might involve more than one step.
B = NewType("B", "C")
C = NewType("C", "B")
reveal_type(B) # revealed: <NewType pseudo-class 'B'>
reveal_type(C) # revealed: <NewType pseudo-class 'C'>
b: B = cast(B, 42)
c: C = C(b)
reveal_type(b) # revealed: B
reveal_type(c) # revealed: C
# Cyclic types behave in surprising ways. These assignments are legal, even though B and C aren't
# the same type, because each of them is a subtype of the other.
b = c
c = b
# Another newtype could inherit from a cyclic one.
D = NewType("D", C)
reveal_type(D) # revealed: <NewType pseudo-class 'D'>
d: D
d = D(42) # error: [invalid-argument-type] "Argument is incorrect: Expected `C`, found `Literal[42]`"
d = D(c)
d = D(b) # Allowed, the same surprise as above. B and C are subtypes of each other.
reveal_type(d) # revealed: D
```
Normal classes can't inherit from newtypes, but generic classes can be parametrized with them, so we
also need to detect "ordinary" type cycles that happen to involve a newtype.
```py
E = NewType("E", list["E"])
reveal_type(E) # revealed: <NewType pseudo-class 'E'>
e: E = E([])
reveal_type(e) # revealed: E
reveal_type(E(E(E(E(E([])))))) # revealed: E
reveal_type(E([E([E([]), E([E([])])]), E([])])) # revealed: E
E(["foo"]) # error: [invalid-argument-type]
E(E(E(["foo"]))) # error: [invalid-argument-type]
```
## `NewType` wrapping preserves singleton-ness and single-valued-ness
```py
from typing_extensions import NewType
from ty_extensions import is_singleton, is_single_valued, static_assert
from types import EllipsisType
A = NewType("A", EllipsisType)
static_assert(is_singleton(A))
static_assert(is_single_valued(A))
reveal_type(type(A(...)) is EllipsisType) # revealed: Literal[True]
# TODO: This should be `Literal[True]` also.
reveal_type(A(...) is ...) # revealed: bool
B = NewType("B", int)
static_assert(not is_singleton(B))
static_assert(not is_single_valued(B))
```
## `NewType`s of tuples can be iterated/unpacked
```py
from typing import NewType
N = NewType("N", tuple[int, str])
a, b = N((1, "foo"))
reveal_type(a) # revealed: int
reveal_type(b) # revealed: str
```
## `isinstance` of a `NewType` instance and its base class is inferred as `Literal[True]`
```py
from typing import NewType
N = NewType("N", int)
def f(x: N):
reveal_type(isinstance(x, int)) # revealed: Literal[True]
```
However, a `NewType` isn't a real class, so it isn't a valid second argument to `isinstance`:
```py
def f(x: N):
# error: [invalid-argument-type] "Argument to function `isinstance` is incorrect"
reveal_type(isinstance(x, N)) # revealed: bool
```
Because of that, we don't generate any narrowing constraints for it:
```py
def f(x: N | str):
if isinstance(x, N): # error: [invalid-argument-type]
reveal_type(x) # revealed: N | str
else:
reveal_type(x) # revealed: N | str
```
## Trying to subclass a `NewType` produces an error matching CPython
<!-- snapshot-diagnostics -->
```py
from typing import NewType
X = NewType("X", int)
class Foo(X): ... # error: [invalid-base]
```
## Don't narrow `NewType`-wrapped `Enum`s inside of match arms
`Literal[Foo.X]` is actually disjoint from `N` here:
```py
from enum import Enum
from typing import NewType
class Foo(Enum):
X = 0
Y = 1
N = NewType("N", Foo)
def f(x: N):
match x:
case Foo.X:
reveal_type(x) # revealed: N
case Foo.Y:
reveal_type(x) # revealed: N
case _:
reveal_type(x) # revealed: N
```
## We don't support `NewType` on Python 3.9
We implement `typing.NewType` as a `KnownClass`, but in Python 3.9 it's actually a function, so all
we get is the `Any` annotations from typeshed. However, `typing_extensions.NewType` is always a
class. This could be improved in the future, but Python 3.9 is now end-of-life, so it's not
high-priority.
```toml
[environment]
python-version = "3.9"
```
```py
from typing import NewType
Foo = NewType("Foo", int)
reveal_type(Foo) # revealed: Any
reveal_type(Foo(42)) # revealed: Any
from typing_extensions import NewType
Bar = NewType("Bar", int)
reveal_type(Bar) # revealed: <NewType pseudo-class 'Bar'>
reveal_type(Bar(42)) # revealed: Bar
```
## The base of a `NewType` can't be a protocol class or a `TypedDict`
<!-- snapshot-diagnostics -->
```py
from typing import NewType, Protocol, TypedDict
class Id(Protocol):
code: int
UserId = NewType("UserId", Id) # error: [invalid-newtype]
class Foo(TypedDict):
a: int
Bar = NewType("Bar", Foo) # error: [invalid-newtype]
```
## TODO: A `NewType` cannot be generic
```py
from typing import Any, NewType, TypeVar
# All of these are allowed.
A = NewType("A", list)
B = NewType("B", list[int])
B = NewType("B", list[Any])
# But a free typevar is not allowed.
T = TypeVar("T")
C = NewType("C", list[T]) # TODO: should be "error: [invalid-newtype]"
```

View file

@ -66,7 +66,7 @@ synthesized `Protocol`s that cannot be upcast to, or interpreted as, a non-`obje
```py
import types
from typing_extensions import Callable, TypeIs, Literal, TypedDict
from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict
def f(): ...
@ -81,6 +81,8 @@ class SomeTypedDict(TypedDict):
x: int
y: bytes
N = NewType("N", int)
# revealed: <super: <class 'object'>, FunctionType>
reveal_type(super(object, f))
# revealed: <super: <class 'object'>, WrapperDescriptorType>
@ -95,6 +97,8 @@ reveal_type(super(object, Alias))
reveal_type(super(object, Foo().method))
# revealed: <super: <class 'object'>, property>
reveal_type(super(object, Foo.some_property))
# revealed: <super: <class 'object'>, int>
reveal_type(super(object, N(42)))
def g(x: object) -> TypeIs[list[object]]:
return isinstance(x, list)

View file

@ -0,0 +1,58 @@
---
source: crates/ty_test/src/lib.rs
expression: snapshot
---
---
mdtest name: new_types.md - NewType - The base of a `NewType` can't be a protocol class or a `TypedDict`
mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md
---
# Python source files
## mdtest_snippet.py
```
1 | from typing import NewType, Protocol, TypedDict
2 |
3 | class Id(Protocol):
4 | code: int
5 |
6 | UserId = NewType("UserId", Id) # error: [invalid-newtype]
7 |
8 | class Foo(TypedDict):
9 | a: int
10 |
11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype]
```
# Diagnostics
```
error[invalid-newtype]: invalid base for `typing.NewType`
--> src/mdtest_snippet.py:6:28
|
4 | code: int
5 |
6 | UserId = NewType("UserId", Id) # error: [invalid-newtype]
| ^^ type `Id`
7 |
8 | class Foo(TypedDict):
|
info: The base of a `NewType` is not allowed to be a protocol class.
info: rule `invalid-newtype` is enabled by default
```
```
error[invalid-newtype]: invalid base for `typing.NewType`
--> src/mdtest_snippet.py:11:22
|
9 | a: int
10 |
11 | Bar = NewType("Bar", Foo) # error: [invalid-newtype]
| ^^^ type `Foo`
|
info: The base of a `NewType` is not allowed to be a `TypedDict`.
info: rule `invalid-newtype` is enabled by default
```

View file

@ -0,0 +1,37 @@
---
source: crates/ty_test/src/lib.rs
expression: snapshot
---
---
mdtest name: new_types.md - NewType - Trying to subclass a `NewType` produces an error matching CPython
mdtest path: crates/ty_python_semantic/resources/mdtest/annotations/new_types.md
---
# Python source files
## mdtest_snippet.py
```
1 | from typing import NewType
2 |
3 | X = NewType("X", int)
4 |
5 | class Foo(X): ... # error: [invalid-base]
```
# Diagnostics
```
error[invalid-base]: Cannot subclass an instance of NewType
--> src/mdtest_snippet.py:5:11
|
3 | X = NewType("X", int)
4 |
5 | class Foo(X): ... # error: [invalid-base]
| ^
|
info: Perhaps you were looking for: `Foo = NewType('Foo', X)`
info: Definition of class `Foo` will raise `TypeError` at runtime
info: rule `invalid-base` is enabled by default
```

View file

@ -46,7 +46,7 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md
32 | reveal_type(super(C, C()).aa) # revealed: int
33 | reveal_type(super(C, C()).bb) # revealed: int
34 | import types
35 | from typing_extensions import Callable, TypeIs, Literal, TypedDict
35 | from typing_extensions import Callable, TypeIs, Literal, NewType, TypedDict
36 |
37 | def f(): ...
38 |
@ -61,59 +61,63 @@ mdtest path: crates/ty_python_semantic/resources/mdtest/class/super.md
47 | x: int
48 | y: bytes
49 |
50 | # revealed: <super: <class 'object'>, FunctionType>
51 | reveal_type(super(object, f))
52 | # revealed: <super: <class 'object'>, WrapperDescriptorType>
53 | reveal_type(super(object, types.FunctionType.__get__))
54 | # revealed: <super: <class 'object'>, GenericAlias>
55 | reveal_type(super(object, Foo[int]))
56 | # revealed: <super: <class 'object'>, _SpecialForm>
57 | reveal_type(super(object, Literal))
58 | # revealed: <super: <class 'object'>, TypeAliasType>
59 | reveal_type(super(object, Alias))
60 | # revealed: <super: <class 'object'>, MethodType>
61 | reveal_type(super(object, Foo().method))
62 | # revealed: <super: <class 'object'>, property>
63 | reveal_type(super(object, Foo.some_property))
64 |
65 | def g(x: object) -> TypeIs[list[object]]:
66 | return isinstance(x, list)
67 |
68 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]):
69 | if hasattr(x, "bar"):
70 | # revealed: <Protocol with members 'bar'>
71 | reveal_type(x)
72 | # error: [invalid-super-argument]
73 | # revealed: Unknown
74 | reveal_type(super(object, x))
75 |
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, z))
50 | N = NewType("N", int)
51 |
52 | # revealed: <super: <class 'object'>, FunctionType>
53 | reveal_type(super(object, f))
54 | # revealed: <super: <class 'object'>, WrapperDescriptorType>
55 | reveal_type(super(object, types.FunctionType.__get__))
56 | # revealed: <super: <class 'object'>, GenericAlias>
57 | reveal_type(super(object, Foo[int]))
58 | # revealed: <super: <class 'object'>, _SpecialForm>
59 | reveal_type(super(object, Literal))
60 | # revealed: <super: <class 'object'>, TypeAliasType>
61 | reveal_type(super(object, Alias))
62 | # revealed: <super: <class 'object'>, MethodType>
63 | reveal_type(super(object, Foo().method))
64 | # revealed: <super: <class 'object'>, property>
65 | reveal_type(super(object, Foo.some_property))
66 | # revealed: <super: <class 'object'>, int>
67 | reveal_type(super(object, N(42)))
68 |
69 | def g(x: object) -> TypeIs[list[object]]:
70 | return isinstance(x, list)
71 |
72 | def _(x: object, y: SomeTypedDict, z: Callable[[int, str], bool]):
73 | if hasattr(x, "bar"):
74 | # revealed: <Protocol with members 'bar'>
75 | reveal_type(x)
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, x))
79 |
80 | is_list = g(x)
81 | # revealed: TypeIs[list[object] @ x]
82 | reveal_type(is_list)
83 | # revealed: <super: <class 'object'>, bool>
84 | reveal_type(super(object, is_list))
85 |
86 | # revealed: <super: <class 'object'>, dict[Literal["x", "y"], int | bytes]>
87 | reveal_type(super(object, y))
88 |
89 | # The first argument to `super()` must be an actual class object;
90 | # instances of `GenericAlias` are not accepted at runtime:
91 | #
92 | # error: [invalid-super-argument]
93 | # revealed: Unknown
94 | reveal_type(super(list[int], []))
95 | class Super:
96 | def method(self) -> int:
97 | return 42
98 |
99 | class Sub(Super):
100 | def method(self: Sub) -> int:
101 | # revealed: <super: <class 'Sub'>, Sub>
102 | return reveal_type(super(self.__class__, self)).method()
80 | # error: [invalid-super-argument]
81 | # revealed: Unknown
82 | reveal_type(super(object, z))
83 |
84 | is_list = g(x)
85 | # revealed: TypeIs[list[object] @ x]
86 | reveal_type(is_list)
87 | # revealed: <super: <class 'object'>, bool>
88 | reveal_type(super(object, is_list))
89 |
90 | # revealed: <super: <class 'object'>, dict[Literal["x", "y"], int | bytes]>
91 | reveal_type(super(object, y))
92 |
93 | # The first argument to `super()` must be an actual class object;
94 | # instances of `GenericAlias` are not accepted at runtime:
95 | #
96 | # error: [invalid-super-argument]
97 | # revealed: Unknown
98 | reveal_type(super(list[int], []))
99 | class Super:
100 | def method(self) -> int:
101 | return 42
102 |
103 | class Sub(Super):
104 | def method(self: Sub) -> int:
105 | # revealed: <super: <class 'Sub'>, Sub>
106 | return reveal_type(super(self.__class__, self)).method()
```
# Diagnostics
@ -206,14 +210,14 @@ info: rule `unresolved-attribute` is enabled by default
```
error[invalid-super-argument]: `<Protocol with members 'bar'>` is an abstract/structural type in `super(<class 'object'>, <Protocol with members 'bar'>)` call
--> src/mdtest_snippet.py:74:21
--> src/mdtest_snippet.py:78:21
|
72 | # error: [invalid-super-argument]
73 | # revealed: Unknown
74 | reveal_type(super(object, x))
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, x))
| ^^^^^^^^^^^^^^^^
75 |
76 | # error: [invalid-super-argument]
79 |
80 | # error: [invalid-super-argument]
|
info: rule `invalid-super-argument` is enabled by default
@ -221,14 +225,14 @@ info: rule `invalid-super-argument` is enabled by default
```
error[invalid-super-argument]: `(int, str, /) -> bool` is an abstract/structural type in `super(<class 'object'>, (int, str, /) -> bool)` call
--> src/mdtest_snippet.py:78:17
--> src/mdtest_snippet.py:82:17
|
76 | # error: [invalid-super-argument]
77 | # revealed: Unknown
78 | reveal_type(super(object, z))
80 | # error: [invalid-super-argument]
81 | # revealed: Unknown
82 | reveal_type(super(object, z))
| ^^^^^^^^^^^^^^^^
79 |
80 | is_list = g(x)
83 |
84 | is_list = g(x)
|
info: rule `invalid-super-argument` is enabled by default
@ -236,15 +240,15 @@ info: rule `invalid-super-argument` is enabled by default
```
error[invalid-super-argument]: `types.GenericAlias` instance `list[int]` is not a valid class
--> src/mdtest_snippet.py:94:13
|
92 | # error: [invalid-super-argument]
93 | # revealed: Unknown
94 | reveal_type(super(list[int], []))
| ^^^^^^^^^^^^^^^^^^^^
95 | class Super:
96 | def method(self) -> int:
|
--> src/mdtest_snippet.py:98:13
|
96 | # error: [invalid-super-argument]
97 | # revealed: Unknown
98 | reveal_type(super(list[int], []))
| ^^^^^^^^^^^^^^^^^^^^
99 | class Super:
100 | def method(self) -> int:
|
info: rule `invalid-super-argument` is enabled by default
```