[ty] Handle explicit variance in legacy typevars (#17897)

We now track the variance of each typevar, and obey the `covariant` and
`contravariant` parameters to the legacy `TypeVar` constructor. We still
don't yet infer variance for PEP-695 typevars or for the
`infer_variance` legacy constructor parameter.

---------

Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
Co-authored-by: Carl Meyer <carl@astral.sh>
This commit is contained in:
Douglas Creager 2025-05-07 08:44:51 -04:00 committed by GitHub
parent c5e299e796
commit 0d9b6a0975
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 368 additions and 25 deletions

View file

@ -123,4 +123,32 @@ from typing import TypeVar
T = TypeVar("T", int)
```
### Cannot be both covariant and contravariant
> To facilitate the declaration of container types where covariant or contravariant type checking is
> acceptable, type variables accept keyword arguments `covariant=True` or `contravariant=True`. At
> most one of these may be passed.
```py
from typing import TypeVar
# error: [invalid-legacy-type-variable]
T = TypeVar("T", covariant=True, contravariant=True)
```
### Variance parameters must be unambiguous
```py
from typing import TypeVar
def cond() -> bool:
return True
# error: [invalid-legacy-type-variable]
T = TypeVar("T", covariant=cond())
# error: [invalid-legacy-type-variable]
U = TypeVar("U", contravariant=cond())
```
[generics]: https://typing.python.org/en/latest/spec/generics.html

View file

@ -0,0 +1,207 @@
# Variance: Legacy syntax
Type variables have a property called _variance_ that affects the subtyping and assignability
relations. Much more detail can be found in the [spec]. To summarize, each typevar is either
**covariant**, **contravariant**, **invariant**, or **bivariant**. (Note that bivariance is not
currently mentioned in the typing spec, but is a fourth case that we must consider.)
For all of the examples below, we will consider a typevar `T`, a generic class using that typevar
`C[T]`, and two types `A` and `B`.
(Note that dynamic types like `Any` never participate in subtyping, so `C[Any]` is neither a subtype
nor supertype of any other specialization of `C`, regardless of `T`'s variance. It is, however,
assignable to any specialization of `C`, regardless of variance, via materialization.)
## Covariance
With a covariant typevar, subtyping and assignability are in "alignment": if `A <: B`, then
`C[A] <: C[B]`.
Types that "produce" data on demand are covariant in their typevar. If you expect a sequence of
`int`s, someone can safely provide a sequence of `bool`s, since each `bool` element that you would
get from the sequence is a valid `int`.
```py
from ty_extensions import is_assignable_to, is_equivalent_to, is_gradual_equivalent_to, is_subtype_of, static_assert, Unknown
from typing import Any, Generic, TypeVar
class A: ...
class B(A): ...
T = TypeVar("T", covariant=True)
class C(Generic[T]):
def receive(self) -> T:
raise ValueError
static_assert(is_assignable_to(C[B], C[A]))
static_assert(not is_assignable_to(C[A], C[B]))
static_assert(is_assignable_to(C[A], C[Any]))
static_assert(is_assignable_to(C[B], C[Any]))
static_assert(is_assignable_to(C[Any], C[A]))
static_assert(is_assignable_to(C[Any], C[B]))
static_assert(is_subtype_of(C[B], C[A]))
static_assert(not is_subtype_of(C[A], C[B]))
static_assert(not is_subtype_of(C[A], C[Any]))
static_assert(not is_subtype_of(C[B], C[Any]))
static_assert(not is_subtype_of(C[Any], C[A]))
static_assert(not is_subtype_of(C[Any], C[B]))
static_assert(is_equivalent_to(C[A], C[A]))
static_assert(is_equivalent_to(C[B], C[B]))
static_assert(not is_equivalent_to(C[B], C[A]))
static_assert(not is_equivalent_to(C[A], C[B]))
static_assert(not is_equivalent_to(C[A], C[Any]))
static_assert(not is_equivalent_to(C[B], C[Any]))
static_assert(not is_equivalent_to(C[Any], C[A]))
static_assert(not is_equivalent_to(C[Any], C[B]))
static_assert(is_gradual_equivalent_to(C[A], C[A]))
static_assert(is_gradual_equivalent_to(C[B], C[B]))
static_assert(is_gradual_equivalent_to(C[Any], C[Any]))
static_assert(is_gradual_equivalent_to(C[Any], C[Unknown]))
static_assert(not is_gradual_equivalent_to(C[B], C[A]))
static_assert(not is_gradual_equivalent_to(C[A], C[B]))
static_assert(not is_gradual_equivalent_to(C[A], C[Any]))
static_assert(not is_gradual_equivalent_to(C[B], C[Any]))
static_assert(not is_gradual_equivalent_to(C[Any], C[A]))
static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
```
## Contravariance
With a contravariant typevar, subtyping and assignability are in "opposition": if `A <: B`, then
`C[B] <: C[A]`.
Types that "consume" data are contravariant in their typevar. If you expect a consumer that receives
`bool`s, someone can safely provide a consumer that expects to receive `int`s, since each `bool`
that you pass into the consumer is a valid `int`.
```py
from ty_extensions import is_assignable_to, is_equivalent_to, is_gradual_equivalent_to, is_subtype_of, static_assert, Unknown
from typing import Any, Generic, TypeVar
class A: ...
class B(A): ...
T = TypeVar("T", contravariant=True)
class C(Generic[T]):
def send(self, value: T): ...
static_assert(not is_assignable_to(C[B], C[A]))
static_assert(is_assignable_to(C[A], C[B]))
static_assert(is_assignable_to(C[A], C[Any]))
static_assert(is_assignable_to(C[B], C[Any]))
static_assert(is_assignable_to(C[Any], C[A]))
static_assert(is_assignable_to(C[Any], C[B]))
static_assert(not is_subtype_of(C[B], C[A]))
static_assert(is_subtype_of(C[A], C[B]))
static_assert(not is_subtype_of(C[A], C[Any]))
static_assert(not is_subtype_of(C[B], C[Any]))
static_assert(not is_subtype_of(C[Any], C[A]))
static_assert(not is_subtype_of(C[Any], C[B]))
static_assert(is_equivalent_to(C[A], C[A]))
static_assert(is_equivalent_to(C[B], C[B]))
static_assert(not is_equivalent_to(C[B], C[A]))
static_assert(not is_equivalent_to(C[A], C[B]))
static_assert(not is_equivalent_to(C[A], C[Any]))
static_assert(not is_equivalent_to(C[B], C[Any]))
static_assert(not is_equivalent_to(C[Any], C[A]))
static_assert(not is_equivalent_to(C[Any], C[B]))
static_assert(is_gradual_equivalent_to(C[A], C[A]))
static_assert(is_gradual_equivalent_to(C[B], C[B]))
static_assert(is_gradual_equivalent_to(C[Any], C[Any]))
static_assert(is_gradual_equivalent_to(C[Any], C[Unknown]))
static_assert(not is_gradual_equivalent_to(C[B], C[A]))
static_assert(not is_gradual_equivalent_to(C[A], C[B]))
static_assert(not is_gradual_equivalent_to(C[A], C[Any]))
static_assert(not is_gradual_equivalent_to(C[B], C[Any]))
static_assert(not is_gradual_equivalent_to(C[Any], C[A]))
static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
```
## Invariance
With an invariant typevar, only equivalent specializations of the generic class are subtypes of or
assignable to each other.
This often occurs for types that are both producers _and_ consumers, like a mutable `list`.
Iterating over the elements in a list would work with a covariant typevar, just like with the
"producer" type above. Appending elements to a list would work with a contravariant typevar, just
like with the "consumer" type above. However, a typevar cannot be both covariant and contravariant
at the same time!
If you expect a mutable list of `int`s, it's not safe for someone to provide you with a mutable list
of `bool`s, since you might try to add an element to the list: if you try to add an `int`, the list
would no longer only contain elements that are subtypes of `bool`.
Conversely, if you expect a mutable list of `bool`s, it's not safe for someone to provide you with a
mutable list of `int`s, since you might try to extract elements from the list: you expect every
element that you extract to be a subtype of `bool`, but the list can contain any `int`.
In the end, if you expect a mutable list, you must always be given a list of exactly that type,
since we can't know in advance which of the allowed methods you'll want to use.
```py
from ty_extensions import is_assignable_to, is_equivalent_to, is_gradual_equivalent_to, is_subtype_of, static_assert, Unknown
from typing import Any, Generic, TypeVar
class A: ...
class B(A): ...
T = TypeVar("T")
class C(Generic[T]):
def send(self, value: T): ...
def receive(self) -> T:
raise ValueError
static_assert(not is_assignable_to(C[B], C[A]))
static_assert(not is_assignable_to(C[A], C[B]))
static_assert(is_assignable_to(C[A], C[Any]))
static_assert(is_assignable_to(C[B], C[Any]))
static_assert(is_assignable_to(C[Any], C[A]))
static_assert(is_assignable_to(C[Any], C[B]))
static_assert(not is_subtype_of(C[B], C[A]))
static_assert(not is_subtype_of(C[A], C[B]))
static_assert(not is_subtype_of(C[A], C[Any]))
static_assert(not is_subtype_of(C[B], C[Any]))
static_assert(not is_subtype_of(C[Any], C[A]))
static_assert(not is_subtype_of(C[Any], C[B]))
static_assert(is_equivalent_to(C[A], C[A]))
static_assert(is_equivalent_to(C[B], C[B]))
static_assert(not is_equivalent_to(C[B], C[A]))
static_assert(not is_equivalent_to(C[A], C[B]))
static_assert(not is_equivalent_to(C[A], C[Any]))
static_assert(not is_equivalent_to(C[B], C[Any]))
static_assert(not is_equivalent_to(C[Any], C[A]))
static_assert(not is_equivalent_to(C[Any], C[B]))
static_assert(is_gradual_equivalent_to(C[A], C[A]))
static_assert(is_gradual_equivalent_to(C[B], C[B]))
static_assert(is_gradual_equivalent_to(C[Any], C[Any]))
static_assert(is_gradual_equivalent_to(C[Any], C[Unknown]))
static_assert(not is_gradual_equivalent_to(C[B], C[A]))
static_assert(not is_gradual_equivalent_to(C[A], C[B]))
static_assert(not is_gradual_equivalent_to(C[A], C[Any]))
static_assert(not is_gradual_equivalent_to(C[B], C[Any]))
static_assert(not is_gradual_equivalent_to(C[Any], C[A]))
static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
```
## Bivariance
With a bivariant typevar, _all_ specializations of the generic class are assignable to (and in fact,
gradually equivalent to) each other, and all fully static specializations are subtypes of (and
equivalent to) each other.
It is not possible to construct a legacy typevar that is explicitly bivariant.
[spec]: https://typing.python.org/en/latest/spec/generics.html#variance

View file

@ -13,9 +13,13 @@ currently mentioned in the typing spec, but is a fourth case that we must consid
For all of the examples below, we will consider a typevar `T`, a generic class using that typevar
`C[T]`, and two types `A` and `B`.
(Note that dynamic types like `Any` never participate in subtyping, so `C[Any]` is neither a subtype
nor supertype of any other specialization of `C`, regardless of `T`'s variance.)
## Covariance
With a covariant typevar, subtyping is in "alignment": if `A <: B`, then `C[A] <: C[B]`.
With a covariant typevar, subtyping and assignability are in "alignment": if `A <: B`, then
`C[A] <: C[B]`.
Types that "produce" data on demand are covariant in their typevar. If you expect a sequence of
`int`s, someone can safely provide a sequence of `bool`s, since each `bool` element that you would
@ -73,7 +77,8 @@ static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
## Contravariance
With a contravariant typevar, subtyping is in "opposition": if `A <: B`, then `C[B] <: C[A]`.
With a contravariant typevar, subtyping are assignability are in "opposition": if `A <: B`, then
`C[B] <: C[A]`.
Types that "consume" data are contravariant in their typevar. If you expect a consumer that receives
`bool`s, someone can safely provide a consumer that expects to receive `int`s, since each `bool`
@ -130,7 +135,8 @@ static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
## Invariance
With an invariant typevar, _no_ specializations of the generic class are subtypes of each other.
With an invariant typevar, _no_ specializations of the generic class are subtypes of or assignable
to each other.
This often occurs for types that are both producers _and_ consumers, like a mutable `list`.
Iterating over the elements in a list would work with a covariant typevar, just like with the
@ -198,7 +204,8 @@ static_assert(not is_gradual_equivalent_to(C[Any], C[B]))
## Bivariance
With a bivariant typevar, _all_ specializations of the generic class are subtypes of (and in fact,
With a bivariant typevar, _all_ specializations of the generic class are assignable to (and in fact,
gradually equivalent to) each other, and all fully static specializations are subtypes of (and
equivalent to) each other.
This is a bit of pathological case, which really only happens when the class doesn't use the typevar

View file

@ -932,6 +932,7 @@ impl<'db> Type<'db> {
typevar.name(db).clone(),
typevar.definition(db),
Some(TypeVarBoundOrConstraints::UpperBound(bound.normalized(db))),
typevar.variance(db),
typevar.default_ty(db),
typevar.kind(db),
))
@ -942,6 +943,7 @@ impl<'db> Type<'db> {
typevar.name(db).clone(),
typevar.definition(db),
Some(TypeVarBoundOrConstraints::Constraints(union.normalized(db))),
typevar.variance(db),
typevar.default_ty(db),
typevar.kind(db),
))
@ -5618,6 +5620,9 @@ pub struct TypeVarInstance<'db> {
/// The upper bound or constraint on the type of this TypeVar
bound_or_constraints: Option<TypeVarBoundOrConstraints<'db>>,
/// The variance of the TypeVar
variance: TypeVarVariance,
/// The default type for this TypeVar
default_ty: Option<Type<'db>>,
@ -5646,7 +5651,15 @@ impl<'db> TypeVarInstance<'db> {
}
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, salsa::Update)]
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, salsa::Update)]
pub enum TypeVarVariance {
Invariant,
Covariant,
Contravariant,
Bivariant,
}
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, salsa::Update)]
pub enum TypeVarBoundOrConstraints<'db> {
UpperBound(Type<'db>),
Constraints(UnionType<'db>),

View file

@ -5,7 +5,7 @@ use crate::semantic_index::SemanticIndex;
use crate::types::signatures::{Parameter, Parameters, Signature};
use crate::types::{
declaration_type, KnownInstanceType, Type, TypeVarBoundOrConstraints, TypeVarInstance,
UnionType,
TypeVarVariance, UnionType,
};
use crate::{Db, FxOrderSet};
@ -260,7 +260,7 @@ impl<'db> Specialization<'db> {
return false;
}
for ((_typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
for ((typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
.zip(self.types(db))
.zip(other.types(db))
{
@ -268,13 +268,19 @@ impl<'db> Specialization<'db> {
return false;
}
// TODO: We currently treat all typevars as invariant. Once we track the actual
// variance of each typevar, these checks should change:
// Subtyping of each type in the specialization depends on the variance of the
// corresponding typevar:
// - covariant: verify that self_type <: other_type
// - contravariant: verify that other_type <: self_type
// - invariant: verify that self_type == other_type
// - bivariant: skip, can't make subtyping false
if !self_type.is_equivalent_to(db, *other_type) {
let compatible = match typevar.variance(db) {
TypeVarVariance::Invariant => self_type.is_equivalent_to(db, *other_type),
TypeVarVariance::Covariant => self_type.is_subtype_of(db, *other_type),
TypeVarVariance::Contravariant => other_type.is_subtype_of(db, *self_type),
TypeVarVariance::Bivariant => true,
};
if !compatible {
return false;
}
}
@ -288,7 +294,7 @@ impl<'db> Specialization<'db> {
return false;
}
for ((_typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
for ((typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
.zip(self.types(db))
.zip(other.types(db))
{
@ -296,13 +302,19 @@ impl<'db> Specialization<'db> {
return false;
}
// TODO: We currently treat all typevars as invariant. Once we track the actual
// variance of each typevar, these checks should change:
// Equivalence of each type in the specialization depends on the variance of the
// corresponding typevar:
// - covariant: verify that self_type == other_type
// - contravariant: verify that other_type == self_type
// - invariant: verify that self_type == other_type
// - bivariant: skip, can't make equivalence false
if !self_type.is_equivalent_to(db, *other_type) {
let compatible = match typevar.variance(db) {
TypeVarVariance::Invariant
| TypeVarVariance::Covariant
| TypeVarVariance::Contravariant => self_type.is_equivalent_to(db, *other_type),
TypeVarVariance::Bivariant => true,
};
if !compatible {
return false;
}
}
@ -316,7 +328,7 @@ impl<'db> Specialization<'db> {
return false;
}
for ((_typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
for ((typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
.zip(self.types(db))
.zip(other.types(db))
{
@ -324,13 +336,19 @@ impl<'db> Specialization<'db> {
continue;
}
// TODO: We currently treat all typevars as invariant. Once we track the actual
// variance of each typevar, these checks should change:
// Assignability of each type in the specialization depends on the variance of the
// corresponding typevar:
// - covariant: verify that self_type <: other_type
// - contravariant: verify that other_type <: self_type
// - invariant: verify that self_type == other_type
// - bivariant: skip, can't make assignability false
if !self_type.is_gradual_equivalent_to(db, *other_type) {
let compatible = match typevar.variance(db) {
TypeVarVariance::Invariant => self_type.is_gradual_equivalent_to(db, *other_type),
TypeVarVariance::Covariant => self_type.is_assignable_to(db, *other_type),
TypeVarVariance::Contravariant => other_type.is_assignable_to(db, *self_type),
TypeVarVariance::Bivariant => true,
};
if !compatible {
return false;
}
}
@ -348,17 +366,25 @@ impl<'db> Specialization<'db> {
return false;
}
for ((_typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
for ((typevar, self_type), other_type) in (generic_context.variables(db).into_iter())
.zip(self.types(db))
.zip(other.types(db))
{
// TODO: We currently treat all typevars as invariant. Once we track the actual
// variance of each typevar, these checks should change:
// Equivalence of each type in the specialization depends on the variance of the
// corresponding typevar:
// - covariant: verify that self_type == other_type
// - contravariant: verify that other_type == self_type
// - invariant: verify that self_type == other_type
// - bivariant: skip, can't make equivalence false
if !self_type.is_gradual_equivalent_to(db, *other_type) {
let compatible = match typevar.variance(db) {
TypeVarVariance::Invariant
| TypeVarVariance::Covariant
| TypeVarVariance::Contravariant => {
self_type.is_gradual_equivalent_to(db, *other_type)
}
TypeVarVariance::Bivariant => true,
};
if !compatible {
return false;
}
}

View file

@ -89,8 +89,8 @@ use crate::types::{
MemberLookupPolicy, MetaclassCandidate, Parameter, ParameterForm, Parameters, Signature,
Signatures, SliceLiteralType, StringLiteralType, SubclassOfType, Symbol, SymbolAndQualifiers,
Truthiness, TupleType, Type, TypeAliasType, TypeAndQualifiers, TypeArrayDisplay,
TypeQualifiers, TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind, UnionBuilder,
UnionType,
TypeQualifiers, TypeVarBoundOrConstraints, TypeVarInstance, TypeVarKind, TypeVarVariance,
UnionBuilder, UnionType,
};
use crate::unpack::{Unpack, UnpackPosition};
use crate::util::subscript::{PyIndex, PySlice};
@ -2504,6 +2504,7 @@ impl<'db> TypeInferenceBuilder<'db> {
name.id.clone(),
definition,
bound_or_constraint,
TypeVarVariance::Invariant, // TODO: infer this
default_ty,
TypeVarKind::Pep695,
)));
@ -4990,12 +4991,72 @@ impl<'db> TypeInferenceBuilder<'db> {
continue;
};
let [Some(name_param), constraints, bound, default, _contravariant, _covariant, _infer_variance] =
let [Some(name_param), constraints, bound, default, contravariant, covariant, _infer_variance] =
overload.parameter_types()
else {
continue;
};
let covariant = match covariant {
Some(ty) => ty.bool(self.db()),
None => Truthiness::AlwaysFalse,
};
let contravariant = match contravariant {
Some(ty) => ty.bool(self.db()),
None => Truthiness::AlwaysFalse,
};
let variance = match (contravariant, covariant) {
(Truthiness::Ambiguous, _) => {
if let Some(builder) = self.context.report_lint(
&INVALID_LEGACY_TYPE_VARIABLE,
call_expression,
) {
builder.into_diagnostic(format_args!(
"The `contravariant` parameter of \
a legacy `typing.TypeVar` cannot have \
an ambiguous value",
));
}
continue;
}
(_, Truthiness::Ambiguous) => {
if let Some(builder) = self.context.report_lint(
&INVALID_LEGACY_TYPE_VARIABLE,
call_expression,
) {
builder.into_diagnostic(format_args!(
"The `covariant` parameter of \
a legacy `typing.TypeVar` cannot have \
an ambiguous value",
));
}
continue;
}
(Truthiness::AlwaysTrue, Truthiness::AlwaysTrue) => {
if let Some(builder) = self.context.report_lint(
&INVALID_LEGACY_TYPE_VARIABLE,
call_expression,
) {
builder.into_diagnostic(format_args!(
"A legacy `typing.TypeVar` cannot be \
both covariant and contravariant",
));
}
continue;
}
(Truthiness::AlwaysTrue, Truthiness::AlwaysFalse) => {
TypeVarVariance::Contravariant
}
(Truthiness::AlwaysFalse, Truthiness::AlwaysTrue) => {
TypeVarVariance::Covariant
}
(Truthiness::AlwaysFalse, Truthiness::AlwaysFalse) => {
TypeVarVariance::Invariant
}
};
let name_param = name_param
.into_string_literal()
.map(|name| name.value(self.db()).as_ref());
@ -5062,6 +5123,7 @@ impl<'db> TypeInferenceBuilder<'db> {
target.id.clone(),
containing_assignment,
bound_or_constraint,
variance,
*default,
TypeVarKind::Legacy,
)),