[red-knot] A FunctionType can be a subtype of Callable (but never the other way around) (#16970)

## Summary

Partially fixes #16953

## Test Plan

Update is_subtype_of.md

---------

Co-authored-by: Carl Meyer <carl@astral.sh>

crates/red_knot_python_semantic/resources/mdtest/call/function.md

@@ -37,8 +37,6 @@ def foo() -> int:
     return 42
 
 def decorator(func) -> Callable[[], int]:
-    # TODO: no error
-    # error: [invalid-return-type]
     return foo
 
 @decorator

crates/red_knot_python_semantic/resources/mdtest/type_properties/is_subtype_of.md

@@ -494,12 +494,34 @@ Return types are covariant.
 
 ```py
 from typing import Callable
-from knot_extensions import is_subtype_of, static_assert
+from knot_extensions import is_subtype_of, static_assert, TypeOf
 
 static_assert(is_subtype_of(Callable[[], int], Callable[[], float]))
 static_assert(not is_subtype_of(Callable[[], float], Callable[[], int]))
 ```
 
+### Optional return type
+
+```py
+from typing import Callable
+from knot_extensions import is_subtype_of, static_assert, TypeOf
+
+flag: bool = True
+
+def optional_return_type() -> int | None:
+    if flag:
+        return 1
+    return None
+
+def required_return_type() -> int:
+    return 1
+
+static_assert(not is_subtype_of(TypeOf[optional_return_type], TypeOf[required_return_type]))
+# TypeOf[some_function] is a singleton function-literal type,  not a general callable type
+static_assert(not is_subtype_of(TypeOf[required_return_type], TypeOf[optional_return_type]))
+static_assert(is_subtype_of(TypeOf[optional_return_type], Callable[[], int | None]))
+```
+
 ### Parameter types
 
 Parameter types are contravariant.
@@ -507,13 +529,20 @@ Parameter types are contravariant.
 #### Positional-only
 
 ```py
-from knot_extensions import CallableTypeFromFunction, is_subtype_of, static_assert
+from typing import Callable
+from knot_extensions import CallableTypeFromFunction, is_subtype_of, static_assert, TypeOf
 
 def float_param(a: float, /) -> None: ...
 def int_param(a: int, /) -> None: ...
 
 static_assert(is_subtype_of(CallableTypeFromFunction[float_param], CallableTypeFromFunction[int_param]))
 static_assert(not is_subtype_of(CallableTypeFromFunction[int_param], CallableTypeFromFunction[float_param]))
+
+static_assert(is_subtype_of(TypeOf[int_param], Callable[[int], None]))
+static_assert(is_subtype_of(TypeOf[float_param], Callable[[float], None]))
+
+static_assert(not is_subtype_of(Callable[[int], None], TypeOf[int_param]))
+static_assert(not is_subtype_of(Callable[[float], None], TypeOf[float_param]))
 ```
 
 Parameter name is not required to be the same for positional-only parameters at the same position:
@@ -533,6 +562,10 @@ def multi_param2(b: int, c: bool, a: str, /) -> None: ...
 
 static_assert(is_subtype_of(CallableTypeFromFunction[multi_param1], CallableTypeFromFunction[multi_param2]))
 static_assert(not is_subtype_of(CallableTypeFromFunction[multi_param2], CallableTypeFromFunction[multi_param1]))
+
+static_assert(is_subtype_of(TypeOf[multi_param1], Callable[[float, int, str], None]))
+
+static_assert(not is_subtype_of(Callable[[float, int, str], None], TypeOf[multi_param1]))
 ```
 
 #### Positional-only with default value
@@ -541,7 +574,8 @@ If the parameter has a default value, it's treated as optional. This means that
 corresponding position in the supertype does not need to have a default value.
 
 ```py
-from knot_extensions import CallableTypeFromFunction, is_subtype_of, static_assert
+from typing import Callable
+from knot_extensions import CallableTypeFromFunction, is_subtype_of, static_assert, TypeOf
 
 def float_with_default(a: float = 1, /) -> None: ...
 def int_with_default(a: int = 1, /) -> None: ...
@@ -552,6 +586,13 @@ static_assert(not is_subtype_of(CallableTypeFromFunction[int_with_default], Call
 
 static_assert(is_subtype_of(CallableTypeFromFunction[int_with_default], CallableTypeFromFunction[int_without_default]))
 static_assert(not is_subtype_of(CallableTypeFromFunction[int_without_default], CallableTypeFromFunction[int_with_default]))
+
+static_assert(is_subtype_of(TypeOf[int_with_default], Callable[[int], None]))
+static_assert(is_subtype_of(TypeOf[int_with_default], Callable[[], None]))
+static_assert(is_subtype_of(TypeOf[float_with_default], Callable[[float], None]))
+
+static_assert(not is_subtype_of(Callable[[int], None], TypeOf[int_with_default]))
+static_assert(not is_subtype_of(Callable[[float], None], TypeOf[float_with_default]))
 ```
 
 As the parameter itself is optional, it can be omitted in the supertype:

crates/red_knot_python_semantic/src/types.rs

@@ -633,10 +633,14 @@ impl<'db> Type<'db> {
                 KnownClass::Slice.to_instance(db).is_subtype_of(db, target)
             }
 
+            (Type::FunctionLiteral(self_function_literal), Type::Callable(_)) => {
+                self_function_literal
+                    .into_callable_type(db)
+                    .is_subtype_of(db, target)
+            }
+
             // A `FunctionLiteral` type is a single-valued type like the other literals handled above,
             // so it also, for now, just delegates to its instance fallback.
-            // This will change in a way similar to the `LiteralString`/`StringLiteral()` case above
-            // when we add support for `typing.Callable`.
             (Type::FunctionLiteral(_), _) => KnownClass::FunctionType
                 .to_instance(db)
                 .is_subtype_of(db, target),