Make join of callables a bit more principled (#22296)

Fixes #9756
Fixes #10160
Fixes #13182
Fixes #14443

This is needed to de-risk #22286

Author: ilevkivskyi

mypy/join.py

@@ -8,7 +8,16 @@
 import mypy.typeops
 from mypy.expandtype import expand_type
 from mypy.maptype import map_instance_to_supertype
-from mypy.nodes import CONTRAVARIANT, COVARIANT, INVARIANT, VARIANCE_NOT_READY, TypeInfo
+from mypy.nodes import (
+    ARG_NAMED,
+    ARG_POS,
+    CONTRAVARIANT,
+    COVARIANT,
+    INVARIANT,
+    VARIANCE_NOT_READY,
+    ArgKind,
+    TypeInfo,
+)
 from mypy.state import state
 from mypy.subtypes import (
     SubtypeContext,
@@ -328,7 +337,7 @@ def visit_unpack_type(self, t: UnpackType) -> UnpackType:
 
     def visit_parameters(self, t: Parameters) -> ProperType:
         if isinstance(self.s, Parameters):
-            if not is_similar_params(t, self.s):
+            if not is_similar_callables(t, self.s):
                 # TODO: it would be prudent to return [*object, **object] instead of Any.
                 return self.default(self.s)
             from mypy.meet import meet_types
@@ -337,6 +346,7 @@ def visit_parameters(self, t: Parameters) -> ProperType:
                 arg_types=[
                     meet_types(s_a, t_a) for s_a, t_a in zip(self.s.arg_types, t.arg_types)
                 ],
+                arg_kinds=combine_arg_kinds(self.s, t),
                 arg_names=combine_arg_names(self.s, t),
             )
         else:
@@ -752,24 +762,29 @@ def normalize_callables(s: ProperType, t: ProperType) -> tuple[ProperType, Prope
     return s, t
 
 
-def is_similar_callables(t: CallableType, s: CallableType) -> bool:
-    """Return True if t and s have identical numbers of
-    arguments, default arguments and varargs.
-    """
-    return (
-        len(t.arg_types) == len(s.arg_types)
-        and t.min_args == s.min_args
-        and t.is_var_arg == s.is_var_arg
-    )
-
+def is_similar_callables(
+    t: CallableType | Parameters, s: CallableType | Parameters, for_meet: bool = False
+) -> bool:
+    """Return True if t and s have identical numbers of arguments with similar shape.
 
-def is_similar_params(t: Parameters, s: Parameters) -> bool:
-    # This matches the logic in is_similar_callables() above.
-    return (
-        len(t.arg_types) == len(s.arg_types)
-        and t.min_args == s.min_args
-        and (t.var_arg() is not None) == (s.var_arg() is not None)
-    )
+    If for_meet is True, the argument names must always match.
+    """
+    if len(t.variables) != len(s.variables):
+        return False
+    if len(t.arg_types) != len(s.arg_types):
+        return False
+    for i, (t_kind, s_kind) in enumerate(zip(t.arg_kinds, s.arg_kinds)):
+        if t.arg_names[i] != s.arg_names[i]:
+            if for_meet or t_kind.is_named():
+                return False
+        if t_kind == s_kind:
+            continue
+        if t_kind.is_positional() and s_kind.is_positional():
+            continue
+        if t_kind.is_named() and s_kind.is_named():
+            continue
+        return False
+    return True
 
 
 def update_callable_ids(c: CallableType, ids: list[TypeVarId]) -> CallableType:
@@ -803,7 +818,12 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
     t, s = match_generic_callables(t, s)
     arg_types: list[Type] = []
     for i in range(len(t.arg_types)):
-        arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i]))
+        if isinstance(get_proper_type(t.arg_types[i]), AnyType):
+            arg_types.append(t.arg_types[i])
+        elif isinstance(get_proper_type(s.arg_types[i]), AnyType):
+            arg_types.append(s.arg_types[i])
+        else:
+            arg_types.append(safe_meet(t.arg_types[i], s.arg_types[i]))
     # TODO in combine_similar_callables also applies here (names and kinds; user metaclasses)
     # The fallback type can be either 'function', 'type', or some user-provided metaclass.
     # The result should always use 'function' as a fallback if either operands are using it.
@@ -816,6 +836,7 @@ def join_similar_callables(t: CallableType, s: CallableType) -> CallableType:
         instance_type = join_types(t.instance_type, s.instance_type)
     return t.copy_modified(
         arg_types=arg_types,
+        arg_kinds=combine_arg_kinds(t, s),
         arg_names=combine_arg_names(t, s),
         ret_type=join_types(t.ret_type, s.ret_type),
         instance_type=instance_type,
@@ -874,6 +895,7 @@ def combine_similar_callables(t: CallableType, s: CallableType) -> CallableType:
         instance_type = join_types(t.instance_type, s.instance_type)
     return t.copy_modified(
         arg_types=arg_types,
+        arg_kinds=combine_arg_kinds(t, s),
         arg_names=combine_arg_names(t, s),
         ret_type=join_types(t.ret_type, s.ret_type),
         instance_type=instance_type,
@@ -899,20 +921,35 @@ def combine_arg_names(
     Note that the third argument's name is omitted and 't' and 's'
     are both valid subtypes of this inferred signature.
 
-    Precondition: is_similar_types(t, s) is true.
+    Precondition: is_similar_callables(t, s) is true.
     """
-    num_args = len(t.arg_types)
     new_names = []
-    for i in range(num_args):
-        t_name = t.arg_names[i]
-        s_name = s.arg_names[i]
-        if t_name == s_name or t.arg_kinds[i].is_named() or s.arg_kinds[i].is_named():
+    for t_name, s_name in zip(t.arg_names, s.arg_names):
+        if t_name == s_name:
             new_names.append(t_name)
         else:
             new_names.append(None)
     return new_names
 
 
+def combine_arg_kinds(t: CallableType | Parameters, s: CallableType | Parameters) -> list[ArgKind]:
+    """Produces a list of argument kinds compatible with both callables.
+
+    Essentially we make all arguments non-optional (both positional and named).
+    Precondition: is_similar_callables(t, s) is true.
+    """
+    new_kinds = []
+    for t_kind, s_kind in zip(t.arg_kinds, s.arg_kinds):
+        if t_kind == s_kind:
+            new_kinds.append(t_kind)
+        elif t_kind.is_positional():
+            new_kinds.append(ARG_POS)
+        else:
+            assert t_kind.is_named()
+            new_kinds.append(ARG_NAMED)
+    return new_kinds
+
+
 def object_from_instance(instance: Instance) -> Instance:
     """Construct the type 'builtins.object' from an instance type."""
     # Use the fact that 'object' is always the last class in the mro.

mypy/meet.py

@@ -5,6 +5,7 @@
 from mypy import join
 from mypy.erasetype import erase_type
 from mypy.maptype import map_instance_to_supertype
+from mypy.nodes import ARG_NAMED_OPT, ARG_OPT, ArgKind
 from mypy.state import state
 from mypy.subtypes import (
     are_parameters_compatible,
@@ -957,7 +958,9 @@ def visit_instance(self, t: Instance) -> ProperType:
         return self.default(self.s)
 
     def visit_callable_type(self, t: CallableType) -> ProperType:
-        if isinstance(self.s, CallableType) and join.is_similar_callables(t, self.s):
+        if isinstance(self.s, CallableType) and join.is_similar_callables(
+            t, self.s, for_meet=True
+        ):
             if is_equivalent(t, self.s):
                 return join.combine_similar_callables(t, self.s)
             result = meet_similar_callables(t, self.s)
@@ -1259,13 +1262,27 @@ def meet_similar_callables(t: CallableType, s: CallableType) -> CallableType:
         instance_type = meet_types(t.instance_type, s.instance_type)
     return t.copy_modified(
         arg_types=arg_types,
+        arg_kinds=combine_arg_kinds_for_meet(t, s),
         ret_type=meet_types(t.ret_type, s.ret_type),
         instance_type=instance_type,
         fallback=fallback,
         name=None,
     )
 
 
+def combine_arg_kinds_for_meet(t: CallableType, s: CallableType) -> list[ArgKind]:
+    new_kinds = []
+    for t_kind, s_kind in zip(t.arg_kinds, s.arg_kinds):
+        if t_kind == s_kind:
+            new_kinds.append(t_kind)
+        elif t_kind.is_positional():
+            new_kinds.append(ARG_OPT)
+        else:
+            assert t_kind.is_named()
+            new_kinds.append(ARG_NAMED_OPT)
+    return new_kinds
+
+
 def meet_type_list(types: list[Type]) -> Type:
     if not types:
         # This should probably be builtins.object but that is hard to get and

test-data/unit/check-classes.test

@@ -9819,3 +9819,32 @@ class A:
     def p(self, value: int) -> None:
         self.x = value
 [builtins fixtures/property.pyi]
+
+[case testJoinOfTypeObjects]
+from typing import TypeVar
+
+class B:
+    def __init__(self, *, y=None) -> None: ...
+class C1(B):
+    def __init__(self, *, x) -> None: ...
+class C2(B): ...
+
+x = [C1, C2]
+reveal_type(x)  # N: Revealed type is "builtins.list[builtins.type]"
+
+T = TypeVar("T")
+def join(a: T, b: T) -> T: ...
+reveal_type(join(C1, C2))  # N: Revealed type is "builtins.type"
+
+[case testLambdaInferredInCollection]
+from typing import Union
+
+def f(x: Union[int, None] = None) -> int: ...
+fs = [f, lambda x: 1, f]
+reveal_type(fs)  # N: Revealed type is "builtins.list[def (x: Any) -> builtins.int]"
+
+[case testLambdaInferredInCollection2]
+def f(x: int, y: int)-> int: ...
+
+fs = [lambda df, x: df.foo, f]
+reveal_type(fs)  # N: Revealed type is "builtins.list[def (Any, Any) -> Any]"

test-data/unit/check-inference.test

@@ -1109,8 +1109,10 @@ def f(*, x: int) -> int: ...
 def g(*, y: int) -> int: ...
 def h(*, x: int) -> int: ...
 
-list_1 = [f, g]  # E: List item 0 has incompatible type "def f(*, x: int) -> int"; expected "def g(*, y: int) -> int"
+list_1 = [f, g]
+reveal_type(list_1)  # N: Revealed type is "builtins.list[builtins.function]"
 list_2 = [f, h]
+reveal_type(list_2)  # N: Revealed type is "builtins.list[def (*, x: builtins.int) -> builtins.int]"
 [builtins fixtures/list.pyi]