Fix overload oscillation for generic type arguments, tuples, and hashes

Extend the uninformative-args bail-out in resolve_overloads to
recursively check type parameter vertices (e.g., Array[T], Hash[K,V],
tuples). Previously only top-level empty vertices and splat array
element vertices were checked, missing cases like Foo.f([@x]) where
@x is empty and overloads differ in the element type.

To avoid false bail-outs for legitimate empty containers (e.g., {}
passed to Hash#merge!), the recursive check is only applied when
overloads actually differ in their positional parameter types.
Overloads that differ only in blocks or keywords use the simpler
top-level empty check.

Author: mame

lib/typeprof/core/graph/box.rb

@@ -255,29 +255,21 @@ def resolve_overloads(changes, genv, node, param_map, a_args, ret, &blk)
       # If any positional argument has no type information, we cannot
       # determine which overload to select. Return silently (untyped)
       # rather than attempting to match. This prevents oscillation in
-      # cyclic cases like @x = Foo.transform(@x), and avoids false
-      # "failed to resolve overloads" diagnostics for untyped arguments.
-      # We still set up dependency edges so the box re-runs when the
-      # empty arguments later receive types.
+      # cyclic cases and avoids false "failed to resolve overloads"
+      # diagnostics for untyped arguments.
       #
-      # For splat arguments, the positional vertex itself holds Array
-      # types (non-empty), but the array *element* vertex may be empty.
-      # The same oscillation occurs when match_arguments? extracts
-      # elements via get_rest_args and the universal typecheck on the
-      # flattened element list fails due to conflicting array sources.
-      # We detect this by checking element vertices of splatted arrays.
-      has_uninformative_args = a_args.positionals.any? {|vtx| vtx.types.empty? }
-      unless has_uninformative_args
-        a_args.positionals.each_with_index do |vtx, i|
-          next unless a_args.splat_flags[i]
-          vtx.each_type do |ty|
-            base = ty.base_type(genv)
-            if base.is_a?(Type::Instance) && base.mod == genv.mod_ary && base.args[0]
-              has_uninformative_args = true if base.args[0].types.empty?
-            end
-          end
-          break if has_uninformative_args
-        end
+      # Top-level empty vertices are always uninformative. For type
+      # parameter vertices (e.g., Array[T], Hash[K,V], tuples), we
+      # only recurse when overloads differ in their positional parameter
+      # types -- otherwise empty type params (like those of `{}`) cannot
+      # cause oscillation and should not trigger bail-out.
+      overloads_differ_in_positionals = !@method_types.each_cons(2).all? {|mt1, mt2|
+        positionals_match?(mt1, mt2)
+      }
+      has_uninformative_args = if overloads_differ_in_positionals
+        a_args.positionals.any? {|vtx| vertex_uninformative?(genv, vtx) }
+      else
+        a_args.positionals.any? {|vtx| vtx.types.empty? }
       end
       if has_uninformative_args
         a_args.positionals.each do |vtx|
@@ -298,6 +290,61 @@ def resolve_overloads(changes, genv, node, param_map, a_args, ret, &blk)
       end
     end
 
+    def vertex_uninformative?(genv, vtx, depth = 0)
+      return true if vtx.types.empty?
+      return false if depth > 3
+      vtx.each_type do |ty|
+        base = ty.base_type(genv)
+        next unless base.is_a?(Type::Instance) && !base.args.empty?
+        base.args.each do |arg_vtx|
+          return true if arg_vtx && vertex_uninformative?(genv, arg_vtx, depth + 1)
+        end
+      end
+      false
+    end
+
+    # Check if two method types have structurally identical positional
+    # parameter types (req, opt, rest).
+    def positionals_match?(mt1, mt2)
+      return false unless mt1.req_positionals.size == mt2.req_positionals.size
+      return false unless mt1.opt_positionals.size == mt2.opt_positionals.size
+      return false unless mt1.rest_positionals.nil? == mt2.rest_positionals.nil?
+      mt1.req_positionals.zip(mt2.req_positionals).all? {|a, b| sig_types_match?(a, b) } &&
+        mt1.opt_positionals.zip(mt2.opt_positionals).all? {|a, b| sig_types_match?(a, b) } &&
+        (mt1.rest_positionals.nil? || sig_types_match?(mt1.rest_positionals, mt2.rest_positionals))
+    end
+
+    # Structural equality check for two SigTyNode objects.
+    def sig_types_match?(a, b)
+      return false unless a.class == b.class
+      case a
+      when AST::SigTyInstanceNode, AST::SigTyInterfaceNode
+        a.cpath == b.cpath &&
+          a.args.size == b.args.size &&
+          a.args.zip(b.args).all? {|x, y| sig_types_match?(x, y) }
+      when AST::SigTySingletonNode
+        a.cpath == b.cpath
+      when AST::SigTyTupleNode, AST::SigTyUnionNode, AST::SigTyIntersectionNode
+        a.types.size == b.types.size &&
+          a.types.zip(b.types).all? {|x, y| sig_types_match?(x, y) }
+      when AST::SigTyRecordNode
+        a.fields.size == b.fields.size &&
+          a.fields.all? {|k, v| b.fields[k] && sig_types_match?(v, b.fields[k]) }
+      when AST::SigTyOptionalNode, AST::SigTyProcNode
+        sig_types_match?(a.type, b.type)
+      when AST::SigTyVarNode
+        a.var == b.var
+      when AST::SigTyLiteralNode
+        a.lit == b.lit
+      when AST::SigTyAliasNode
+        a.cpath == b.cpath && a.name == b.name &&
+          a.args.size == b.args.size &&
+          a.args.zip(b.args).all? {|x, y| sig_types_match?(x, y) }
+      else
+        true # Leaf types (bool, nil, self, void, untyped, etc.)
+      end
+    end
+
     def show
       @method_types.map do |method_type|
         args = []

scenario/regressions/generic-arg-overload-oscillation.rb

@@ -0,0 +1,28 @@
+## update: test.rbs
+class Foo
+  def self.f: (Array[Integer]) -> String | (Array[String]) -> Symbol
+end
+
+## update: test.rb
+# Generic type arguments in overload selection cause oscillation.
+#
+# typecheck_for_module (sig_type.rb) recursively checks type parameter
+# vertices. When an element vertex is empty, typecheck returns true
+# (via !found_any), making all overloads match. The resulting disjoint
+# return types feed back and cause the element types to oscillate.
+#
+# Variants that exhibit the same issue:
+#   - Hash[Symbol, Integer] vs Hash[Symbol, String]
+#   - Array[Array[Integer]] vs Array[Array[String]]
+#   - Custom generic: Box[Integer] vs Box[String]
+#   - Tuple: [Integer] vs [String]
+def check
+  @x = Foo.f([@x])
+end
+
+## assert
+class Object
+  def check: -> untyped
+end
+
+## diagnostics

scenario/regressions/hash-arg-overload-oscillation.rb

@@ -0,0 +1,18 @@
+## update: test.rbs
+class Foo
+  def self.f: (Hash[Symbol, Integer]) -> String | (Hash[Symbol, String]) -> Symbol
+end
+
+## update: test.rb
+# Hash value type argument causes overload oscillation via the same
+# mechanism: empty value type vertex in typecheck_for_module.
+def check
+  @x = Foo.f({ a: @x })
+end
+
+## assert
+class Object
+  def check: -> untyped
+end
+
+## diagnostics

scenario/regressions/tuple-overload-oscillation.rb

@@ -0,0 +1,19 @@
+## update: test.rbs
+class Foo
+  def self.f: ([Integer]) -> String | ([String]) -> Symbol
+end
+
+## update: test.rb
+# Tuple element typecheck causes oscillation via the same mechanism
+# as generic type argument oscillation: empty element vertex makes
+# typecheck return true for all overloads.
+def check
+  @x = Foo.f([@x])
+end
+
+## assert
+class Object
+  def check: -> untyped
+end
+
+## diagnostics