Fix varargs overload resolution with wildcard types

Fixes #24072

When comparing overloaded methods where one is non-varargs with wildcard types (e.g., `Class[? <: T]`) and another is varargs, the non-varargs method should be preferred.
Previously, the compiler failed to distinguish these methods , and results in an ambiguity error.

```scala
def blub[T](a: Class[? <: T]): Unit // m1
def blub[T](a: Class[T], ints: Int*): Unit // m2
blub(classOf[Object]) // m1 should be picked, but fails to resolve
````

The problem is `compare(m1, m2)` returned 0 because:

- (1). `m2` (varargs) is correctly considered "not as good" as `m1`.
- (2). `m1` (non-varargs) was also considered "not as good" as `m2`. (but `m1` should be as good as `m2`! because `Class[Concrete]` can be applied to both m1 and m2).

The (2) occurred because `Class[? <: T]` is not a subtype of `Class[T]` (due to invariance). Consequently, `isApplicableMethodRef(m2, Class[? <: T])` returned `false` because `isCompatible(Class[? <: T], Class[T])` returned `false` during the applicability check against the method.

This commit adds special handling in `TestApplication.argOK` to check if wildcard upper bounds are compatible with their formal types during overload resolution, in addition to `isCompatible`.

Author: tanishiking

compiler/src/dotty/tools/dotc/typer/Applications.scala

@@ -883,7 +883,27 @@ trait Applications extends Compatibility {
             case SAMType(samMeth, samParent) => argtpe <:< samMeth.toFunctionType(isJava = samParent.classSymbol.is(JavaDefined))
             case _ => false
 
-        isCompatible(argtpe, formal)
+        // For overload resolution, allow wildcard upper bounds to match their bound type.
+        // For example, given:
+        //   def blub[T](a: Class[? <: T]): String = "a"
+        //   def blub[T](a: Class[T], ints: Int*): String = "b"
+        //   blub(classOf[Object])
+        //
+        // The non-varargs overload should be preferred. While Class[? <: T] is not a
+        // subtype of Class[T] (Class is invariant), for overload resolution we consider
+        // Class[? <: T] "applicable" where Class[T] is expected by checking if the
+        // wildcard's upper bound is a subtype of the formal type parameter.
+        def wildcardArgOK =
+          (argtpe, formal) match
+            case (AppliedType(tycon1, args1), AppliedType(tycon2, args2))
+            if tycon1 =:= tycon2 && args1.length == args2.length =>
+              args1.lazyZip(args2).forall {
+                case (TypeBounds(_, hi), formal) => hi relaxed_<:< formal
+                case (arg, formal) => arg =:= formal
+              }
+            case _ => false
+
+        isCompatible(argtpe, formal) || wildcardArgOK
         // Only allow SAM-conversion to PartialFunction if implicit conversions
         // are enabled. This is necessary to avoid ambiguity between an overload
         // taking a PartialFunction and one taking a Function1 because

tests/run/overload_repeated/B_2.scala

@@ -13,6 +13,13 @@ object Test {
   def bar4[T](x: T): Int = 1
   def bar4[T](x: T, xs: T*): Int = 2
 
+  // https://github.com/scala/scala3/issues/24072
+  def bar5[T](a: Class[? <: T]): Int = 1
+  def bar5[T](a: Class[T], ints: Int*): Int = 2
+
+  def bar6[T](a: Int): Int = 1
+  def bar6[T](a: Class[T], ints: Int*): Int = 2
+
   def main(args: Array[String]): Unit = {
     // In Java, varargs are always less specific than non-varargs (see
     // https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.12.2),
@@ -28,5 +35,7 @@ object Test {
     assert(bar2("") == 1) // same in Scala 2
     assert(bar3("") == 1) // same in Scala 2
     assert(bar4("") == 1) // same in Scala 2
+    assert(bar5(classOf[Object]) == 1) // same in Scala2
+    assert(bar6(classOf[Object]) == 2) // same in Scala2
   }
 }