Don't flag match type aliases as unreducible

If `M[_]` is a match type alias, it was rejected for being an unreducible
application of wildcard types. It is now accepted. I believe that is sound
- this is siply an unreducible match type. The situation is not the same
as `F[_]` where `F` is an abstract type that can be refined in several ways
in subclasses.

Fixes #9999

Author: odersky

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4004,7 +4004,7 @@ object Types {
         case nil => x
       foldArgs(op(x, tycon), args)
 
-    override def tryNormalize(using Context): Type = tycon match {
+    override def tryNormalize(using Context): Type = tycon.stripTypeVar match {
       case tycon: TypeRef =>
         def tryMatchAlias = tycon.info match {
           case MatchAlias(alias) =>
@@ -4014,13 +4014,29 @@ object Types {
           case _ =>
             NoType
         }
-
         tryCompiletimeConstantFold.orElse(tryMatchAlias)
-
       case _ =>
         NoType
     }
 
+    /** Does this application expand to a match type? */
+    def isMatchAlias(using Context): Boolean = tycon.stripTypeVar match
+      case tycon: TypeRef =>
+        tycon.info match
+          case _: MatchAlias => true
+          case _ => false
+      case _ => false
+
+    /** Is this an unreducible application to wildcard arguments?
+     *  This is the case if tycon is higher-kinded. This means
+     *  it is a subtype of a hk-lambda, but not a match alias.
+     *  (normal parameterized aliases are removed in `appliedTo`).
+     *  Applications of hgher-kinded type constructors to wildcard arguments
+     *  are equivalent to existential types, which are not supported.
+     */
+    def isUnreducibleWild(using Context): Boolean =
+      tycon.isLambdaSub && hasWildcardArg && !isMatchAlias
+
     def tryCompiletimeConstantFold(using Context): Type = tycon match {
       case tycon: TypeRef if defn.isCompiletimeAppliedType(tycon.symbol) =>
         def constValue(tp: Type): Option[Any] = tp.dealias match {

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -113,7 +113,7 @@ object Checking {
 
     def checkWildcardApply(tp: Type): Unit = tp match {
       case tp @ AppliedType(tycon, _) =>
-        if (tycon.isLambdaSub && tp.hasWildcardArg)
+        if tp.isUnreducibleWild then
           report.errorOrMigrationWarning(
             showInferred(UnreducibleApplication(tycon), tp, tpt),
             tree.srcPos)

compiler/test/dotc/pos-test-pickling.blacklist

@@ -35,6 +35,7 @@ i7872.scala
 i11236.scala
 i11247.scala
 i11250
+i9999.scala
 
 # Opaque type
 i5720.scala

tests/pos/i9999.scala

@@ -0,0 +1,47 @@
+case class A();
+case class B();
+
+type M2[X <: A|B] = X match {
+  case A => A
+  case B => B
+}
+
+def f2(x: A|B): M2[x.type] = x match {
+  case _: A => A()
+  case _: B => B()
+}
+
+type M1[X <: A|B] = X match {
+  case A => A
+  case B => (x: A|B) => M2[x.type]
+}
+
+def f1(x: A|B): M1[x.type] = x match {
+  case _: A => A()
+  case _: B => (x: A|B) => f2(x)
+}
+
+case class More(); case class Stop();
+
+sealed abstract class DSL
+case class Fun[F <: More|Stop => DSL](cont: F) extends DSL
+case class Nop() extends DSL
+
+type Match2[X <: More|Stop] <: DSL = X match {
+  case More => Fun[(y: More|Stop) => Match1[y.type]]
+  case Stop => Nop
+}
+type Match1[X] <: DSL = X match {
+  case More => Nop
+  case Stop => Nop
+}
+
+def fun2(x: More|Stop): Match2[x.type] = x match {
+  case _: More => Fun(fun1)  // error
+  case _: Stop => Nop()
+}
+
+def fun1(y: More|Stop): Match1[y.type] = y match {
+  case _: More => Nop()
+  case _: Stop => Nop()
+}