Fix the visibility check in markFree (#20221)

Fixes #20169. Fixes #20224.

It turns out that the type argument in #20169 is properly boxed. It is
just that when doing box adaptation, when charging `cap` into
environments it gets discarded by the visibility check.

Right now, a symbol is considered visible in an environment only when
the owner of the environment is contained in the symbol. This is not
right: when there is not containment relation between the symbol and the
environment owner the symbol is also visible from the environment, which
is the case here for `cap`.

Author: odersky

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -358,9 +358,17 @@ class CheckCaptures extends Recheck, SymTransformer:
     def markFree(cs: CaptureSet, pos: SrcPos)(using Context): Unit =
       if !cs.isAlwaysEmpty then
         forallOuterEnvsUpTo(ctx.owner.topLevelClass): env =>
-          def isVisibleFromEnv(sym: Symbol) =
-            (env.kind == EnvKind.NestedInOwner || env.owner != sym)
-            && env.owner.isContainedIn(sym)
+          // Whether a symbol is defined inside the owner of the environment?
+          inline def isContainedInEnv(sym: Symbol) =
+            if env.kind == EnvKind.NestedInOwner then
+              sym.isProperlyContainedIn(env.owner)
+            else
+              sym.isContainedIn(env.owner)
+          // A captured reference with the symbol `sym` is visible from the environment
+          // if `sym` is not defined inside the owner of the environment
+          inline def isVisibleFromEnv(sym: Symbol) = !isContainedInEnv(sym)
+          // Only captured references that are visible from the environment
+          // should be included.
           val included = cs.filter:
             case ref: TermRef => isVisibleFromEnv(ref.symbol.owner)
             case ref: ThisType => isVisibleFromEnv(ref.cls)
@@ -378,6 +386,7 @@ class CheckCaptures extends Recheck, SymTransformer:
           // there won't be an apply; need to include call captures now
           includeCallCaptures(tree.symbol, tree.srcPos)
       else
+        //debugShowEnvs()
         markFree(tree.symbol, tree.srcPos)
       super.recheckIdent(tree, pt)
 
@@ -946,6 +955,19 @@ class CheckCaptures extends Recheck, SymTransformer:
           expected
     end addOuterRefs
 
+    /** A debugging method for showing the envrionments during capture checking. */
+    private def debugShowEnvs()(using Context): Unit =
+      def showEnv(env: Env): String = i"Env(${env.owner}, ${env.kind}, ${env.captured})"
+      val sb = StringBuilder()
+      @annotation.tailrec def walk(env: Env | Null): Unit =
+        if env != null then
+          sb ++= showEnv(env)
+          sb ++= "\n"
+          walk(env.outer0)
+      sb ++= "===== Current Envs ======\n"
+      walk(curEnv)
+      sb ++= "===== End          ======\n"
+      println(sb.result())
 
     /** Adapt `actual` type to `expected` type by inserting boxing and unboxing conversions
      *
@@ -1085,6 +1107,7 @@ class CheckCaptures extends Recheck, SymTransformer:
                   pos)
                 }
               if !insertBox then  // unboxing
+                //debugShowEnvs()
                 markFree(criticalSet, pos)
               adaptedType(!boxed)
           else

compiler/src/dotty/tools/dotc/cc/Setup.scala

@@ -411,8 +411,12 @@ class Setup extends PreRecheck, SymTransformer, SetupAPI:
 
         case tree @ TypeApply(fn, args) =>
           traverse(fn)
-          for case arg: TypeTree <- args do
-            transformTT(arg, boxed = true, exact = false) // type arguments in type applications are boxed
+          fn match
+            case Select(qual, nme.asInstanceOf_) =>
+              // No need to box type arguments of an asInstanceOf call. See #20224.
+            case _ =>
+              for case arg: TypeTree <- args do
+                transformTT(arg, boxed = true, exact = false) // type arguments in type applications are boxed
 
         case tree: TypeDef if tree.symbol.isClass =>
           inContext(ctx.withOwner(tree.symbol)):

tests/neg-custom-args/captures/i16725.scala

@@ -0,0 +1,15 @@
+import language.experimental.captureChecking
+@annotation.capability
+class IO:
+  def brewCoffee(): Unit = ???
+def usingIO[T](op: IO => T): T = ???
+
+type Wrapper[T] = [R] -> (f: T => R) -> R
+def mk[T](x: T): Wrapper[T] = [R] => f => f(x)
+def useWrappedIO(wrapper: Wrapper[IO]): () -> Unit =
+  () =>
+    wrapper: io => // error
+      io.brewCoffee()
+def main(): Unit =
+  val escaped = usingIO(io => useWrappedIO(mk(io)))
+  escaped()  // boom

tests/neg-custom-args/captures/i20169.scala

@@ -0,0 +1,8 @@
+case class Box[T](x: T):
+  def foreach(f: T => Unit): Unit = f(x)
+
+def runOps(ops: Box[() => Unit]): () -> Unit =
+  val applyFn: (() => Unit) -> Unit = f => f()
+  val fn: () -> Unit = () =>
+    ops.foreach(applyFn)  // error
+  fn