Introduce best-effort compilation for IDEs

2 new experimental options are introduces for the compiler:
`-Ybest-effort` and `-Ywith-best-effort-tasty`. A related
Best Effort TASTy (.betasty) format, a TASTy aligned file format able
to hold some errored trees was also added. Behaviour of the options and
the format is documented as part of this commit in the
`best-effort-compilation.md` docs file.

`-Ybest-effort` is used to produce `.betasty` files in the
`<output>/META-INF/best-effort`. `-Ywith-best-effort-tasty` allows to
use them during compilation, limiting it to the frontend phases if such
file is used. If any .betasty is used, transparent inline macros also
cease to be expanded by the compiler.

Since best-effort compilation can fail (e.g. due to cyclic reference
errors which sometimes are not able to be pickled or unpickled),
the crashes caused by it are wrapped into an additional descriptive
error message in the aim to fail more gracefully (and not pollute our
issue tracker with known problems).

The feature is tested in two ways:
 * with a set of pairs of dependent projects, one of which is
 meant to produce .betasty by using `-Ybest-effort`, and the other
 tries to consume it using `-Ywith-best-effort-tasty`.
 * by reusing the compiler nonbootstrapped neg tests, first by running
 them with `-Ybest-effort` option, and then by running read-tasty tests
 on the produced betasty files to thest best-effort tastt unpickling

Additionally, `-Ywith-best-effort-tasty` allows to print `.betasty` via
`-print-tasty`.

Author: jchyb

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -17,6 +17,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usesBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -39,6 +39,8 @@ class Driver {
       catch
         case ex: FatalError =>
           report.error(ex.getMessage.nn) // signals that we should fail compilation.
+        case ex: Throwable if ctx.usesBestEffortTasty =>
+          report.bestEffortError(ex, "Some best-effort tasty files were not able to be read.")
         case ex: TypeError if !runOrNull.enrichedErrorMessage =>
           println(runOrNull.enrichErrorMessage(s"${ex.toMessage} while compiling ${files.map(_.path).mkString(", ")}"))
           throw ex
@@ -100,8 +102,8 @@ class Driver {
             None
           else file.extension match
             case "jar" => Some(file.path)
-            case "tasty" =>
-              TastyFileUtil.getClassPath(file) match
+            case "tasty" | "betasty" =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -297,6 +297,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       ctx.settings.Yskip.value, ctx.settings.YstopBefore.value, stopAfter, ctx.settings.Ycheck.value)
     ctx.base.usePhases(phases)
 
+    var forceReachPhaseMaybe =
+      if (ctx.isBestEffort && phases.exists(_.phaseName == "typer")) Some("typer")
+      else None
+
     if ctx.settings.YnoDoubleBindings.value then
       ctx.base.checkNoDoubleBindings = true
 
@@ -307,7 +311,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 
       for phase <- allPhases do
         doEnterPhase(phase)
-        val phaseWillRun = phase.isRunnable
+        val phaseWillRun = phase.isRunnable || forceReachPhaseMaybe.nonEmpty
         if phaseWillRun then
           Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
@@ -320,6 +324,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
                 def printCtx(unit: CompilationUnit) = phase.printingContext(
                   ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
                 lastPrintedTree = printTree(lastPrintedTree)(using printCtx(unit))
+
+            forceReachPhaseMaybe match {
+              case Some(forceReachPhase) if phase.phaseName == forceReachPhase =>
+                forceReachPhaseMaybe = None
+              case _ =>
+            }
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -929,12 +929,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      if !ctx.isBestEffort then assert(sym.isClass)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      if !ctx.isBestEffort then assert(sym.is(Module))
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -49,7 +49,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -58,7 +58,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -288,7 +288,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
   protected def isMatchingFile(f: JFile): Boolean =
-    f.isTasty || (f.isClass && f.classToTasty.isEmpty)
+    f.isTasty || f.isBestEffortTasty || (f.isClass && f.classToTasty.isEmpty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -24,8 +24,12 @@ object FileUtils {
 
     def hasTastyExtension: Boolean = file.hasExtension("tasty")
 
+    def hasBetastyExtension: Boolean = file.hasExtension("betasty")
+
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
+    def isBestEffortTasty: Boolean = !file.isDirectory && hasBetastyExtension
+
     def isScalaBinary: Boolean = file.isClass || file.isTasty
 
     def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
@@ -54,6 +58,8 @@ object FileUtils {
 
     def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_BETASTY)
+
     /** Returns the tasty file associated with this class file */
     def classToTasty: Option[JFile] =
       assert(file.isClass, s"non-class: $file")
@@ -66,6 +72,7 @@ object FileUtils {
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
   private val SUFFIX_TASTY = ".tasty"
+  private val SUFFIX_BETASTY = ".betasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -406,6 +406,9 @@ private sealed trait YSettings:
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting("-Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
 
+  val YbestEffort: Setting[Boolean] = BooleanSetting("-Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -472,6 +472,18 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is best-effort-dir option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffort.value
+
+    /** Is the from-best-effort-tasty option set to true? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usesBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsesBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 
@@ -959,6 +971,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()