Merge branch 'main' into implement/Xlint/shadow_warnings

Author: schuetzcarl

.github/workflows/ci.yaml

@@ -134,7 +134,7 @@ jobs:
 
       - name: Cmd Tests
         run: |
-          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
+          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped-tasty-tests/run ;stdlib-bootstrapped-tasty-tests/test; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 
@@ -255,6 +255,11 @@ jobs:
         run: |
           ./project/scripts/sbt ";scala3-interfaces/mimaReportBinaryIssues ;scala3-library-bootstrapped/mimaReportBinaryIssues ;scala3-library-bootstrappedJS/mimaReportBinaryIssues; tasty-core-bootstrapped/mimaReportBinaryIssues; stdlib-bootstrapped/mimaReportBinaryIssues"
 
+      - name: TASTy MiMa
+        run: |
+          # This script cleans the compiler and recompiles it from scratch (keep as last run)
+          ./project/scripts/stdlib-bootstrapped-tasty-mima.sh
+
   community_build_a:
     runs-on: [self-hosted, Linux]
     container:
@@ -488,7 +493,7 @@ jobs:
 
       - name: Test
         run: |
-          ./project/scripts/sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped/test:run ;stdlib-bootstrapped-tasty-tests/test"
+          ./project/scripts/sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;stdlib-bootstrapped-tasty-tests/run ;stdlib-bootstrapped-tasty-tests/test"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 

.github/workflows/spec.yml

@@ -0,0 +1,55 @@
+name: Specification
+
+on:
+  push:
+    tags:
+      - '*'
+    branches-ignore:
+      - 'gh-readonly-queue/**'
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+env:
+  DOTTY_CI_RUN: true
+
+jobs:
+  specification:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./docs/_spec
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Keep in sync with ./docs/_spec/Dockerfile
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+      - name: Install required gems
+        run: |
+          gem update --system
+          gem install sass-embedded -v 1.58.0
+          gem install bundler:1.17.2 jekyll
+          bundle install
+          npm install bower
+
+      - name: Build the specification
+        run: |
+          bundle exec jekyll build
+
+      # Deploy
+      - name: Deployment
+        env:
+          USER_FOR_TEST: ${{ secrets.SPEC_DEPLOY_USER }}
+        if: ${{ env.USER_FOR_TEST != '' }}
+        uses: burnett01/rsync-deployments@6.0.0
+        with:
+          switches: -rzv
+          path: docs/_spec/_site/
+          remote_path: ${{ secrets.SPEC_DEPLOY_PATH }}
+          remote_host: ${{ secrets.SPEC_DEPLOY_HOST }}
+          remote_user: ${{ secrets.SPEC_DEPLOY_USER }}
+          remote_key: ${{ secrets.SPEC_DEPLOY_KEY }}
+          remote_key_pass: ${{ secrets.SPEC_DEPLOY_PASS }}

.gitignore

@@ -99,3 +99,6 @@ docs/_spec/.jekyll-metadata
 # scaladoc related
 scaladoc/output/
 
+#coverage
+coverage/
+

.gitmodules

@@ -28,10 +28,6 @@
 [submodule "community-build/community-projects/scala-xml"]
 	path = community-build/community-projects/scala-xml
 	url = https://github.com/dotty-staging/scala-xml
-[submodule "community-build/community-projects/shapeless"]
-	path = community-build/community-projects/shapeless
-	url = https://github.com/dotty-staging/shapeless
-	branch = shapeless-3-staging
 [submodule "community-build/community-projects/xml-interpolator"]
 	path = community-build/community-projects/xml-interpolator
 	url = https://github.com/dotty-staging/xml-interpolator.git
@@ -222,3 +218,6 @@
 [submodule "community-build/community-projects/parboiled2"]
 	path = community-build/community-projects/parboiled2
 	url = https://github.com/dotty-staging/parboiled2.git
+[submodule "community-build/community-projects/shapeless-3"]
+	path = community-build/community-projects/shapeless-3
+	url = https://github.com/dotty-staging/shapeless-3.git

NOTICE.md

@@ -89,15 +89,19 @@ major authors were omitted by oversight.
     details.
 
   * dotty.tools.dotc.coverage: Coverage instrumentation utilities have been
-    adapted from the scoverage plugin for scala 2 [5], which is under the
+    adapted from the scoverage plugin for scala 2 [4], which is under the
     Apache 2.0 license.
 
+  * dooty.tools.pc: Presentation compiler implementation adapted from
+    scalameta/metals [5] mtags module, which is under the Apache 2.0 license.
+
   * The Dotty codebase contains parts which are derived from
-    the ScalaPB protobuf library [4], which is under the Apache 2.0 license.
+    the ScalaPB protobuf library [6], which is under the Apache 2.0 license.
 
 
 [1] https://github.com/scala/scala
 [2] https://github.com/adriaanm/scala/tree/sbt-api-consolidate/src/compiler/scala/tools/sbt
 [3] https://github.com/sbt/sbt/tree/0.13/compile/interface/src/main/scala/xsbt
-[4] https://github.com/lampepfl/dotty/pull/5783/files
-[5] https://github.com/scoverage/scalac-scoverage-plugin
+[4] https://github.com/scoverage/scalac-scoverage-plugin
+[5] https://github.com/scalameta/metals
+[6] https://github.com/lampepfl/dotty/pull/5783/files

bench-run/src/main/scala/dotty/tools/benchmarks/Main.scala

@@ -68,7 +68,7 @@ object Bench {
     println()
     println("Usage:")
     println()
-    println("dotty-bench-run/jmh:run [<warmup>] [<iterations>] [<forks>] <regexp> [<input>|--] [<output>]")
+    println("scala3-bench-run/jmh:run [<warmup>] [<iterations>] [<forks>] <regexp> [<input>|--] [<output>]")
     println()
     println("warmup: warmup iterations. defaults to 20.")
     println("iterations: benchmark iterations. defaults to 20.")

bench/scripts/collection-vector.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-sbt "dotty-bench-bootstrapped/jmh:run 40 40 3 bench/tests/Vector.scala"
+sbt "scala3-bench-bootstrapped/jmh:run 40 40 3 bench/tests/Vector.scala"

bench/scripts/compiler-cold.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
+find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "scala3-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt

bench/scripts/compiler.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 5 10" {} + | sbt
+find compiler/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "scala3-bench-bootstrapped/jmh:run 5 10" {} + | sbt

bench/scripts/library-cold.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt
+find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "scala3-bench-bootstrapped/jmh:run 0 1 10" {} + | sbt

bench/scripts/library.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "dotty-bench-bootstrapped/jmh:run 40 30" {} + | sbt
+find library/src/ -type f \( -name "*.scala" -or -name "*.java" \) -exec echo "scala3-bench-bootstrapped/jmh:run 40 30" {} + | sbt

build.sbt

@@ -15,6 +15,7 @@ val `scala3-bench` = Build.`scala3-bench`
 val `scala3-bench-bootstrapped` = Build.`scala3-bench-bootstrapped`
 val `scala3-bench-micro` = Build.`scala3-bench-micro`
 val `stdlib-bootstrapped` = Build.`stdlib-bootstrapped`
+val `stdlib-bootstrapped-tasty` = Build.`stdlib-bootstrapped-tasty`
 val `stdlib-bootstrapped-tasty-tests` = Build.`stdlib-bootstrapped-tasty-tests`
 val `tasty-core` = Build.`tasty-core`
 val `tasty-core-bootstrapped` = Build.`tasty-core-bootstrapped`
@@ -28,6 +29,8 @@ val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
+val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`
+val `scala3-presentation-compiler-bootstrapped` = Build.`scala3-presentation-compiler-bootstrapped`
 
 val sjsSandbox = Build.sjsSandbox
 val sjsJUnitTests = Build.sjsJUnitTests

community-build/community-projects/izumi-reflect

@@ -140,7 +140,7 @@ final case class SbtCommunityProject(
       case Some(ivyHome) => List(s"-Dsbt.ivy.home=$ivyHome")
       case _ => Nil
     extraSbtArgs ++ sbtProps ++ List(
-      "-sbt-version", "1.8.2",
+      "-sbt-version", "1.9.3",
       "-Dsbt.supershell=false",
       s"-Ddotty.communitybuild.dir=$communitybuildDir",
       s"--addPluginSbtFile=$sbtPluginFilePath"
@@ -366,13 +366,11 @@ object projects:
     // sbtDocCommand = "library/doc" // Does no compile? No idea :/
   )
 
-
-  lazy val shapeless = SbtCommunityProject(
-    project       = "shapeless",
-    sbtTestCommand = """set deriving/scalacOptions -= "-Xfatal-warnings"; set typeable/scalacOptions -= "-Xfatal-warnings"; test""",
-      // selectively disable -Xfatal-warnings due to deprecations
-    sbtDocCommand = forceDoc("typeable", "deriving", "data"),
-    scalacOptions = Nil // disable -Ysafe-init, due to -Xfatal-warnings
+  lazy val shapeless3 = SbtCommunityProject(
+    project = "shapeless-3",
+    sbtTestCommand = "testJVM; testJS",
+    sbtDocCommand = forceDoc("typeable", "deriving"),
+    scalacOptions = SbtCommunityProject.scalacOptions.filter(_ != "-Ysafe-init"), // due to -Xfatal-warnings
   )
 
   lazy val xmlInterpolator = SbtCommunityProject(
@@ -682,7 +680,7 @@ object projects:
     sbtTestCommand = "runCommunityBuild",
     sbtPublishCommand = "publishLocal",
     dependencies = List(scalatest),
-    scalacOptions = List("-language:implicitConversions"), // disabled -Ysafe-init, due to bug in macro
+    scalacOptions = SbtCommunityProject.scalacOptions.filter(_ != "-Xcheck-macros") :+ "-language:implicitConversions", // disabled -Xcheck-macros, due to bug in macro
   )
 
   lazy val onnxScala = SbtCommunityProject(
@@ -796,7 +794,7 @@ def allProjects = List(
   projects.minitest,
   projects.fastparse,
   projects.stdLib213,
-  projects.shapeless,
+  projects.shapeless3,
   projects.xmlInterpolator,
   projects.effpi,
   projects.sconfig,

community-build/community-projects/perspective

@@ -91,9 +91,10 @@ class CommunityBuildTestC:
   @Test def scalaz = projects.scalaz.run()
   @Test def scas = projects.scas.run()
   @Test def sconfig = projects.sconfig.run()
-  @Test def shapeless = projects.shapeless.run()
+  @Test def shapeless3 = projects.shapeless3.run()
   @Test def sourcecode = projects.sourcecode.run()
   @Test def specs2 = projects.specs2.run()
+
   @Test def stdLib213 = projects.stdLib213.run()
   @Test def ujson = projects.ujson.run()
   @Test def upickle = projects.upickle.run()

community-build/community-projects/shapeless

@@ -148,7 +148,7 @@ object MainGenericRunner {
     case (o @ javaOption(striped)) :: tail =>
       processArgs(tail, settings.withJavaArgs(striped).withScalaArgs(o))
     case (o @ scalaOption(_*)) :: tail =>
-      val remainingArgs = (CommandLineParser.expandArg(o) ++ tail).toList
+      val remainingArgs = CommandLineParser.expandArg(o) ++ tail
       processArgs(remainingArgs, settings)
     case (o @ colorOption(_*)) :: tail =>
       processArgs(tail, settings.withScalaArgs(o))

community-build/community-projects/shapeless-3

@@ -763,9 +763,14 @@ trait BCodeSkelBuilder extends BCodeHelpers {
       for (p <- params) { locals.makeLocal(p.symbol) }
       // debug assert((params.map(p => locals(p.symbol).tk)) == asmMethodType(methSymbol).getArgumentTypes.toList, "debug")
 
-      if (params.size > MaximumJvmParameters) {
+      val paramsSize = params.map { param =>
+        val tpeTym = param.symbol.info.typeSymbol
+        if tpeTym == defn.LongClass || tpeTym == defn.DoubleClass then 2 else 1
+      }.sum
+      if (paramsSize > MaximumJvmParameters) {
         // SI-7324
-        report.error(em"Platform restriction: a parameter list's length cannot exceed $MaximumJvmParameters.", ctx.source.atSpan(methSymbol.span))
+        val info = if paramsSize == params.length then "" else " (Long and Double count as 2)" // https://docs.oracle.com/javase/specs/jvms/se8/html/jvms-4.html#jvms-4.3.3
+        report.error(em"Platform restriction: a parameter list's length cannot exceed $MaximumJvmParameters$info.", ctx.source.atSpan(methSymbol.span))
         return
       }
 

community-build/community-projects/stdLib213

@@ -30,6 +30,7 @@ import scala.tools.asm
 import scala.tools.asm.tree._
 import tpd._
 import dotty.tools.io.AbstractFile
+import dotty.tools.dotc.util
 import dotty.tools.dotc.util.NoSourcePosition
 
 
@@ -106,7 +107,7 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
   }
 
   // Creates a callback that will be evaluated in PostProcessor after creating a file
-  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: interfaces.SourceFile): AbstractFile => Unit = clsFile => {
+  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: util.SourceFile): AbstractFile => Unit = clsFile => {
     val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
       (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
     }
@@ -115,12 +116,9 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
     if (ctx.compilerCallback != null)
       ctx.compilerCallback.onClassGenerated(sourceFile, convertAbstractFile(clsFile), className)
 
-    if (ctx.sbtCallback != null) {
-      val jSourceFile = sourceFile.jfile.orElse(null)
-      val cb = ctx.sbtCallback
-      if (isLocal) cb.generatedLocalClass(jSourceFile, clsFile.file)
-      else cb.generatedNonLocalClass(jSourceFile, clsFile.file, className, fullClassName)
-    }
+    ctx.withIncCallback: cb =>
+      if (isLocal) cb.generatedLocalClass(sourceFile, clsFile.jpath)
+      else cb.generatedNonLocalClass(sourceFile, clsFile.jpath, className, fullClassName)
   }
 
   /** Convert a `dotty.tools.io.AbstractFile` into a

community-build/src/scala/dotty/communitybuild/projects.scala

@@ -23,6 +23,12 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
 
   def postProcessAndSendToDisk(generatedDefs: GeneratedDefs): Unit = {
     val GeneratedDefs(classes, tasty) = generatedDefs
+    if !ctx.settings.YoutputOnlyTasty.value then
+      postProcessClassesAndSendToDisk(classes)
+    postProcessTastyAndSendToDisk(tasty)
+  }
+
+  private def postProcessClassesAndSendToDisk(classes: List[GeneratedClass]): Unit = {
     for (GeneratedClass(classNode, sourceFile, isArtifact, onFileCreated) <- classes) {
       val bytes =
         try
@@ -46,8 +52,10 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
         if clsFile != null then onFileCreated(clsFile)
       }
     }
+  }
 
-    for (GeneratedTasty(classNode, binaryGen) <- tasty){
+  private def postProcessTastyAndSendToDisk(tasty: List[GeneratedTasty]): Unit = {
+    for (GeneratedTasty(classNode, binaryGen) <- tasty) {
       classfileWriter.writeTasty(classNode.name.nn, binaryGen())
     }
   }

community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala

@@ -5,6 +5,7 @@ import core._
 import Contexts._
 import SymDenotations.ClassDenotation
 import Symbols._
+import Comments.Comment
 import util.{FreshNameCreator, SourceFile, NoSource}
 import util.Spans.Span
 import ast.{tpd, untpd}
@@ -69,11 +70,23 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Can this compilation unit be suspended */
   def isSuspendable: Boolean = true
 
+  /** List of all comments present in this compilation unit */
+  var comments: List[Comment] = Nil
+
+  /** This is used to record dependencies to invalidate during incremental
+   *  compilation, but only if `ctx.runZincPhases` is true.
+   */
+  val depRecorder: sbt.DependencyRecorder = sbt.DependencyRecorder()
+
   /** Suspends the compilation unit by thowing a SuspendException
    *  and recording the suspended compilation unit
    */
   def suspend()(using Context): Nothing =
     assert(isSuspendable)
+    // Clear references to symbols that may become stale. No need to call
+    // `depRecorder.sendToZinc()` since all compilation phases will be rerun
+    // when this unit is unsuspended.
+    depRecorder.clear()
     if !suspended then
       if (ctx.settings.XprintSuspension.value)
         report.echo(i"suspended: $this")

compiler/src/dotty/tools/MainGenericRunner.scala

@@ -240,15 +240,16 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 
       for (phase <- ctx.base.allPhases)
         if (phase.isRunnable)
-          Stats.trackTime(s"$phase ms ") {
+          Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
             units = phase.runOn(units)
             profiler.afterPhase(phase, profileBefore)
             if (ctx.settings.Xprint.value.containsPhase(phase))
               for (unit <- units)
-                lastPrintedTree =
-                  printTree(lastPrintedTree)(using ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
+                def printCtx(unit: CompilationUnit) = phase.printingContext(
+                  ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
+                lastPrintedTree = printTree(lastPrintedTree)(using printCtx(unit))
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)
@@ -308,7 +309,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
 
   private def printTree(last: PrintedTree)(using Context): PrintedTree = {
     val unit = ctx.compilationUnit
-    val fusedPhase = ctx.phase.prevMega
+    val fusedPhase = ctx.phase.prev.megaPhase
     val echoHeader = f"[[syntax trees at end of $fusedPhase%25s]] // ${unit.source}"
     val tree = if ctx.isAfterTyper then unit.tpdTree else unit.untpdTree
     val treeString = fusedPhase.show(tree)