[WIP] Defer loading package.class to allow package.scala to be named

build.sbt

@@ -1041,6 +1041,10 @@ lazy val junit = project.in(file("test") / "junit")
     javaOptions in Test += "-Xss1M",
     (forkOptions in Test) := (forkOptions in Test).value.withWorkingDirectory((baseDirectory in ThisBuild).value),
     (forkOptions in Test in testOnly) := (forkOptions in Test in testOnly).value.withWorkingDirectory((baseDirectory in ThisBuild).value),
+    (forkOptions in Test in run) := {
+      val existing = (forkOptions in Test in run).value
+      existing.withRunJVMOptions(existing.runJVMOptions ++ List("-Xmx2G", "-Xss1M"))
+    },
     libraryDependencies ++= Seq(junitDep, junitInterfaceDep, jolDep),
     testOptions += Tests.Argument(TestFrameworks.JUnit, "-a", "-v"),
     unmanagedSourceDirectories in Compile := Nil,

src/compiler/scala/tools/nsc/Global.scala

@@ -390,7 +390,9 @@ class Global(var currentSettings: Settings, reporter0: Reporter)
 
 // ------------ Phases -------------------------------------------}
 
-  var globalPhase: Phase = NoPhase
+  private[this] var _globalPhase: Phase = NoPhase
+  override final def globalPhase: Phase = _globalPhase
+  final def globalPhase_=(ph: Phase): Unit = _globalPhase = ph
 
   abstract class GlobalPhase(prev: Phase) extends Phase(prev) {
     phaseWithId(id) = this

src/compiler/scala/tools/nsc/backend/jvm/BTypesFromSymbols.scala

@@ -17,6 +17,7 @@ import scala.reflect.internal.Flags.{DEFERRED, SYNTHESIZE_IMPL_IN_SUBCLASS}
 import scala.tools.asm
 import scala.tools.nsc.backend.jvm.BTypes._
 import scala.tools.nsc.backend.jvm.BackendReporting._
+import scala.tools.nsc.symtab.Flags
 
 /**
  * This class mainly contains the method classBTypeFromSymbol, which extracts the necessary
@@ -701,7 +702,7 @@ abstract class BTypesFromSymbols[G <: Global](val global: G) extends BTypes {
     // new instances via outerClassInstance.new InnerModuleClass$().
     // TODO: do this early, mark the symbol private.
     val privateFlag =
-      sym.isPrivate || (sym.isPrimaryConstructor && isTopLevelModuleClass(sym.owner))
+      (sym.isPrivate || sym.hasFlag(Flags.notPRIVATE)) || (sym.isPrimaryConstructor && isTopLevelModuleClass(sym.owner))
 
     // Symbols marked in source as `final` have the FINAL flag. (In the past, the flag was also
     // added to modules and module classes, not anymore since 296b706).

src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

@@ -47,9 +47,13 @@ abstract class SymbolLoaders {
   protected def compileLate(srcfile: AbstractFile): Unit
 
   protected def enterIfNew(owner: Symbol, member: Symbol, completer: SymbolLoader): Symbol = {
-    assert(owner.info.decls.lookup(member.name) == NoSymbol, owner.fullName + "." + member.name)
-    owner.info.decls enter member
-    member
+    owner.info.decls.lookup(member.name) match {
+      case NoSymbol =>
+        owner.info.decls enter member
+        member
+      case member =>
+        member
+    }
   }
 
   protected def signalError(root: Symbol, ex: Throwable) {
@@ -280,10 +284,11 @@ abstract class SymbolLoaders {
       val shownPackageName = if (packageName == ClassPath.RootPackage) "<root package>" else packageName
       s"package loader $shownPackageName"
     }
+    override val decls = newScope
 
     protected def doComplete(root: Symbol) {
       assert(root.isPackageClass, root)
-      root.setInfo(new PackageClassInfoType(newScope, root))
+      root.setInfo(new PackageClassInfoType(decls, root))
 
       val classPathEntries = classPath.list(packageName)
 

src/compiler/scala/tools/nsc/typechecker/Namers.scala

@@ -762,7 +762,8 @@ trait Namers extends MethodSynthesis {
     def enterPackage(tree: PackageDef) {
       val sym = createPackageSymbol(tree.pos, tree.pid)
       tree.symbol = sym
-      newNamer(context.make(tree, sym.moduleClass, sym.info.decls)) enterSyms tree.stats
+      val scope = sym.rawInfo.typeSymbolDirect.rawInfo.decls
+      newNamer(context.make(tree, sym.moduleClass, scope)) enterSyms tree.stats
     }
 
     private def enterImport(tree: Import) = {
@@ -2161,7 +2162,7 @@ trait Namers extends MethodSynthesis {
     // scala/bug#7264 Force the info of owners from previous compilation runs.
     //         Doing this generally would trigger cycles; that's what we also
     //         use the lower-level scan through the current Context as a fall back.
-    if (!currentRun.compiles(owner)) owner.initialize
+    if (!owner.isPackageClass && !currentRun.compiles(owner)) owner.initialize
 
     if (original.isModuleClass) original.sourceModule
     else if (!owner.isTerm && owner.hasCompleteInfo)

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -1863,9 +1863,14 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
     def typedModuleDef(mdef: ModuleDef): Tree = {
       // initialize all constructors of the linked class: the type completer (Namer.methodSig)
       // might add default getters to this object. example: "object T; class T(x: Int = 1)"
-      val linkedClass = companionSymbolOf(mdef.symbol, context)
-      if (linkedClass != NoSymbol)
+      val linkedClass = companionSymbolOf(mdef.symbol, context).suchThat { sym =>
+        val isStale = sym.rawInfo.isInstanceOf[loaders.ClassfileLoader]
+        if (isStale) sym.owner.info.decls.unlink(sym)
+        !isStale
+      }
+      if (linkedClass != NoSymbol) {
         linkedClass.info.decl(nme.CONSTRUCTOR).alternatives foreach (_.initialize)
+      }
 
       val clazz = mdef.symbol.moduleClass
       currentRun.profiler.beforeTypedImplDef(clazz)

src/reflect/scala/reflect/internal/SymbolTable.scala

@@ -244,6 +244,7 @@ abstract class SymbolTable extends macros.Universe
   final def phase: Phase = {
     ph
   }
+  def globalPhase: Phase = phase
 
   def atPhaseStackMessage = atPhaseStack match {
     case Nil    => ""
@@ -352,6 +353,7 @@ abstract class SymbolTable extends macros.Universe
 
   def openPackageModule(container: Symbol, dest: Symbol) {
     // unlink existing symbols in the package
+    assert(globalPhase.name != "namer")
     for (member <- container.info.decls.iterator) {
       if (!member.isPrivate && !member.isConstructor) {
         // todo: handle overlapping definitions in some way: mark as errors

src/reflect/scala/reflect/internal/Types.scala

@@ -3541,7 +3541,7 @@ trait Types
 // Creators ---------------------------------------------------------------
 
   /** Rebind symbol `sym` to an overriding member in type `pre`. */
-  private def rebind(pre: Type, sym: Symbol): Symbol = {
+  private[scala] def rebind(pre: Type, sym: Symbol): Symbol = {
     if (!sym.isOverridableMember || sym.owner == pre.typeSymbol) sym
     else pre.nonPrivateMember(sym.name).suchThat { sym =>
       // scala/bug#7928 `isModuleNotMethod` is here to avoid crashing with spuriously "overloaded" module accessor and module symbols.

test/junit/scala/tools/nsc/DeterminismTest.scala

@@ -1,19 +1,16 @@
 package scala.tools.nsc
 
-import java.io.OutputStreamWriter
-import java.nio.charset.Charset
-import java.nio.file.attribute.BasicFileAttributes
-import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor}
+import java.nio.file.{Files, Path, Paths}
 
-import javax.tools.ToolProvider
-import org.junit.Test
+import org.junit.{Ignore, Test}
 
-import scala.collection.JavaConverters.seqAsJavaListConverter
 import scala.reflect.internal.util.{BatchSourceFile, SourceFile}
-import scala.tools.nsc.reporters.StoreReporter
-import FileUtils._
+import scala.reflect.io.AbstractFile
 
 class DeterminismTest {
+  private val tester = new DeterminismTester
+  import tester.test
+
   @Test def testLambdaLift(): Unit = {
     def code = List[SourceFile](
       source("a.scala",
@@ -300,65 +297,29 @@ class DeterminismTest {
     test(List(code))
   }
 
-  def source(name: String, code: String): SourceFile = new BatchSourceFile(name, code)
-  private def test(groups: List[List[SourceFile]]): Unit = {
-    val referenceOutput = Files.createTempDirectory("reference")
-
-    def compile(output: Path, files: List[SourceFile]): Unit = {
-      val g = new Global(new Settings)
-      g.settings.usejavacp.value = true
-      g.settings.classpath.value = output.toAbsolutePath.toString
-      g.settings.outputDirs.setSingleOutput(output.toString)
-      val storeReporter = new StoreReporter
-      g.reporter = storeReporter
-      import g._
-      val r = new Run
-      // println("scalac " + files.mkString(" "))
-      r.compileSources(files)
-      Predef.assert(!storeReporter.hasErrors, storeReporter.infos.mkString("\n"))
-      files.filter(_.file.name.endsWith(".java")) match {
-        case Nil =>
-        case javaSources =>
-          def tempFileFor(s: SourceFile): Path = {
-            val f = output.resolve(s.file.name)
-            Files.write(f, new String(s.content).getBytes(Charset.defaultCharset()))
-          }
-          val options = List("-d", output.toString)
-          val javac = ToolProvider.getSystemJavaCompiler
-          assert(javac != null, "No javac from getSystemJavaCompiler. If the java on your path isn't a JDK version, but $JAVA_HOME is, launch sbt with --java-home \"$JAVA_HOME\"")
-          val fileMan = javac.getStandardFileManager(null, null, null)
-          val javaFileObjects = fileMan.getJavaFileObjects(javaSources.map(s => tempFileFor(s).toAbsolutePath.toString): _*)
-          val task = javac.getTask(new OutputStreamWriter(System.out), fileMan, null, options.asJava, Nil.asJava, javaFileObjects)
-          val result = task.call()
-          Predef.assert(result)
-      }
-    }
-
-    for (group <- groups.init) {
-      compile(referenceOutput, group)
-    }
-    compile(referenceOutput, groups.last)
-
-    class CopyVisitor(src: Path, dest: Path) extends SimpleFileVisitor[Path] {
-      override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = {
-        Files.createDirectories(dest.resolve(src.relativize(dir)))
-        super.preVisitDirectory(dir, attrs)
-      }
-      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
-        Files.copy(file, dest.resolve(src.relativize(file)))
-        super.visitFile(file, attrs)
-      }
-    }
-    for (permutation <- permutationsWithSubsets(groups.last)) {
-      val recompileOutput = Files.createTempDirectory("recompileOutput")
-      copyRecursive(referenceOutput, recompileOutput)
-      compile(recompileOutput, permutation)
-      assertDirectorySame(referenceOutput, recompileOutput, permutation.toString)
-      deleteRecursive(recompileOutput)
-    }
-    deleteRecursive(referenceOutput)
+  @Test def testReferenceToInnerClassMadeNonPrivate(): Unit = {
+    def code = List[SourceFile](
+      source("t.scala",
+             """
+               | trait T {
+               |   private class Inner
+               |   class OtherInner { new Inner } // triggers makeNotPrivate of Inner
+               |   private val v: Option[Inner] = None
+               | }
+        """.stripMargin),
+      source("c.scala","""class C extends T""")
+      )
+    test(List(code))
+  }
 
+  @Test def testPackageObjectUserLand(): Unit = {
+    def code = List[SourceFile](
+      source("package.scala", "package userland; object `package` { type Throwy = java.lang.Throwable }"),
+      source("th.scala", "package userland; class th[T <: Throwy](cause: T = null)")
+    )
+    test(code :: Nil, permuter = _.reverse :: Nil)
   }
-  def permutationsWithSubsets[A](as: List[A]): List[List[A]] =
-    as.permutations.toList.flatMap(_.inits.filter(_.nonEmpty)).distinct
+
+  def source(name: String, code: String): SourceFile = new BatchSourceFile(name, code)
+  def source(path: Path): SourceFile = new BatchSourceFile(AbstractFile.getFile(path.toFile))
 }

test/junit/scala/tools/nsc/DeterminismTester.scala

@@ -0,0 +1,110 @@
+package scala.tools.nsc
+
+import java.io.OutputStreamWriter
+import java.nio.charset.Charset
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor}
+import scala.collection.JavaConverters._
+import javax.tools.ToolProvider
+
+import scala.reflect.internal.util.SourceFile
+import scala.reflect.io.AbstractFile
+import scala.tools.nsc.FileUtils._
+import scala.tools.nsc.reporters.StoreReporter
+import scala.reflect.internal.util.BatchSourceFile
+
+object DeterminismTester extends DeterminismTester {
+  def main(args: Array[String]): Unit = {
+    val (scalacOpts, sourceFilesPaths) = args.indexOf("--") match {
+      case -1 => (Nil, args.toList)
+      case i =>
+        val tuple = args.toList.splitAt(i)
+        (tuple._1, tuple._2.drop(1))
+    }
+    def isJavaOrScala(p: Path) = {
+      val name = p.getFileName.toString
+      name.endsWith(".java") || name.endsWith(".scala")
+    }
+    def expand(path: Path): Seq[Path] = {
+      if (Files.isDirectory(path))
+        Files.walk(path).iterator().asScala.filter(isJavaOrScala).toList
+      else path :: Nil
+    }
+    val sourceFiles = sourceFilesPaths.map(Paths.get(_)).flatMap(expand).map(path => new BatchSourceFile(AbstractFile.getFile(path.toFile)))
+    test(sourceFiles :: Nil, scalacOpts)
+  }
+}
+
+class DeterminismTester {
+
+  def test(groups: List[List[SourceFile]]): Unit = test(groups, Nil)
+  def test(groups: List[List[SourceFile]], scalacOptions: List[String] = Nil,
+           permuter: List[SourceFile] => List[List[SourceFile]] = defaultPermuter(_)): Unit = {
+    val referenceOutput = Files.createTempDirectory("reference")
+
+    def compile(output: Path, files: List[SourceFile]): Unit = {
+       println("====== compile: " + files.map(_.file.absolute).map("\"" + _ + "\"").mkString(",\n"))
+      val g = new Global(new Settings)
+      g.settings.usejavacp.value = true
+      g.settings.classpath.value = output.toAbsolutePath.toString
+      g.settings.outputDirs.setSingleOutput(output.toString)
+      g.settings.processArguments(scalacOptions, true)
+      val storeReporter = new StoreReporter
+      g.reporter = storeReporter
+      import g._
+      val r = new Run
+      // println("scalac " + files.mkString(" "))
+      r.compileSources(files)
+      Predef.assert(!storeReporter.hasErrors, storeReporter.infos.mkString("\n"))
+      files.filter(_.file.name.endsWith(".java")) match {
+        case Nil =>
+        case javaSources =>
+          def tempFileFor(s: SourceFile): Path = {
+            val f = output.resolve(s.file.name)
+            Files.write(f, new String(s.content).getBytes(Charset.defaultCharset()))
+          }
+          val options = List("-d", output.toString)
+          val javac = ToolProvider.getSystemJavaCompiler
+          assert(javac != null, "No javac from getSystemJavaCompiler. If the java on your path isn't a JDK version, but $JAVA_HOME is, launch sbt with --java-home \"$JAVA_HOME\"")
+          val fileMan = javac.getStandardFileManager(null, null, null)
+          val javaFileObjects = fileMan.getJavaFileObjects(javaSources.map(s => tempFileFor(s).toAbsolutePath.toString): _*)
+          val task = javac.getTask(new OutputStreamWriter(System.out), fileMan, null, options.asJava, Nil.asJava, javaFileObjects)
+          val result = task.call()
+          Predef.assert(result)
+      }
+    }
+
+    for (group <- groups.init) {
+      compile(referenceOutput, group)
+    }
+    compile(referenceOutput, groups.last)
+
+    class CopyVisitor(src: Path, dest: Path) extends SimpleFileVisitor[Path] {
+      override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = {
+        Files.createDirectories(dest.resolve(src.relativize(dir)))
+        super.preVisitDirectory(dir, attrs)
+      }
+      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+        Files.copy(file, dest.resolve(src.relativize(file)))
+        super.visitFile(file, attrs)
+      }
+    }
+    val permutations: List[List[SourceFile]] = permuter(groups.last)
+    for (permutation <- permutations) {
+      val recompileOutput = Files.createTempDirectory("recompileOutput")
+      copyRecursive(referenceOutput, recompileOutput)
+      compile(recompileOutput, permutation)
+      assertDirectorySame(referenceOutput, recompileOutput, permutation.toString)
+      deleteRecursive(recompileOutput)
+    }
+    deleteRecursive(referenceOutput)
+  }
+  def defaultPermuter(sources: List[SourceFile]): List[List[SourceFile]] = {
+    if (sources.size > 32) {
+      sources.reverse :: sources.map(_ :: Nil)
+    } else permutationsWithSubsets(sources)
+  }
+  def permutationsWithSubsets[A](as: List[A]): List[List[A]] =
+    as.permutations.toList.flatMap(_.inits.filter(_.nonEmpty)).distinct
+
+}