Tests: Support two-stage compilation (src -> klib -> kexe)

HACKING.md

@@ -93,6 +93,11 @@ To update the blackbox compiler tests set TeamCity build number in `gradle.prope
 * **-Ptest_verbose** enables printing compiler args and other helpful information during a test execution.
 
         ./gradlew -Ptest_verbose :backend.native:tests:mpp_optional_expectation
+        
+* **-Ptest_two_stage** enables two-stage compilation of tests. If two-stage compilation is enabled, test sources are compiled into a klibrary
+and then a final native binary is produced from this klibrary using the -Xsource-library compiler flag.
+
+        ./gradlew -Ptest_two_stage backend.native:tests:array0
        
  ## Performance measurement
 

Interop/StubGenerator/build.gradle

@@ -31,7 +31,7 @@ repositories {
 
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib:$buildKotlinVersion"
-    compile "org.jetbrains.kotlin:kotlin-compiler:$buildKotlinVersion"
+    compile "org.jetbrains.kotlin:kotlin-compiler:$kotlinVersion"
     compile project(':Interop:Indexer')
     compile "org.jetbrains.kotlin:kotlin-native-shared:$konanVersion"
     compile project(path: ":endorsedLibraries:kliopt", configuration: "jvmRuntimeElements")

backend.native/tests/build.gradle

@@ -3,6 +3,7 @@
  * that can be found in the LICENSE file.
  */
 
+import shadow.org.jetbrains.kotlin.gradle.plugin.tasks.KonanCompileNativeBinary
 import org.jetbrains.kotlin.*
 import org.jetbrains.kotlin.konan.target.KonanTarget
 
@@ -16,7 +17,7 @@ buildscript {
         maven {
             url buildKotlinCompilerRepo
         }
-	maven {
+	    maven {
             url kotlinCompilerRepo
         }
     }
@@ -87,6 +88,13 @@ project.convention.plugins.executor = ExecutorServiceKt.create(project)
 // Do not generate run tasks for konan built artifacts
 ext.konanNoRun = true
 
+// Enable two-stage compilation if the test_two_stage property is set.
+if (project.hasProperty("test_two_stage")) {
+    tasks.withType(KonanCompileNativeBinary.class) {
+        enableTwoStageCompilation = true
+    }
+}
+
 allprojects {
     // Root directories for test output (logs, compiled files, statistics etc). Only single path must be in each set.
     // backend.native/tests

gradle.properties

@@ -26,3 +26,6 @@ testKotlinCompilerVersion=1.3.60-dev-1210
 konanVersion=1.3.60
 org.gradle.jvmargs='-Dfile.encoding=UTF-8'
 org.gradle.workers.max=4
+
+# Uncomment to enable composite build
+#kotlinProjectPath=<insert the path to Kotlin project root here>

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanBuildingTask.kt

@@ -47,16 +47,8 @@ abstract class KonanBuildingTask: KonanArtifactWithLibrariesTask(), KonanBuildin
     val konanVersion
         @Input get() = project.konanVersion.toString(true, true)
 
-    protected abstract fun buildArgs(): List<String>
-
     @TaskAction
-    open fun run() {
-        destinationDir.mkdirs()
-        if (dumpParameters) {
-            dumpProperties(this)
-        }
-        toolRunner.run(buildArgs())
-    }
+    abstract fun run()
 
     // DSL.
 

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanCompileTask.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright 2010-2017 JetBrains s.r.o.
+ * Copyright 2010-2019 JetBrains s.r.o.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,8 +25,6 @@ import org.gradle.process.CommandLineArgumentProvider
 import org.jetbrains.kotlin.gradle.plugin.konan.*
 import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifact
 import org.jetbrains.kotlin.gradle.plugin.model.KonanModelArtifactImpl
-import org.jetbrains.kotlin.konan.CURRENT
-import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.library.defaultResolver
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
 import org.jetbrains.kotlin.konan.target.Distribution
@@ -97,16 +95,100 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
     val apiVersion      : String?
         @Optional @Input get() = project.konanExtension.apiVersion
 
+    /**
+     * Is the two-stage compilation enabled.
+     *
+     * In regular (one-stage) compilation, sources are directly compiled into a final native binary.
+     * In two-stage compilation, sources are compiled into a klib first and then a final native binary is produced from this klib.
+     */
+    abstract val enableTwoStageCompilation: Boolean
+
     protected fun directoryToKt(dir: Any) = project.fileTree(dir).apply {
         include("**/*.kt")
         exclude { it.file.startsWith(project.buildDir) }
     }
 
     // Command line  ------------------------------------------------------------
 
-    override fun buildArgs() = mutableListOf<String>().apply {
+    // Exclude elements matching the predicate.
+    private fun List<String>.excludeFlags(predicate: (String) -> Boolean) = filterNot(predicate)
+
+    // Exclude the listed elements.
+    private fun List<String>.excludeFlags(vararg keys: String) = keys.toSet().let { keysToExclude ->
+        excludeFlags { it in keysToExclude }
+    }
+
+    // Exclude the arguments passed by the given keys.
+    // E.g. if the list contains the following elements: ["-l", "foo", "-r", "bar"],
+    // call exclude("-r") returns the following list: ["-l", "foo"].
+    private fun List<String>.excludeArguments(vararg args: String): List<String> {
+        val argsToExclude = args.toSet()
+        val xPrefixesToExclude = argsToExclude.filter { it.startsWith("-X") }.map { "$it=" }
+        val result = mutableListOf<String>()
+
+        var i = 0
+        while (i < size) {
+            val key = this[i]
+            when {
+                key in argsToExclude -> {
+                    // Skip the key and the following arg.
+                    i++
+                }
+                // Support args passed as -X<arg>=<value>.
+                xPrefixesToExclude.any { key.startsWith(it) } -> { /* Skip the key. */ }
+                else -> result += key
+            }
+            i++
+        }
+        return result
+    }
+
+    // Don't include coverage flags into the first stage because they are not supported when compiling a klib.
+    private fun firstStageExtraOpts() = extraOpts
+        .excludeFlags("-Xcoverage")
+        .excludeArguments("-Xcoverage-file", "-Xlibrary-to-cover")
+
+    // Don't include the -Xemit-lazy-objc-header flag into
+    // the second stage because this stage have no sources.
+    private fun secondStageExtraOpts() = extraOpts
+        .excludeArguments("-Xemit-lazy-objc-header")
+
+    /** Args passed to the compiler at the first stage of two-stage compilation (klib building). */
+    protected fun buildFirstStageArgs(klibPath: String) = mutableListOf<String>().apply {
+        addArg("-output", klibPath)
+        addArg("-produce", CompilerOutputKind.LIBRARY.name.toLowerCase())
+
+        addAll(buildCommonArgs())
+
+        addAll(firstStageExtraOpts())
+
+        allSourceFiles.mapTo(this) { it.absolutePath }
+        commonSrcFiles
+            .flatMap { it.files }
+            .mapTo(this) { "-Xcommon-sources=${it.absolutePath}" }
+    }
+
+    /** Args passed to the compiler at the second stage of two-stage compilation (producing a final binary from the klib).  */
+    protected fun buildSecondStageArgs(klibPath: String) = mutableListOf<String>().apply {
         addArg("-output", artifact.canonicalPath)
+        addArg("-produce", produce.name.toLowerCase())
+        addArgIfNotNull("-entry", entryPoint)
+
+        addAll(buildCommonArgs())
+
+        addFileArgs("-native-library", nativeLibraries)
+        linkerOpts.forEach {
+            addArg("-linker-option", it)
+        }
 
+        addAll(secondStageExtraOpts())
+
+        addArg("-l", klibPath)
+        add("-Xsource-library=${klibPath}")
+    }
+
+    /** Args passed to the compiler at both stages of the two-stage compilation and during the singe-stage compilation. */
+    protected fun buildCommonArgs() = mutableListOf<String>().apply {
         addArgs("-repo", libraries.repos.map { it.canonicalPath })
 
         if (platformConfiguration.files.isNotEmpty()) {
@@ -120,12 +202,6 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
         // The library's directory is added in libraries.repos.
         addArgs("-library", libraries.artifacts.map { it.artifact.nameWithoutExtension })
 
-        addFileArgs("-native-library", nativeLibraries)
-        addArg("-produce", produce.name.toLowerCase())
-        linkerOpts.forEach {
-            addArg("-linker-option", it)
-        }
-
         addArgIfNotNull("-target", konanTarget.visibleName)
         addArgIfNotNull("-language-version", languageVersion)
         addArgIfNotNull("-api-version", apiVersion)
@@ -143,6 +219,21 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
 
         if (libraries.friends.isNotEmpty())
             addArg("-friend-modules", libraries.friends.joinToString(File.pathSeparator))
+    }
+
+    /** Args passed to the compiler if the two-stage compilation is disabled. */
+    fun buildSingleStageArgs() = mutableListOf<String>().apply {
+        addArg("-output", artifact.canonicalPath)
+        addArg("-produce", produce.name.toLowerCase())
+        addArgIfNotNull("-entry", entryPoint)
+
+        addAll(buildCommonArgs())
+
+        addFileArgs("-native-library", nativeLibraries)
+
+        linkerOpts.forEach {
+            addArg("-linker-option", it)
+        }
 
         addAll(extraOpts)
 
@@ -253,9 +344,37 @@ abstract class KonanCompileTask: KonanBuildingTask(), KonanCompileSpec {
         )
     }
     // endregion
+
+    override fun run() {
+        destinationDir.mkdirs()
+        if (dumpParameters) {
+            dumpProperties(this)
+        }
+        if (enableTwoStageCompilation) {
+            logger.info("Start two-stage compilation")
+            val intermediateDir = project.konanBuildRoot
+                .resolve("intermediate")
+                .targetSubdir(konanTarget)
+                .apply { mkdirs() }
+            val klibPrefix = CompilerOutputKind.LIBRARY.prefix(konanTarget)
+            val klibSuffix = CompilerOutputKind.LIBRARY.suffix(konanTarget)
+            val intermediateKlib = intermediateDir.resolve("$klibPrefix$artifactName$klibSuffix").absolutePath
+            logger.info("Start first stage")
+            toolRunner.run(buildFirstStageArgs(intermediateKlib))
+            logger.info("Start second stage")
+            toolRunner.run(buildSecondStageArgs(intermediateKlib))
+        } else {
+            toolRunner.run(buildSingleStageArgs())
+        }
+    }
+}
+
+abstract class KonanCompileNativeBinary: KonanCompileTask() {
+    @Input
+    override var enableTwoStageCompilation: Boolean = false
 }
 
-open class KonanCompileProgramTask: KonanCompileTask() {
+open class KonanCompileProgramTask: KonanCompileNativeBinary() {
     override val produce: CompilerOutputKind get() = CompilerOutputKind.PROGRAM
 
     var runTask: Exec? = null
@@ -287,14 +406,14 @@ open class KonanCompileProgramTask: KonanCompileTask() {
 
 }
 
-open class KonanCompileDynamicTask: KonanCompileTask() {
+open class KonanCompileDynamicTask: KonanCompileNativeBinary() {
     override val produce: CompilerOutputKind get() = CompilerOutputKind.DYNAMIC
 
     val headerFile: File
         @OutputFile get() = destinationDir.resolve("$artifactPrefix${artifactName}_api.h")
 }
 
-open class KonanCompileFrameworkTask: KonanCompileTask() {
+open class KonanCompileFrameworkTask: KonanCompileNativeBinary() {
     override val produce: CompilerOutputKind get() = CompilerOutputKind.FRAMEWORK
 
     override val artifact
@@ -303,8 +422,9 @@ open class KonanCompileFrameworkTask: KonanCompileTask() {
 
 open class KonanCompileLibraryTask: KonanCompileTask() {
     override val produce: CompilerOutputKind get() = CompilerOutputKind.LIBRARY
+    override val enableTwoStageCompilation: Boolean = false
 }
 
-open class KonanCompileBitcodeTask: KonanCompileTask() {
+open class KonanCompileBitcodeTask: KonanCompileNativeBinary() {
     override val produce: CompilerOutputKind get() = CompilerOutputKind.BITCODE
 }

tools/kotlin-native-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/konan/tasks/KonanInteropTask.kt

@@ -75,7 +75,7 @@ open class KonanInteropTask @Inject constructor(val workerExecutor: WorkerExecut
     @InputFiles val headers   = mutableSetOf<FileCollection>()
     @InputFiles val linkFiles = mutableSetOf<FileCollection>()
 
-    override fun buildArgs() = mutableListOf<String>().apply {
+    fun buildArgs() = mutableListOf<String>().apply {
         addArg("-o", artifact.canonicalPath)
 
         addArgIfNotNull("-target", konanTarget.visibleName)