Made benchmark application flexible to run separate benchmarks (JetBr…

…ains#2649)

INTEROP.md

@@ -257,7 +257,7 @@ All the supported C types have corresponding representations in Kotlin:
 *   Pointers and arrays are mapped to `CPointer<T>?`.
 *   Enums can be mapped to either Kotlin enum or integral values, depending on
     heuristics and the [definition file hints](#definition-file-hints).
-*   Structs are mapped to types having fields available via the dot notation,
+*   Structs / unions are mapped to types having fields available via the dot notation,
     i.e. `someStructInstance.field1`.
 *   `typedef` are represented as `typealias`.
 
@@ -527,7 +527,7 @@ it belongs to. Once the control flow leaves the `memScoped` scope the C pointers
 
 ### Passing and receiving structs by value ###
 
-When a C function takes or returns a struct `T` by value, the corresponding
+When a C function takes or returns a struct / union `T` by value, the corresponding
 argument type or return type is represented as `CValue<T>`.
 
 `CValue<T>` is an opaque type, so the structure fields cannot be accessed with

Interop/Runtime/src/main/kotlin/kotlinx/cinterop/Utils.kt

@@ -327,6 +327,9 @@ public inline fun <reified T : CVariable> createValues(count: Int, initializer:
 }
 
 // TODO: optimize other [cValuesOf] methods:
+/**
+ * Returns sequence of immutable values [CValues] to pass them to C code.
+ */
 fun cValuesOf(vararg elements: Byte): CValues<ByteVar> = object : CValues<ByteVar>() {
     // Optimization to avoid unneeded virtual calls in base class implementation.
     override fun getPointer(scope: AutofreeScope): CPointer<ByteVar> {

performance/build.gradle

@@ -17,141 +17,47 @@ buildscript {
     }
 }
 
-apply plugin: 'kotlin-multiplatform'
-
-repositories {
-    maven {
-        url 'https://cache-redirector.jetbrains.com/jcenter'
-    }
-    maven {
-        url kotlinCompilerRepo
-    }
-    maven {
-        url buildKotlinCompilerRepo
-    }
-
-}
-
-defaultTasks 'bench'
-
-private def determinePreset() {
-    def preset = MPPTools.defaultHostPreset(project)
-    println("$project has been configured for ${preset.name} platform.")
-    preset
-}
-
-def hostPreset = determinePreset()
-def applicationName = 'Ring'
-
-kotlin {
-    sourceSets {
-        commonMain {
-            dependencies {
-                implementation "org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinVersion"
-            }
-            kotlin.srcDir '../tools/benchmarks/shared/src'
-            kotlin.srcDir 'src/main/kotlin'
-
-        }
-        nativeMain {
-            kotlin.srcDir 'src/main/kotlin-native'
-        }
-        jvmMain {
-            dependencies {
-                implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
-            }
-            kotlin.srcDir 'src/main/kotlin-jvm'
-        }
-    }
-
-    targets {
-        fromPreset(presets.jvm, 'jvm') {
-            compilations.all {
-                tasks[compileKotlinTaskName].kotlinOptions {
-                    jvmTarget = '1.8'
-                }
-                tasks[compileKotlinTaskName].kotlinOptions.suppressWarnings = true
-            }
-        }
-
-        fromPreset(hostPreset, 'native') {
-            compilations.main.outputKinds('EXECUTABLE')
-            compilations.main.extraOpts '-opt'
-            compilations.main.buildTypes = [RELEASE]
-        }
+task konanRun {
+    subprojects.each {
+       dependsOn it.getTasksByName('konanRun', true)[0]
     }
 }
 
-MPPTools.addTimeListener(project)
-
-MPPTools.createRunTask(project, 'konanRun', kotlin.targets.native) {
-    workingDir = project.provider {
-        kotlin.targets.native.compilations.main.getBinary('EXECUTABLE', buildType).parentFile
+task jvmRun {
+    subprojects.each {
+        dependsOn it.getTasksByName('jvmRun', true)[0]
     }
-    depends("build")
-    args("$nativeWarmup", "$attempts", "${buildDir.absolutePath}/${nativeBenchResults}")
-}
-
-task jvmRun(type: JavaExec)  {
-    dependsOn 'build'
-    def output = new ByteArrayOutputStream()
-    def runtimeClasspath = files(
-            kotlin.targets.jvm.compilations.main.output.allOutputs,
-            project.configurations.getByName(kotlin.targets.jvm.compilations.main.runtimeDependencyConfigurationName)
-    )
-    classpath runtimeClasspath
-    main = "MainKt"
-    args "$jvmWarmup", "$attempts",  "${buildDir.absolutePath}/${jvmBenchResults}"
 }
 
-task konanJsonReport {
-    doLast {
-        def nativeExecutable = MPPTools.getKotlinNativeExecutable(kotlin.targets.native, "RELEASE")
-        def nativeCompileTime = MPPTools.getNativeCompileTime(applicationName)
-        String benchContents = new File("${buildDir.absolutePath}/${nativeBenchResults}").text
-        def properties = getCommonProperties() + ['type'           : 'native',
-                                                  'compilerVersion': "${konanVersion}".toString(),
-                                                  'flags'          : kotlin.targets.native.compilations.main.extraOpts.collect{ "\"$it\"" },
-                                                  'benchmarks'     : benchContents,
-                                                  'compileTime'    : nativeCompileTime,
-                                                  'codeSize'       : MPPTools.getCodeSizeBenchmark(applicationName, nativeExecutable) ]
-        def output = MPPTools.createJsonReport(properties)
-        new File("${buildDir.absolutePath}/${nativeJson}").write(output)
-        uploadBenchmarkResultToBintray(nativeJson)
+task clean {
+    subprojects.each {
+        dependsOn it.getTasksByName('clean', true)[0]
     }
+    delete "${buildDir.absolutePath}"
 }
 
-task jvmJsonReport {
-    doLast {
-        def jarPath = project.getTasks().getByName("jvmJar").archivePath
-        def jvmCompileTime = MPPTools.getJvmCompileTime(applicationName)
-        String benchContents = new File("${buildDir.absolutePath}/${jvmBenchResults}").text
-        def properties = getCommonProperties() + ['type'           : 'jvm',
-                                                  'compilerVersion': "${buildKotlinVersion}".toString(),
-                                                  'benchmarks'     : benchContents,
-                                                  'compileTime'    : jvmCompileTime,
-                                                  'codeSize'       : MPPTools.getCodeSizeBenchmark(applicationName, "${jarPath}") ]
-        def output = MPPTools.createJsonReport(properties)
-        new File("${buildDir.absolutePath}/${jvmJson}").write(output)
-        uploadBenchmarkResultToBintray(jvmJson)
-    }
-}
-
-jvmRun.finalizedBy jvmJsonReport
-konanRun.finalizedBy konanJsonReport
+defaultTasks 'konanRun'
 
-private void dumpReport(String name, ByteArrayOutputStream output) {
-    new File("${name}").withOutputStream {
-        it.write(output.toByteArray())
-    }
+// Produce and send slack report.
+task slackReport(type: RegressionsReporter) {
+    // Create folder for report (root Kotlin project settings make create report in separate folder).
+    def reportDirectory = new File(outputReport).parentFile
+    mkdir reportDirectory
+    def targetsResults = new File(new File("${rootBuildDirectory}"), "targetsResults").toString()
+    mkdir targetsResults
+    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
+    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
+            "${rootBuildDirectory}/${analyzerToolDirectory}")
+    htmlReport = outputReport
+    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
+    def target = System.getProperty("os.name")
+    summaryFile = "${targetsResults}/${target}.txt"
 }
 
-private def getCommonProperties() {
-    return ['cpu': System.getProperty("os.arch"),
-            'os': System.getProperty("os.name"), // OperatingSystem.current().getName()
-            'jdkVersion': System.getProperty("java.version"), // org.gradle.internal.jvm.Jvm.current().javaVersion
-            'jdkVendor': System.getProperty("java.vendor"),
-            'kotlinVersion': "${kotlinVersion}".toString()]
+task slackSummary(type: RegressionsSummaryReporter) {
+    targetsResultFiles = ["Linux": "${rootBuildDirectory}/targetsResults/Linux.txt",
+                          "MacOSX": "${rootBuildDirectory}/targetsResults/Mac OS X.txt",
+                          "Windows": "${rootBuildDirectory}/targetsResults/Windows 10.txt"]
 }
 
 private def uploadBenchmarkResultToBintray(String fileName) {
@@ -169,35 +75,45 @@ private def uploadBenchmarkResultToBintray(String fileName) {
     }
 }
 
-task bench(type:Exec) {
-    dependsOn jvmRun
-    dependsOn konanRun
+def mergeReports(String fileName) {
+    def reports = []
+    subprojects.each {
+        def reportFile = new File("${it.buildDir.absolutePath}/${fileName}")
+        if (reportFile.exists()) {
+            reports.add(reportFile)
+        }
+    depends("build")
+    }
+    def output = MPPTools.mergeReports(reports)
+    mkdir buildDir.absolutePath
+    new File("${buildDir.absolutePath}/${fileName}").write(output)
+}
+
+task mergeNativeReports {
+    doLast {
+        mergeReports(nativeJson)
+        uploadBenchmarkResultToBintray(nativeJson)
+    }
+}
+
+task mergeJvmReports {
+    doLast {
+        mergeReports(jvmJson)
+        uploadBenchmarkResultToBintray(jvmJson)
+    }
+}
+
+subprojects.each {
+    it.getTasksByName('jvmJsonReport', true)[0].finalizedBy mergeJvmReports
+    it.getTasksByName('konanJsonReport', true)[0].finalizedBy mergeNativeReports
+}
+
+task teamCityStat(type:Exec) {
     def extension = MPPTools.getNativeProgramExtension()
     def analyzer = MPPTools.findFile("${analyzerTool}${extension}", "${rootBuildDirectory}/${analyzerToolDirectory}")
     if (analyzer != null) {
-        commandLine "${analyzer}", "-r", "text", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}", "${buildDir.absolutePath}/${jvmJson}"
+        commandLine "${analyzer}", "-r", "teamcity", "${buildDir.absolutePath}/${nativeJson}"
     } else {
         println("No analyzer $analyzerTool found in subdirectories of ${rootBuildDirectory}/${analyzerToolDirectory}")
     }
-}
-
-// Produce and send slack report.
-task slackReport(type: RegressionsReporter) {
-    // Create folder for report (root Kotlin project settings make create report in separate folder).
-    def reportDirectory = outputReport.substring(0, outputReport.lastIndexOf("/"))
-    mkdir reportDirectory
-    mkdir "../targetsResults"
-    currentBenchmarksReportFile = "${buildDir.absolutePath}/${nativeJson}"
-    analyzer = MPPTools.findFile("${analyzerTool}${MPPTools.getNativeProgramExtension()}",
-            "${rootBuildDirectory}/${analyzerToolDirectory}")
-    htmlReport = outputReport
-    defaultBranch = project.findProperty('kotlin.native.default.branch') ?: "master"
-    def target = System.getProperty("os.name")
-    summaryFile = "../targetsResults/${target}.txt"
-}
-
-task slackSummary(type: RegressionsSummaryReporter) {
-    targetsResultFiles = ["Linux": "../targetsResults/Linux.txt",
-                          "MacOSX": "../targetsResults/Mac OS X.txt",
-                          "Windows": "../targetsResults/Windows 10.txt"]
 }

performance/buildSrc/src/main/kotlin/MPPTools.kt

@@ -110,6 +110,16 @@ fun createJsonReport(projectProperties: Map<String, Any>): String {
     return report.toJson()
 }
 
+fun mergeReports(reports: List<File>): String {
+    val reportsToMerge = reports.map {
+        val json = it.inputStream().bufferedReader().use { it.readText() }
+        val reportElement = JsonTreeParser.parse(json)
+        BenchmarksReport.create(reportElement)
+
+    }
+    return reportsToMerge.reduce { result, it -> result + it }.toJson()
+}
+
 // Find file with set name in directory.
 fun findFile(fileName: String, directory: String): String? =
     File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()

performance/buildSrc/src/main/kotlin/RunJvmTask.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+import groovy.lang.Closure
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.Task
+import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
+import org.gradle.api.tasks.Input
+import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
+import javax.inject.Inject
+import java.io.File
+
+open class RunJvmTask: JavaExec() {
+    var outputFileName: String? = null
+    @Input
+    @Option(option = "filter", description = "filter")
+    var filter: String = ""
+
+    override fun configure(configureClosure: Closure<Any>): Task {
+        return super.configure(configureClosure)
+    }
+
+    private fun executeTask(output: java.io.OutputStream? = null) {
+        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
+        args(filterArgs)
+        exec()
+    }
+
+    @TaskAction
+    fun run() {
+        if (outputFileName != null)
+            File(outputFileName).outputStream().use { output ->  executeTask(output)}
+        else
+            executeTask()
+    }
+}

performance/buildSrc/src/main/kotlin/RunKotlinNativeTask.kt

@@ -7,6 +7,8 @@ import groovy.lang.Closure
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
+import org.gradle.api.tasks.Input
 import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
 import javax.inject.Inject
 import java.io.File
@@ -18,6 +20,10 @@ open class RunKotlinNativeTask @Inject constructor(
     var buildType = "RELEASE"
     var workingDir: Any = project.projectDir
     var outputFileName: String? = null
+    @Input
+    @Option(option = "filter", description = "filter")
+    var filter: String = ""
+
     private var curArgs: List<String> = emptyList()
     private val curEnvironment: MutableMap<String, Any> = mutableMapOf()
 
@@ -40,9 +46,11 @@ open class RunKotlinNativeTask @Inject constructor(
     }
 
     private fun executeTask(output: java.io.OutputStream? = null) {
+        val filterArgs = filter.split("\\s*,\\s*".toRegex())
+                .map{ if (it.isNotEmpty()) listOf("-f", it) else listOf(null) }.flatten().filterNotNull()
         project.exec {
             it.executable = curTarget.compilations.main.getBinary("EXECUTABLE", buildType).toString()
-            it.args = curArgs
+            it.args = curArgs + filterArgs
             it.environment = curEnvironment
             it.workingDir(workingDir)
             if (output != null)