Skip to content

Commit

Permalink
Adding CompilationDatabase generation (JetBrains#4091)
Browse files Browse the repository at this point in the history
  • Loading branch information
projedi authored Apr 22, 2020
1 parent 627904d commit ffe2fcf
Show file tree
Hide file tree
Showing 7 changed files with 200 additions and 30 deletions.
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,9 @@ llvmCoverageMappingC/CMakeLists.txt
tools/performance-server/node_modules
tools/performance-server/server
tools/performance-server/ui/js

# compilation database
compile_commands.json

# clangd caches
.clangd/
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/

package org.jetbrains.kotlin

import java.io.File
import javax.inject.Inject
import kotlinx.serialization.Serializable
import kotlinx.serialization.builtins.list
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonConfiguration
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.konan.target.HostManager

@Serializable
private data class Entry(
val directory: String,
val file: String,
val arguments: List<String>,
val output: String
) {
companion object {
fun create(
directory: File,
file: File,
args: List<String>,
outputDir: File
): Entry {
return Entry(
directory.absolutePath,
file.absolutePath,
args + listOf(file.absolutePath),
File(outputDir, file.name + ".o").absolutePath
)
}

fun writeListTo(file: File, entries: List<Entry>) {
val json = Json(JsonConfiguration.Stable.copy(prettyPrint = true))
file.writeText(json.stringify(Entry.serializer().list, entries))
}

fun readListFrom(file: File): List<Entry> {
val json = Json(JsonConfiguration.Stable)
return json.parse(Entry.serializer().list, file.readText())
}
}
}

open class GenerateCompilationDatabase @Inject constructor(@Input val target: String,
@Input val srcRoot: File,
@Input val files: Iterable<File>,
@Input val executable: String,
@Input val compilerFlags: List<String>,
@Input val outputDir: File
): DefaultTask() {
@OutputFile
var outputFile = File(outputDir, "compile_commands.json")

@TaskAction
fun run() {
val plugin = project.convention.getPlugin(ExecClang::class.java)
val executable = plugin.resolveExecutable(executable)
val args = listOf(executable) + compilerFlags + plugin.konanArgs(target)
val entries: List<Entry> = files.map { Entry.create(srcRoot, it, args, outputDir) }
Entry.writeListTo(outputFile, entries)
}
}

open class MergeCompilationDatabases @Inject constructor(): DefaultTask() {
@InputFiles
val inputFiles = mutableListOf<File>()

@OutputFile
var outputFile = File(project.buildDir, "compile_commands.json")

@TaskAction
fun run() {
val entries = mutableListOf<Entry>()
for (file in inputFiles) {
entries.addAll(Entry.readListFrom(file))
}
Entry.writeListTo(outputFile, entries)
}
}

fun mergeCompilationDatabases(project: Project, name: String, paths: List<String>): Task {
val subtasks: List<MergeCompilationDatabases> = paths.map {
val task = project.tasks.getByPath(it)
if (task !is MergeCompilationDatabases) {
throw GradleException("Unknown task type for compdb merging: $task")
}
task
}
return project.tasks.create(name, MergeCompilationDatabases::class.java) { task ->
task.dependsOn(subtasks)
task.inputFiles.addAll(subtasks.map { it.outputFile })
}
}

fun createCompilationDatabaseFromCompileToBitcodeTasks(project: Project, name: String): Task {
val compileTasks = project.tasks.withType(CompileToBitcode::class.java).toList()
val compdbTasks = compileTasks.mapNotNull { task ->
// TODO: consider generating databases for more than just current host target.
if (task.target != HostManager.hostName) {
null
} else {
project.tasks.create("${task.name}_CompilationDatabase",
GenerateCompilationDatabase::class.java,
task.target,
task.srcRoot,
task.inputFiles,
task.executable,
task.compilerFlags,
task.objDir)
}
}
return project.tasks.create(name, MergeCompilationDatabases::class.java) { task ->
task.dependsOn(compdbTasks)
task.inputFiles.addAll(compdbTasks.map { it.outputFile })
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,45 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil

private val targetDir by lazy { File(project.buildDir, target) }

private val objDir by lazy { File(targetDir, folderName) }
val objDir by lazy { File(targetDir, folderName) }

private val KonanTarget.isMINGW
get() = this.family == Family.MINGW

val executable
get() = when (language) {
Language.C -> "clang"
Language.CPP -> "clang++"
}

val compilerFlags: List<String>
get() {
val commonFlags = listOf("-c", "-emit-llvm", "-I$headersDir")
val languageFlags = when (language) {
Language.C ->
// Used flags provided by original build of allocator C code.
listOf("-std=gnu11", "-O3", "-Wall", "-Wextra", "-Wno-unknown-pragmas",
"-Werror", "-ftls-model=initial-exec", "-Wno-unused-function")
Language.CPP ->
listOfNotNull("-std=c++14", "-Werror", "-O2",
"-fPIC".takeIf { !HostManager().targetByName(target).isMINGW })
}
return commonFlags + languageFlags + compilerArgs
}

val inputFiles: Iterable<File>
get() {
val srcFilesPatterns =
when (language) {
Language.C -> listOf("**/*.c")
Language.CPP -> listOf("**/*.cpp", "**/*.mm")
}
return project.fileTree(srcDir) {
it.include(srcFilesPatterns)
it.exclude(excludeFiles)
}.files
}

@OutputFile
val outFile = File(targetDir, "${folderName}.bc")

Expand All @@ -59,28 +93,11 @@ open class CompileToBitcode @Inject constructor(@InputDirectory val srcRoot: Fil
if (target in excludedTargets) return
objDir.mkdirs()
val plugin = project.convention.getPlugin(ExecClang::class.java)
val commonFlags = listOf("-c", "-emit-llvm", "-I$headersDir")
val (executable, defaultFlags, srcFilesPatterns) =
when (language) {
Language.C -> Triple("clang",
// Used flags provided by original build of allocator C code.
commonFlags + listOf("-std=gnu11", "-O3", "-Wall", "-Wextra", "-Wno-unknown-pragmas",
"-Werror", "-ftls-model=initial-exec", "-Wno-unused-function"),
listOf("**/*.c"))
Language.CPP -> Triple("clang++",
commonFlags + listOfNotNull("-std=c++14", "-Werror", "-O2",
"-fPIC".takeIf { !HostManager().targetByName(target).isMINGW }),
listOf("**/*.cpp", "**/*.mm"))
}

plugin.execKonanClang(target, Action {
it.workingDir = objDir
it.executable = executable
it.args = defaultFlags + compilerArgs +
project.fileTree(srcDir) {
it.include(srcFilesPatterns)
it.exclude(excludeFiles)
}.files.map { it.absolutePath }
it.args = compilerFlags + inputFiles.map { it.absolutePath }
})

if (!skipLinkagePhase) {
Expand Down
24 changes: 13 additions & 11 deletions build-tools/src/main/kotlin/org/jetbrains/kotlin/ExecClang.kt
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,22 @@ class ExecClang(private val project: Project) {
return platformManager.platform(target).clang.clangArgsForKonanSources.asList()
}

private fun konanArgs(targetName: String?): List<String> {
fun konanArgs(targetName: String?): List<String> {
val target = platformManager.targetManager(targetName).target
return konanArgs(target)
}

fun resolveExecutable(executable: String?): String {
val executable = executable ?: "clang"

if (listOf("clang", "clang++").contains(executable)) {
val llvmDir = project.findProperty("llvmDir")
return "${llvmDir}/bin/$executable"
} else {
throw GradleException("unsupported clang executable: $executable")
}
}

// The bare ones invoke clang with system default sysroot.

fun execBareClang(action: Action<in ExecSpec>): ExecResult {
Expand Down Expand Up @@ -82,16 +93,7 @@ class ExecClang(private val project: Project) {
action.execute(execSpec)

execSpec.apply {
if (executable == null) {
executable = "clang"
}

if (listOf("clang", "clang++").contains(executable)) {
val llvmDir = project.findProperty("llvmDir")
executable = "${llvmDir}/bin/$executable" }
else {
throw GradleException("unsupported clang executable: $executable")
}
executable = resolveExecutable(executable)

val hostPlatform = project.findProperty("hostPlatform") as Platform
environment["PATH"] = project.files(hostPlatform.clang.clangPaths).asPath +
Expand Down
9 changes: 9 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.PlatformInfo
import org.jetbrains.kotlin.KotlinBuildPusher
import org.jetbrains.kotlin.CollisionDetector
import org.jetbrains.kotlin.CollisionTransformer
import org.jetbrains.kotlin.CompilationDatabaseKt
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import org.apache.tools.ant.filters.ReplaceTokens
import org.jetbrains.kotlin.UtilsKt
Expand Down Expand Up @@ -738,6 +739,7 @@ task clean {
doLast {
delete distDir
delete bundle.outputs.files
delete tasks.findByName("compdb").outputFile
}
}

Expand All @@ -749,3 +751,10 @@ task pusher(type: KotlinBuildPusher){
overrideConfigurationId = System.getenv("TEAMCITY_OVERRIDE_NATIVE_ID") ?: "Kotlin_KotlinDev_DeployMavenArtifacts_OverrideNative"
token = project.findProperty("teamcityBearToken") ?: System.getenv("TEAMCITY_BEAR_TOKEN")
}

CompilationDatabaseKt.mergeCompilationDatabases(project, "compdb", [
":common:compdb",
":runtime:compdb"
]).configure {
outputFile = file("$projectDir/compile_commands.json")
}
3 changes: 3 additions & 0 deletions common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* that can be found in the LICENSE file.
*/
import org.jetbrains.kotlin.CompileToBitcode
import org.jetbrains.kotlin.CompilationDatabaseKt

// TODO: consider using some Gradle plugins to build and test

Expand All @@ -14,6 +15,8 @@ targetList.each { targetName ->
tasks.create("${targetName}Files", CompileToBitcode, file("src/files"), 'files', targetName)
}

CompilationDatabaseKt.createCompilationDatabaseFromCompileToBitcodeTasks(project, "compdb")

task build {
dependsOn "${hostName}Hash"
dependsOn "${hostName}Files"
Expand Down
3 changes: 3 additions & 0 deletions runtime/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* that can be found in the LICENSE file.
*/
import org.jetbrains.kotlin.CompileToBitcode
import org.jetbrains.kotlin.CompilationDatabaseKt
import org.jetbrains.kotlin.UtilsKt

// TODO: consider using some Gradle plugins to build and test
Expand Down Expand Up @@ -77,6 +78,8 @@ targetList.each { targetName ->
}
}

CompilationDatabaseKt.createCompilationDatabaseFromCompileToBitcodeTasks(project, "compdb")

task hostRuntime(dependsOn: "${hostName}Runtime")

task clean {
Expand Down

0 comments on commit ffe2fcf

Please sign in to comment.