Skip to content

Commit

Permalink
Move to Artifactory usage (JetBrains#3513)
Browse files Browse the repository at this point in the history
  • Loading branch information
LepilkinaElena authored Oct 29, 2019
1 parent 1f33b27 commit 85a8120
Show file tree
Hide file tree
Showing 10 changed files with 81 additions and 78 deletions.
8 changes: 4 additions & 4 deletions HACKING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,15 +153,15 @@ and then a final native binary is produced from this klibrary using the -Xinclud
Output can be redirected to file with flag `--output/-o`.
To get detailed information about supported options, please use `--help/-h`.

Analyzer tool can compare both local files and files placed on Bintray/TeamCity.
Analyzer tool can compare both local files and files placed on Artifactory/TeamCity.

File description stored on Bintray
File description stored on Artifactory

bintray:<build number>:<target (Linux|Windows10|MacOSX)>:<filename>
artifactory:<build number>:<target (Linux|Windows10|MacOSX)>:<filename>

Example

bintray:1.2-dev-7942:Windows10:nativeReport.json
artifactory:1.2-dev-7942:Windows10:nativeReport.json

File description stored on TeamCity

Expand Down
12 changes: 5 additions & 7 deletions build-tools/src/main/kotlin/org/jetbrains/kotlin/BuildRegister.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,10 @@ open class BuildRegister : DefaultTask() {
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val bintrayUser = buildProperties.getProperty("bintray.user")
val bintrayApiKey = buildProperties.getProperty("bintray.apikey")
val teamCityUser = buildProperties.getProperty("teamcity.auth.userId")
val teamCityPassword = buildProperties.getProperty("teamcity.auth.password")
val buildNumber = buildProperties.getProperty("build.number")
val apiKey = buildProperties.getProperty("artifactory.apikey")

// Get branch.
val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
Expand All @@ -104,8 +103,8 @@ open class BuildRegister : DefaultTask() {
// Get summary information.
val output = arrayOf("$analyzer", "summary", "--exec-samples", "all", "--compile", "samples",
"--compile-samples", "HelloWorld,Videoplayer", "--codesize-samples", "all",
"--exec-normalize", "bintray:goldenResults.csv",
"--codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
"--exec-normalize", "artifactory:builds/goldenResults.csv",
"--codesize-normalize", "artifactory:builds/goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()

// Postprocess information.
Expand All @@ -131,7 +130,7 @@ open class BuildRegister : DefaultTask() {

val frameworkOutput = arrayOf("$analyzer", "summary", "--compile", "samples",
"--compile-samples", "FrameworkBenchmarksAnalyzer", "--codesize-samples", "FrameworkBenchmarksAnalyzer",
"--codesize-normalize", "bintray:goldenResults.csv", "$currentBenchmarksReportFile")
"--codesize-normalize", "artifactory:builds/goldenResults.csv", "$currentBenchmarksReportFile")
.runCommand()

val buildInfoPartsFramework = frameworkOutput.split(',')
Expand All @@ -154,8 +153,7 @@ open class BuildRegister : DefaultTask() {
append("{\"buildId\":\"$buildId\",")
append("\"teamCityUser\":\"$teamCityUser\",")
append("\"teamCityPassword\":\"$teamCityPassword\",")
append("\"bintrayUser\": \"$bintrayUser\", ")
append("\"bintrayPassword\":\"$bintrayApiKey\", ")
append("\"artifactoryApiKey\":\"$apiKey\",")
append("\"target\": \"$target\",")
append("\"buildType\": \"$buildType\",")
append("\"failuresNumber\": $failures,")
Expand Down
15 changes: 9 additions & 6 deletions build-tools/src/main/kotlin/org/jetbrains/kotlin/MPPTools.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,13 +141,14 @@ fun getCompileOnlyBenchmarksOpts(project: Project, defaultCompilerOpts: List<Str
fun findFile(fileName: String, directory: String): String? =
File(directory).walkBottomUp().find { it.name == fileName }?.getAbsolutePath()

fun uploadFileToBintray(url: String, project: String, version: String, packageName: String, bintrayFilePath: String,
filePath: String, username: String? = null, password: String? = null) {
val uploadUrl = "$url/$project/$packageName/$version/$bintrayFilePath?publish=1"
sendUploadRequest(uploadUrl, filePath, username, password)
fun uploadFileToArtifactory(url: String, project: String, artifactoryFilePath: String,
filePath: String, password: String) {
val uploadUrl = "$url/$project/$artifactoryFilePath"
sendUploadRequest(uploadUrl, filePath, extraHeaders = listOf(Pair("X-JFrog-Art-Api", password)))
}

fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null) {
fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null,
extraHeaders: List<Pair<String, String>> = emptyList()) {
val uploadingFile = File(fileName)
val connection = URL(url).openConnection() as HttpURLConnection
connection.doOutput = true
Expand All @@ -158,7 +159,9 @@ fun sendUploadRequest(url: String, fileName: String, username: String? = null, p
val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}

extraHeaders.forEach {
connection.addRequestProperty(it.first, it.second)
}
try {
connection.connect()
BufferedOutputStream(connection.outputStream).use { output ->
Expand Down
14 changes: 7 additions & 7 deletions build-tools/src/main/kotlin/org/jetbrains/kotlin/RegressionsReporter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ open class RegressionsReporter : DefaultTask() {
}
}

// File name on bintray is the same as current.
val bintrayFileName = currentBenchmarksReportFile.substringAfterLast("/")
// File name on Artifactory is the same as current.
val artifactoryFileName = currentBenchmarksReportFile.substringAfterLast("/")

// Get compare to build.
val compareToBuild = getBuild(previousBuildLocator(buildTypeId, defaultBranch), user, password)
Expand All @@ -135,21 +135,21 @@ open class RegressionsReporter : DefaultTask() {
val target = System.getProperty("os.name").replace("\\s".toRegex(), "")

// Generate comparison report.
val output = arrayOf("$analyzer", "-r", "html", "$currentBenchmarksReportFile", "bintray:$compareToBuildNumber:$target:$bintrayFileName", "-o", "$htmlReport")
val output = arrayOf("$analyzer", "-r", "html", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$htmlReport")
.runCommand()

if (output.contains("Uncaught exception")) {
error("Error during comparasion of $currentBenchmarksReportFile and " +
"bintray:$compareToBuildNumber:$target:$bintrayFileName with $analyzer! " +
"artifactory:$compareToBuildNumber:$target:$artifactoryFileName with $analyzer! " +
"Please check files existance and their correctness.")
}
arrayOf("$analyzer", "-r", "statistics", "$currentBenchmarksReportFile", "bintray:$compareToBuildNumber:$target:$bintrayFileName", "-o", "$summaryFile")
arrayOf("$analyzer", "-r", "statistics", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$summaryFile")
.runCommand()

val reportLink = "https://kotlin-native-perf-summary.labs.jb.gg/?target=$target&build=$buildNumber"
val detailedReportLink = "https://kotlin-native-performance.labs.jb.gg/?" +
"report=bintray:$buildNumber:$target:$bintrayFileName&" +
"compareTo=bintray:$compareToBuildNumber:$target:$bintrayFileName"
"report=artifactory:$buildNumber:$target:$artifactoryFileName&" +
"compareTo=artifactory:$compareToBuildNumber:$target:$artifactoryFileName"

val title = "\n*Performance report for target $target (build $buildNumber)* - $reportLink\n" +
"*Detailed info - * $detailedReportLink"
Expand Down
4 changes: 2 additions & 2 deletions performance/KotlinVsSwift/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ defaultTasks 'konanRun'
task mergeNativeReports {
doLast {
mergeReports(nativeJson)
uploadBenchmarkResultToBintray(nativeJson)
uploadBenchmarkResultToArtifactory(nativeJson)
}
}

task mergeJvmReports {
doLast {
mergeReports(jvmJson)
uploadBenchmarkResultToBintray(jvmJson)
uploadBenchmarkResultToArtifactory(jvmJson)
}
}

Expand Down
16 changes: 7 additions & 9 deletions performance/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,16 @@ task slackSummary(type: RegressionsSummaryReporter) {
'Windows': "${rootBuildDirectory}/targetsResults/Windows 10.txt".toString()]
}

private def uploadBenchmarkResultToBintray(String fileName) {
private def uploadBenchmarkResultToArtifactory(String fileName) {
def teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
if (teamcityConfig) {
def buildProperties = new Properties()
buildProperties.load(new FileInputStream(teamcityConfig))
def user = buildProperties.getProperty("bintray.user")
def password = buildProperties.getProperty("bintray.apikey")
def password = buildProperties.getProperty("artifactory.apikey")
def buildNumber = buildProperties.getProperty("build.number")
def target = System.getProperty("os.name").replaceAll("\\s", "")
MPPTools.uploadFileToBintray("${bintrayUrl}", "${bintrayRepo}",
buildNumber, "${bintrayPackage}", "${target}/${buildNumber}/${fileName}",
"${buildDir.absolutePath}/${fileName}", user, password)
MPPTools.uploadFileToArtifactory("${artifactoryUrl}", "${artifactoryRepo}",
"${target}/${buildNumber}/${fileName}", "${buildDir.absolutePath}/${fileName}", password)
}
}

Expand Down Expand Up @@ -126,7 +124,7 @@ task registerExternalBenchmarks {
if (!merged.isEmpty()) {
mkdir buildDir.absolutePath
new File(buildDir, externalBenchmarksReport).write(merged)
uploadBenchmarkResultToBintray(externalBenchmarksReport)
uploadBenchmarkResultToArtifactory(externalBenchmarksReport)
}
}
}
Expand Down Expand Up @@ -164,14 +162,14 @@ def mergeReports(String fileName) {
task mergeNativeReports {
doLast {
mergeReports(nativeJson)
uploadBenchmarkResultToBintray(nativeJson)
uploadBenchmarkResultToArtifactory(nativeJson)
}
}

task mergeJvmReports {
doLast {
mergeReports(jvmJson)
uploadBenchmarkResultToBintray(jvmJson)
uploadBenchmarkResultToArtifactory(jvmJson)
}
}

Expand Down
5 changes: 2 additions & 3 deletions performance/gradle.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ jvmJson = jvmReport.json
analyzerTool = benchmarksAnalyzer
analyzerToolDirectory = tools/benchmarksAnalyzer/build/bin
outputReport = ../report/report.html
bintrayUrl = https://api.bintray.com/content/lepilkinaelena
bintrayRepo = KotlinNativePerformance
bintrayPackage = jsonReports
artifactoryUrl = https://repo.labs.intellij.net
artifactoryRepo = kotlin-native-benchmarks
externalReports = coroutinesReport.txt
externalBenchmarksReport = externalReport.json
18 changes: 9 additions & 9 deletions tools/benchmarksAnalyzer/src/main/kotlin/main.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,27 +31,27 @@ abstract class Connector {
abstract fun getFileContent(fileLocation: String, user: String? = null): String
}

object BintrayConnector : Connector() {
override val connectorPrefix = "bintray:"
val bintrayUrl = "https://dl.bintray.com/content/lepilkinaelena/KotlinNativePerformance"
object ArtifactoryConnector : Connector() {
override val connectorPrefix = "artifactory:"
val artifactoryUrl = "https://repo.labs.intellij.net/kotlin-native-benchmarks"

override fun getFileContent(fileLocation: String, user: String?): String {
val fileParametersSize = 3
val fileDescription = fileLocation.substringAfter(connectorPrefix)
val fileParameters = fileDescription.split(':', limit = fileParametersSize)

// Right link to bintray file.
// Right link to Artifactory file.
if (fileParameters.size == 1) {
val accessFileUrl = "$bintrayUrl/${fileParameters[0]}"
val accessFileUrl = "$artifactoryUrl/${fileParameters[0]}"
return sendGetRequest(accessFileUrl, followLocation = true)
}
// Used builds description format.
if (fileParameters.size != fileParametersSize) {
error("To get file from bintray, please, specify, build number from TeamCity and target" +
" in format bintray:build_number:target:filename")
error("To get file from Artifactory, please, specify, build number from TeamCity and target" +
" in format artifactory:build_number:target:filename")
}
val (buildNumber, target, fileName) = fileParameters
val accessFileUrl = "$bintrayUrl/$target/$buildNumber/$fileName"
val accessFileUrl = "$artifactoryUrl/$target/$buildNumber/$fileName"
return sendGetRequest(accessFileUrl, followLocation = true)
}
}
Expand All @@ -78,7 +78,7 @@ object TeamCityConnector: Connector() {

fun getFileContent(fileName: String, user: String? = null): String {
return when {
BintrayConnector.isCompatible(fileName) -> BintrayConnector.getFileContent(fileName, user)
ArtifactoryConnector.isCompatible(fileName) -> ArtifactoryConnector.getFileContent(fileName, user)
TeamCityConnector.isCompatible(fileName) -> TeamCityConnector.getFileContent(fileName, user)
else -> readFile(fileName)
}
Expand Down
Loading

0 comments on commit 85a8120

Please sign in to comment.