Test improvements

.github/workflows/gradle_task.yml

@@ -47,10 +47,20 @@ jobs:
           distribution: temurin
           java-version: 11
 
+      - name: Cache Gradle TestKit
+        id: cache-gradle-testkit
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ runner.temp }}/.gradle-test-kit/caches
+          key: gradle-testkit-${{ runner.os }}
+
       - uses: gradle/gradle-build-action@v2
+        env:
+          GRADLE_TESTKIT_DIR: ${{ runner.temp }}/.gradle-test-kit/caches
         with:
           gradle-home-cache-cleanup: true
-          arguments: ${{ inputs.gradle-task }}
+          arguments: ${{ inputs.gradle-task }} --scan
 
       - name: Upload build reports
         if: failure()

buildSrc/src/main/kotlin/buildsrc/conventions/kotlin-gradle-plugin-tests.gradle.kts

@@ -34,3 +34,12 @@ testing.suites {
     }
   }
 }
+
+tasks.withType<Test>().configureEach {
+  // Help speed up TestKit tests by re-using dependencies cache
+  // https://docs.gradle.org/current/userguide/dependency_resolution.html#sub:shared-readonly-cache
+  environment("GRADLE_RO_DEP_CACHE", gradle.gradleUserHomeDir.resolve("caches"))
+  providers.environmentVariable("GRADLE_TESTKIT_DIR").orNull?.let {
+    systemProperty("gradleTestKitDir", it)
+  }
+}

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/AndroidLibraryTest.kt

@@ -8,7 +8,7 @@ import org.gradle.testkit.runner.TaskOutcome.SUCCESS
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 
-@Disabled("Leftovers after revert of #94")
+@Disabled("Leftovers after revert of https://github.com/Kotlin/binary-compatibility-validator/issues/94")
 internal class AndroidLibraryTest : BaseKotlinGradleTest() {
 
   // region Kotlin Android Library
@@ -25,7 +25,7 @@ internal class AndroidLibraryTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":kotlin-library:apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":kotlin-library:apiDump", SUCCESS)
     }
   }
 
@@ -38,7 +38,7 @@ internal class AndroidLibraryTest : BaseKotlinGradleTest() {
         arguments.add(":kotlin-library:apiCheck")
       }
     }.build {
-      task(":kotlin-library:apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":kotlin-library:apiCheck", SUCCESS)
     }
   }
 
@@ -58,7 +58,7 @@ internal class AndroidLibraryTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":java-library:apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":java-library:apiDump", SUCCESS)
     }
   }
 
@@ -71,7 +71,7 @@ internal class AndroidLibraryTest : BaseKotlinGradleTest() {
         arguments.add(":java-library:apiCheck")
       }
     }.build {
-      task(":java-library:apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":java-library:apiCheck", SUCCESS)
     }
   }
 

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/DefaultConfigTests.kt

@@ -26,7 +26,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
 
     runner.buildAndFail {
       output shouldContain "Please ensure that task ':apiDump' was executed"
-      task(":apiCheck") shouldHaveOutcome FAILED
+      shouldHaveRunTask(":apiCheck", FAILED)
     }
   }
 
@@ -43,7 +43,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.buildAndFail {
-      task(":apiCheck") shouldHaveOutcome FAILED
+      shouldHaveRunTask(":apiCheck", FAILED)
       shouldNotHaveRunTask(":check") // apiCheck fails before we can run check
     }
   }
@@ -63,7 +63,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -86,7 +86,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -109,7 +109,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -132,7 +132,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.buildAndFail {
-      task(":apiCheck") shouldHaveOutcome FAILED
+      shouldHaveRunTask(":apiCheck", FAILED)
 
       // note that tabs are used as function indents!
       val dumpOutput = /*language=TEXT*/ """
@@ -160,7 +160,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiDump", SUCCESS)
 
       assertTrue(
         rootProjectApiDump.exists(),
@@ -187,7 +187,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiDump", SUCCESS)
 
       val apiDumpFile = rootProjectDir.resolve("api/testproject.api")
       assertTrue(apiDumpFile.exists(), "api dump file ${apiDumpFile.path} should exist")
@@ -217,7 +217,7 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiDump", SUCCESS)
 
       assertTrue(rootProjectApiDump.exists(), "api dump file should exist")
 
@@ -242,8 +242,8 @@ internal class DefaultConfigTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":check") shouldHaveOutcome SUCCESS
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":check", SUCCESS)
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 }

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/IgnoredClassesTests.kt

@@ -2,7 +2,7 @@ package kotlinx.validation.test
 
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.api.*
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.build
-import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveOutcome
+import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveRunTask
 import io.kotest.matchers.comparables.shouldBeEqualComparingTo
 import org.gradle.testkit.runner.TaskOutcome.SUCCESS
 import org.junit.jupiter.api.Assertions.assertTrue
@@ -30,7 +30,7 @@ internal class IgnoredClassesTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -54,7 +54,7 @@ internal class IgnoredClassesTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -78,7 +78,7 @@ internal class IgnoredClassesTests : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiDump", SUCCESS)
 
       assertTrue(rootProjectApiDump.exists(), "api dump file should exist")
 

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/InputJarTest.kt

@@ -2,7 +2,7 @@ package kotlinx.validation.test
 
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.api.*
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.build
-import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveOutcome
+import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveRunTask
 import org.gradle.testkit.runner.TaskOutcome.SUCCESS
 import org.junit.jupiter.api.Test
 
@@ -30,8 +30,8 @@ class InputJarTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":jar") shouldHaveOutcome SUCCESS
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":jar", SUCCESS)
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 }

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/JavaTestFixturesTest.kt

@@ -22,7 +22,7 @@ class JavaTestFixturesTest : FunSpec({
       test("expect :apiDump task passes") {
         project.runner.withArguments("apiDump").build {
           withClue(output) {
-            task(":apiDump") shouldHaveOutcome SUCCESS
+            shouldHaveRunTask(":apiDump", SUCCESS)
           }
         }
       }
@@ -36,7 +36,7 @@ class JavaTestFixturesTest : FunSpec({
           )
           .build {
             withClue(output) {
-              task(":apiCheck") shouldHaveOutcome SUCCESS
+              shouldHaveRunTask(":apiCheck", SUCCESS)
             }
           }
       }
@@ -74,7 +74,7 @@ class JavaTestFixturesTest : FunSpec({
       test("expect :apiDump task passes") {
         project.runner.withArguments("apiDump").build {
           withClue(output) {
-            task(":apiDump") shouldHaveOutcome SUCCESS
+            shouldHaveRunTask(":apiDump", SUCCESS)
           }
         }
       }
@@ -88,7 +88,7 @@ class JavaTestFixturesTest : FunSpec({
           )
           .build {
             withClue(output) {
-              task(":apiCheck") shouldHaveOutcome SUCCESS
+              shouldHaveRunTask(":apiCheck", SUCCESS)
             }
           }
       }

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/MultiPlatformSingleJvmTargetTest.kt

@@ -3,7 +3,8 @@ package kotlinx.validation.test
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.*
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.api.*
 import io.kotest.assertions.withClue
-import io.kotest.matchers.comparables.shouldBeEqualComparingTo
+import io.kotest.matchers.file.shouldBeAFile
+import io.kotest.matchers.shouldBe
 import io.kotest.matchers.string.shouldContain
 import java.io.File
 import org.gradle.testkit.runner.TaskOutcome.FAILED
@@ -46,7 +47,7 @@ internal class MultiPlatformSingleJvmTargetTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -119,7 +120,7 @@ internal class MultiPlatformSingleJvmTargetTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiDump", SUCCESS)
 
       val mainExpectedApi = """
         |${readResourceFile("/examples/classes/Subsub1Class.dump").trim()}
@@ -129,9 +130,8 @@ internal class MultiPlatformSingleJvmTargetTest : BaseKotlinGradleTest() {
         |
       """.trimMargin()
 
-      val actual = jvmApiDump.readText().invariantNewlines()
-
-      actual.shouldBeEqualComparingTo(mainExpectedApi)
+      jvmApiDump.shouldBeAFile()
+      jvmApiDump.readText().invariantNewlines() shouldBe mainExpectedApi
     }
   }
 

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/MultipleJvmTargetsTest.kt

@@ -53,7 +53,7 @@ internal class MultipleJvmTargetsTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveRunTask(":apiCheck", SUCCESS)
     }
   }
 
@@ -92,7 +92,7 @@ internal class MultipleJvmTargetsTest : BaseKotlinGradleTest() {
 
     runner.buildAndFail {
       withClue(output) {
-        shouldHaveRunTask(":apiCheck") shouldHaveOutcome FAILED
+        shouldHaveRunTask(":apiCheck", FAILED)
         output shouldContain "API check failed for project :testproject"
         shouldNotHaveRunTask(":check")
       }
@@ -118,22 +118,21 @@ internal class MultipleJvmTargetsTest : BaseKotlinGradleTest() {
 
     }
     runner.build {
-      task(":apiDump") shouldHaveOutcome SUCCESS
-
-      System.err.println(output)
+      withClue(output) {
+        shouldHaveRunTask(":apiDump", SUCCESS)
 
-      val anotherExpectedApi = readResourceFile("/examples/classes/Subsub1Class.dump")
-      anotherApiDump.shouldBeAFile()
-      anotherApiDump.readText().invariantNewlines().shouldBe(anotherExpectedApi)
+        val anotherExpectedApi = readResourceFile("/examples/classes/Subsub1Class.dump")
+        anotherApiDump.shouldBeAFile()
+        anotherApiDump.readText().invariantNewlines() shouldBe anotherExpectedApi
 
-      val mainExpectedApi =
-        anotherExpectedApi + readResourceFile("/examples/classes/Subsub2Class.dump")
-      jvmApiDump.shouldBeAFile()
-      jvmApiDump.readText().invariantNewlines().shouldBe(mainExpectedApi)
+        val mainExpectedApi =
+          anotherExpectedApi + readResourceFile("/examples/classes/Subsub2Class.dump")
+        jvmApiDump.shouldBeAFile()
+        jvmApiDump.readText().invariantNewlines() shouldBe mainExpectedApi
+      }
     }
   }
 
   private val jvmApiDump: File get() = rootProjectDir.resolve("api/jvm/testproject.api")
   private val anotherApiDump: File get() = rootProjectDir.resolve("api/anotherJvm/testproject.api")
-
 }

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/NonPublicMarkersTest.kt

@@ -3,6 +3,7 @@ package kotlinx.validation.test
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.api.*
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.build
 import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveOutcome
+import dev.adamko.kotlin.binary_compatibility_validator.test.utils.shouldHaveTaskWithOutcome
 import org.gradle.testkit.runner.TaskOutcome.SUCCESS
 import org.junit.jupiter.api.Test
 
@@ -30,7 +31,7 @@ class NonPublicMarkersTest : BaseKotlinGradleTest() {
     }
 
     runner.build {
-      task(":apiCheck") shouldHaveOutcome SUCCESS
+      shouldHaveTaskWithOutcome(":apiCheck", SUCCESS)
     }
   }
 }

modules/bcv-gradle-plugin-functional-tests/src/functionalTest/kotlin/kotlinx/validation/test/SettingsPluginDslTest.kt

@@ -9,7 +9,7 @@ import io.kotest.matchers.paths.shouldExist
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.string.shouldContain
 import kotlin.io.path.readText
-import org.gradle.testkit.runner.TaskOutcome
+import org.gradle.testkit.runner.TaskOutcome.SUCCESS
 import org.intellij.lang.annotations.Language
 
 internal class SettingsPluginDslTest : FunSpec({
@@ -71,8 +71,8 @@ internal class SettingsPluginDslTest : FunSpec({
                 shouldNotHaveRunTask(":apiDump")
               }
 
-              task(":sub1:apiDump") shouldHaveOutcome TaskOutcome.SUCCESS
-              task(":sub2:apiDump") shouldHaveOutcome TaskOutcome.SUCCESS
+              shouldHaveRunTask(":sub1:apiDump", SUCCESS)
+              shouldHaveRunTask(":sub2:apiDump", SUCCESS)
             }
 
           testCase.project.projectDir.resolve("sub1/api/sub1.api").asClue { apiDump ->