Allow reverse testing failure if installation failed. (#409)

Author: maleadt

.buildkite/pipeline.yml

@@ -6,19 +6,21 @@ steps:
       - JuliaCI/julia-coverage#v1:
           codecov: true
     command: |
-      julia -e 'using Pkg
-
-                println("--- :julia: Instantiating project")
+      julia -e 'println("--- :julia: Instantiating project")
+                using Pkg
                 Pkg.develop(; path=pwd())
-                Pkg.develop(; name="CUDA")
+                Pkg.develop(; name="CUDA")' || exit 3
 
-                println("+++ :julia: Running tests")
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
                 Pkg.test("CUDA"; coverage=true)'
     agents:
       queue: "juliagpu"
       cuda: "*"
     if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
     timeout_in_minutes: 120
+    soft_fail:
+      - exit_status: 3
 
   - label: "oneAPI.jl"
     plugins:
@@ -34,16 +36,18 @@ steps:
                 Pkg.develop(; name="oneAPI")
 
                 println("+++ :julia: Building support library")
-                include(joinpath(Pkg.devdir(), "oneAPI", "deps", "build_ci.jl"))
-                Pkg.activate()
+                include(joinpath(Pkg.devdir(), "oneAPI", "deps", "build_ci.jl"))' || exit 3
 
-                println("+++ :julia: Running tests")
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
                 Pkg.test("oneAPI"; coverage=true)'
     agents:
       queue: "juliagpu"
       intel: "*"
     if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
     timeout_in_minutes: 60
+    soft_fail:
+      - exit_status: 3
 
   - label: "Metal.jl"
     plugins:
@@ -52,20 +56,22 @@ steps:
       - JuliaCI/julia-coverage#v1:
           codecov: true
     command: |
-      julia -e 'using Pkg
-
-                println("--- :julia: Instantiating project")
+      julia -e 'println("--- :julia: Instantiating project")
+                using Pkg
                 Pkg.develop(; path=pwd())
-                Pkg.develop(; name="Metal")
+                Pkg.develop(; name="Metal")' || exit 3
 
-                println("+++ :julia: Running tests")
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
                 Pkg.test("Metal"; coverage=true)'
     agents:
       queue: "juliaecosystem"
       os: "macos"
       arch: "aarch64"
     if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
     timeout_in_minutes: 120
+    soft_fail:
+      - exit_status: 3
 
 #  - label: "AMDGPU.jl"
 #    plugins:
@@ -74,22 +80,21 @@ steps:
 #      - JuliaCI/julia-coverage#v1:
 #          codecov: true
 #    command: |
-#      julia -e 'using Pkg;
+#       julia -e 'println("--- :julia: Instantiating project")
+#                 using Pkg
+#                 Pkg.develop(; path=pwd())
+#                 Pkg.develop(; name="AMDGPU")' || exit 3
 #
-#                println("--- :julia: Instantiating project")
-#                Pkg.develop(; path=pwd())
-#                Pkg.develop(; name="AMDGPU")
-#
-#                println("+++ :julia: Running tests")
-#                Pkg.activate()
-#                Pkg.test("AMDGPU"; coverage=true)'
+#       julia -e 'println("+++ :julia: Running tests")
+#                 using Pkg
+#                 Pkg.test("AMDGPU"; coverage=true)'
 #    agents:
 #      queue: "juliagpu"
 #      rocm: "*"
 #    if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
 #    timeout_in_minutes: 120
 #    soft_fail:
-#      - exit_status: 1
+#      - exit_status: 3
 
   - label: "Enzyme.jl"
     plugins:
@@ -98,18 +103,20 @@ steps:
       - JuliaCI/julia-coverage#v1:
           codecov: true
     command: |
-      julia -e 'using Pkg
-
-                println("--- :julia: Instantiating project")
+      julia -e 'println("--- :julia: Instantiating project")
+                using Pkg
                 Pkg.develop(; path=pwd())
-                Pkg.develop(; name="Enzyme")
+                Pkg.develop(; name="Enzyme")' || exit 3
 
-                println("+++ :julia: Running tests")
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
                 Pkg.test("Enzyme"; coverage=true)'
     agents:
       queue: "juliagpu"
     if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
     timeout_in_minutes: 60
+    soft_fail:
+      - exit_status: 3
 
 env:
   JULIA_PKG_SERVER: "" # it often struggles with our large artifacts