Buildkite Test Analytics: fix failure_expanded (#53706)

This pull request changes `failure_expanded` from `Dict{String, Any}` to
`Vector{Dict{String, Any}}` to fix a JSON schema issue in the [Bootleg
JSON
writer](https://github.com/JuliaLang/julia/blob/7eb5cb89fb938a1dc67efa3861b25562767a7bbe/test/buildkitetestjson.jl#L13)
🏴‍☠️ which is responsible for producing JSON test results for [Buildkite
Test Analytics](https://buildkite.com/test-analytics).

The [`failure_expanded` attribute of a test
result](https://buildkite.com/docs/test-analytics/importing-json#json-test-results-data-reference-test-result-objects)
is a JSON array of objects, rather than a single object. I believe this
is to support the possibility of multiple failure reasons in a single
test case, although I'm not sure if that's used anywhere.

I believe the associated uploads (batches of up to 5,000 results) are
currently getting a successful HTTP 202 Accepted response, but the
response body will contain an error for each test case that had a
non-array `failure_expanded`, meaning those test cases will be dropped:

```http
HTTP/1.1 202 Accepted
Content-Type: application/json; charset=utf-8
Date: Tue, 12 Mar 2024 13:11:46 GMT
X-Request-Id: a35322f6-9990-4b8e-8895-d62bd6e10935

{
  "id": "55ac3b92-…",
  "run_id": "bfa6de98-…",
  "queued": 4998,
  "skipped": 2,
  "errors": [
    "Validation failed: Failure expanded must be an Array",
    "Validation failed: Failure expanded must be an Array"
  ],
  "run_url": "http://buildkite.com/…"
}
```

Rather than make the Buildkite API more permissive, I figured I'd come
and fix it upstream, and write my first few tiny lines of Julia while
I'm at it 😁

I've verified that the adjusted JSON output it accepted by our ingestion
system.

---

For verification, I added an error to an arbitrarily selected test
(because [workers don't return information about passing/broken tests,
only errors or
failure](https://github.com/JuliaLang/julia/blob/7eb5cb89fb938a1dc67efa3861b25562767a7bbe/test/runtests.jl#L363-L365)):

```patch
diff --git a/test/char.jl b/test/char.jl
index 1d35790..582423e8a3 100644
--- a/test/char.jl
+++ b/test/char.jl
@@ -1,6 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license

 @testset "basic properties" begin
+    @test throw(ErrorException("testing Buildkite JSON"))
     @test typemax(Char) == reinterpret(Char, typemax(UInt32))
     @test typemin(Char) == Char(0)
     @test typemax(Char) == reinterpret(Char, 0xffffffff)
```

… and then `CI=true ./julia test/runtests.jl char` which produces
`test/results_1.json`.

Before:

```json
[
  {
    "file_name": "/Users/pda/code/julia/test/char.jl",
    "history": {
      "duration": 2.123565912246704,
      "start_at": 1.710245465232599e9,
      "end_at": 1.710245467356165e9
    },
    "name": "test_error: throw(ErrorException(\"testing Buildkite JSON\"))",
    "location": "/Users/pda/code/julia/test/char.jl:4",
    "failure_reason": "Exception (unexpectedly) thrown during test",
    "scope": "/Overall/char",
    "failure_expanded": {
      "backtrace": [
        " [1] top-level scope",
        "   @ ~/code/julia/test/char.jl:4",
        " [2] macro expansion",
        "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
        " [3] macro expansion",
        "   @ ~/code/julia/test/char.jl:4 [inlined]",
        " [4] macro expansion",
        "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
        " [5] macro expansion",
        "   @ ~/code/julia/test/char.jl:4 [inlined]"
      ],
      "expanded": [
        "testing Buildkite JSON"
      ]
    },
    "id": "e9272117-d5f5-f542-039b-cfd3d2e8f33a",
    "result": "failed"
  }
]
```

After:

```json
[
  {
    "file_name": "/Users/pda/code/julia/test/char.jl",
    "history": {
      "duration": 2.123565912246704,
      "start_at": 1.710245465232599e9,
      "end_at": 1.710245467356165e9
    },
    "name": "test_error: throw(ErrorException(\"testing Buildkite JSON\"))",
    "location": "/Users/pda/code/julia/test/char.jl:4",
    "failure_reason": "Exception (unexpectedly) thrown during test",
    "scope": "/Overall/char",
    "failure_expanded": [
      {
        "backtrace": [
          " [1] top-level scope",
          "   @ ~/code/julia/test/char.jl:4",
          " [2] macro expansion",
          "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
          " [3] macro expansion",
          "   @ ~/code/julia/test/char.jl:4 [inlined]",
          " [4] macro expansion",
          "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
          " [5] macro expansion",
          "   @ ~/code/julia/test/char.jl:4 [inlined]"
        ],
        "expanded": [
          "testing Buildkite JSON"
        ]
      }
    ],
    "id": "e9272117-d5f5-f542-039b-cfd3d2e8f33a",
    "result": "failed"
  }
]
```

Diff:

```patch
--- buildkite-before.json	2024-03-12 23:08:26
+++ buildkite-after.json	2024-03-12 23:07:58
@@ -10,23 +10,25 @@
     "location": "/Users/pda/code/julia/test/char.jl:4",
     "failure_reason": "Exception (unexpectedly) thrown during test",
     "scope": "/Overall/char",
-    "failure_expanded": {
-      "backtrace": [
-        " [1] top-level scope",
-        "   @ ~/code/julia/test/char.jl:4",
-        " [2] macro expansion",
-        "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
-        " [3] macro expansion",
-        "   @ ~/code/julia/test/char.jl:4 [inlined]",
-        " [4] macro expansion",
-        "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
-        " [5] macro expansion",
-        "   @ ~/code/julia/test/char.jl:4 [inlined]"
-      ],
-      "expanded": [
-        "testing Buildkite JSON"
-      ]
-    },
+    "failure_expanded": [
+      {
+        "backtrace": [
+          " [1] top-level scope",
+          "   @ ~/code/julia/test/char.jl:4",
+          " [2] macro expansion",
+          "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
+          " [3] macro expansion",
+          "   @ ~/code/julia/test/char.jl:4 [inlined]",
+          " [4] macro expansion",
+          "   @ ~/code/julia/usr/share/julia/stdlib/v1.12/Test/src/Test.jl:164 [inlined]",
+          " [5] macro expansion",
+          "   @ ~/code/julia/test/char.jl:4 [inlined]"
+        ],
+        "expanded": [
+          "testing Buildkite JSON"
+        ]
+      }
+    ],
     "id": "e9272117-d5f5-f542-039b-cfd3d2e8f33a",
     "result": "failed"
   }
```

Author: vchuravy

test/buildkitetestjson.jl

@@ -1,9 +1,9 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# Convert test(set) results to a Buildkit-compatible JSON representation.
+# Convert test(set) results to a Buildkite-compatible JSON representation.
 # Based on <https://buildkite.com/docs/test-analytics/importing-json#json-test-results-data-reference>.
 
-module BuildKiteTestJSON
+module BuildkiteTestJSON
 
 using Test
 using Dates
@@ -105,9 +105,9 @@ function add_failure_info!(data::Dict{String, Any}, result::Test.Result)
         data["failure_reason"] = if result.test_type === :test_error
             if occursin("\nStacktrace:\n", result.backtrace)
                 err, trace = split(result.backtrace, "\nStacktrace:\n", limit=2)
-                data["failure_expanded"] = Dict{String, Any}(
-                    "expanded" => split(err, '\n'),
-                    "backtrace" => split(trace, '\n'))
+                data["failure_expanded"] =
+                    [Dict{String,Any}("expanded" => split(err, '\n'),
+                                      "backtrace" => split(trace, '\n'))]
             end
             "Exception (unexpectedly) thrown during test"
         elseif result.test_type === :test_nonbool

test/runtests.jl

@@ -11,7 +11,7 @@ include("choosetests.jl")
 include("testenv.jl")
 include("buildkitetestjson.jl")
 
-using .BuildKiteTestJSON
+using .BuildkiteTestJSON
 
 (; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS)
 tests = unique(tests)