ci: improve test CI (#5661)

Author: DaniPopes

.github/INTEGRATION_FAILURE.md

@@ -3,8 +3,8 @@ title: "bug: long-running integration tests failed"
 labels: P-high, T-bug
 ---
 
-The heavy (long-running) integration tests have failed. This indicates a regression in foundry.
+The heavy (long-running) integration tests have failed. This indicates a regression in Foundry.
 
-Check the [heavy integration tests workflow page]({{env.WORKFLOW_URL}}) for details.
+Check the [heavy integration tests workflow page]({{ env.WORKFLOW_URL }}) for details.
 
-This issue was raised by the workflow at `.github/workflows/heavy-integration.yml`.
+This issue was raised by the workflow at `.github/workflows/heavy-integration.yml`.

.github/RELEASE_FAILURE_ISSUE_TEMPLATE.md

@@ -5,6 +5,6 @@ labels: P-high, T-bug
 
 The release workflow has failed. Some or all binaries might have not been published correctly.
 
-Check the [release workflow page]({{env.WORKFLOW_URL}}) for details.
+Check the [release workflow page]({{ env.WORKFLOW_URL }}) for details.
 
-This issue was raised by the workflow at `.github/workflows/release.yml`.
+This issue was raised by the workflow at `.github/workflows/release.yml`.

.github/scripts/matrices.py

@@ -0,0 +1,135 @@
+import json
+import os
+
+
+class Target:
+    # GitHub runner OS
+    os_id: str
+    # Rust target triple
+    target: str
+
+    def __init__(self, os_id: str, target: str):
+        self.os_id = os_id
+        self.target = target
+
+
+class Case:
+    name: str
+    filter: str
+    n_partitions: int
+    xplatform: bool
+
+    def __init__(self, name: str, filter: str, n_partitions: int, xplatform: bool):
+        self.name = name
+        self.filter = filter
+        self.n_partitions = n_partitions
+        self.xplatform = xplatform
+
+
+class Expanded:
+    os: str
+    target: str
+    name: str
+    flags: str
+    partition: int
+
+    def __init__(self, os: str, target: str, name: str, flags: str, partition: int):
+        self.os = os
+        self.target = target
+        self.name = name
+        self.flags = flags
+        self.partition = partition
+
+
+default_target = Target("ubuntu-latest", "x86_64-unknown-linux-gnu")
+if os.environ.get("EVENT_NAME") == "pull_request":
+    targets = [default_target]
+else:
+    targets = [
+        default_target,
+        Target("ubuntu-latest", "aarch64-unknown-linux-gnu"),
+        Target("macos-latest", "x86_64-apple-darwin"),
+        Target("macos-latest", "aarch64-apple-darwin"),
+        Target("windows-latest", "x86_64-pc-windows-msvc"),
+    ]
+
+config = [
+    Case(
+        name="unit",
+        filter="kind(lib) | kind(bench) | kind(proc-macro)",
+        n_partitions=1,
+        xplatform=True,
+    ),
+    Case(
+        name="integration",
+        filter="kind(test) & !test(/issue|forge_std|ext_integration/)",
+        n_partitions=3,
+        xplatform=True,
+    ),
+    Case(
+        name="integration/issue-repros",
+        filter="package(=forge) & test(~issue)",
+        n_partitions=2,
+        xplatform=False,
+    ),
+    Case(
+        name="integration/forge-std",
+        filter="package(=forge) & test(~forge_std)",
+        n_partitions=1,
+        xplatform=False,
+    ),
+    Case(
+        name="integration/external",
+        filter="package(=forge) & test(~ext_integration)",
+        n_partitions=2,
+        xplatform=False,
+    ),
+]
+
+
+def build_matrix():
+    os_ids = []
+    targets_ = []
+    for target in targets:
+        os_ids.append(target.os_id)
+        targets_.append(target.target)
+    print(json.dumps({"os": os_ids, "target": targets_}))
+
+
+def test_matrix():
+    expanded = []
+    for target in targets:
+        for case in config:
+            if not case.xplatform and target != default_target:
+                continue
+
+            for partition in range(1, case.n_partitions + 1):
+                os_str = ""
+                if len(targets) > 1:
+                    os_str = f" ({target.target})"
+
+                name = case.name
+                flags = f"-E '{case.filter}'"
+                if case.n_partitions > 1:
+                    s = f"{partition}/{case.n_partitions}"
+                    name += f" ({s})"
+                    flags += f" --partition count:{s}"
+                name += os_str
+
+                obj = Expanded(
+                    os=target.os_id,
+                    target=target.target,
+                    name=name,
+                    flags=flags,
+                    partition=partition,
+                )
+                expanded.append(vars(obj))
+
+    print(json.dumps({"include": expanded}), end="", flush=True)
+
+
+if __name__ == "__main__":
+    if int(os.environ.get("TEST", "0")) == 0:
+        build_matrix()
+    else:
+        test_matrix()

.github/workflows/cross-platform.yml

@@ -15,6 +15,7 @@ jobs:
     cargo-deny:
         name: cargo deny check
         runs-on: ubuntu-latest
+        timeout-minutes: 30
         steps:
             - uses: actions/checkout@v3
             - uses: EmbarkStudios/cargo-deny-action@v1