CI (Buildkite): Update all rootfs images to version v3.1, add some do…

…cumentation, and reorganize/rename/move a bunch of configuration files (#41698)

.buildkite/README.md

@@ -0,0 +1,5 @@
+# Buildkite
+
+This directory contains the Buildkite configuration files for Base Julia CI.
+
+The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository.

.buildkite/experimental/0_webui.yml → .buildkite/pipelines/experimental/0_webui.yml

@@ -15,4 +15,10 @@ steps:
           # but some of our signed pipelines can wait upon the completion of these unsigned
           # pipelines.
           unsigned_pipelines:
-            - .buildkite/experimental/pipeline.yml
+            - .buildkite/pipelines/experimental/launch_unsigned_builders.yml
+
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          # signed_pipelines:
+          #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
+          #     signature: "my_signature"

.buildkite/pipelines/experimental/README.md

@@ -0,0 +1,7 @@
+## Experimental pipeline (`master` branch only)
+
+This is the [`julia-master->experimental`](https://buildkite.com/julialang/julia-master-experimental) pipeline.
+
+We use this pipeline for builders that are not yet stable enough to go into the main pipeline.
+
+These builders are triggered by GitHub webhook events, such as pushes and pull requests.

.buildkite/pipelines/experimental/launch_unsigned_builders.yml

@@ -0,0 +1,6 @@
+steps:
+  - label: ":buildkite: Launch unsigned pipelines"
+    commands: |
+      buildkite-agent pipeline upload .buildkite/pipelines/experimental/misc/sanitizers.yml
+    agents:
+      queue: julia

.buildkite/experimental/sanitizers.yml → .buildkite/pipelines/experimental/misc/sanitizers.yml

@@ -1,6 +1,3 @@
-# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
-# since we need nestable sandboxing.  The rootfs images being used here are built from
-# the `.buildkite/rootfs_images/llvm-passes.jl` file.
 agents:
   queue: "julia"
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
@@ -14,8 +11,8 @@ steps:
       - JuliaCI/julia#v1:
           version: 1.6
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v2.0/llvm_passes.tar.gz
-          rootfs_treehash: "0b5a80c1d0ab110a57fbdb7f4bc042a5ede310f3"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
           uid: 1000
           gid: 1000
           workspaces:
@@ -29,6 +26,6 @@ steps:
       echo "--- Test that ASAN is enabled"
       contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug
     timeout_in_minutes: 120
-    # notify:
-    #   - github_commit_status:
-    #       context: "asan"
+    # notify:                   # TODO: uncomment this line
+    #   - github_commit_status: # TODO: uncomment this line
+    #       context: "asan"     # TODO: uncomment this line

.buildkite/0_webui.yml → .buildkite/pipelines/main/0_webui.yml

@@ -15,10 +15,10 @@ steps:
           # but some of our signed pipelines can wait upon the completion of these unsigned
           # pipelines.
           unsigned_pipelines:
-            - .buildkite/pipeline.yml
+            - .buildkite/pipelines/main/launch_unsigned_builders.yml
 
           # Our signed pipelines must have a `signature` or `signature_file` parameter that
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           signed_pipelines:
-            - pipeline: .buildkite/signed_pipeline_test.yml
+            - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml
               signature: "U2FsdGVkX18ZrMlORSIi0TvW99eZ3JyUEhntMyXjrOSJ9jAtUohgHc8+dMnWUc6qWjYt8k63wfPrth33SGWDiF814Bm1W5Zk3+R6HgVu6UCrQEI5sLm+SJPbrvET+Jkn"

.buildkite/pipelines/main/README.md

@@ -0,0 +1,15 @@
+## Main pipeline
+
+This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests.
+
+We have a different main pipeline for each permanent branch.
+
+For example:
+
+| Permanent Branch | Pipeline                                                                         |
+| ---------------- | -------------------------------------------------------------------------------- |
+| `master`         | [`julia-master`](https://buildkite.com/julialang/julia-master)                   |
+| `release-1.6`    | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) |
+| `release-1.7`    | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) |
+
+(This is not a complete list.)

.buildkite/pipelines/main/launch_unsigned_builders.yml

@@ -0,0 +1,29 @@
+# This file launches all the build jobs that _don't_ require secrets access.
+# These jobs can pass their output off to jobs that do require secrets access,
+# but those privileged steps require signing before they can be run.
+#
+# Yes, this is creating another layer of indirection; the flow now looks like:
+#
+#   [webui] -> launch_unsigned_builders.yml -> misc/whitespace.yml
+#
+# when we could theoretically just have the `webui` launch `misc/whitespace.yml`,
+# however this raises the bar for contributors to add new (unsigned) steps to
+# our CI configuration, so I'd rather live with an extra layer of indirection
+# and only need to touch the webui configuration when we need to alter
+# something about the privileged steps.
+
+steps:
+  - label: ":buildkite: Launch unsigned builders"
+    commands: |
+      # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
+
+      # Next, we launch the miscellaneous builders in alphabetical order.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+
+      # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
+    agents:
+      queue: julia

.buildkite/misc/doctest.yml → .buildkite/pipelines/main/misc/doctest.yml

@@ -1,6 +1,3 @@
-# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
-# since we need nestable sandboxing.  The rootfs images being used here are built from
-# the `.buildkite/rootfs_images/llvm-passes.jl` file.
 agents:
   queue: "julia"
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
@@ -14,8 +11,8 @@ steps:
       - JuliaCI/julia#v1:
           version: 1.6
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
-          rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           uid: 1000
           gid: 1000
           workspaces:
@@ -33,7 +30,7 @@ steps:
 
       echo "--- Run Julia doctests"
       JULIA_NUM_THREADS=1 make -C doc doctest=true
-    timeout_in_minutes: 120
+    timeout_in_minutes: 45
     notify:
       - github_commit_status:
           context: "doctest"

.buildkite/misc/embedding.yml → .buildkite/pipelines/main/misc/embedding.yml

@@ -1,6 +1,3 @@
-# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
-# since we need nestable sandboxing.  The rootfs images being used here are built from
-# the `.buildkite/rootfs_images/llvm-passes.jl` file.
 agents:
   queue: "julia"
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
@@ -14,8 +11,8 @@ steps:
       - JuliaCI/julia#v1:
           version: 1.6
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
-          rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/misc/llvmpasses.yml → .buildkite/pipelines/main/misc/llvmpasses.yml

@@ -1,6 +1,3 @@
-# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
-# since we need nestable sandboxing.  The rootfs images being used here are built from
-# the `.buildkite/rootfs_images/llvm-passes.jl` file.
 agents:
   queue: "julia"
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
@@ -36,8 +33,8 @@ steps:
       - JuliaCI/julia#v1:
           version: 1.6
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
-          rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/misc/whitespace.yml → .buildkite/pipelines/main/misc/whitespace.yml

@@ -1,6 +1,3 @@
-# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
-# since we need nestable sandboxing.  The rootfs images being used here are built from
-# the `.buildkite/rootfs_images/llvm-passes.jl` file.
 agents:
   queue: "julia"
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
@@ -14,8 +11,8 @@ steps:
       - JuliaCI/julia#v1:
           version: 1.6
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz
-          rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           workspaces:
             - "/cache/repos:/cache/repos"
     commands: |