release-1.7: Backports for 1.7.0/1.7.0-rc3

.buildkite/pipelines/main/misc/doctest.yml

@@ -12,8 +12,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/pipelines/main/misc/embedding.yml

@@ -12,8 +12,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/pipelines/main/misc/llvmpasses.yml

@@ -12,8 +12,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
-          rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
@@ -32,8 +32,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.8/package_linux.x86_64.tar.gz
-          rootfs_treehash: "84a323ae8fcc724f8ea5aca5901bbbf4bda3e519"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/pipelines/main/misc/sanitizers.yml

@@ -12,8 +12,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
-          rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
           uid: 1000
           gid: 1000
           workspaces:
@@ -25,3 +25,23 @@ steps:
     commands: |
       echo "--- Build julia-debug with ASAN"
       contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
+  - label: "tsan"
+    key: "tsan"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    if: | # We only run the `tsan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
+    commands: |
+      echo "--- Build julia-debug runtime with TSAN"
+      contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug

.buildkite/pipelines/main/misc/whitespace.yml

@@ -12,8 +12,8 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           workspaces:
             - "/cache/repos:/cache/repos"
     timeout_in_minutes: 10

.buildkite/pipelines/main/platforms/package_linux.arches

@@ -1,7 +1,7 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v3.2          0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v3.2          fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-linux         32          false         32          i686           none           60         no       no       no       v3.2          209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v3.2          c03a0158b19d48ac84b426834fce0d3584cdd0c7
-linux         64          false         64          x86_64         none           60         no       no       no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
-musl          64          false         64          x86_64         none           60         no       no       no       v3.19         e6a2730e37c386c46915b2650d6aaaa398195152
+# PLATFORM    LABEL       GROUP    ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         32          .        .             32          i686           .              .             .              .          .        .        .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux         64          .        .             64          x86_64         .              .             .              .          .        .        .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# linux       aarch64     .        .             aarch64     aarch64        .              .             .              .          .        .        .        ....          ........................................
+# linux       armv7l      .        .             armv7l      armv7l         .              .             .              .          .        .        .        ....          ........................................
+# linux       ppc64le     .        .             ppc64le     powerpc64le    .              .             .              .          .        .        .        ....          ........................................
+musl          64          .        .             64          x86_64         .              .             .              .          .        .        .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90

.buildkite/pipelines/main/platforms/package_linux.yml

@@ -19,7 +19,7 @@ steps:
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
-    timeout_in_minutes: ${TIMEOUT?}
+    timeout_in_minutes: ${TIMEOUT_BK?}
     commands: |
       echo "--- Print the full and short commit hashes"
       SHORT_COMMIT_LENGTH=10

.buildkite/pipelines/main/platforms/tester_linux.arches

@@ -1,9 +1,25 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v3.2          0566841e29f0f9880541c26a6595fd5ce0beb5ff
-# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v3.2          fb359370b052a47ce5c84cc6b4a7a03ed7053b25
-linux         32          false         32          i686           none           60         no       no       no       v3.2          209c4db679a515befd7fb50ecc6bfbecf7ec3d32
-# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v3.2          c03a0158b19d48ac84b426834fce0d3584cdd0c7
-linux         64_rr       false         64          x86_64         none           60         yes      no       no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
-linux         64_st       false         64          x86_64         none           60         no       yes      no       v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
-linux         64_mt       false         64          x86_64         none           60         no       no       yes      v3.2          474bf61a926b2d7fcf202284d59d4b11a04601d7
-musl          64          true          64          x86_64         none           60         no       no       no       v3.19         e6a2730e37c386c46915b2650d6aaaa398195152
+# PLATFORM    LABEL         GROUP    ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         32_g1         g1       .             32          i686           .              .             .              .          .        .        .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux         32_g2         g2       .             32          i686           .              .             .              3          .        .        .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+
+linux         64_g1_mt      g1       .             64          x86_64         .              .             .              .          .        .        yes      v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_g2_mt      g2       .             64          x86_64         .              .             .              3          .        .        yes      v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+linux         64_g1_st      g1       .             64          x86_64         .              .             .              .          .        yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_g2_st      g2       .             64          x86_64         .              .             .              3          .        yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+linux         64_g1_rrst    g1       .             64          x86_64         .              300           240            .          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_g2_rrst    g2       .             64          x86_64         .              180           120            3          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_g3_st      g3       .             64          x86_64         .              .             .              3          .        yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+# linux       aarch64_g1    g1       true          aarch64     aarch64        .              .             .              .          .        .        .        ----          ----------------------------------------
+# linux       aarch64_g2    g2       true          aarch64     aarch64        .              .             .              .          .        .        .        ----          ----------------------------------------
+
+# linux       armv7l_g1     g1       true          armv7l      armv7l         .              .             .              .          .        .        .        ----          ----------------------------------------
+# linux       armv7l_g2     g2       true          armv7l      armv7l         .              .             .              .          .        .        .        ----          ----------------------------------------
+
+# linux       ppc64le_g1    g1       true          ppc64le     powerpc64le    .              .             .              .          .        .        .        ----          ----------------------------------------
+# linux       ppc64le_g2    g2       true          ppc64le     powerpc64le    .              .             .              .          .        .        .        ----          ----------------------------------------
+
+musl          64_g1         g1       true          64          x86_64         .              .             .              .          .        .        .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+musl          64_g2         g2       true          64          x86_64         .              .             .              .          .        .        .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90

.buildkite/pipelines/main/platforms/tester_linux.yml

@@ -23,7 +23,11 @@ steps:
             - "/cache/repos:/cache/repos"
     env:
       JULIA_SHELL: "/bin/bash"
-    timeout_in_minutes: ${TIMEOUT?}
+    timeout_in_minutes: ${TIMEOUT_BK?}
+    retry:
+      automatic:
+        - exit_status: "*"
+          limit: ${RETRIES?}
     soft_fail: ${ALLOW_FAIL?}
     commands: |
       echo "--- Print the full and short commit hashes"
@@ -52,48 +56,65 @@ steps:
       echo "JULIA_CPU_THREADS is: $${JULIA_CPU_THREADS:?}"
       $${JULIA_BINARY:?} -e '@info "" Sys.CPU_THREADS'
 
-      echo "--- Set some environment variables in preparation for running the Julia test suite"
-      unset JULIA_DEPOT_PATH
+      echo "--- Set some environment variables"
       export OPENBLAS_NUM_THREADS=8
-      export TESTS="[\"all\"]"
+      unset JULIA_DEPOT_PATH
+      unset JULIA_PKG_SERVER
 
-      if [[ "${IS_RR?}" == "yes" ]]; then
-        export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+      # Make sure that temp files and temp directories are created in a location that is
+      # backed by real storage.
+      export TMPDIR="$(pwd)/tmp"
+      mkdir -p $${TMPDIR:?}
 
-        export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
-        export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
+      export NETWORK_RELATED_TESTS="Artifacts Downloads download LazyArtifacts LibGit2/online Pkg"
 
-        # For the `rr` job, we disable multi-threading.
-        export JULIA_NUM_THREADS=1
+      if [[   "${GROUP?}" == "all" ]]; then
+        export TESTS="all LibGit2/online --force-net"
+      elif [[   "${GROUP?}" == "all_except_pkg" ]]; then
+        export TESTS="all LibGit2/online --force-net --skip Pkg"
+      elif [[   "${GROUP?}" == "g1" ]]; then
+        # Group 1: ALL tests EXCEPT the network-related tests.
+        export TESTS="all --force-net --skip $${NETWORK_RELATED_TESTS:?}"
+      elif [[ "${GROUP?}" == "g2" ]]; then
+        # Group 2: ONLY the network-related tests.
+        # In Group 2, we use whatever the default setting is with regards to the Pkg server.
+        export TESTS="$${NETWORK_RELATED_TESTS:?} --force-net"
+      elif [[ "${GROUP?}" == "g3" ]]; then
+        # Group 3: only Pkg.
+        # In Group 3, we explicitly opt-out of the Pkg server.
+        # The purpose of group 3 is to test the non-Pkg-server codepaths of Pkg.
+        export TESTS="Pkg --force-net"
+        export JULIA_PKG_SERVER=""
+      else
+        echo "Invalid value for GROUP: ${GROUP?}"
+        exit 1
+      fi
+
+      export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
+
+      if [[ "${IS_RR?}" == "yes" ]]; then
+        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+        export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
       else
-        export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
+        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}"
         export NCORES_FOR_TESTS="Sys.CPU_THREADS"
+      fi
 
-        if [[ "${IS_ST?}"   == "yes" ]]; then
-          # "ST" = single-threaded
-          export JULIA_NUM_THREADS=1
-        elif [[ "${IS_MT?}" == "yes" ]]; then
-          # "MT" = multi-threaded
-          export JULIA_NUM_THREADS=16
+      if [[ "${IS_ST?}"   == "yes" ]]; then
+        export JULIA_NUM_THREADS=1
+      fi
 
-          if [[ "$${BUILDKITE_PIPELINE_SLUG:?}"   == "julia-release-1-dot-6" ]]; then
-            # On Julia 1.6, we skip the Distributed test suite if multithreading is enabled.
-            export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
-          elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-7" ]]; then
-            # On Julia 1.7, we skip the Distributed test suite if multithreading is enabled.
-            export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
-          else
-            # On Julia 1.8 and later, we do not skip any test suites.
-            export TESTS="[\"all\"]"
-          fi
-        fi
+      if [[ "${IS_MT?}" == "yes" ]]; then
+        export JULIA_NUM_THREADS=16
       fi
 
-      echo "JULIA_BINARY_FOR_TESTS is: $${JULIA_BINARY_FOR_TESTS:?}"
-      echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # this variable might not be set
+      echo "--- Print the test group, list of test sets, and other useful environment variables"
+      echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
+      echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
       echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
       echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
+      echo "GROUP is:                  ${GROUP?}"
       echo "TESTS is:                  $${TESTS:?}"
 
       echo "--- Run the Julia test suite"
-      $${JULIA_BINARY_FOR_TESTS:?} -e "Base.runtests($${TESTS:?}; ncores = $${NCORES_FOR_TESTS:?})"
+      $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"

.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml

@@ -19,10 +19,11 @@ steps:
           persist_depot_dirs: packages,artifacts,compiled
           version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
+    timeout_in_minutes: 360 # 360 minutes = 6 hours
     commands: |
       echo "--- Build Julia from source"
       make --output-sync -j 6
@@ -41,5 +42,3 @@ steps:
 
       echo "--- Process and upload coverage information"
       ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl
-    timeout_in_minutes: 240 # 240 minutes = 4 hours
-

.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml.signature

@@ -1 +1 @@
-Salted__��@P=j���R���U(�,~�p @Q������'h7����OMJ���g���t<�A�(�v?ɴ<,:�j���Y'o��ڥσdٛ
+Salted__�I�y�֌��>y�NckB�v�n�+�Hvrލ��Ƽ�r����/�uY�����u����I�iiE��(��v�L��!��?��v�

.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches

@@ -1,2 +1,2 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-linux         64_no_bb    false         64_no_bb    x86_64         USE_BINARYBUILDER=0    240        no       no       no       v4.4          ec3873fc1756ecb4c815e8785e9ca45cf54c68b6
+# PLATFORM    LABEL       GROUP    ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         64src       .        .             64src       x86_64         USE_BINARYBUILDER=0    180           .              .          .        .        .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a