CI: add the sanitizers pipelines (e.g. ASAN) to Buildkite (#41530)

* Setup CI for ASAN

* Launch the `sanitizers.yml` unsigned pipeline

* Use a workspace directory in ./tmp

* Add some log group headers to make the logs easier to navigate

* Install `julia` binary inside sandbox

* Double timeout

* More descriptive message from sanitizer CI

* Fix the path to the binary

* Use addenv

* Apply suggestions from code review

Co-authored-by: Elliot Saba <staticfloat@gmail.com>

* Group ASAN related files under contrib/asan/

* Remove redundant JULIA_PRECOMPILE=1

Co-authored-by: Dilum Aluthge <dilum@aluthge.com>
Co-authored-by: Elliot Saba <staticfloat@gmail.com>

.buildkite/pipeline.yml

@@ -14,8 +14,11 @@
 steps:
   - label: ":buildkite: Launch unsigned pipelines"
     commands: |
+      # We launch whitespace first, because we want that pipeline to finish as quickly as possible.
+      # The remaining unsigned pipelines are launched in alphabetical order.
       buildkite-agent pipeline upload .buildkite/whitespace.yml
-      buildkite-agent pipeline upload .buildkite/llvm_passes.yml
       buildkite-agent pipeline upload .buildkite/embedding.yml
+      buildkite-agent pipeline upload .buildkite/llvm_passes.yml
+      buildkite-agent pipeline upload .buildkite/sanitizers.yml
     agents:
       queue: julia

.buildkite/sanitizers.yml

@@ -0,0 +1,34 @@
+# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
+# since we need nestable sandboxing.  The rootfs images being used here are built from
+# the `.buildkite/rootfs_images/llvm-passes.jl` file.
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+
+steps:
+  - label: "asan"
+    key: asan
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
+          rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+      # `contrib/check-asan.jl` needs a `julia` binary:
+      - JuliaCI/julia#v1:
+          version: 1.6
+    commands: |
+      echo "--- Build julia-debug with ASAN"
+      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_NUM_CORES} debug
+      echo "--- Test that ASAN is enabled"
+      contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug
+    timeout_in_minutes: 120
+    notify:
+      - github_commit_status:
+          context: "asan"

contrib/asan/Make.user.asan

@@ -0,0 +1,24 @@
+TOOLCHAIN=$(BUILDROOT)/../toolchain/usr/tools
+
+# use our new toolchain
+USECLANG=1
+override CC=$(TOOLCHAIN)/clang
+override CXX=$(TOOLCHAIN)/clang++
+export ASAN_SYMBOLIZER_PATH=$(TOOLCHAIN)/llvm-symbolizer
+
+USE_BINARYBUILDER_LLVM=1
+
+override SANITIZE=1
+override SANITIZE_ADDRESS=1
+
+# make the GC use regular malloc/frees, which are hooked by ASAN
+override WITH_GC_DEBUG_ENV=1
+
+# default to a debug build for better line number reporting
+override JULIA_BUILD_MODE=debug
+
+# make ASAN consume less memory
+export ASAN_OPTIONS=detect_leaks=0:fast_unwind_on_malloc=0:allow_user_segv_handler=1:malloc_context_size=2
+
+# tell libblastrampoline to not use RTLD_DEEPBIND
+export LBT_USE_RTLD_DEEPBIND=0

contrib/asan/Make.user.tools

@@ -0,0 +1,2 @@
+USE_BINARYBUILDER_LLVM=1
+BUILD_LLVM_CLANG=1

contrib/asan/build.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+#
+# Usage:
+#     contrib/asan/build.sh <path> [<make_targets>...]
+#
+# Build ASAN-enabled julia.  Given a workspace directory <path>, build
+# ASAN-enabled julia in <path>/asan.  Required toolss are install under
+# <path>/toolchain.  This scripts also takes optional <make_targets> arguments
+# which are passed to `make`.  The default make target is `debug`.
+
+set -ue
+
+# `$WORKSPACE` is a directory in which we create `toolchain` and `asan`
+# sub-directories.
+WORKSPACE="$1"
+shift
+if [ "$WORKSPACE" = "" ]; then
+    echo "Workspace directory must be specified as the first argument" >&2
+    exit 2
+fi
+
+mkdir -pv "$WORKSPACE"
+WORKSPACE="$(cd "$WORKSPACE" && pwd)"
+if [ "$WORKSPACE" = "" ]; then
+    echo "Failed to create the workspace directory." >&2
+    exit 2
+fi
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JULIA_HOME="$HERE/../../"
+
+echo
+echo "Installing toolchain..."
+
+TOOLCHAIN="$WORKSPACE/toolchain"
+if [ ! -d "$TOOLCHAIN" ]; then
+    make -C "$JULIA_HOME" configure O=$TOOLCHAIN
+    cp "$HERE/Make.user.tools"  "$TOOLCHAIN/Make.user"
+fi
+
+make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools
+
+echo
+echo "Building Julia..."
+
+BUILD="$WORKSPACE/asan"
+if [ ! -d "$BUILD" ]; then
+    make -C "$JULIA_HOME" configure O="$BUILD"
+    cp "$HERE/Make.user.asan"  "$BUILD/Make.user"
+fi
+
+make -C "$BUILD" "$@"

contrib/asan/check.jl

@@ -0,0 +1,92 @@
+#!/bin/bash
+# -*- mode: julia -*-
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+#
+# Usage:
+#     contrib/asan/check.jl <julia>
+#
+# Check that <julia> is built with ASAN.
+#
+#=
+JULIA="${JULIA:-julia}"
+exec "$JULIA" --startup-file=no --compile=min "${BASH_SOURCE[0]}" "$@"
+=#
+
+function main(args = ARGS)::Int
+    if length(args) != 1
+        @error "Expect a single argument" args
+        return 2
+    end
+    julia, = args
+
+    # It looks like double-free is easy to robustly trigger.
+    code = """
+    @info "Testing a pattern that would trigger ASAN"
+    write(ARGS[1], "started")
+
+    ptr = ccall(:malloc, Ptr{UInt}, (Csize_t,), 256)
+    ccall(:free, Cvoid, (Ptr{UInt},), ptr)
+    ccall(:free, Cvoid, (Ptr{UInt},), ptr)
+
+    @error "Failed to trigger ASAN"
+    """
+
+    local proc
+    timeout = Threads.Atomic{Bool}(false)
+    isstarted = false
+    mktemp() do tmppath, tmpio
+        cmd = addenv(
+            `$julia -e $code $tmppath`,
+            "ASAN_OPTIONS" =>
+                "detect_leaks=0:fast_unwind_on_malloc=0:allow_user_segv_handler=1:malloc_context_size=2",
+            "LBT_USE_RTLD_DEEPBIND" => "0",
+        )
+        # Note: Ideally, we set ASAN_SYMBOLIZER_PATH here. But there is no easy
+        # way to find out the path from just a Julia binary.
+
+        @debug "Starting a process" cmd
+        proc = run(pipeline(cmd; stdout, stderr); wait = false)
+        timer = Timer(10)
+        @sync try
+            @async begin
+                try
+                    wait(timer)
+                    true
+                catch err
+                    err isa EOFError || rethrow()
+                    false
+                end && begin
+                    timeout[] = true
+                    kill(proc)
+                end
+            end
+            wait(proc)
+        finally
+            close(timer)
+        end
+
+        # At the very beginning of the process, the `julia` subprocess put a
+        # marker that it is successfully started. This is to avoid mixing
+        # non-functional `julia` binary (or even non-`julia` command) and
+        # correctly working `julia` with ASAN:
+        isstarted = read(tmpio, String) == "started"
+    end
+
+    if timeout[]
+        @error "Timeout waiting for the subprocess"
+        return 1
+    elseif success(proc)
+        @error "ASAN was not triggered"
+        return 1
+    elseif !isstarted
+        @error "Failed to start the process"
+        return 1
+    else
+        @info "ASAN is functional in the Julia binary `$julia`"
+        return 0
+    end
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    exit(main())
+end