Add a way to run tests on non-linux machines

tgross35 · tgross35 · commit 56a404ff6cf1 · 2024-04-13T04:24:45.000-04:00
Allow using the `rust-lang/rust:nightly` docker image to run tests in
cases where the host rust and cargo cannot be used, such as non-linux
hosts.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
 *.rs.bk
 Cargo.lock
 target
+target-docker
+compiler-rt
+*.tar.gz
diff --git a/README.md b/README.md
@@ -78,6 +78,25 @@ features = ["c"]
 [8]: http://en.cppreference.com/w/cpp/language/implicit_conversion
 [9]: https://doc.rust-lang.org/std/primitive.i32.html
 
+## Testing
+
+The easiest way to test locally is using Docker. This can be done by running
+`./ci/run-docker.sh [target]`.
+
+In order to run the full test suite, you will also need the C compiler runtime
+to test against, located in a directory called `compiler-rt`. This can be
+obtained with the following:
+
+```sh
+curl -L -o rustc-llvm-18.0.tar.gz https://github.com/rust-lang/llvm-project/archive/rustc/18.0-2024-02-13.tar.gz
+tar xzf rustc-llvm-18.0.tar.gz --strip-components 1 llvm-project-rustc-18.0-2024-02-13/compiler-rt
+````
+
+Local targets may also be tested with `./ci/run.sh [target]`.
+
+Note that testing may not work on all hosts, in which cases it is acceptable to
+rely on CI.
+
 ## Progress
 
 - [x] adddf3.c
diff --git a/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/ci/docker/arm-unknown-linux-gnueabi/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/i586-unknown-linux-gnu/Dockerfile b/ci/docker/i586-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc-multilib libc6-dev ca-certificates
diff --git a/ci/docker/i686-unknown-linux-gnu/Dockerfile b/ci/docker/i686-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc-multilib libc6-dev ca-certificates
diff --git a/ci/docker/mips-unknown-linux-gnu/Dockerfile b/ci/docker/mips-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates \
diff --git a/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile b/ci/docker/mips64el-unknown-linux-gnuabi64/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     ca-certificates \
diff --git a/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/ci/docker/mipsel-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
diff --git a/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
diff --git a/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
diff --git a/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
diff --git a/ci/docker/thumbv6m-none-eabi/Dockerfile b/ci/docker/thumbv6m-none-eabi/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/thumbv7em-none-eabi/Dockerfile b/ci/docker/thumbv7em-none-eabi/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/thumbv7em-none-eabihf/Dockerfile b/ci/docker/thumbv7em-none-eabihf/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/thumbv7m-none-eabi/Dockerfile b/ci/docker/thumbv7m-none-eabi/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates \
diff --git a/ci/docker/wasm32-unknown-unknown/Dockerfile b/ci/docker/wasm32-unknown-unknown/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:20.04
+ARG IMAGE=ubuntu:20.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc clang libc6-dev ca-certificates
diff --git a/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/ci/docker/x86_64-unknown-linux-gnu/Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:18.04
+ARG IMAGE=ubuntu:18.04
+FROM $IMAGE
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     gcc libc6-dev ca-certificates
diff --git a/ci/run-docker.sh b/ci/run-docker.sh
@@ -1,37 +1,76 @@
+#!/bin/bash
+
 # Small script to run tests for a target (or all targets) inside all the
 # respective docker images.
 
 set -ex
 
 run() {
-    local target=$1
+    local target="$1"
 
-    echo $target
+    echo "TESTING TARGET: $target"
 
     # This directory needs to exist before calling docker, otherwise docker will create it but it
     # will be owned by root
     mkdir -p target
 
-    docker build -t $target ci/docker/$target
+    if [ $(uname -s) = "Linux" ] && [ -z "$DOCKER_BASE_NAME" ]; then
+      # Share the host rustc and target. Do this only on Linux and if the image
+      # isn't overridden
+      run_args=(
+           --user "$(id -u):$(id -g)"
+           -e "CARGO_HOME=/cargo"
+           -v "${HOME}/.cargo:/cargo"
+           -v "$(pwd)/target:/builtins-target" 
+           -v "$(rustc --print sysroot):/rust:ro"
+      )
+      run_cmd="HOME=/tmp PATH=\$PATH:/rust/bin ci/run.sh $target"
+    else
+      # Use rustc provided by a docker image
+      docker volume create compiler-builtins-cache
+      build_args=(
+        "--build-arg" "IMAGE=${DOCKER_BASE_IMAGE:-rustlang/rust:nightly}"
+      )
+      run_args=(
+        -v "compiler-builtins-cache:/builtins-target"
+      )
+      run_cmd="HOME=/tmp USING_CONTAINER_RUSTC=1 ci/run.sh $target"
+    fi
+
+    if [ -d compiler-rt ]; then
+      export RUST_COMPILER_RT_ROOT=./compiler-rt
+    fi
+
+    docker build \
+           -t "builtins-$target" \
+           ${build_args[@]:-} \
+           "ci/docker/$target"
     docker run \
            --rm \
-           --user $(id -u):$(id -g) \
-           -e CARGO_HOME=/cargo \
-           -e CARGO_TARGET_DIR=/target \
            -e RUST_COMPILER_RT_ROOT \
-           -v "${HOME}/.cargo":/cargo \
-           -v `pwd`/target:/target \
-           -v `pwd`:/checkout:ro \
-           -v `rustc --print sysroot`:/rust:ro \
+           -e "CARGO_TARGET_DIR=/builtins-target" \
+           -v "$(pwd):/checkout:ro" \
            -w /checkout \
+           ${run_args[@]:-} \
            --init \
-           $target \
-           sh -c "HOME=/tmp PATH=\$PATH:/rust/bin ci/run.sh $target"
+           "builtins-$target" \
+           sh -c "$run_cmd"
 }
 
+if [ "${1:-}" = "--help" ] || [ "$#" -gt 1 ]; then
+  set +x
+  echo "\
+    usage: ./ci/run-docker.sh [target]
+
+    you can also set DOCKER_BASE_IMAGE to use something other than the default
+    ubuntu:18.04 (or rustlang/rust:nightly).
+  "
+  exit
+fi
+
 if [ -z "$1" ]; then
-  for d in `ls ci/docker/`; do
-    run $d
+  for d in ci/docker/*; do
+    run $(basename "$d")
   done
 else
   run $1
diff --git a/ci/run.sh b/ci/run.sh
@@ -1,8 +1,15 @@
 set -ex
 
+if [ "$USING_CONTAINER_RUSTC" = 1 ]; then
+    # Install nonstandard components if we have control of the environment
+    rustup target list --installed |
+        grep -E "^$1\$" ||
+        rustup target add "$1"
+fi
+
 # Test our implementation
 if [ "$NO_STD" = "1" ]; then
-    echo nothing to do
+    echo "nothing to do for no_std"
 else
     run="cargo test --manifest-path testcrate/Cargo.toml --target $1"
     $run
@@ -13,8 +20,8 @@ else
     $run --features no-asm --release
 fi
 
-if [ -d /target ]; then
-    path=/target/${1}/debug/deps/libcompiler_builtins-*.rlib
+if [ -d /builtins-target ]; then
+    path=/builtins-target/${1}/debug/deps/libcompiler_builtins-*.rlib
 else
     path=target/${1}/debug/deps/libcompiler_builtins-*.rlib
 fi
@@ -100,11 +107,15 @@ for rlib in $(echo $path); do
     echo checking $rlib for references to core
     echo "================================================================"
 
-    $NM --quiet -U $rlib | grep 'T _ZN4core' | awk '{print $3}' | sort | uniq > defined_symbols.txt
-    $NM --quiet -u $rlib | grep 'U _ZN4core' | awk '{print $2}' | sort | uniq > undefined_symbols.txt
-    grep -v -F -x -f defined_symbols.txt undefined_symbols.txt
+    $NM --quiet -U $rlib | grep 'T _ZN4core' | awk '{print $3}' |
+        sort | uniq > "$CARGO_TARGET_DIR/defined_symbols.txt"
+    $NM --quiet -u $rlib | grep 'U _ZN4core' | awk '{print $2}' |
+        sort | uniq > "$CARGO_TARGET_DIR/undefined_symbols.txt"
+    grep -v -F -x -f "$CARGO_TARGET_DIR/defined_symbols.txt" \
+                     "$CARGO_TARGET_DIR/undefined_symbols.txt"
 
     if test $? = 0; then
+        echo "error: found unexpected references to core"
         exit 1
     fi
     set -ex
