[antithesis] Add test setup for xsvm

Author: maru-ava

.github/workflows/ci.yml

@@ -265,8 +265,8 @@ jobs:
       - name: Check image build
         shell: bash
         run: bash -x scripts/tests.build_image.sh
-  test_build_antithesis_avalanchego_image:
-    name: Antithesis avalanchego build
+  test_build_antithesis_avalanchego_images:
+    name: Build Antithesis avalanchego images
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -275,6 +275,16 @@ jobs:
         run: bash -x scripts/tests.build_antithesis_images.sh
         env:
           TEST_SETUP: avalanchego
+  test_build_antithesis_xsvm_images:
+    name: Build Antithesis xsvm images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check image build for xsvm test setup
+        shell: bash
+        run: bash -x scripts/tests.build_antithesis_images.sh
+        env:
+          TEST_SETUP: xsvm
   govulncheck:
     runs-on: ubuntu-latest
     name: govulncheck

.github/workflows/publish_antithesis_images.yml

@@ -31,3 +31,10 @@ jobs:
         IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
         TAG: latest
         TEST_SETUP: avalanchego
+
+    - name: Build images for xsvm test setup
+      run: bash -x ./scripts/build_antithesis_images.sh
+      env:
+        IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
+        TAG: latest
+        TEST_SETUP: xsvm

scripts/build_antithesis_images.sh

@@ -5,8 +5,9 @@ set -euo pipefail
 # Builds docker images for antithesis testing.
 
 # e.g.,
-# ./scripts/build_antithesis_images.sh                                           # Build local images
-# IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
+# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh                                    # Build local images for avalanchego
+# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh                                           # Build local images for xsvm
+# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
 
 # Directory above this script
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
@@ -28,11 +29,14 @@ GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
 function build_images {
   local test_setup=$1
   local uninstrumented_node_dockerfile=$2
+  local node_only=${3:-}
 
   # Define image names
   local base_image_name="antithesis-${test_setup}"
+  local avalanchego_node_image_name="antithesis-avalanchego-node:${TAG}"
   if [[ -n "${IMAGE_PREFIX}" ]]; then
     base_image_name="${IMAGE_PREFIX}/${base_image_name}"
+    avalanchego_node_image_name="${IMAGE_PREFIX}/${avalanchego_node_image_name}"
   fi
   local node_image_name="${base_image_name}-node:${TAG}"
   local workload_image_name="${base_image_name}-workload:${TAG}"
@@ -49,18 +53,34 @@ function build_images {
   fi
 
   # Define default build command
-  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION}"
+  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"
+
+  if [[ "${test_setup}" == "xsvm" ]]; then
+    # The xsvm node image is build on the avalanchego node image
+    docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_NODE_IMAGE=${avalanchego_node_image_name}"
+  fi
 
   # Build node image first to allow the config and workload image builds to use it.
   ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg NODE_IMAGE="${node_image_name}" -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+
+  if [[ -z "${node_only}" || "${node_only}" == "0" ]]; then
+    # Skip building the config and workload images. Supports building the avalanchego
+    # node image as the base image for the xsvm node image.
+    ${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+    ${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
+  fi
 }
 
 TEST_SETUP="${TEST_SETUP:-}"
 if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
   build_images avalanchego "${AVALANCHE_PATH}/Dockerfile"
+elif [[ "${TEST_SETUP}" == "xsvm" ]]; then
+  # Only build the node image to use as the base for the xsvm image
+  NODE_ONLY=1
+  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "${NODE_ONLY}"
+
+  build_images xsvm "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
 else
-  echo "TEST_SETUP must be set. Valid values are 'avalanchego'"
+  echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
   exit 255
 fi

scripts/build_antithesis_xsvm_workload.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Directory above this script
+AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
+# Load the constants
+source "$AVALANCHE_PATH"/scripts/constants.sh
+
+echo "Building Workload..."
+go build -o "$AVALANCHE_PATH/build/antithesis-xsvm-workload" "$AVALANCHE_PATH/tests/antithesis/xsvm/"*.go

scripts/tests.build_antithesis_images.sh

@@ -10,6 +10,10 @@ set -euo pipefail
 #   4. Stopping the workload and its target network
 #
 
+# e.g.,
+# TEST_SETUP=avalanchego ./scripts/tests.build_antithesis_images.sh                 # Test build of images for avalanchego test setup
+# DEBUG=1 TEST_SETUP=avalanchego ./scripts/tests.build_antithesis_images.sh         # Retain the temporary compose path for troubleshooting
+
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 
 # Discover the default tag that will be used for the image
@@ -27,6 +31,8 @@ docker create --name "${CONTAINER_NAME}" "${IMAGE_NAME}:${TAG}" /bin/true
 
 # Create a temporary directory to write the compose configuration to
 TMPDIR="$(mktemp -d)"
+echo "using temporary directory ${TMPDIR} as the docker-compose path"
+
 COMPOSE_FILE="${TMPDIR}/docker-compose.yml"
 COMPOSE_CMD="docker-compose -f ${COMPOSE_FILE}"
 
@@ -36,8 +42,10 @@ function cleanup {
   docker rm "${CONTAINER_NAME}"
   echo "stopping and removing the docker compose project"
   ${COMPOSE_CMD} down --volumes
-  echo "removing temporary dir"
-  rm -rf "${TMPDIR}"
+  if [[ -z "${DEBUG:-}" ]]; then
+    echo "removing temporary dir"
+    rm -rf "${TMPDIR}"
+  fi
 }
 trap cleanup EXIT
 
@@ -47,9 +55,12 @@ docker cp "${CONTAINER_NAME}":/docker-compose.yml "${COMPOSE_FILE}"
 # Copy the volume paths out of the container
 docker cp "${CONTAINER_NAME}":/volumes "${TMPDIR}/"
 
-# Run the docker compose project for one minute without error
+# Run the docker compose project for 2 minutes without error. 2
+# minutes is suggested because the way docker-compose brings all
+# containers up simultaneously and the lack of coordination results in
+# exponential back-off on the nodes trying to bootstrap.
 ${COMPOSE_CMD} up -d
-sleep 60
+sleep 120
 if ${COMPOSE_CMD} ps -q | xargs docker inspect -f '{{ .State.Status }}' | grep -v 'running'; then
   echo "An error occurred."
   exit 255

tests/antithesis/README.md

@@ -8,11 +8,14 @@ enables discovery and reproduction of anomalous behavior.
 
 ## Package details
 
-| Filename     | Purpose                                                                           |
-|:-------------|:----------------------------------------------------------------------------------|
-| compose.go   | Enables generation of Docker Compose project files for antithesis testing.        |
-| avalanchego/ | Contains resources supporting antithesis testing of avalanchego's primary chains. |
-
+| Filename       | Purpose                                                                         |
+|:---------------|:--------------------------------------------------------------------------------|
+| compose.go     | Generates Docker Compose project files and volume paths for antithesis testing. |
+| config.go      | Defines common flags for the workload binary.                                   |
+| init_db.go     | Initializes db state for subnet testing.                                        |
+| node_health.go | Helper to check node health.                                                    |
+| avalanchego/   | Defines an antithesis test setup for avalanchego's primary chains.              |
+| xsvm/          | Defines an antithesis test setup for the xsvm VM.                               |
 
 ## Instrumentation
 
@@ -45,3 +48,55 @@ a test setup:
 In addition, github workflows are suggested to ensure
 `scripts/tests.build_antithesis_images.sh` runs against PRs and
 `scripts/build_antithesis_images.sh` runs against pushes.
+
+## Troubleshooting a test setup
+
+### Running a workload directly
+
+The workload of the 'avalanchego' test setup can be invoked against an
+arbitrary network:
+
+```bash
+$ AVAWL_URIS="http://10.0.20.3:9650 http://10.0.20.4:9650" go run ./tests/antithesis/avalanchego
+```
+
+The workload of a subnet test setup like 'xsvm' additionally requires
+a network with a configured chain for the xsvm VM and the ID for that
+chain needs to be provided to the workload:
+
+```bash
+$ AVAWL_URIS=... CHAIN_IDS="2S9ypz...AzMj9" go run ./tests/antithesis/xsvm
+```
+
+### Running a workload with docker-compose
+
+Running the test script for a given test setup with the `DEBUG` flag
+set will avoid cleaning up the the temporary directory where the
+docker-compose setup is written to. This will allow manual invocation of
+docker-compose to see the log output of the workload.
+
+```bash
+$ DEBUG=1 ./scripts/tests.build_antithesis_images.sh
+```
+
+After the test script has terminated, the name of the temporary
+directory will appear in the output of the script:
+
+```
+...
+using temporary directory /tmp/tmp.E6eHdDr4ln as the docker-compose path"
+...
+```
+
+Running compose from the temporary directory will ensure the workload
+output appears on stdout for inspection:
+
+```bash
+$ cd [temporary directory]
+
+# Start the compose project
+$ docker-compose up
+
+# Cleanup the compose project
+$ docker-compose down --volumes
+```

tests/antithesis/avalanchego/Dockerfile.node

@@ -56,6 +56,9 @@ RUN mkdir -p /symbols
 COPY --from=builder /avalanchego_instrumented/symbols /symbols
 COPY --from=builder /opt/antithesis/lib/libvoidstar.so /usr/lib/libvoidstar.so
 
+# Use the same path as the uninstrumented node image for consistency
+WORKDIR /avalanchego/build
+
 # Copy the executable into the container
 COPY --from=builder /avalanchego_instrumented/customer/build/avalanchego ./avalanchego
 

tests/antithesis/avalanchego/main.go

@@ -11,11 +11,11 @@ import (
 	"os"
 	"time"
 
-	"github.com/ava-labs/avalanchego/api/health"
 	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/genesis"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/snow/choices"
+	"github.com/ava-labs/avalanchego/tests/antithesis"
 	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/crypto/secp256k1"
 	"github.com/ava-labs/avalanchego/utils/set"
@@ -38,13 +38,13 @@ import (
 const NumKeys = 5
 
 func main() {
-	c, err := NewConfig(os.Args)
+	c, err := antithesis.NewConfig(os.Args)
 	if err != nil {
 		log.Fatalf("invalid config: %s", err)
 	}
 
 	ctx := context.Background()
-	awaitHealthyNodes(ctx, c.URIs)
+	antithesis.AwaitHealthyNodes(ctx, c.URIs)
 
 	kc := secp256k1fx.NewKeychain(genesis.EWOQKey)
 	walletSyncStartTime := time.Now()
@@ -133,39 +133,6 @@ func main() {
 	genesisWorkload.run(ctx)
 }
 
-func awaitHealthyNodes(ctx context.Context, uris []string) {
-	for _, uri := range uris {
-		awaitHealthyNode(ctx, uri)
-	}
-	log.Println("all nodes reported healthy")
-}
-
-func awaitHealthyNode(ctx context.Context, uri string) {
-	client := health.NewClient(uri)
-	ticker := time.NewTicker(100 * time.Millisecond)
-	defer ticker.Stop()
-
-	log.Printf("awaiting node health at %s", uri)
-	for {
-		res, err := client.Health(ctx, nil)
-		switch {
-		case err != nil:
-			log.Printf("node couldn't be reached at %s", uri)
-		case res.Healthy:
-			log.Printf("node reported healthy at %s", uri)
-			return
-		default:
-			log.Printf("node reported unhealthy at %s", uri)
-		}
-
-		select {
-		case <-ticker.C:
-		case <-ctx.Done():
-			log.Printf("node health check cancelled at %s", uri)
-		}
-	}
-}
-
 type workload struct {
 	id     int
 	wallet primary.Wallet