[antithesis] Add test setup for xsvm

Author: maru-ava

.github/workflows/ci.yml

@@ -265,8 +265,8 @@ jobs:
       - name: Check image build
         shell: bash
         run: bash -x scripts/tests.build_image.sh
-  test_build_antithesis_avalanchego_image:
-    name: Antithesis avalanchego build
+  test_build_antithesis_images:
+    name: Build Antithesis images
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -275,6 +275,11 @@ jobs:
         run: bash -x scripts/tests.build_antithesis_images.sh
         env:
           TEST_SETUP: avalanchego
+      - name: Check image build for xsvm test setup
+        shell: bash
+        run: bash -x scripts/tests.build_antithesis_images.sh
+        env:
+          TEST_SETUP: xsvm
   govulncheck:
     runs-on: ubuntu-latest
     name: govulncheck

.github/workflows/publish_antithesis_images.yml

@@ -31,3 +31,10 @@ jobs:
         IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
         TAG: latest
         TEST_SETUP: avalanchego
+
+    - name: Build images for xsvm test setup
+      run: bash -x ./scripts/build_antithesis_images.sh
+      env:
+        IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
+        TAG: latest
+        TEST_SETUP: xsvm

scripts/build_antithesis_images.sh

@@ -5,8 +5,9 @@ set -euo pipefail
 # Builds docker images for antithesis testing.
 
 # e.g.,
-# ./scripts/build_antithesis_images.sh                                           # Build local images
-# IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
+# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh                                    # Build local images for avalanchego
+# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh                                           # Build local images for xsvm
+# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
 
 # Directory above this script
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
@@ -28,11 +29,14 @@ GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
 function build_images {
   local test_setup=$1
   local uninstrumented_node_dockerfile=$2
+  local node_only=${3:-}
 
   # Define image names
   local base_image_name="antithesis-${test_setup}"
+  local avalanchego_node_image_name="antithesis-avalanchego-node:${TAG}"
   if [[ -n "${IMAGE_PREFIX}" ]]; then
     base_image_name="${IMAGE_PREFIX}/${base_image_name}"
+    avalanchego_node_image_name="${IMAGE_PREFIX}/${avalanchego_node_image_name}"
   fi
   local node_image_name="${base_image_name}-node:${TAG}"
   local workload_image_name="${base_image_name}-workload:${TAG}"
@@ -49,18 +53,34 @@ function build_images {
   fi
 
   # Define default build command
-  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION}"
+  local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"
+
+  if [[ "${test_setup}" == "xsvm" ]]; then
+    # The xsvm node image is build on the avalanchego node image
+    docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_NODE_IMAGE=${avalanchego_node_image_name}"
+  fi
 
   # Build node image first to allow the config and workload image builds to use it.
   ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg NODE_IMAGE="${node_image_name}" -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
-  ${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+
+  if [[ -z "${node_only}" || "${node_only}" == "0" ]]; then
+    # Skip building the config and workload images. Supports building the avalanchego
+    # node image as the base image for the xsvm node image.
+    ${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+    ${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
+  fi
 }
 
 TEST_SETUP="${TEST_SETUP:-}"
 if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
   build_images avalanchego "${AVALANCHE_PATH}/Dockerfile"
+elif [[ "${TEST_SETUP}" == "xsvm" ]]; then
+  # Only build the node image to use as the base for the xsvm image
+  NODE_ONLY=1
+  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "${NODE_ONLY}"
+
+  build_images xsvm "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
 else
-  echo "TEST_SETUP must be set. Valid values are 'avalanchego'"
+  echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
   exit 255
 fi

scripts/build_antithesis_xsvm_workload.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Directory above this script
+AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
+# Load the constants
+source "$AVALANCHE_PATH"/scripts/constants.sh
+
+echo "Building Workload..."
+go build -o "$AVALANCHE_PATH/build/antithesis-xsvm-workload" "$AVALANCHE_PATH/tests/antithesis/xsvm/"*.go

tests/antithesis/README.md

@@ -8,11 +8,12 @@ enables discovery and reproduction of anomalous behavior.
 
 ## Package details
 
-| Filename     | Purpose                                                                           |
-|:-------------|:----------------------------------------------------------------------------------|
-| compose.go   | Enables generation of Docker Compose project files for antithesis testing.        |
-| avalanchego/ | Contains resources supporting antithesis testing of avalanchego's primary chains. |
-
+| Filename     | Purpose                                                                         |
+|:-------------|:--------------------------------------------------------------------------------|
+| compose.go   | Generates Docker Compose project files and volume paths for antithesis testing. |
+| initdb.go    | Initializes db state for subnet testing                                         |
+| avalanchego/ | Defines an antithesis test setup for avalanchego's primary chains.              |
+| xsvm/        | Defines an antithesis test setup for the xsvm VM.                               |
 
 ## Instrumentation
 

tests/antithesis/avalanchego/Dockerfile.node

@@ -56,6 +56,9 @@ RUN mkdir -p /symbols
 COPY --from=builder /avalanchego_instrumented/symbols /symbols
 COPY --from=builder /opt/antithesis/lib/libvoidstar.so /usr/lib/libvoidstar.so
 
+# Use the same path as the uninstrumented node image for consistency
+WORKDIR /avalanchego/build
+
 # Copy the executable into the container
 COPY --from=builder /avalanchego_instrumented/customer/build/avalanchego ./avalanchego
 

tests/antithesis/compose.go

@@ -104,13 +104,34 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
 			config.StakingSignerKeyContentKey: signerKey,
 		}
 
-		nodeName := "avalanche"
+		serviceName := getServiceName(i)
+
+		volumes := []types.ServiceVolumeConfig{
+			{
+				Type:   types.VolumeTypeBind,
+				Source: fmt.Sprintf("./volumes/%s/logs", serviceName),
+				Target: "/root/.avalanchego/logs",
+			},
+		}
+
+		trackSubnets, err := node.Flags.GetStringVal(config.TrackSubnetsKey)
+		if err != nil {
+			return nil, err
+		}
+		if len(trackSubnets) > 0 {
+			env[config.TrackSubnetsKey] = trackSubnets
+			// DB volume will need to initialized with the subnet
+			volumes = append(volumes, types.ServiceVolumeConfig{
+				Type:   types.VolumeTypeBind,
+				Source: fmt.Sprintf("./volumes/%s/db", serviceName),
+				Target: "/root/.avalanchego/db",
+			})
+		}
+
 		if i == 0 {
-			nodeName += "-bootstrap-node"
 			bootstrapIP = address + ":9651"
 			bootstrapIDs = node.NodeID.String()
 		} else {
-			nodeName = fmt.Sprintf("%s-node-%d", nodeName, i+1)
 			env[config.BootstrapIPsKey] = bootstrapIP
 			env[config.BootstrapIDsKey] = bootstrapIDs
 		}
@@ -120,18 +141,12 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
 		env = keyMapToEnvVarMap(env)
 
 		services[i+1] = types.ServiceConfig{
-			Name:          nodeName,
-			ContainerName: nodeName,
-			Hostname:      nodeName,
+			Name:          serviceName,
+			ContainerName: serviceName,
+			Hostname:      serviceName,
 			Image:         nodeImageName,
-			Volumes: []types.ServiceVolumeConfig{
-				{
-					Type:   types.VolumeTypeBind,
-					Source: fmt.Sprintf("./volumes/%s/logs", nodeName),
-					Target: "/root/.avalanchego/logs",
-				},
-			},
-			Environment: env.ToMappingWithEquals(),
+			Volumes:       volumes,
+			Environment:   env.ToMappingWithEquals(),
 			Networks: map[string]*types.ServiceNetworkConfig{
 				networkName: {
 					Ipv4Address: address,
@@ -146,6 +161,9 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
 	workloadEnv := types.Mapping{
 		"AVAWL_URIS": strings.Join(uris, " "),
 	}
+	if len(network.Subnets) > 0 && len(network.Subnets[0].Chains) > 0 {
+		workloadEnv["AVAWL_CHAIN_ID"] = network.Subnets[0].Chains[0].ChainID.String()
+	}
 
 	workloadName := "workload"
 	services[0] = types.ServiceConfig{
@@ -191,3 +209,14 @@ func keyMapToEnvVarMap(keyMap types.Mapping) types.Mapping {
 	}
 	return envVarMap
 }
+
+// Retrieve the service name for a node at the given index. Common to
+// GenerateComposeConfig and InitDBVolumes to ensure consistency
+// between db volumes configuration and volume paths.
+func getServiceName(index int) string {
+	baseName := "avalanche"
+	if index == 0 {
+		return baseName + "-bootstrap-node"
+	}
+	return fmt.Sprintf("%s-node-%d", baseName, index+1)
+}

tests/antithesis/initdb.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package antithesis
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
+	"github.com/ava-labs/avalanchego/utils/perms"
+)
+
+// Initialize the db volumes for a docker-compose configuration. The returned network will be updated with
+// subnet and chain IDs and the target path will be configured with volumes for each node containing the
+// initialized db state.
+func InitDBVolumes(network *tmpnet.Network, avalancheGoPath string, pluginDir string, targetPath string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
+	defer cancel()
+	if err := tmpnet.StartNewNetwork(
+		ctx,
+		os.Stdout,
+		network,
+		"",
+		avalancheGoPath,
+		pluginDir,
+		0, // nodeCount is 0 because nodes should already be configured for subnets
+	); err != nil {
+		return fmt.Errorf("failed to start network: %w", err)
+	}
+	// Since the goal is to initialize the DB, we can stop the network after it has been started successfully
+	if err := network.Stop(ctx); err != nil {
+		return fmt.Errorf("failed to stop network: %w", err)
+	}
+
+	absPath, err := filepath.Abs(targetPath)
+	if err != nil {
+		return fmt.Errorf("failed to convert target path to absolute path: %w", err)
+	}
+
+	// Create volumes in the target path and copy the db state from the nodes
+	for i, node := range network.Nodes {
+		sourcePath := filepath.Join(node.GetDataDir(), "db")
+		destPath := filepath.Join(absPath, "volumes", getServiceName(i))
+		if err := os.MkdirAll(destPath, perms.ReadWriteExecute); err != nil {
+			return fmt.Errorf("failed to create volume path %q: %w", destPath, err)
+		}
+		cmd := exec.Command("cp", "-r", sourcePath, destPath)
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("failed to copy db state from %q to %q: %w", sourcePath, destPath, err)
+		}
+	}
+
+	return nil
+}

tests/antithesis/xsvm/Dockerfile.config

@@ -0,0 +1,39 @@
+# The version is supplied as a build argument rather than hard-coded
+# to minimize the cost of version changes.
+ARG GO_VERSION
+
+# NODE_IMAGE needs to identify an existing xsvm node image and should include the tag
+ARG NODE_IMAGE
+
+FROM $NODE_IMAGE AS xsvm_node
+
+# ============= Compilation Stage ================
+FROM golang:$GO_VERSION-bullseye AS builder
+
+WORKDIR /build
+# Copy and download avalanche dependencies using go mod
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+# Copy the code into the container
+COPY . .
+
+# IMAGE_TAG should be set to the tag for the images in the generated docker compose file.
+ARG IMAGE_TAG=latest
+
+# Copy the avalanchego binary and plugin from the node image
+RUN mkdir -p ./build/plugins
+COPY --from=xsvm_node /avalanchego/build/avalanchego ./build
+COPY --from=xsvm_node /root/.avalanchego/plugins/* ./build/plugins/
+
+# Generate docker compose configuration
+RUN AVALANCHEGO_PATH=./build/avalanchego AVALANCHEGO_PLUGIN_DIR=./build/plugins\
+ TARGET_PATH=./build IMAGE_TAG="$IMAGE_TAG" go run ./tests/antithesis/xsvm/gencomposeconfig
+
+# ============= Cleanup Stage ================
+FROM scratch AS execution
+
+# Copy the docker compose file and volumes into the container
+COPY --from=builder /build/build/docker-compose.yml /docker-compose.yml
+COPY --from=builder /build/build/volumes /volumes

tests/antithesis/xsvm/Dockerfile.node

@@ -0,0 +1,61 @@
+# The version is supplied as a build argument rather than hard-coded
+# to minimize the cost of version changes.
+ARG GO_VERSION
+
+# AVALANCHEGO_NODE_IMAGE needs to identify an existing avalanchego node image and should include the tag
+ARG AVALANCHEGO_NODE_IMAGE
+
+# Antithesis: Getting the Antithesis golang instrumentation library
+FROM docker.io/antithesishq/go-instrumentor AS instrumentor
+
+# ============= Compilation Stage ================
+FROM golang:$GO_VERSION-bullseye AS builder
+
+WORKDIR /build
+# Copy and download avalanche dependencies using go mod
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+# Copy the code into the container
+COPY . .
+
+# Keep the commit hash to easily verify the exact version that is running
+RUN git rev-parse HEAD > ./commit_hash.txt
+
+# Copy the instrumentor and supporting files to their correct locations
+COPY --from=instrumentor /opt/antithesis /opt/antithesis
+COPY --from=instrumentor /opt/antithesis/lib /lib
+
+# Create the destination output directory for the instrumented code
+RUN mkdir -p /avalanchego_instrumented
+
+# Park the .git file in a safe location
+RUN mkdir -p /opt/tmp/
+RUN cp -r .git /opt/tmp/
+
+# Instrument avalanchego
+RUN /opt/antithesis/bin/goinstrumentor \
+    -stderrthreshold=INFO \
+    -antithesis /opt/antithesis/instrumentation \
+    . \
+    /avalanchego_instrumented
+
+WORKDIR /avalanchego_instrumented/customer
+RUN go mod download
+RUN ln -s /opt/tmp/.git .git
+
+# Build xsvm VM
+RUN ./scripts/build_xsvm.sh
+
+# ============= Cleanup Stage ================
+FROM $AVALANCHEGO_NODE_IMAGE AS execution
+
+# The commit hash and antithesis dependencies should be part of the base image.
+
+# Copy the executable into the container
+RUN mkdir -p /root/.avalanchego/plugins
+COPY --from=builder /avalanchego_instrumented/customer/build/xsvm \
+    /root/.avalanchego/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH
+
+# The node image's entrypoint will be reused.