Skip to content

Commit ce90247

Browse files
maru-avamaru-ava
committed
[antithesis] Enable workload instrumentation
The Antithesis test setups currently use regex matches on log output to trigger fault injection (by matching on output indicating that test setup was completed) and record desired test properties (e.g. by matching on output indicating that an X-Chain transaction was verified). The configuration of this matching is currently stored on Antithesis infrastructure and only accessible to Antithesis personal. This changeset uses the Antithesis SDK to explicitly signal events so that the use of potentially unreliable regex matching can be avoided and that more control over test execution and analysis can be given to code in the avalanchego repo.
1 parent e3d889c commit ce90247

13 files changed

+191
-120
lines changed

‎go.mod‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ go 1.21.10
99
require (
1010
github.com/DataDog/zstd v1.5.2
1111
github.com/NYTimes/gziphandler v1.1.1
12+
github.com/antithesishq/antithesis-sdk-go v0.3.8
1213
github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2
1314
github.com/ava-labs/ledger-avalanche/go v0.0.0-20231102202641-ae2ebdaeac34
1415
github.com/btcsuite/btcd/btcutil v1.1.3
@@ -160,7 +161,7 @@ require (
160161
go.uber.org/multierr v1.10.0 // indirect
161162
golang.org/x/sys v0.18.0 // indirect
162163
golang.org/x/text v0.14.0 // indirect
163-
golang.org/x/tools v0.16.0 // indirect
164+
golang.org/x/tools v0.17.0 // indirect
164165
google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect
165166
gopkg.in/ini.v1 v1.67.0 // indirect
166167
gopkg.in/yaml.v2 v2.4.0 // indirect

‎go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
5959
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
6060
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
6161
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
62+
github.com/antithesishq/antithesis-sdk-go v0.3.8 h1:OvGoHxIcOXFJLyn9IJQ5DzByZ3YVAWNBc394ObzDRb8=
63+
github.com/antithesishq/antithesis-sdk-go v0.3.8/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
6264
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
6365
github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2 h1:RX9DcvgWxq42B2aiGzk77Y8w2bcB7ApO/Cdj9hA6QoE=
6466
github.com/ava-labs/coreth v0.13.5-remove-optional-gatherer.2/go.mod h1:cm5c12xo5NiTgtbmeduv8i2nYdzgkczz9Wm3yiwwTRU=
@@ -700,8 +702,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
700702
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
701703
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
702704
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
703-
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
704-
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
705705
golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
706706
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
707707
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -916,8 +916,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f
916916
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
917917
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
918918
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
919-
golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
920-
golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
919+
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
920+
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
921921
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
922922
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
923923
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

‎scripts/build_antithesis_images.sh‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,32 @@ function build_images {
4141
local node_image_name="${base_image_name}-node:${TAG}"
4242
local workload_image_name="${base_image_name}-workload:${TAG}"
4343
local config_image_name="${base_image_name}-config:${TAG}"
44+
# The same builder image is used to build node and workload images for all test
45+
# setups. It is not intended to be pushed.
46+
local builder_image_name="antithesis-avalanchego-builder:${TAG}"
4447

4548
# Define dockerfiles
4649
local base_dockerfile="${AVALANCHE_PATH}/tests/antithesis/${test_setup}/Dockerfile"
50+
local builder_dockerfile="${base_dockerfile}.builder-instrumented"
4751
local node_dockerfile="${base_dockerfile}.node"
52+
# Working directory for instrumented builds
53+
local builder_workdir="/avalanchego_instrumented/customer"
4854
if [[ "$(go env GOARCH)" == "arm64" ]]; then
49-
# Antithesis instrumentation is only supported on amd64. On apple silicon (arm64), the
50-
# uninstrumented Dockerfile will be used to build the node image to enable local test
51-
# development.
55+
# Antithesis instrumentation is only supported on amd64. On apple silicon (arm64),
56+
# uninstrumented Dockerfiles will be used to enable local test development.
57+
builder_dockerfile="${base_dockerfile}.builder-uninstrumented"
5258
node_dockerfile="${uninstrumented_node_dockerfile}"
59+
# Working directory for uninstrumented builds
60+
builder_workdir="/build"
5361
fi
5462

5563
# Define default build command
56-
local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"
64+
local docker_cmd="docker buildx build\
65+
--build-arg GO_VERSION=${GO_VERSION}\
66+
--build-arg NODE_IMAGE=${node_image_name}\
67+
--build-arg BUILDER_IMAGE=${builder_image_name}\
68+
--build-arg BUILDER_WORKDIR=${builder_workdir}\
69+
--build-arg TAG=${TAG}"
5770

5871
if [[ "${test_setup}" == "xsvm" ]]; then
5972
# The xsvm node image is built on the avalanchego node image, which is assumed to have already been
@@ -69,6 +82,13 @@ function build_images {
6982
docker_cmd="${docker_cmd} --push"
7083
fi
7184

85+
if [[ "${test_setup}" == "avalanchego" ]]; then
86+
# Build the image that enables compiling golang binaries for the node and workload
87+
# image builds. The builder image is intended to enable building instrumented binaries
88+
# if built on amd64 and non-instrumented binaries if built on arm64.
89+
${docker_cmd} -t "${builder_image_name}" -f "${builder_dockerfile}" "${AVALANCHE_PATH}"
90+
fi
91+
7292
# Build node image first to allow the workload image to use it.
7393
${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
7494

‎scripts/build_test.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ source "$AVALANCHE_PATH"/scripts/constants.sh
99

1010
EXCLUDED_TARGETS="| grep -v /mocks | grep -v proto | grep -v tests/e2e | grep -v tests/upgrade"
1111

12+
if [[ "$(go env GOOS)" == "windows" ]]; then
13+
# Test discovery for the antithesis test setups is broken due to
14+
# their dependence on the linux-only Antithesis SDK.
15+
EXCLUDED_TARGETS="${EXCLUDED_TARGETS} | grep -v tests/antithesis"
16+
fi
17+
1218
TEST_TARGETS="$(eval "go list ./... ${EXCLUDED_TARGETS}")"
1319

1420
# shellcheck disable=SC2086

‎tests/antithesis/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,21 @@ In addition, github workflows are suggested to ensure
4949
`scripts/tests.build_antithesis_images.sh` runs against PRs and
5050
`scripts/build_antithesis_images.sh` runs against pushes.
5151

52+
### Use of a builder image
53+
54+
To simplify building instrumented (for running in CI) and
55+
non-instrumented (for running locally) versions of the workload and
56+
node images, a common builder image is used. If on an amd64 host,
57+
`tests/antithesis/avalanchego/Dockerfile.builder-instrumented` is used
58+
to create an instrumented builder. On an arm64 host,
59+
`tests/antithesis/avalanchego/Dockerfile.builder-uninstrumented` is
60+
used to create an uninstrumented builder. In both cases, the builder
61+
image is based on the default golang image and will include the source
62+
code necessary to build the node and workload binaries. The
63+
alternative would require duplicating builder setup for instrumented
64+
and non-instrumented builds for the workload and node images of each
65+
test setup.
66+
5267
## Troubleshooting a test setup
5368

5469
### Running a workload directly

‎tests/antithesis/avalanchego/Dockerfile.builder-instrumented‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# The version is supplied as a build argument rather than hard-coded
2+
# to minimize the cost of version changes.
3+
ARG GO_VERSION
4+
5+
# Antithesis: Getting the Antithesis golang instrumentation library
6+
FROM docker.io/antithesishq/go-instrumentor AS instrumentor
7+
8+
# ============= Instrumentation Stage ================
9+
FROM golang:$GO_VERSION-bullseye
10+
11+
WORKDIR /build
12+
# Copy and download avalanche dependencies using go mod
13+
COPY go.mod .
14+
COPY go.sum .
15+
RUN go mod download
16+
17+
# Copy the code into the container
18+
COPY . .
19+
20+
# Ensure pre-existing builds are not available for inclusion in the final image
21+
RUN [ -d ./build ] && rm -rf ./build/* || true
22+
23+
# Keep the commit hash to easily verify the exact version that is running
24+
RUN git rev-parse HEAD > ./commit_hash.txt
25+
26+
# Copy the instrumentor and supporting files to their correct locations
27+
COPY --from=instrumentor /opt/antithesis /opt/antithesis
28+
COPY --from=instrumentor /opt/antithesis/lib /lib
29+
30+
# Create the destination output directory for the instrumented code
31+
RUN mkdir -p /avalanchego_instrumented
32+
33+
# Park the .git file in a safe location
34+
RUN mkdir -p /opt/tmp/
35+
RUN cp -r .git /opt/tmp/
36+
37+
# Instrument avalanchego
38+
RUN /opt/antithesis/bin/goinstrumentor \
39+
-stderrthreshold=INFO \
40+
-antithesis /opt/antithesis/instrumentation \
41+
. \
42+
/avalanchego_instrumented
43+
44+
WORKDIR /avalanchego_instrumented/customer
45+
RUN go mod download
46+
RUN ln -s /opt/tmp/.git .git

‎tests/antithesis/avalanchego/Dockerfile.builder-uninstrumented‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# The version is supplied as a build argument rather than hard-coded
2+
# to minimize the cost of version changes.
3+
ARG GO_VERSION
4+
5+
FROM golang:$GO_VERSION-bullseye
6+
7+
WORKDIR /build
8+
# Copy and download avalanche dependencies using go mod
9+
COPY go.mod .
10+
COPY go.sum .
11+
RUN go mod download
12+
13+
# Copy the code into the container
14+
COPY . .
15+
16+
# Ensure pre-existing builds are not available for inclusion in the final image
17+
RUN [ -d ./build ] && rm -rf ./build/* || true

‎tests/antithesis/avalanchego/Dockerfile.node‎

Lines changed: 4 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,11 @@
1-
# The version is supplied as a build argument rather than hard-coded
2-
# to minimize the cost of version changes.
3-
ARG GO_VERSION
4-
5-
# Antithesis: Getting the Antithesis golang instrumentation library
6-
FROM docker.io/antithesishq/go-instrumentor AS instrumentor
1+
# TAG should identify the builder image
2+
ARG TAG
73

84
# ============= Compilation Stage ================
9-
FROM golang:$GO_VERSION-bullseye AS builder
10-
11-
WORKDIR /build
12-
# Copy and download avalanche dependencies using go mod
13-
COPY go.mod .
14-
COPY go.sum .
15-
RUN go mod download
16-
17-
# Copy the code into the container
18-
COPY . .
19-
20-
# Keep the commit hash to easily verify the exact version that is running
21-
RUN git rev-parse HEAD > ./commit_hash.txt
22-
23-
# Copy the instrumentor and supporting files to their correct locations
24-
COPY --from=instrumentor /opt/antithesis /opt/antithesis
25-
COPY --from=instrumentor /opt/antithesis/lib /lib
26-
27-
# Create the destination output directory for the instrumented code
28-
RUN mkdir -p /avalanchego_instrumented
29-
30-
# Park the .git file in a safe location
31-
RUN mkdir -p /opt/tmp/
32-
RUN cp -r .git /opt/tmp/
33-
34-
# Instrument avalanchego
35-
RUN /opt/antithesis/bin/goinstrumentor \
36-
-stderrthreshold=INFO \
37-
-antithesis /opt/antithesis/instrumentation \
38-
. \
39-
/avalanchego_instrumented
5+
FROM antithesis-avalanchego-builder:$TAG AS builder
406

7+
# The workdir is hard-coded since this Dockerfile is only intended for instrumented builds.
418
WORKDIR /avalanchego_instrumented/customer
42-
RUN go mod download
43-
RUN ln -s /opt/tmp/.git .git
449

4510
# Build avalanchego with race detection (-r) enabled.
4611
RUN ./scripts/build.sh -r

‎tests/antithesis/avalanchego/Dockerfile.workload‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,16 @@
1-
# The version is supplied as a build argument rather than hard-coded
2-
# to minimize the cost of version changes.
3-
ARG GO_VERSION
1+
# TAG should identify the builder image
2+
ARG TAG
43

54
# NODE_IMAGE needs to identify an existing node image and should include the tag
65
ARG NODE_IMAGE
76

87
# ============= Compilation Stage ================
9-
FROM golang:$GO_VERSION-bullseye AS builder
8+
FROM antithesis-avalanchego-builder:$TAG AS builder
109

11-
WORKDIR /build
12-
# Copy and download avalanche dependencies using go mod
13-
COPY go.mod .
14-
COPY go.sum .
15-
RUN go mod download
10+
# The builder workdir will vary between instrumented and non-instrumented builders
11+
ARG BUILDER_WORKDIR
1612

17-
# Copy the code into the container
18-
COPY . .
13+
WORKDIR $BUILDER_WORKDIR
1914

2015
# Build the workload
2116
RUN ./scripts/build_antithesis_avalanchego_workload.sh
@@ -24,7 +19,10 @@ RUN ./scripts/build_antithesis_avalanchego_workload.sh
2419
# Base the workflow on the node image to support bootstrap testing
2520
FROM $NODE_IMAGE AS execution
2621

22+
# The builder workdir will vary between instrumented and non-instrumented builders
23+
ARG BUILDER_WORKDIR
24+
2725
# Copy the executable into the container
28-
COPY --from=builder /build/build/antithesis-avalanchego-workload ./workload
26+
COPY --from=builder $BUILDER_WORKDIR/build/antithesis-avalanchego-workload ./workload
2927

3028
CMD [ "./workload" ]

‎tests/antithesis/avalanchego/main.go‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@ package main
66
import (
77
"context"
88
"crypto/rand"
9+
"fmt"
910
"log"
1011
"math/big"
1112
"os"
1213
"time"
1314

15+
"github.com/antithesishq/antithesis-sdk-go/assert"
16+
"github.com/antithesishq/antithesis-sdk-go/lifecycle"
17+
1418
"github.com/ava-labs/avalanchego/database"
1519
"github.com/ava-labs/avalanchego/genesis"
1620
"github.com/ava-labs/avalanchego/ids"
@@ -128,6 +132,8 @@ func main() {
128132
}
129133
}
130134

135+
lifecycle.SetupComplete(fmt.Sprintf("Initialized %d workers", NumKeys))
136+
131137
for _, w := range workloads[1:] {
132138
go w.run(ctx)
133139
}
@@ -514,38 +520,54 @@ func (w *workload) makeOwner() secp256k1fx.OutputOwners {
514520

515521
func (w *workload) confirmXChainTx(ctx context.Context, tx *xtxs.Tx) {
516522
txID := tx.ID()
523+
confirmed := true
517524
for _, uri := range w.uris {
518525
client := avm.NewClient(uri, "X")
519526
status, err := client.ConfirmTx(ctx, txID, 100*time.Millisecond)
520527
if err != nil {
521528
log.Printf("failed to confirm X-chain transaction %s on %s: %s", txID, uri, err)
522-
return
529+
confirmed = false
530+
break
523531
}
524532
if status != choices.Accepted {
525533
log.Printf("failed to confirm X-chain transaction %s on %s: status == %s", txID, uri, status)
526-
return
534+
confirmed = false
535+
break
527536
}
528537
log.Printf("confirmed X-chain transaction %s on %s", txID, uri)
529538
}
530-
log.Printf("confirmed X-chain transaction %s on all nodes", txID)
539+
if confirmed {
540+
log.Printf("confirmed X-chain transaction %s on all nodes", txID)
541+
}
542+
assert.Always(confirmed, "X-chain transactions can be confirmed on all nodes", map[string]any{
543+
"txID": txID,
544+
})
531545
}
532546

533547
func (w *workload) confirmPChainTx(ctx context.Context, tx *ptxs.Tx) {
534548
txID := tx.ID()
549+
confirmed := true
535550
for _, uri := range w.uris {
536551
client := platformvm.NewClient(uri)
537552
s, err := client.AwaitTxDecided(ctx, txID, 100*time.Millisecond)
538553
if err != nil {
539554
log.Printf("failed to confirm P-chain transaction %s on %s: %s", txID, uri, err)
540-
return
555+
confirmed = false
556+
break
541557
}
542558
if s.Status != status.Committed {
543559
log.Printf("failed to confirm P-chain transaction %s on %s: status == %s", txID, uri, s.Status)
544-
return
560+
confirmed = false
561+
break
545562
}
546563
log.Printf("confirmed P-chain transaction %s on %s", txID, uri)
547564
}
548-
log.Printf("confirmed P-chain transaction %s on all nodes", txID)
565+
if confirmed {
566+
log.Printf("confirmed P-chain transaction %s on all nodes", txID)
567+
}
568+
assert.Always(confirmed, "P-chain transactions can be confirmed on all nodes", map[string]any{
569+
"txID": txID,
570+
})
549571
}
550572

551573
func (w *workload) verifyXChainTxConsumedUTXOs(ctx context.Context, tx *xtxs.Tx) {

0 commit comments

Comments
 (0)