Skip to content

Commit 3126b84

Browse files
maru-avamaru-ava
committed
[tmpnet] Enable deployment to kube
1 parent a713bbc commit 3126b84

30 files changed

+1672
-346
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,34 @@ jobs:
6565
prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
6666
loki_username: ${{ secrets.LOKI_ID || '' }}
6767
loki_password: ${{ secrets.LOKI_PASSWORD || '' }}
68+
e2e_kube:
69+
runs-on: ubuntu-latest
70+
steps:
71+
- uses: actions/checkout@v4
72+
- uses: ./.github/actions/setup-go-for-project
73+
- name: Run e2e tests
74+
shell: bash
75+
run: bash -x ./scripts/tests.e2e.kube.sh
76+
env:
77+
PROMETHEUS_USERNAME: ${{ secrets.PROMETHEUS_ID || '' }}
78+
PROMETHEUS_PASSWORD: ${{ secrets.PROMETHEUS_PASSWORD || '' }}
79+
LOKI_USERNAME: ${{ secrets.LOKI_ID || '' }}
80+
LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD || '' }}
81+
GH_REPO: ${{ github.repository_owner }}/${{ github.event.repository.name }}
82+
GH_WORKFLOW: ${{ github.workflow }}
83+
GH_RUN_ID: ${{ github.run_id }}
84+
GH_RUN_NUMBER: ${{ github.run_number }}
85+
GH_RUN_ATTEMPT: ${{ github.run_attempt }}
86+
GH_JOB_ID: ${{ github.job }}
87+
- name: Export kind logs
88+
shell: bash
89+
run: kind export logs /tmp/kind-logs
90+
- name: Upload kind logs
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: kind-logs
94+
path: /tmp/kind-logs
95+
if-no-files-found: error
6896
e2e_existing_network:
6997
runs-on: ubuntu-latest
7098
steps:

‎scripts/build_antithesis_images.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,5 +87,5 @@ else
8787
"${AVALANCHE_PATH}/build/antithesis/xsvm" \
8888
"AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVAGO_PLUGIN_DIR=${AVALANCHE_PATH}/build/plugins"
8989

90-
build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
90+
build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/Dockerfile.node"
9191
fi

‎scripts/build_xsvm_image.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
if ! [[ "$0" =~ scripts/build_xsvm_image.sh ]]; then
6+
echo "must be run from repository root"
7+
exit 255
8+
fi
9+
10+
# Directory above this script
11+
AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
12+
13+
# TODO(marun) This image name should be configurable
14+
DOCKER_IMAGE="localhost:5001/avalanchego"
15+
16+
# Build the avalancehgo node image
17+
FORCE_TAG_LATEST=1 SKIP_BUILD_RACE=1 DOCKER_IMAGE="${DOCKER_IMAGE}" ./scripts/build_image.sh
18+
19+
# TODO(marun) conditionally push the image to the registry
20+
GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
21+
docker buildx build --build-arg GO_VERSION="${GO_VERSION}" --build-arg AVALANCHEGO_NODE_IMAGE="${DOCKER_IMAGE}" \
22+
--push -t "${DOCKER_IMAGE}-xsvm" -f "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile" .

‎scripts/tests.e2e.kube.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
# Run e2e tests against nodes deployed to a kind cluster.
6+
7+
# TODO(marun)
8+
# - Support testing against a remote cluster
9+
# - Convert to golang to simplify reuse
10+
# - Make idempotent to simplify development and debugging
11+
12+
if ! [[ "$0" =~ scripts/tests.e2e.kube.sh ]]; then
13+
echo "must be run from repository root"
14+
exit 255
15+
fi
16+
17+
./scripts/ensure_kube_cluster.sh
18+
19+
KUBE_CONTEXT="${KUBE_CONTEXT:-kind-kind}"
20+
# TODO(marun) Make the namespace configurable
21+
NAMESPACE=tmpnet
22+
PATH="${PWD}/bin:$PATH" kubectl create namespace "${NAMESPACE}" || true
23+
24+
if [[ -z "${SKIP_BUILD_IMAGE:-}" ]]; then
25+
bash -x ./scripts/build_xsvm_image.sh
26+
fi
27+
28+
MONITORING_NAMESPACE=ci-monitoring
29+
PATH="${PWD}/bin:$PATH" kubectl create namespace "${MONITORING_NAMESPACE}" || true
30+
31+
# Deploy promtail if credentials are available
32+
if [[ -n "${LOKI_USERNAME:-}" && -n "${LOKI_PASSWORD:-}" ]]; then
33+
kubectl --namespace="${MONITORING_NAMESPACE}" create secret generic loki-credentials \
34+
--from-literal=username="${LOKI_USERNAME}" \
35+
--from-literal=password="${LOKI_PASSWORD}" \
36+
--dry-run=client -o yaml | kubectl apply -f -
37+
kubectl --namespace="${MONITORING_NAMESPACE}" apply -f ./tests/fixture/tmpnet/yaml/promtail-daemonset.yaml
38+
else
39+
echo "LOKI_USERNAME and LOKI_PASSWORD not set, skipping deployment of promtail"
40+
fi
41+
42+
# Deploy prometheus agent if credentials are available
43+
if [[ -n "${PROMETHEUS_USERNAME:-}" && -n "${PROMETHEUS_PASSWORD:-}" ]]; then
44+
kubectl --namespace="${MONITORING_NAMESPACE}" create secret generic prometheus-credentials \
45+
--from-literal=username="${PROMETHEUS_USERNAME}" \
46+
--from-literal=password="${PROMETHEUS_PASSWORD}" \
47+
--dry-run=client -o yaml | kubectl apply -f -
48+
kubectl --namespace="${MONITORING_NAMESPACE}" apply -f ./tests/fixture/tmpnet/yaml/prometheus-agent.yaml
49+
else
50+
echo "PROMETHEUS_USERNAME and PROMETHEUS_PASSWORD not set, skipping deployment of prometheus agent"
51+
fi
52+
53+
E2E_SERIAL=1 KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" PATH="${PWD}/bin:$PATH" \
54+
bash -x ./scripts/tests.e2e.sh --runtime=kube --image-name=localhost:5001/avalanchego-xsvm

‎scripts/tests.e2e.sh‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,17 @@ fi
2020
# the instructions to build non-portable BLST.
2121
source ./scripts/constants.sh
2222

23-
# Enable subnet testing by building xsvm
24-
./scripts/build_xsvm.sh
25-
echo ""
26-
2723
# Ensure an absolute path to avoid dependency on the working directory
2824
# of script execution.
29-
AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
30-
E2E_ARGS="--avalanchego-path=${AVALANCHEGO_PATH}"
25+
E2E_ARGS="${*:-}"
26+
if ! [[ "${E2E_ARGS}" =~ "--runtime=kube" ]]; then
27+
# If not running in kubernetes, use the local avalanchego binary
28+
AVALANCHEGO_PATH="$(realpath "${AVALANCHEGO_PATH:-./build/avalanchego}")"
29+
E2E_ARGS+=" --avalanchego-path=${AVALANCHEGO_PATH}"
30+
31+
# Enable subnet testing by building the xsvm binary
32+
./scripts/build_xsvm.sh
33+
fi
3134

3235
#################################
3336
# Determine ginkgo args
@@ -58,5 +61,5 @@ else
5861
fi
5962

6063
#################################
61-
# shellcheck disable=SC2086
62-
./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- "${E2E_ARGS[@]}" "${@}"
64+
# shellcheck disable=SC2086,SC2068
65+
./bin/ginkgo ${GINKGO_ARGS} -v ./tests/e2e -- ${E2E_ARGS[@]}

‎tests/antithesis/compose.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ var (
3838
// simplify usage by main entrypoints. If the provided network includes a subnet, the initial DB state for
3939
// the subnet will be created and written to the target path.
4040
func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error {
41+
// TODO(marun) Is there a better way to ensure parity between the configuration that initializes the database and the configuration used at runtime?
42+
if network.DefaultFlags == nil {
43+
network.DefaultFlags = tmpnet.FlagsMap{}
44+
}
45+
network.DefaultFlags.SetDefaults(tmpnet.DefaultTestFlags())
46+
4147
targetPath := os.Getenv(targetPathEnvName)
4248
if len(targetPath) == 0 {
4349
return errTargetPathEnvVarNotSet
@@ -66,7 +72,16 @@ func GenerateComposeConfig(network *tmpnet.Network, baseImageName string) error
6672
return fmt.Errorf("failed to get bootstrap volume path: %w", err)
6773
}
6874

69-
if err := initBootstrapDB(network, avalancheGoPath, pluginDir, bootstrapVolumePath); err != nil {
75+
network.DefaultRuntimeConfig = tmpnet.NodeRuntimeConfig{
76+
AvalancheGoPath: avalancheGoPath,
77+
}
78+
// TODO(marun) Need to have a standard way of initializing a network
79+
if network.DefaultFlags == nil {
80+
network.DefaultFlags = tmpnet.FlagsMap{}
81+
}
82+
network.DefaultFlags[config.PluginDirKey] = pluginDir
83+
84+
if err := initBootstrapDB(network, bootstrapVolumePath); err != nil {
7085
return fmt.Errorf("failed to initialize db volumes: %w", err)
7186
}
7287
}

‎tests/antithesis/init_db.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,14 @@ func getBootstrapVolumePath(targetPath string) (string, error) {
2828
// Bootstraps a local process-based network, creates its subnets and chains, and copies
2929
// the resulting db state from one of the nodes to the provided path. The path will be
3030
// created if it does not already exist.
31-
func initBootstrapDB(network *tmpnet.Network, avalancheGoPath string, pluginDir string, destPath string) error {
31+
func initBootstrapDB(network *tmpnet.Network, destPath string) error {
3232
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
3333
defer cancel()
3434
if err := tmpnet.BootstrapNewNetwork(
3535
ctx,
3636
tests.NewDefaultLogger(""),
3737
network,
3838
"",
39-
avalancheGoPath,
40-
pluginDir,
4139
); err != nil {
4240
return fmt.Errorf("failed to bootstrap network: %w", err)
4341
}

‎tests/e2e/c/dynamic_fees.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ var _ = e2e.DescribeCChain("[Dynamic Fees]", func() {
3535

3636
ginkgo.It("should ensure that the gas price is affected by load", func() {
3737
tc.By("creating a new private network to ensure isolation from other tests")
38-
privateNetwork := tmpnet.NewDefaultNetwork("avalanchego-e2e-dynamic-fees")
38+
privateNetwork := tmpnet.NewDefaultNetwork(tc.Log(), "avalanchego-e2e-dynamic-fees")
3939
e2e.GetEnv(tc).StartPrivateNetwork(privateNetwork)
4040

4141
// Avoid emitting a spec-scoped metrics link for the shared

‎tests/e2e/e2e_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,5 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
4949
// Run in every ginkgo process
5050

5151
// Initialize the local test environment from the global state
52-
e2e.InitSharedTestEnvironment(ginkgo.GinkgoT(), envBytes)
52+
e2e.InitSharedTestEnvironment(e2e.NewTestContext(), envBytes)
5353
})

‎tests/e2e/faultinjection/duplicate_node_id.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
2626
ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
2727
network := e2e.GetEnv(tc).GetNetwork()
2828

29+
if network.DefaultRuntimeConfig.KubeRuntimeConfig != nil {
30+
// Enabling this test for kube requires supporting a flexible name mapping
31+
ginkgo.Skip("This test is not supported on kube because it relies on the mapping of network uuid + nodeid -> statefulset name")
32+
}
33+
2934
tc.By("creating new node")
3035
node1 := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
3136
e2e.WaitForHealthy(tc, node1)
@@ -43,10 +48,10 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
4348
// the same node ID.
4449
config.DataDirKey: fmt.Sprintf("%s-second", node1Flags[config.DataDirKey]),
4550
}
46-
node2 := e2e.AddEphemeralNode(tc, network, node2Flags)
51+
node2 := e2e.AddEphemeralNodeWithWaitForHealth(tc, network, node2Flags, false /* waitForHealth */)
4752

4853
tc.By("checking that the second new node fails to become healthy before timeout")
49-
err := tmpnet.WaitForHealthy(tc.DefaultContext(), node2)
54+
err := tmpnet.WaitForHealthyNode(tc.DefaultContext(), tc.Log(), node2)
5055
require.ErrorIs(err, context.DeadlineExceeded)
5156

5257
tc.By("stopping the first new node")
@@ -68,7 +73,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
6873
require := require.New(tc)
6974

7075
// Collect the node ids of the new node's peers
71-
infoClient := info.NewClient(newNode.URI)
76+
uri := e2e.GetLocalURI(tc, newNode)
77+
infoClient := info.NewClient(uri)
7278
peers, err := infoClient.Peers(tc.DefaultContext(), nil)
7379
require.NoError(err)
7480
peerIDs := set.NewSet[ids.NodeID](len(existingNodes))
@@ -81,7 +87,8 @@ func checkConnectedPeers(tc tests.TestContext, existingNodes []*tmpnet.Node, new
8187
require.True(peerIDs.Contains(existingNode.NodeID))
8288

8389
// Check that the new node is a peer
84-
infoClient := info.NewClient(existingNode.URI)
90+
uri := e2e.GetLocalURI(tc, existingNode)
91+
infoClient := info.NewClient(uri)
8592
peers, err := infoClient.Peers(tc.DefaultContext(), nil)
8693
require.NoError(err)
8794
isPeer := false

0 commit comments

Comments
 (0)