Skip to content

Commit 2429fd9

Browse files
committed
e2e: Migrate duplicate node id test from kurtosis
1 parent 6b9a223 commit 2429fd9

File tree

7 files changed

+300
-22
lines changed

7 files changed

+300
-22
lines changed

tests/e2e/e2e.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55
package e2e
66

77
import (
8+
"os"
89
"time"
910

1011
ginkgo "github.com/onsi/ginkgo/v2"
1112
"github.com/stretchr/testify/require"
1213

14+
cfg "github.com/ava-labs/avalanchego/config"
15+
"github.com/ava-labs/avalanchego/tests"
1316
"github.com/ava-labs/avalanchego/tests/fixture"
1417
"github.com/ava-labs/avalanchego/tests/fixture/testnet"
1518
"github.com/ava-labs/avalanchego/tests/fixture/testnet/local"
@@ -23,6 +26,10 @@ const (
2326
// Defines default tx confirmation timeout.
2427
// Enough for test/custom networks.
2528
DefaultConfirmTxTimeout = 20 * time.Second
29+
30+
// Defines default node bootstrap timeout.
31+
// Enough for test/custom networks.
32+
DefaultBootstrapTimeout = 20 * time.Second
2633
)
2734

2835
// Env is used to access shared test fixture. Intended to be
@@ -46,3 +53,27 @@ func (te *TestEnvironment) AllocateTestKeys(count int) []*secp256k1.PrivateKey {
4653
require.NoError(ginkgo.GinkgoT(), err)
4754
return keys
4855
}
56+
57+
// Add a temporary node that is only intended to be used by a single
58+
// test. Its ID and URI are not intended to be returned from The
59+
// Network instance to avoid accessibility from other tests.
60+
func AddTemporaryNode(network testnet.Network, flags testnet.FlagsMap, waitForHealth bool) testnet.Node {
61+
require := require.New(ginkgo.GinkgoT())
62+
63+
// A temporary location ensures the node won't be accessible from
64+
// the Network instance.
65+
nodeDir, err := os.MkdirTemp("", "")
66+
require.NoError(err)
67+
68+
flags[cfg.DataDirKey] = nodeDir
69+
node, err := network.AddNode(ginkgo.GinkgoWriter, flags, waitForHealth)
70+
require.NoError(err)
71+
72+
// Ensure node is stopped and its configuration removed
73+
ginkgo.DeferCleanup(func() {
74+
tests.Outf("{{yellow}}Shutting down temporary node %s{{/}}\n", node.GetID())
75+
require.NoError(node.Remove())
76+
})
77+
78+
return node
79+
}

tests/e2e/e2e_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
// ensure test packages are scanned by ginkgo
2323
_ "github.com/ava-labs/avalanchego/tests/e2e/banff"
24+
_ "github.com/ava-labs/avalanchego/tests/e2e/faultinjection"
2425
_ "github.com/ava-labs/avalanchego/tests/e2e/p"
2526
_ "github.com/ava-labs/avalanchego/tests/e2e/static-handlers"
2627
_ "github.com/ava-labs/avalanchego/tests/e2e/x/transfer"
@@ -107,7 +108,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
107108
// after the test run.
108109
ginkgo.DeferCleanup(func() {
109110
tests.Outf("{{red}}Shutting down network{{/}}\n")
110-
network.Stop()
111+
require.NoError(network.Stop())
111112
require.NoError(os.RemoveAll(network.Dir))
112113
})
113114
}
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
2+
// See the file LICENSE for licensing terms.
3+
4+
package faultinjection
5+
6+
import (
7+
"context"
8+
"time"
9+
10+
ginkgo "github.com/onsi/ginkgo/v2"
11+
"github.com/stretchr/testify/require"
12+
13+
"github.com/ava-labs/avalanchego/api/health"
14+
"github.com/ava-labs/avalanchego/api/info"
15+
cfg "github.com/ava-labs/avalanchego/config"
16+
"github.com/ava-labs/avalanchego/ids"
17+
"github.com/ava-labs/avalanchego/tests/e2e"
18+
"github.com/ava-labs/avalanchego/tests/fixture/testnet"
19+
"github.com/ava-labs/avalanchego/utils/set"
20+
)
21+
22+
var _ = ginkgo.Describe("Duplicate node handling", func() {
23+
require := require.New(ginkgo.GinkgoT())
24+
25+
ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
26+
network := e2e.Env.GetNetwork()
27+
nodes := network.GetNodes()
28+
29+
ginkgo.By("creating new node")
30+
newNode1 := e2e.AddTemporaryNode(network, testnet.FlagsMap{}, true /* waitForHealthy */)
31+
32+
ginkgo.By("checking that the new node is connected to its peers")
33+
checkConnectedPeers(nodes, newNode1)
34+
35+
ginkgo.By("creating a second new node with the same staking keypair as the first new node")
36+
node1Flags := newNode1.GetConfig().Flags
37+
node2Flags := testnet.FlagsMap{
38+
cfg.StakingTLSKeyContentKey: node1Flags[cfg.StakingTLSKeyContentKey],
39+
cfg.StakingCertContentKey: node1Flags[cfg.StakingCertContentKey],
40+
}
41+
newNode2 := e2e.AddTemporaryNode(network, node2Flags, false /* waitForHealthy */)
42+
healthClient2 := health.NewClient(newNode2.GetRuntimeState().URI)
43+
44+
ginkgo.By("checking that the second new node fails to become healthy within timeout")
45+
require.Never(func() bool {
46+
health, err := healthClient2.Health(context.Background(), nil)
47+
require.NoError(err)
48+
return health.Healthy
49+
}, e2e.DefaultBootstrapTimeout, time.Second, "node2 was able to bootstrap")
50+
51+
ginkgo.By("stopping the first new node")
52+
require.NoError(newNode1.Stop())
53+
54+
ginkgo.By("checking that the second new node becomes healthy within timeout")
55+
require.Eventually(func() bool {
56+
health, err := healthClient2.Health(context.Background(), nil)
57+
require.NoError(err)
58+
return health.Healthy
59+
}, e2e.DefaultBootstrapTimeout, time.Second, "node2 was not able to bootstrap")
60+
61+
ginkgo.By("checking that the second new node is connected to its peers")
62+
checkConnectedPeers(nodes, newNode2)
63+
})
64+
})
65+
66+
// Check that a new node is connected to existing nodes and vice versa
67+
func checkConnectedPeers(existingNodes []testnet.Node, newNode testnet.Node) {
68+
require := require.New(ginkgo.GinkgoT())
69+
70+
// Collect the node ids of the new node's peers
71+
infoClient := info.NewClient(newNode.GetRuntimeState().URI)
72+
peers, err := infoClient.Peers(context.Background())
73+
require.NoError(err)
74+
peerIDs := set.NewSet[ids.NodeID](len(existingNodes))
75+
for _, peer := range peers {
76+
peerIDs.Add(peer.ID)
77+
}
78+
79+
newNodeID := newNode.GetID().String()
80+
for _, existingNode := range existingNodes {
81+
// Check that the existing node is a peer of the new node
82+
require.True(peerIDs.Contains(existingNode.GetID()))
83+
84+
// Check that the new node is a peer
85+
infoClient := info.NewClient(existingNode.GetRuntimeState().URI)
86+
peers, err := infoClient.Peers(context.Background())
87+
require.NoError(err)
88+
isPeer := false
89+
for _, peer := range peers {
90+
if peer.ID.String() == newNodeID {
91+
isPeer = true
92+
break
93+
}
94+
}
95+
require.True(isPeer)
96+
}
97+
}

tests/fixture/testnet/cmd/main.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,13 +72,36 @@ func main() {
7272
if err != nil {
7373
return err
7474
}
75-
network.Stop()
75+
if err := network.Stop(); err != nil {
76+
return err
77+
}
7678
fmt.Fprintf(os.Stdout, "Stopped network configured at: %s\n", networkDir)
7779
return nil
7880
},
7981
}
8082
rootCmd.AddCommand(stopNetworkCmd)
8183

84+
addNodeCmd := &cobra.Command{
85+
Use: "add-node",
86+
Short: "Adds a node to a local network",
87+
RunE: func(cmd *cobra.Command, args []string) error {
88+
if len(networkDir) == 0 {
89+
return fmt.Errorf("--network-dir or %s are required", local.NetworkDirEnvName)
90+
}
91+
network, err := local.LoadNetwork(networkDir)
92+
if err != nil {
93+
return err
94+
}
95+
node, err := network.AddLocalNode(os.Stdout, nil, true /* waitForHealth */)
96+
if err != nil {
97+
return err
98+
}
99+
fmt.Fprintf(os.Stdout, "Added node %q to network configured at: %s\n", node.NodeID, network.Dir)
100+
return nil
101+
},
102+
}
103+
rootCmd.AddCommand(addNodeCmd)
104+
82105
if err := rootCmd.Execute(); err != nil {
83106
fmt.Fprintf(os.Stderr, "testnetctl failed %v\n", err)
84107
os.Exit(1)

tests/fixture/testnet/interfaces.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,24 @@
33

44
package testnet
55

6-
import "github.com/ava-labs/avalanchego/ids"
6+
import (
7+
"io"
8+
9+
"github.com/ava-labs/avalanchego/ids"
10+
)
711

812
// Defines network capabilities supportable regardless of how a network is orchestrated.
913
type Network interface {
1014
GetConfig() NetworkConfig
1115
GetNodes() []Node
16+
AddNode(w io.Writer, flags FlagsMap, waitForHealth bool) (Node, error)
1217
}
1318

1419
// Defines node capabilities supportable regardless of how a network is orchestrated.
1520
type Node interface {
1621
GetID() ids.NodeID
1722
GetConfig() NodeConfig
1823
GetRuntimeState() NodeRuntimeState
24+
Stop() error
25+
Remove() error
1926
}

tests/fixture/testnet/local/network.go

Lines changed: 117 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import (
1111
"io"
1212
"os"
1313
"path/filepath"
14+
"strconv"
15+
"strings"
1416
"time"
1517

1618
cfg "github.com/ava-labs/avalanchego/config"
@@ -56,6 +58,22 @@ func (ln *LocalNetwork) GetNodes() []testnet.Node {
5658
return nodes
5759
}
5860

61+
// Adds a backend-agnostic node to the network
62+
func (ln *LocalNetwork) AddNode(w io.Writer, flags testnet.FlagsMap, waitForHealth bool) (testnet.Node, error) {
63+
if flags == nil {
64+
flags = testnet.FlagsMap{}
65+
}
66+
node, err := ln.AddLocalNode(w, &LocalNode{
67+
NodeConfig: testnet.NodeConfig{
68+
Flags: flags,
69+
},
70+
}, waitForHealth)
71+
if err != nil {
72+
return nil, err
73+
}
74+
return node, nil
75+
}
76+
5977
// Starts a new network stored under the provided root dir. Required
6078
// configuration will be defaulted if not provided.
6179
func StartNetwork(
@@ -110,8 +128,7 @@ func StopNetwork(dir string) error {
110128
if err != nil {
111129
return err
112130
}
113-
network.Stop()
114-
return nil
131+
return network.Stop()
115132
}
116133

117134
// Ensure the network has the configuration it needs to start.
@@ -342,12 +359,20 @@ func (ln *LocalNetwork) GetURIs() []string {
342359
}
343360

344361
// Stop all nodes in the network.
345-
func (ln *LocalNetwork) Stop() {
362+
func (ln *LocalNetwork) Stop() error {
346363
// Assume the nodes are loaded and the pids are current
364+
errs := []error{}
347365
for _, node := range ln.Nodes {
348-
node.Stop()
366+
if err := node.Stop(); err != nil {
367+
errs = append(errs, err)
368+
}
349369
}
350-
// TODO(marun) Collect and return errors
370+
if len(errs) > 0 {
371+
// TODO(marun) Update to use errors.Join once 1.20 becomes the
372+
// minimum golang version
373+
return fmt.Errorf("failed to stop all nodes: %v", errs)
374+
}
375+
return nil
351376
}
352377

353378
func (ln *LocalNetwork) GetGenesisPath() string {
@@ -569,3 +594,90 @@ func (ln *LocalNetwork) ReadAll() error {
569594
}
570595
return ln.ReadNodes()
571596
}
597+
598+
func (ln *LocalNetwork) AddLocalNode(w io.Writer, node *LocalNode, waitForHealth bool) (*LocalNode, error) {
599+
// Assume network configuration has been written to disk and is current in memory
600+
601+
if err := ln.ReadNodes(); err != nil {
602+
return nil, err
603+
}
604+
605+
maxPort := 0
606+
bootstrapIPs := []string{}
607+
bootstrapIDs := []string{}
608+
for _, node := range ln.Nodes {
609+
if len(node.StakingAddress) == 0 {
610+
// Node is not running
611+
continue
612+
}
613+
614+
bootstrapIPs = append(bootstrapIPs, node.StakingAddress)
615+
bootstrapIDs = append(bootstrapIDs, node.NodeID.String())
616+
617+
// Find the maximum port if using static ports
618+
if ln.UseStaticPorts {
619+
addressParts := strings.Split(node.StakingAddress, ":")
620+
rawPort := addressParts[len(addressParts)-1]
621+
port, err := strconv.Atoi(rawPort)
622+
if err != nil {
623+
return nil, err
624+
}
625+
if port > maxPort {
626+
maxPort = port
627+
}
628+
}
629+
}
630+
631+
if len(bootstrapIDs) == 0 {
632+
// TODO(marun) Make sure to relax this for the initial node
633+
// when building a network node-by-node
634+
return nil, errors.New("failed to add node due to missing bootstrap nodes")
635+
}
636+
637+
if node == nil {
638+
// TODO(marun) Simplify this instantiation
639+
node = &LocalNode{
640+
NodeConfig: *testnet.NewNodeConfig(),
641+
}
642+
}
643+
644+
err := ln.PopulateNodeConfig(node)
645+
if err != nil {
646+
return nil, err
647+
}
648+
649+
// Configure the provided node for this network
650+
httpPort := 0
651+
stakingPort := 0
652+
if ln.UseStaticPorts {
653+
httpPort = maxPort
654+
stakingPort = maxPort + 1
655+
}
656+
node.SetNetworkingConfig(httpPort, stakingPort, bootstrapIDs, bootstrapIPs)
657+
658+
err = node.WriteConfig()
659+
if err != nil {
660+
return nil, err
661+
}
662+
process, err := node.Start(w, ln.ExecPath, ln.UseStaticPorts)
663+
if err != nil {
664+
return nil, err
665+
}
666+
667+
if waitForHealth {
668+
// TODO(marun) Use a timeout
669+
ctx := context.Background()
670+
671+
// Health check
672+
for {
673+
if healthy, err := isHealthy(ctx, process); err != nil {
674+
return nil, err
675+
} else if healthy {
676+
break
677+
}
678+
time.Sleep(time.Millisecond * 500)
679+
}
680+
}
681+
682+
return node, nil
683+
}

0 commit comments

Comments
 (0)