e2e: Migrate duplicate node id test from kurtosis

Author: maru-ava

tests/e2e/e2e.go

@@ -5,11 +5,14 @@
 package e2e
 
 import (
+	"os"
 	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/require"
 
+	cfg "github.com/ava-labs/avalanchego/config"
+	"github.com/ava-labs/avalanchego/tests"
 	"github.com/ava-labs/avalanchego/tests/fixture"
 	"github.com/ava-labs/avalanchego/tests/fixture/testnet"
 	"github.com/ava-labs/avalanchego/tests/fixture/testnet/local"
@@ -23,6 +26,10 @@ const (
 	// Defines default tx confirmation timeout.
 	// Enough for test/custom networks.
 	DefaultConfirmTxTimeout = 20 * time.Second
+
+	// Defines default node bootstrap timeout.
+	// Enough for test/custom networks.
+	DefaultBootstrapTimeout = 10 * time.Second
 )
 
 // Env is used to access shared test fixture. Intended to be
@@ -46,3 +53,27 @@ func (te *TestEnvironment) AllocateTestKeys(count int) []*secp256k1.PrivateKey {
 	require.NoError(ginkgo.GinkgoT(), err)
 	return keys
 }
+
+// Add a temporary node that is only intended to be used by a single
+// test. Its ID and URI are not intended to be returned from The
+// Network instance to avoid accessibility from other tests.
+func AddTemporaryNode(network testnet.Network, flags testnet.FlagsMap, waitForHealth bool) testnet.Node {
+	require := require.New(ginkgo.GinkgoT())
+
+	// A temporary location ensures the node won't be accessible from
+	// the Network instance.
+	nodeDir, err := os.MkdirTemp("", "")
+	require.NoError(err)
+
+	flags[cfg.DataDirKey] = nodeDir
+	node, err := network.AddNode(ginkgo.GinkgoWriter, flags, waitForHealth)
+	require.NoError(err)
+
+	// Ensure node is stopped and its configuration removed
+	ginkgo.DeferCleanup(func() {
+		tests.Outf("{{yellow}}Shutting down temporary node %s{{/}}\n", node.GetID())
+		require.NoError(node.Remove())
+	})
+
+	return node
+}

tests/e2e/e2e_test.go

@@ -21,6 +21,7 @@ import (
 
 	// ensure test packages are scanned by ginkgo
 	_ "github.com/ava-labs/avalanchego/tests/e2e/banff"
+	_ "github.com/ava-labs/avalanchego/tests/e2e/faultinjection"
 	_ "github.com/ava-labs/avalanchego/tests/e2e/p"
 	_ "github.com/ava-labs/avalanchego/tests/e2e/static-handlers"
 	_ "github.com/ava-labs/avalanchego/tests/e2e/x/transfer"
@@ -107,7 +108,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 		// after the test run.
 		ginkgo.DeferCleanup(func() {
 			tests.Outf("{{red}}Shutting down network{{/}}\n")
-			network.Stop()
+			require.NoError(network.Stop())
 			require.NoError(os.RemoveAll(network.Dir))
 		})
 	}

tests/e2e/faultinjection/duplicate_node_id.go

@@ -0,0 +1,97 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package faultinjection
+
+import (
+	"context"
+	"time"
+
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/api/health"
+	"github.com/ava-labs/avalanchego/api/info"
+	cfg "github.com/ava-labs/avalanchego/config"
+	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/tests/e2e"
+	"github.com/ava-labs/avalanchego/tests/fixture/testnet"
+	"github.com/ava-labs/avalanchego/utils/set"
+)
+
+var _ = ginkgo.Describe("Duplicate node handling", func() {
+	require := require.New(ginkgo.GinkgoT())
+
+	ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
+		network := e2e.Env.GetNetwork()
+		nodes := network.GetNodes()
+
+		ginkgo.By("creating new node")
+		newNode1 := e2e.AddTemporaryNode(network, testnet.FlagsMap{}, true /* waitForHealthy */)
+
+		ginkgo.By("checking that the new node is connected to its peers")
+		checkConnectedPeers(nodes, newNode1)
+
+		ginkgo.By("creating a second new node with the same staking keypair as the first new node")
+		node1Flags := newNode1.GetConfig().Flags
+		node2Flags := testnet.FlagsMap{
+			cfg.StakingTLSKeyContentKey: node1Flags[cfg.StakingTLSKeyContentKey],
+			cfg.StakingCertContentKey:   node1Flags[cfg.StakingCertContentKey],
+		}
+		newNode2 := e2e.AddTemporaryNode(network, node2Flags, false /* waitForHealthy */)
+		healthClient2 := health.NewClient(newNode2.GetRuntimeState().URI)
+
+		ginkgo.By("checking that the second new node fails to become healthy within timeout")
+		require.Never(func() bool {
+			health, err := healthClient2.Health(context.Background(), nil)
+			require.NoError(err)
+			return health.Healthy
+		}, e2e.DefaultBootstrapTimeout, time.Second, "node2 was able to bootstrap")
+
+		ginkgo.By("stopping the first new node")
+		require.NoError(newNode1.Stop())
+
+		ginkgo.By("checking that the second new node becomes healthy within timeout")
+		require.Eventually(func() bool {
+			health, err := healthClient2.Health(context.Background(), nil)
+			require.NoError(err)
+			return health.Healthy
+		}, e2e.DefaultBootstrapTimeout, time.Second, "node2 was not able to bootstrap")
+
+		ginkgo.By("checking that the second new node is connected to its peers")
+		checkConnectedPeers(nodes, newNode2)
+	})
+})
+
+// Check that a new node is connected to existing nodes and vice versa
+func checkConnectedPeers(existingNodes []testnet.Node, newNode testnet.Node) {
+	require := require.New(ginkgo.GinkgoT())
+
+	// Collect the node ids of the new node's peers
+	infoClient := info.NewClient(newNode.GetRuntimeState().URI)
+	peers, err := infoClient.Peers(context.Background())
+	require.NoError(err)
+	peerIDs := set.NewSet[ids.NodeID](len(existingNodes))
+	for _, peer := range peers {
+		peerIDs.Add(peer.ID)
+	}
+
+	newNodeID := newNode.GetID().String()
+	for _, existingNode := range existingNodes {
+		// Check that the existing node is a peer of the new node
+		require.True(peerIDs.Contains(existingNode.GetID()))
+
+		// Check that the new node is a peer
+		infoClient := info.NewClient(existingNode.GetRuntimeState().URI)
+		peers, err := infoClient.Peers(context.Background())
+		require.NoError(err)
+		isPeer := false
+		for _, peer := range peers {
+			if peer.ID.String() == newNodeID {
+				isPeer = true
+				break
+			}
+		}
+		require.True(isPeer)
+	}
+}

tests/fixture/testnet/cmd/main.go

@@ -72,13 +72,36 @@ func main() {
 			if err != nil {
 				return err
 			}
-			network.Stop()
+			if err := network.Stop(); err != nil {
+				return err
+			}
 			fmt.Fprintf(os.Stdout, "Stopped network configured at: %s\n", networkDir)
 			return nil
 		},
 	}
 	rootCmd.AddCommand(stopNetworkCmd)
 
+	addNodeCmd := &cobra.Command{
+		Use:   "add-node",
+		Short: "Adds a node to a local network",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(networkDir) == 0 {
+				return fmt.Errorf("--network-dir or %s are required", local.NetworkDirEnvName)
+			}
+			network, err := local.LoadNetwork(networkDir)
+			if err != nil {
+				return err
+			}
+			node, err := network.AddLocalNode(os.Stdout, nil, true /* waitForHealth */)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(os.Stdout, "Added node %q to network configured at: %s\n", node.NodeID, network.Dir)
+			return nil
+		},
+	}
+	rootCmd.AddCommand(addNodeCmd)
+
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "testnetctl failed %v\n", err)
 		os.Exit(1)

tests/fixture/testnet/interfaces.go

@@ -3,17 +3,24 @@
 
 package testnet
 
-import "github.com/ava-labs/avalanchego/ids"
+import (
+	"io"
+
+	"github.com/ava-labs/avalanchego/ids"
+)
 
 // Defines network capabilities supportable regardless of how a network is orchestrated.
 type Network interface {
 	GetConfig() NetworkConfig
 	GetNodes() []Node
+	AddNode(w io.Writer, flags FlagsMap, waitForHealth bool) (Node, error)
 }
 
 // Defines node capabilities supportable regardless of how a network is orchestrated.
 type Node interface {
 	GetID() ids.NodeID
 	GetConfig() NodeConfig
 	GetRuntimeState() NodeRuntimeState
+	Stop() error
+	Remove() error
 }

tests/fixture/testnet/local/network.go

@@ -11,6 +11,8 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"time"
 
 	cfg "github.com/ava-labs/avalanchego/config"
@@ -56,6 +58,22 @@ func (ln *LocalNetwork) GetNodes() []testnet.Node {
 	return nodes
 }
 
+// Adds a backend-agnostic node to the network
+func (ln *LocalNetwork) AddNode(w io.Writer, flags testnet.FlagsMap, waitForHealth bool) (testnet.Node, error) {
+	if flags == nil {
+		flags = testnet.FlagsMap{}
+	}
+	node, err := ln.AddLocalNode(w, &LocalNode{
+		NodeConfig: testnet.NodeConfig{
+			Flags: flags,
+		},
+	}, waitForHealth)
+	if err != nil {
+		return nil, err
+	}
+	return node, nil
+}
+
 // Starts a new network stored under the provided root dir. Required
 // configuration will be defaulted if not provided.
 func StartNetwork(
@@ -110,8 +128,7 @@ func StopNetwork(dir string) error {
 	if err != nil {
 		return err
 	}
-	network.Stop()
-	return nil
+	return network.Stop()
 }
 
 // Ensure the network has the configuration it needs to start.
@@ -342,12 +359,18 @@ func (ln *LocalNetwork) GetURIs() []string {
 }
 
 // Stop all nodes in the network.
-func (ln *LocalNetwork) Stop() {
+func (ln *LocalNetwork) Stop() error {
 	// Assume the nodes are loaded and the pids are current
+	errs := []error{}
 	for _, node := range ln.Nodes {
-		node.Stop()
+		if err := node.Stop(); err != nil {
+			errs = append(errs, err)
+		}
 	}
-	// TODO(marun) Collect and return errors
+	if len(errs) > 0 {
+		return errors.Join(errs...)
+	}
+	return nil
 }
 
 func (ln *LocalNetwork) GetGenesisPath() string {
@@ -569,3 +592,90 @@ func (ln *LocalNetwork) ReadAll() error {
 	}
 	return ln.ReadNodes()
 }
+
+func (ln *LocalNetwork) AddLocalNode(w io.Writer, node *LocalNode, waitForHealth bool) (*LocalNode, error) {
+	// Assume network configuration has been written to disk and is current in memory
+
+	if err := ln.ReadNodes(); err != nil {
+		return nil, err
+	}
+
+	maxPort := 0
+	bootstrapIPs := []string{}
+	bootstrapIDs := []string{}
+	for _, node := range ln.Nodes {
+		if len(node.StakingAddress) == 0 {
+			// Node is not running
+			continue
+		}
+
+		bootstrapIPs = append(bootstrapIPs, node.StakingAddress)
+		bootstrapIDs = append(bootstrapIDs, node.NodeID.String())
+
+		// Find the maximum port if using static ports
+		if ln.UseStaticPorts {
+			addressParts := strings.Split(node.StakingAddress, ":")
+			rawPort := addressParts[len(addressParts)-1]
+			port, err := strconv.Atoi(rawPort)
+			if err != nil {
+				return nil, err
+			}
+			if port > maxPort {
+				maxPort = port
+			}
+		}
+	}
+
+	if len(bootstrapIDs) == 0 {
+		// TODO(marun) Make sure to relax this for the initial node
+		// when building a network node-by-node
+		return nil, errors.New("failed to add node due to missing bootstrap nodes")
+	}
+
+	if node == nil {
+		// TODO(marun) Simplify this instantiation
+		node = &LocalNode{
+			NodeConfig: *testnet.NewNodeConfig(),
+		}
+	}
+
+	err := ln.PopulateNodeConfig(node)
+	if err != nil {
+		return nil, err
+	}
+
+	// Configure the provided node for this network
+	httpPort := 0
+	stakingPort := 0
+	if ln.UseStaticPorts {
+		httpPort = maxPort
+		stakingPort = maxPort + 1
+	}
+	node.SetNetworkingConfig(httpPort, stakingPort, bootstrapIDs, bootstrapIPs)
+
+	err = node.WriteConfig()
+	if err != nil {
+		return nil, err
+	}
+	process, err := node.Start(w, ln.ExecPath, ln.UseStaticPorts)
+	if err != nil {
+		return nil, err
+	}
+
+	if waitForHealth {
+		// TODO(marun) Use a timeout
+		ctx := context.Background()
+
+		// Health check
+		for {
+			if healthy, err := isHealthy(ctx, process); err != nil {
+				return nil, err
+			} else if healthy {
+				break
+			}
+			time.Sleep(time.Millisecond * 500)
+		}
+	}
+
+	return node, nil
+}