Add Primary Network Partial Sync Option (#1769)

abi87 · StephenButtolph · web-flow · commit 6a8c0bd09d12 · 2023-08-09T19:57:32.000-04:00
Co-authored-by: Stephen Buttolph &lt;stephen@avalabs.org&gt;
diff --git a/chains/manager.go b/chains/manager.go
@@ -87,10 +87,11 @@ var (
 	// Bootstrapping prefixes for ChainVMs
 	bootstrappingDB = []byte("bs")
 
-	errUnknownVMType          = errors.New("the vm should have type avalanche.DAGVM or snowman.ChainVM")
-	errCreatePlatformVM       = errors.New("attempted to create a chain running the PlatformVM")
-	errNotBootstrapped        = errors.New("subnets not bootstrapped")
-	errNoPrimaryNetworkConfig = errors.New("no subnet config for primary network found")
+	errUnknownVMType           = errors.New("the vm should have type avalanche.DAGVM or snowman.ChainVM")
+	errCreatePlatformVM        = errors.New("attempted to create a chain running the PlatformVM")
+	errNotBootstrapped         = errors.New("subnets not bootstrapped")
+	errNoPrimaryNetworkConfig  = errors.New("no subnet config for primary network found")
+	errPartialSyncAsAValidator = errors.New("partial sync should not be configured for a validator")
 
 	_ Manager = (*manager)(nil)
 )
@@ -185,7 +186,8 @@ type ManagerConfig struct {
 	Validators                  validators.Manager         // Validators validating on this chain
 	NodeID                      ids.NodeID                 // The ID of this node
 	NetworkID                   uint32                     // ID of the network this node is connected to
-	Server                      server.Server              // Handles HTTP API calls
+	PartialSyncPrimaryNetwork   bool
+	Server                      server.Server // Handles HTTP API calls
 	Keystore                    keystore.Keystore
 	AtomicMemory                *atomic.Memory
 	AVAXAssetID                 ids.ID
@@ -1212,6 +1214,7 @@ func (m *manager) createSnowmanChain(
 		Validators:    vdrs,
 		Params:        consensusParams,
 		Consensus:     consensus,
+		PartialSync:   m.PartialSyncPrimaryNetwork && commonCfg.Ctx.ChainID == constants.PlatformChainID,
 	}
 	engine, err := smeng.New(engineConfig)
 	if err != nil {
@@ -1321,6 +1324,32 @@ func (m *manager) registerBootstrappedHealthChecks() error {
 	if err := m.Health.RegisterHealthCheck("bootstrapped", bootstrappedCheck, health.ApplicationTag); err != nil {
 		return fmt.Errorf("couldn't register bootstrapped health check: %w", err)
 	}
+
+	// We should only report unhealthy if the node is partially syncing the
+	// primary network and is a validator.
+	if !m.PartialSyncPrimaryNetwork {
+		return nil
+	}
+
+	partialSyncCheck := health.CheckerFunc(func(ctx context.Context) (interface{}, error) {
+		// Note: The health check is skipped during bootstrapping to allow a
+		// node to sync the network even if it was previously a validator.
+		if !m.IsBootstrapped(constants.PlatformChainID) {
+			return "node is currently bootstrapping", nil
+		}
+		if !validators.Contains(m.Validators, constants.PrimaryNetworkID, m.NodeID) {
+			return "node is not a primary network validator", nil
+		}
+
+		m.Log.Warn("node is a primary network validator",
+			zap.Error(errPartialSyncAsAValidator),
+		)
+		return "node is a primary network validator", errPartialSyncAsAValidator
+	})
+
+	if err := m.Health.RegisterHealthCheck("validation", partialSyncCheck, health.ApplicationTag); err != nil {
+		return fmt.Errorf("couldn't register validation health check: %w", err)
+	}
 	return nil
 }
 
diff --git a/config/config.go b/config/config.go
@@ -777,6 +777,7 @@ func getStakingConfig(v *viper.Viper, networkID uint32) (node.StakingConfig, err
 	config := node.StakingConfig{
 		SybilProtectionEnabled:        v.GetBool(SybilProtectionEnabledKey),
 		SybilProtectionDisabledWeight: v.GetUint64(SybilProtectionDisabledWeightKey),
+		PartialSyncPrimaryNetwork:     v.GetBool(PartialSyncPrimaryNetworkKey),
 		StakingKeyPath:                GetExpandedArg(v, StakingTLSKeyPathKey),
 		StakingCertPath:               GetExpandedArg(v, StakingCertPathKey),
 		StakingSignerPath:             GetExpandedArg(v, StakingSignerKeyPathKey),
diff --git a/config/flags.go b/config/flags.go
@@ -261,6 +261,7 @@ func addNodeFlags(fs *pflag.FlagSet) {
 	fs.String(StakingSignerKeyContentKey, "", "Specifies base64 encoded signer private key for staking")
 	fs.Bool(SybilProtectionEnabledKey, true, "Enables sybil protection. If enabled, Network TLS is required")
 	fs.Uint64(SybilProtectionDisabledWeightKey, 100, "Weight to provide to each peer when sybil protection is disabled")
+	fs.Bool(PartialSyncPrimaryNetworkKey, false, "Only sync the P-chain on the Primary Network. If the node is a Primary Network validator, it will report unhealthy")
 	// Uptime Requirement
 	fs.Float64(UptimeRequirementKey, genesis.LocalParams.UptimeRequirement, "Fraction of time a validator must be online to receive rewards")
 	// Minimum Stake required to validate the Primary Network
diff --git a/config/keys.go b/config/keys.go
@@ -129,6 +129,7 @@ const (
 	SnowOptimalProcessingKey                           = "snow-optimal-processing"
 	SnowMaxProcessingKey                               = "snow-max-processing"
 	SnowMaxTimeProcessingKey                           = "snow-max-time-processing"
+	PartialSyncPrimaryNetworkKey                       = "partial-sync-primary-network"
 	TrackSubnetsKey                                    = "track-subnets"
 	AdminAPIEnabledKey                                 = "api-admin-enabled"
 	InfoAPIEnabledKey                                  = "api-info-enabled"
diff --git a/node/config.go b/node/config.go
@@ -92,6 +92,7 @@ type IPConfig struct {
 type StakingConfig struct {
 	genesis.StakingConfig
 	SybilProtectionEnabled        bool            `json:"sybilProtectionEnabled"`
+	PartialSyncPrimaryNetwork     bool            `json:"partialSyncPrimaryNetwork"`
 	StakingTLSCert                tls.Certificate `json:"-"`
 	StakingSigningKey             *bls.SecretKey  `json:"-"`
 	SybilProtectionDisabledWeight uint64          `json:"sybilProtectionDisabledWeight"`
diff --git a/node/node.go b/node/node.go
@@ -818,6 +818,7 @@ func (n *Node) initChainManager(avaxAssetID ids.ID) error {
 		Router:                                  n.Config.ConsensusRouter,
 		Net:                                     n.Net,
 		Validators:                              n.vdrs,
+		PartialSyncPrimaryNetwork:               n.Config.PartialSyncPrimaryNetwork,
 		NodeID:                                  n.ID,
 		NetworkID:                               n.Config.NetworkID,
 		Server:                                  n.APIServer,
@@ -886,6 +887,7 @@ func (n *Node) initVMs() error {
 				Validators:                    vdrs,
 				UptimeLockedCalculator:        n.uptimeCalculator,
 				SybilProtectionEnabled:        n.Config.SybilProtectionEnabled,
+				PartialSyncPrimaryNetwork:     n.Config.PartialSyncPrimaryNetwork,
 				TrackedSubnets:                n.Config.TrackedSubnets,
 				TxFee:                         n.Config.TxFee,
 				CreateAssetTxFee:              n.Config.CreateAssetTxFee,
diff --git a/snow/engine/snowman/config.go b/snow/engine/snowman/config.go
@@ -16,10 +16,11 @@ import (
 type Config struct {
 	common.AllGetsServer
 
-	Ctx        *snow.ConsensusContext
-	VM         block.ChainVM
-	Sender     common.Sender
-	Validators validators.Set
-	Params     snowball.Parameters
-	Consensus  snowman.Consensus
+	Ctx         *snow.ConsensusContext
+	VM          block.ChainVM
+	Sender      common.Sender
+	Validators  validators.Set
+	Params      snowball.Parameters
+	Consensus   snowman.Consensus
+	PartialSync bool
 }
diff --git a/snow/engine/snowman/transitive.go b/snow/engine/snowman/transitive.go
@@ -455,7 +455,9 @@ func (t *Transitive) GetBlock(ctx context.Context, blkID ids.ID) (snowman.Block,
 
 func (t *Transitive) sendChits(ctx context.Context, nodeID ids.NodeID, requestID uint32) {
 	lastAccepted := t.Consensus.LastAccepted()
-	if t.Ctx.StateSyncing.Get() {
+	// If we aren't fully verifying blocks, only vote for blocks that are widely
+	// preferred by the validator set.
+	if t.Ctx.StateSyncing.Get() || t.Config.PartialSync {
 		t.Sender.SendChits(ctx, nodeID, requestID, lastAccepted, lastAccepted)
 	} else {
 		t.Sender.SendChits(ctx, nodeID, requestID, t.Consensus.Preference(), lastAccepted)
diff --git a/vms/platformvm/blocks/builder/builder.go b/vms/platformvm/blocks/builder/builder.go
@@ -147,8 +147,12 @@ func (b *builder) AddUnverifiedTx(tx *txs.Tx) error {
 		return err
 	}
 
-	if err := b.Mempool.Add(tx); err != nil {
-		return err
+	// If we are partially syncing the Primary Network, we should not be
+	// maintaining the transaction mempool locally.
+	if !b.txExecutorBackend.Config.PartialSyncPrimaryNetwork {
+		if err := b.Mempool.Add(tx); err != nil {
+			return err
+		}
 	}
 	return b.GossipTx(tx)
 }
diff --git a/vms/platformvm/blocks/builder/network.go b/vms/platformvm/blocks/builder/network.go
@@ -37,7 +37,7 @@ type Network interface {
 
 type network struct {
 	ctx        *snow.Context
-	blkBuilder Builder
+	blkBuilder *builder
 
 	// gossip related attributes
 	appSender common.AppSender
@@ -99,6 +99,13 @@ func (n *network) AppGossip(_ context.Context, nodeID ids.NodeID, msgBytes []byt
 		zap.Int("messageLen", len(msgBytes)),
 	)
 
+	if n.blkBuilder.txExecutorBackend.Config.PartialSyncPrimaryNetwork {
+		n.ctx.Log.Debug("dropping AppGossip message",
+			zap.String("reason", "primary network is not being fully synced"),
+		)
+		return nil
+	}
+
 	msgIntf, err := message.Parse(msgBytes)
 	if err != nil {
 		n.ctx.Log.Debug("dropping AppGossip message",
diff --git a/vms/platformvm/blocks/executor/manager.go b/vms/platformvm/blocks/executor/manager.go
@@ -53,7 +53,10 @@ func NewManager(
 			validators:   validatorManager,
 			bootstrapped: txExecutorBackend.Bootstrapped,
 		},
-		rejector: &rejector{backend: backend},
+		rejector: &rejector{
+			backend:         backend,
+			addTxsToMempool: !txExecutorBackend.Config.PartialSyncPrimaryNetwork,
+		},
 	}
 }
 
diff --git a/vms/platformvm/blocks/executor/rejector.go b/vms/platformvm/blocks/executor/rejector.go
@@ -16,6 +16,7 @@ var _ blocks.Visitor = (*rejector)(nil)
 // being shutdown.
 type rejector struct {
 	*backend
+	addTxsToMempool bool
 }
 
 func (r *rejector) BanffAbortBlock(b *blocks.BanffAbortBlock) error {
@@ -66,6 +67,10 @@ func (r *rejector) rejectBlock(b blocks.Block, blockType string) error {
 		zap.Stringer("parentID", b.Parent()),
 	)
 
+	if !r.addTxsToMempool {
+		return nil
+	}
+
 	for _, tx := range b.Txs() {
 		if err := r.Mempool.Add(tx); err != nil {
 			r.ctx.Log.Debug(
diff --git a/vms/platformvm/blocks/executor/rejector_test.go b/vms/platformvm/blocks/executor/rejector_test.go
@@ -134,6 +134,7 @@ func TestRejectBlock(t *testing.T) {
 					Mempool:      mempool,
 					state:        state,
 				},
+				addTxsToMempool: true,
 			}
 
 			// Set expected calls on dependencies.
diff --git a/vms/platformvm/config/config.go b/vms/platformvm/config/config.go
@@ -35,6 +35,9 @@ type Config struct {
 	// True if the node is being run with staking enabled
 	SybilProtectionEnabled bool
 
+	// If true, only the P-chain will be instantiated on the primary network.
+	PartialSyncPrimaryNetwork bool
+
 	// Set of subnets that this node is validating
 	TrackedSubnets set.Set[ids.ID]
 
diff --git a/vms/platformvm/txs/executor/standard_tx_executor.go b/vms/platformvm/txs/executor/standard_tx_executor.go
@@ -9,8 +9,11 @@ import (
 	"fmt"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/ava-labs/avalanchego/chains/atomic"
 	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/utils/constants"
 	"github.com/ava-labs/avalanchego/utils/set"
 	"github.com/ava-labs/avalanchego/vms/components/avax"
 	"github.com/ava-labs/avalanchego/vms/components/verify"
@@ -135,7 +138,9 @@ func (e *StandardTxExecutor) ImportTx(tx *txs.ImportTx) error {
 		utxoIDs[i] = utxoID[:]
 	}
 
-	if e.Bootstrapped.Get() {
+	// Skip verification of the shared memory inputs if the other primary
+	// network chains are not guaranteed to be up-to-date.
+	if e.Bootstrapped.Get() && !e.Config.PartialSyncPrimaryNetwork {
 		if err := verify.SameSubnet(context.TODO(), e.Ctx, tx.SourceChain); err != nil {
 			return err
 		}
@@ -186,6 +191,9 @@ func (e *StandardTxExecutor) ImportTx(tx *txs.ImportTx) error {
 	// Produce the UTXOS
 	avax.Produce(e.State, txID, tx.Outs)
 
+	// Note: We apply atomic requests even if we are not verifying atomic
+	// requests to ensure the shared state will be correct if we later start
+	// verifying the requests.
 	e.AtomicRequests = map[ids.ID]*atomic.Requests{
 		tx.SourceChain: {
 			RemoveRequests: utxoIDs,
@@ -230,6 +238,9 @@ func (e *StandardTxExecutor) ExportTx(tx *txs.ExportTx) error {
 	// Produce the UTXOS
 	avax.Produce(e.State, txID, tx.Outs)
 
+	// Note: We apply atomic requests even if we are not verifying atomic
+	// requests to ensure the shared state will be correct if we later start
+	// verifying the requests.
 	elems := make([]*atomic.Element, len(tx.ExportedOutputs))
 	for i, out := range tx.ExportedOutputs {
 		utxo := &avax.UTXO{
@@ -288,6 +299,14 @@ func (e *StandardTxExecutor) AddValidatorTx(tx *txs.AddValidatorTx) error {
 	avax.Consume(e.State, tx.Ins)
 	avax.Produce(e.State, txID, tx.Outs)
 
+	if e.Config.PartialSyncPrimaryNetwork && tx.Validator.NodeID == e.Ctx.NodeID {
+		e.Ctx.Log.Warn("verified transaction that would cause this node to become unhealthy",
+			zap.String("reason", "primary network is not being fully synced"),
+			zap.Stringer("txID", txID),
+			zap.String("txType", "addValidator"),
+			zap.Stringer("nodeID", tx.Validator.NodeID),
+		)
+	}
 	return nil
 }
 
@@ -434,6 +453,17 @@ func (e *StandardTxExecutor) AddPermissionlessValidatorTx(tx *txs.AddPermissionl
 	avax.Consume(e.State, tx.Ins)
 	avax.Produce(e.State, txID, tx.Outs)
 
+	if e.Config.PartialSyncPrimaryNetwork &&
+		tx.Subnet == constants.PrimaryNetworkID &&
+		tx.Validator.NodeID == e.Ctx.NodeID {
+		e.Ctx.Log.Warn("verified transaction that would cause this node to become unhealthy",
+			zap.String("reason", "primary network is not being fully synced"),
+			zap.Stringer("txID", txID),
+			zap.String("txType", "addPermissionlessValidator"),
+			zap.Stringer("nodeID", tx.Validator.NodeID),
+		)
+	}
+
 	return nil
 }
 
diff --git a/vms/platformvm/vm.go b/vms/platformvm/vm.go
@@ -235,7 +235,9 @@ func (vm *VM) Initialize(
 
 // Create all chains that exist that this node validates.
 func (vm *VM) initBlockchains() error {
-	if err := vm.createSubnet(constants.PrimaryNetworkID); err != nil {
+	if vm.Config.PartialSyncPrimaryNetwork {
+		vm.ctx.Log.Info("skipping primary network chain creation")
+	} else if err := vm.createSubnet(constants.PrimaryNetworkID); err != nil {
 		return err
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -147,8 +147,12 @@ func (b builder) AddUnverifiedTx(tx txs.Tx) error {`
`147`	`147`	`return err`
`148`	`148`	`}`
`149`	`149`
`150`		`- if err := b.Mempool.Add(tx); err != nil {`
`151`		`- return err`
	`150`	`+ // If we are partially syncing the Primary Network, we should not be`
	`151`	`+ // maintaining the transaction mempool locally.`
	`152`	`+ if !b.txExecutorBackend.Config.PartialSyncPrimaryNetwork {`
	`153`	`+ if err := b.Mempool.Add(tx); err != nil {`
	`154`	`+ return err`
	`155`	`+ }`
`152`	`156`	`}`
`153`	`157`	`return b.GossipTx(tx)`
`154`	`158`	`}`