Merge pull request #228 from Fantom-foundation/rapol/feature/modify-yml

Modification to match YML definition

RT.jenkinsfile

@@ -1,10 +1,12 @@
+@Library('shared-library') _
+
 // Norma CI test norma using "make test"
 
 def buildSuccess = false
 def enableA1 = false
 def enableA2 = false
 def enableB1 = false
-def enableB2 = false
+def enableB2 = true
 def enableB3 = false
 def enableB4 = false
 def enableB5 = false
@@ -135,7 +137,7 @@ pipeline {
 			}
 		}
 
-		stage('Test B2 - KillValMidRun') {
+		stage('Test B2 - EndValMidRun') {
 			when {
 				expression {
 					buildSuccess
@@ -144,13 +146,14 @@ pipeline {
 			}
 
 			steps {
-				build/norma run --label b2 -o . release_testing/b2.KillValMidRun.yml
+				build/norma run --label b2 -o . scenarios/release_testing/b2.KillValMidRun.yml
 			}
 
 			post {
-				success {
-					dir('./norma_data_b2_latest') {
-						archiveArtifacts artifacts: '**'
+				always {
+					script {
+						def artifacts = ["*.yml", "*.csv", "*.log", "*.html"]
+						uploadArtifacts(artifacts)
 					}
 				}
 			}

driver/executor/executor.go

@@ -189,10 +189,13 @@ func scheduleNodeEvents(node *parser.Node, queue *eventQueue, net driver.Network
 	if node.End != nil {
 		endTime = Seconds(*node.End)
 	}
+
 	for i := 0; i < instances; i++ {
 		name := fmt.Sprintf("%s-%d", node.Name, i)
 		var instance = new(driver.Node)
-		queue.add(toSingleEvent(startTime, fmt.Sprintf("starting node %s", name), func() error {
+
+		// add initial start
+		queue.add(toSingleEvent(startTime, fmt.Sprintf("Creating node %s", name), func() error {
 			newNode, err := net.CreateNode(&driver.NodeConfig{
 				Name:      name,
 				Validator: node.IsValidator(),
@@ -202,6 +205,75 @@ func scheduleNodeEvents(node *parser.Node, queue *eventQueue, net driver.Network
 			return err
 		}))
 
+		// handle timer
+		if &node.Timer != nil {
+			for timing, evt := range node.Timer {
+				switch evt {
+				case "start":
+					queue.add(toSingleEvent(
+						Seconds(timing),
+						fmt.Sprintf("Starting node %s", name),
+						func() error {
+							_, err := net.StartNode(*instance)
+							return err
+						},
+					))
+				case "end":
+					queue.add(toSingleEvent(
+						Seconds(timing),
+						fmt.Sprintf("Ending node %s", name),
+						func() error {
+							if instance == nil {
+								return nil
+							}
+							if err := net.RemoveNode(*instance); err != nil {
+								return err
+							}
+							if err := (*instance).Stop(); err != nil {
+								return err
+							}
+							return nil
+						},
+					))
+				case "kill":
+					queue.add(toSingleEvent(
+						Seconds(timing),
+						fmt.Sprintf("SigKill to node %s", name),
+						func() error {
+							return net.KillNode(*instance)
+						},
+					))
+				case "restart":
+					queue.add(toEvent(
+						Seconds(timing),
+						fmt.Sprintf("Restart - ending node %s", name),
+						func() ([]event, error) {
+							if instance == nil {
+								return []event{}, nil
+							}
+							if err := net.RemoveNode(*instance); err != nil {
+								return []event{}, err
+							}
+							if err := (*instance).Stop(); err != nil {
+								return []event{}, err
+							}
+							return []event{
+								toSingleEvent(
+									Seconds(timing)+30, // 30 seconds grace period
+									fmt.Sprintf("Restart - starting node %s", name),
+									func() error {
+										_, err := net.StartNode(*instance)
+										return err
+									},
+								),
+							}, nil
+						},
+					))
+				}
+			}
+		}
+
+		// handle genesis
 		if &node.Genesis != nil {
 			if node.Genesis.Import != "" {
 				queue.add(toSingleEvent(
@@ -223,6 +295,7 @@ func scheduleNodeEvents(node *parser.Node, queue *eventQueue, net driver.Network
 			}
 		}
 
+		// handle event
 		if &node.Event != nil {
 			if node.Event.Import != nil {
 				queue.add(toSingleEvent(
@@ -244,6 +317,7 @@ func scheduleNodeEvents(node *parser.Node, queue *eventQueue, net driver.Network
 			}
 		}
 
+		// handle final end
 		queue.add(toSingleEvent(endTime, fmt.Sprintf("stopping node %s", name), func() error {
 			if instance == nil {
 				return nil

driver/network.go

@@ -34,9 +34,15 @@ type Network interface {
 	// nodes to the network as needed.
 	CreateNode(config *NodeConfig) (Node, error)
 
-	// RemoveNode removes node from the network
+	// StartNode starts a node that has already been created.
+	StartNode(Node) (Node, error)
+
+	// RemoveNode ends the client gracefully and removes node from the network
 	RemoveNode(Node) error
 
+	// KillNode disgracefully removes node from the network
+	KillNode(Node) error
+
 	// CreateApplication creates a new application in this network, ready to
 	// produce load as defined by its configuration.
 	CreateApplication(config *ApplicationConfig) (Application, error)

driver/network/local/local.go

@@ -148,14 +148,16 @@ func NewLocalNetwork(config *driver.NetworkConfig) (*LocalNetwork, error) {
 	return net, nil
 }
 
-// createNode is an internal version of CreateNode enabling the creation
-// of validator and non-validator nodes in the network.
-func (n *LocalNetwork) createNode(nodeConfig *node.OperaNodeConfig) (*node.OperaNode, error) {
-	node, err := node.StartOperaDockerNode(n.docker, n.network, nodeConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to start opera docker; %v", err)
+// StartNode starts a node after it has been created.
+func (n *LocalNetwork) StartNode(nd driver.Node) (driver.Node, error) {
+	opera, ok := nd.(*node.OperaNode)
+	if !ok {
+		return nil, fmt.Errorf("trying to start non-sonic node")
 	}
+	return n.startNode(opera)
+}
 
+func (n *LocalNetwork) startNode(node *node.OperaNode) (*node.OperaNode, error) {
 	n.nodesMutex.Lock()
 	id, err := node.GetNodeID()
 	if err != nil {
@@ -179,6 +181,16 @@ func (n *LocalNetwork) createNode(nodeConfig *node.OperaNodeConfig) (*node.Opera
 	return node, nil
 }
 
+// createNode is an internal version of CreateNode enabling the creation
+// of validator and non-validator nodes in the network.
+func (n *LocalNetwork) createNode(nodeConfig *node.OperaNodeConfig) (*node.OperaNode, error) {
+	node, err := node.StartOperaDockerNode(n.docker, n.network, nodeConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to start opera docker; %v", err)
+	}
+	return n.startNode(node)
+}
+
 // CreateNode creates nodes in the network during run.
 func (n *LocalNetwork) CreateNode(config *driver.NodeConfig) (driver.Node, error) {
 	newValId := 0
@@ -292,6 +304,10 @@ func (n *LocalNetwork) RemoveNode(node driver.Node) error {
 	return nil
 }
 
+func (n *LocalNetwork) KillNode(node driver.Node) error {
+	return node.Kill()
+}
+
 func (n *LocalNetwork) SendTransaction(tx *types.Transaction) {
 	n.rpcWorkerPool.SendTransaction(tx)
 }

driver/node.go

@@ -62,6 +62,9 @@ type Node interface {
 	// are expected to succeed.
 	Stop() error
 
+	// Kill shuts down this node disgracefully by using SigKill.
+	Kill() error
+
 	// Cleanup releases all underlying resources. After the cleanup no more
 	// operations on this node are expected to succeed.
 	Cleanup() error

driver/node/opera.go

@@ -69,14 +69,15 @@ const operaDockerImageName = "sonic"
 // OperaNode implements the driver's Node interface by running a go-opera
 // client on a generic host.
 type OperaNode struct {
-	host  network.Host
-	label string
+	host      network.Host
+	container *docker.Container
+	label     string
 }
 
 type OperaNodeConfig struct {
 	// The label to be used to name this node. The label should not be empty.
 	Label string
-	// The ID of the validator, nil if the node should node be a validator.
+	// The ID of the validator, nil if the node should not be a validator.
 	ValidatorId *int
 	// The configuration of the network the configured node should be part of.
 	NetworkConfig *driver.NetworkConfig
@@ -131,8 +132,9 @@ func StartOperaDockerNode(client *docker.Client, dn *docker.Network, config *Ope
 		return nil, err
 	}
 	node := &OperaNode{
-		host:  host,
-		label: config.Label,
+		host:      host,
+		container: host,
+		label:     config.Label,
 	}
 
 	// Wait until the OperaNode inside the Container is ready.
@@ -245,3 +247,8 @@ func (n *OperaNode) RemovePeer(id driver.NodeID) error {
 		return rpcClient.Call(nil, "admin_removePeer", id)
 	})
 }
+
+// Kill sends a SigKill singal to node.
+func (n *OperaNode) Kill() error {
+	return n.container.SendSignal(docker.SigKill)
+}

driver/norma/run.go

@@ -103,6 +103,10 @@ var (
 )
 
 func run(ctx *cli.Context) (err error) {
+	if num := ctx.Int(numValidators.Name); num != 0 {
+		fmt.Printf("[DEPRECATED] --num-validator flag has been deprecated along with NumValidator configuration in scenarios.\n --num-validator %d will not have any effect when running the provided scenarios.", num)
+	}
+
 	db := strings.ToLower(ctx.String(dbImpl.Name))
 	if db == "carmen" || db == "go-file" {
 		db = "go-file"
@@ -135,11 +139,6 @@ func run(ctx *cli.Context) (err error) {
 		return err
 	}
 
-	if num := ctx.Int(numValidators.Name); num > 0 {
-		fmt.Printf("Overriding number of validators to %d (--%s)\n", num, numValidators.Name)
-		scenario.NumValidators = &num
-	}
-
 	if err := scenario.Check(); err != nil {
 		return err
 	}
@@ -175,14 +174,12 @@ func run(ctx *cli.Context) (err error) {
 
 	// Startup network.
 	netConfig := driver.NetworkConfig{
-		NumberOfValidators:    1,
+		NumberOfValidators:    scenario.GetStaticValidatorCount(),
 		StateDbImplementation: db,
 		VmImplementation:      vm,
 	}
-	if scenario.NumValidators != nil {
-		netConfig.NumberOfValidators = *scenario.NumValidators
-	}
-	fmt.Printf("Creating network with %d validator(s) using the `%v` DB and `%v` VM implementation ...\n",
+
+	fmt.Printf("Creating network with %d static validator(s) using the `%v` DB and `%v` VM implementation ...\n",
 		netConfig.NumberOfValidators, netConfig.StateDbImplementation, netConfig.VmImplementation,
 	)
 	netConfig.MaxBlockGas = scenario.GenesisGasLimits.MaxBlockGas