Implemented methods to compute waiting time at the replicas

internal/orchestration/worker.go

@@ -20,8 +20,10 @@ import (
 	"github.com/relab/hotstuff/crypto"
 	"github.com/relab/hotstuff/crypto/keygen"
 	"github.com/relab/hotstuff/eventloop"
+	"github.com/relab/hotstuff/internal/latency"
 	"github.com/relab/hotstuff/internal/proto/orchestrationpb"
 	"github.com/relab/hotstuff/internal/protostream"
+	"github.com/relab/hotstuff/internal/tree"
 	"github.com/relab/hotstuff/logging"
 	"github.com/relab/hotstuff/metrics"
 	"github.com/relab/hotstuff/metrics/types"
@@ -193,6 +195,10 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl
 	}
 	var viewDuration synchronizer.ViewDuration
 	if opts.GetLeaderRotation() == "tree-leader" {
+		// TODO(meling): Temporary default; should be configurable and moved to the appropriate place.
+		opts.SetTreeHeightWaitTime()
+		// create tree only if we are using tree leader (Kauri)
+		builder.Options().SetTree(createTree(opts))
 		viewDuration = synchronizer.NewFixedViewDuration(opts.GetInitialTimeout().AsDuration())
 	} else {
 		viewDuration = synchronizer.NewViewDuration(
@@ -215,7 +221,6 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl
 		logger,
 	)
 	builder.Options().SetSharedRandomSeed(opts.GetSharedSeed())
-	builder.Options().SetTreeConfig(opts.GetBranchFactor(), opts.TreePositionIDs(), opts.TreeDeltaDuration())
 
 	if w.measurementInterval > 0 {
 		replicaMetrics := metrics.GetReplicaMetrics(w.metrics...)
@@ -245,6 +250,20 @@ func (w *Worker) createReplica(opts *orchestrationpb.ReplicaOpts) (*replica.Repl
 	return replica.New(c, builder), nil
 }
 
+// createTree creates a tree based on the given replica options.
+func createTree(replicaOpts *orchestrationpb.ReplicaOpts) tree.Tree {
+	tree := tree.CreateTree(replicaOpts.HotstuffID(), int(replicaOpts.GetBranchFactor()), replicaOpts.TreePositionIDs())
+	switch {
+	case replicaOpts.GetAggregationTime():
+		tree.SetAggregationWaitTime(latency.MatrixFrom(replicaOpts.GetLocations()), replicaOpts.TreeDeltaDuration())
+	case replicaOpts.GetTreeHeightTime():
+		fallthrough
+	default:
+		tree.SetTreeHeightWaitTime(replicaOpts.TreeDeltaDuration())
+	}
+	return tree
+}
+
 func (w *Worker) startReplicas(req *orchestrationpb.StartReplicaRequest) (*orchestrationpb.StartReplicaResponse, error) {
 	for _, id := range req.GetIDs() {
 		replica, ok := w.replicas[hotstuff.ID(id)]

internal/proto/orchestrationpb/orchestration.proto

@@ -59,6 +59,13 @@ message ReplicaOpts {
   uint32 BranchFactor = 24;
   // Tree Duration in tree communication
   google.protobuf.Duration TreeDelta = 25;
+  // DelayType is the type of delay to use for the wait time.
+  oneof DelayType {
+    // AggregationTime computes the wait time based on the latency of links in the tree.
+    bool AggregationTime = 26;
+    // TreeHeightTime computes the wait time only based on the height of the tree.
+    bool TreeHeightTime = 27;
+  }
 }
 
 // ReplicaInfo is the information that the replicas need about each other.

internal/proto/orchestrationpb/replica_opts.go

@@ -38,6 +38,14 @@ func (x *ReplicaOpts) TreeDeltaDuration() time.Duration {
 	return x.GetTreeDelta().AsDuration()
 }
 
+func (x *ReplicaOpts) SetAggregationWaitTime() {
+	x.DelayType = &ReplicaOpts_AggregationTime{AggregationTime: true}
+}
+
+func (x *ReplicaOpts) SetTreeHeightWaitTime() {
+	x.DelayType = &ReplicaOpts_TreeHeightTime{TreeHeightTime: true}
+}
+
 func (x *ReplicaOpts) SetTreeOptions(branchFactor uint32, positions []uint32, treeDelta time.Duration) {
 	x.TreePositions = positions
 	x.BranchFactor = branchFactor

internal/tree/tree.go

@@ -2,28 +2,31 @@ package tree
 
 import (
 	"slices"
+	"time"
 
 	"github.com/relab/hotstuff"
 )
 
 // Tree contains the local replica's ID which must be part of the tree.
+// Once created, the tree is immutable.
 type Tree struct {
 	id           hotstuff.ID
 	height       int
 	branchFactor int
 	treePosToID  []hotstuff.ID
+	waitTime     time.Duration
 }
 
 // CreateTree creates the tree configuration.
-func CreateTree(myID hotstuff.ID, bf int, ids []hotstuff.ID) *Tree {
+func CreateTree(myID hotstuff.ID, bf int, ids []hotstuff.ID) Tree {
 	if bf < 2 {
 		panic("Branch factor must be greater than 1")
 	}
 	if slices.Index(ids, myID) == -1 {
 		panic("Replica ID not found in tree configuration")
 	}
 
-	return &Tree{
+	return Tree{
 		id:           myID,
 		height:       treeHeight(len(ids), bf),
 		branchFactor: bf,
@@ -46,14 +49,14 @@ func treeHeight(numNodes, bf int) (height int) {
 }
 
 // TreeHeight returns the height of the full tree.
-func (t *Tree) TreeHeight() int {
+func (t Tree) TreeHeight() int {
 	return t.height
 }
 
 // Parent returns the ID of the parent of this tree's replica and true.
 // If this tree's replica is the root, the root's ID is returned
 // and false to indicate it does not have a parent.
-func (t *Tree) Parent() (hotstuff.ID, bool) {
+func (t Tree) Parent() (hotstuff.ID, bool) {
 	myPos := t.replicaPosition(t.id)
 	if myPos == 0 {
 		return t.id, false
@@ -62,18 +65,23 @@ func (t *Tree) Parent() (hotstuff.ID, bool) {
 	return t.treePosToID[parentPos], true
 }
 
+// Root returns the ID of the root of the tree.
+func (t Tree) Root() hotstuff.ID {
+	return t.treePosToID[0]
+}
+
 // IsRoot return true if the replica is at root of the tree.
-func (t *Tree) IsRoot(replicaID hotstuff.ID) bool {
+func (t Tree) IsRoot(replicaID hotstuff.ID) bool {
 	return t.replicaPosition(replicaID) == 0
 }
 
 // ReplicaChildren returns the children of this tree's replica, if any.
-func (t *Tree) ReplicaChildren() []hotstuff.ID {
+func (t Tree) ReplicaChildren() []hotstuff.ID {
 	return t.ChildrenOf(t.id)
 }
 
 // ChildrenOf returns the children of a specific replica.
-func (t *Tree) ChildrenOf(replicaID hotstuff.ID) []hotstuff.ID {
+func (t Tree) ChildrenOf(replicaID hotstuff.ID) []hotstuff.ID {
 	replicaPos := t.replicaPosition(replicaID)
 	if replicaPos == -1 {
 		// safe since nil slices are equal to empty slices when iterating.
@@ -93,13 +101,13 @@ func (t *Tree) ChildrenOf(replicaID hotstuff.ID) []hotstuff.ID {
 }
 
 // ReplicaHeight returns the height of this tree's replica.
-func (t *Tree) ReplicaHeight() int {
+func (t Tree) ReplicaHeight() int {
 	return t.heightOf(t.id)
 }
 
 // PeersOf returns the sibling peers of the tree's replica,
 // unless the replica is the root, in which case there are no siblings.
-func (t *Tree) PeersOf() []hotstuff.ID {
+func (t Tree) PeersOf() []hotstuff.ID {
 	parent, ok := t.Parent()
 	if !ok {
 		// the tree's replica is the root, hence it has no siblings.
@@ -109,7 +117,7 @@ func (t *Tree) PeersOf() []hotstuff.ID {
 }
 
 // SubTree returns all subtree replicas of this tree's replica.
-func (t *Tree) SubTree() []hotstuff.ID {
+func (t Tree) SubTree() []hotstuff.ID {
 	children := t.ChildrenOf(t.id)
 	if len(children) == 0 {
 		// safe since nil slices are equal to empty slices when iterating.
@@ -126,7 +134,7 @@ func (t *Tree) SubTree() []hotstuff.ID {
 }
 
 // heightOf returns the height from the given replica's vantage point.
-func (t *Tree) heightOf(replicaID hotstuff.ID) int {
+func (t Tree) heightOf(replicaID hotstuff.ID) int {
 	if t.IsRoot(replicaID) {
 		return t.height
 	}
@@ -162,7 +170,7 @@ func (t *Tree) heightOf(replicaID hotstuff.ID) int {
 	return 0
 }
 
-func (t *Tree) replicaPosition(id hotstuff.ID) int {
+func (t Tree) replicaPosition(id hotstuff.ID) int {
 	return slices.Index(t.treePosToID, id)
 }
 

internal/tree/treelatency.go

@@ -0,0 +1,61 @@
+package tree
+
+import (
+	"slices"
+	"time"
+
+	"github.com/relab/hotstuff"
+	"github.com/relab/hotstuff/internal/latency"
+)
+
+// WaitTime returns the expected time to wait for the aggregation of votes.
+func (t *Tree) WaitTime() time.Duration {
+	return t.waitTime
+}
+
+// SetAggregationWaitTime sets the wait time for the aggregation of votes based on the
+// highest latency path from node id to its leaf nodes.
+// Only one of SetAggregationWaitTime or SetTreeHeightWaitTime should be called.
+func (t *Tree) SetAggregationWaitTime(lm latency.Matrix, delta time.Duration) {
+	t.waitTime = t.aggregationTime(t.id, lm, delta)
+}
+
+// SetTreeHeightWaitTime sets the wait time for the aggregation of votes based on the
+// height of the tree.
+// Only one of SetAggregationWaitTime or SetTreeHeightWaitTime should be called.
+func (t *Tree) SetTreeHeightWaitTime(delta time.Duration) {
+	t.waitTime = t.treeHeightTime(delta)
+}
+
+// aggregationTime returns the time to wait for the aggregation of votes based on the
+// highest latency path from node id to its leaf nodes.
+// The id is required because the function is recursive.
+//
+// If the node is a leaf, it returns 0 as no aggregation is required.
+// For other nodes, the aggregation time for a child includes:
+// - Round-trip time to the child
+// - Aggregation time required by the child node (recursive call)
+func (t *Tree) aggregationTime(id hotstuff.ID, lm latency.Matrix, delta time.Duration) time.Duration {
+	children := t.ChildrenOf(id)
+	if len(children) == 0 {
+		return 0 // base case: leaf nodes have zero aggregation latency.
+	}
+	// calculate aggregation latencies for each child
+	latencies := make([]time.Duration, len(children))
+	for i, child := range children {
+		latencies[i] = 2*lm.Latency(id, child) + t.aggregationTime(child, lm, delta)
+	}
+	return max(latencies) + delta
+}
+
+// treeHeightTime returns a fixed time to wait based on the height of the tree.
+func (t *Tree) treeHeightTime(delta time.Duration) time.Duration {
+	return time.Duration(2*(t.ReplicaHeight()-1)) * delta
+}
+
+func max(latencies []time.Duration) time.Duration {
+	if len(latencies) == 0 {
+		return 0
+	}
+	return slices.Max(latencies)
+}

internal/tree/treelatency_test.go

@@ -0,0 +1,80 @@
+package tree
+
+import (
+	"testing"
+	"time"
+
+	"github.com/relab/hotstuff"
+	"github.com/relab/hotstuff/internal/latency"
+)
+
+var (
+	cities07 = []string{
+		"Melbourne", "Toronto", "Prague", "Paris",
+		"Tokyo", "Amsterdam", "Auckland",
+	}
+	cities15 = []string{
+		"Melbourne", "Melbourne", "Toronto", "Toronto", "Prague",
+		"Prague", "Paris", "Paris", "Tokyo", "Tokyo", "Amsterdam",
+		"Amsterdam", "Auckland", "Auckland", "Melbourne",
+	}
+)
+
+func TestAggregationWaitTime(t *testing.T) {
+	testDataAggregationTime := []struct {
+		id    hotstuff.ID
+		locs  []string
+		delta time.Duration
+		want  time.Duration
+	}{
+		{id: 1, locs: cities07, delta: 0, want: 521775000},
+		{id: 2, locs: cities07, delta: 0, want: 178253000},
+		{id: 3, locs: cities07, delta: 0, want: 279038000},
+		{id: 4, locs: cities07, delta: 0, want: 0},
+		{id: 1, locs: cities15, delta: 0, want: 607507000},
+		{id: 2, locs: cities15, delta: 0, want: 511744000},
+		{id: 3, locs: cities15, delta: 0, want: 388915000},
+		{id: 4, locs: cities15, delta: 0, want: 178253000},
+		{id: 5, locs: cities15, delta: 0, want: 269007000},
+	}
+	for _, test := range testDataAggregationTime {
+		bf := 2
+		treePos := DefaultTreePos(len(test.locs))
+		tree := CreateTree(test.id, bf, treePos)
+		lm := latency.MatrixFrom(test.locs)
+		tree.SetAggregationWaitTime(lm, test.delta)
+		got := tree.WaitTime()
+		if got != test.want {
+			t.Errorf("tree.WaitTime(%v) = %v, want %v", test.delta, got, test.want)
+		}
+	}
+}
+
+func TestTreeHeightWaitTime(t *testing.T) {
+	testDataTreeHeightTime := []struct {
+		id    hotstuff.ID
+		locs  []string
+		delta time.Duration
+		want  time.Duration
+	}{
+		{id: 1, locs: cities15, delta: 10, want: 60},
+		{id: 2, locs: cities15, delta: 10, want: 40},
+		{id: 3, locs: cities15, delta: 10, want: 40},
+		{id: 4, locs: cities15, delta: 10, want: 20},
+		{id: 9, locs: cities15, delta: 10, want: 0},
+		{id: 1, locs: cities07, delta: 10, want: 40},
+		{id: 2, locs: cities07, delta: 10, want: 20},
+		{id: 3, locs: cities07, delta: 10, want: 20},
+		{id: 4, locs: cities07, delta: 10, want: 0},
+	}
+	for _, test := range testDataTreeHeightTime {
+		bf := 2
+		treePos := DefaultTreePos(len(test.locs))
+		tree := CreateTree(test.id, bf, treePos)
+		tree.SetTreeHeightWaitTime(test.delta)
+		got := tree.WaitTime()
+		if got != test.want {
+			t.Errorf("tree.WaitTime(%v) = %v, want %v", test.delta, got, test.want)
+		}
+	}
+}