diff --git a/consensus/consensus.go b/consensus/consensus.go new file mode 100644 index 00000000..faf387ef --- /dev/null +++ b/consensus/consensus.go @@ -0,0 +1,33 @@ +package consensus + +import ( + "fmt" +) + +// Type identifies consensus protocols suported by Blockless. +type Type uint + +const ( + Raft Type = iota + 1 + PBFT +) + +func (t Type) String() string { + switch t { + case Raft: + return "Raft" + case PBFT: + return "PBFT" + default: + return fmt.Sprintf("unknown: %d", t) + } +} + +func (t Type) Valid() bool { + switch t { + case Raft, PBFT: + return true + default: + return false + } +} diff --git a/consensus/pbft/commit.go b/consensus/pbft/commit.go new file mode 100644 index 00000000..e67568c8 --- /dev/null +++ b/consensus/pbft/commit.go @@ -0,0 +1,139 @@ +package pbft + +import ( + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +func (r *Replica) maybeSendCommit(view uint, sequenceNo uint, digest string) error { + + log := r.log.With().Uint("view", view).Uint("sequence_number", sequenceNo).Str("digest", digest).Logger() + + if !r.shouldSendCommit(view, sequenceNo, digest) { + log.Info().Msg("not sending commit") + return nil + } + + log.Info().Msg("request prepared, broadcasting commit") + + err := r.sendCommit(view, sequenceNo, digest) + if err != nil { + return fmt.Errorf("could not send commit message: %w", err) + } + + if !r.committed(view, sequenceNo, digest) { + log.Info().Msg("request is not yet committed") + return nil + } + + log.Info().Msg("request committed, executing") + + return r.execute(view, sequenceNo, digest) +} + +func (r *Replica) shouldSendCommit(view uint, sequenceNo uint, digest string) bool { + + log := r.log.With().Uint("view", view).Uint("sequence_number", sequenceNo).Str("digest", digest).Logger() + + if !r.prepared(view, sequenceNo, digest) { + log.Info().Msg("request not yet prepared, commit not due yet") + return false + } + + // Have we already sent a commit message? + msgID := getMessageID(view, sequenceNo) + commits, ok := r.commits[msgID] + if ok { + _, sent := commits.m[r.id] + if sent { + log.Info().Msg("commit for this request already broadcast") + return false + } + } + + return true +} + +func (r *Replica) sendCommit(view uint, sequenceNo uint, digest string) error { + + log := r.log.With().Uint("view", view).Uint("sequence_number", sequenceNo).Str("digest", digest).Logger() + + log.Info().Msg("broadcasting commit message") + + commit := Commit{ + View: view, + SequenceNumber: sequenceNo, + Digest: digest, + } + + err := r.sign(&commit) + if err != nil { + return fmt.Errorf("could not sign commit message: %w", err) + } + + err = r.broadcast(commit) + if err != nil { + return fmt.Errorf("could not broadcast commit message: %w", err) + } + + log.Info().Msg("commit message successfully broadcast") + + // Record this commit message. + r.recordCommitReceipt(r.id, commit) + + return nil +} + +func (r *Replica) processCommit(replica peer.ID, commit Commit) error { + + log := r.log.With().Str("replica", replica.String()).Uint("view", commit.View).Uint("sequence_no", commit.SequenceNumber).Str("digest", commit.Digest).Logger() + + log.Info().Msg("received commit message") + + if commit.View != r.view { + return fmt.Errorf("commit has an invalid view value (received: %v, current: %v)", commit.View, r.view) + } + + err := r.verifySignature(&commit, replica) + if err != nil { + return fmt.Errorf("could not validate commit signature: %w", err) + } + + r.recordCommitReceipt(replica, commit) + + if !r.committed(commit.View, commit.SequenceNumber, commit.Digest) { + log.Info().Msg("request is not yet committed") + return nil + } + + err = r.execute(commit.View, commit.SequenceNumber, commit.Digest) + if err != nil { + return fmt.Errorf("request execution failed: %w", err) + + } + + return nil +} + +func (r *Replica) recordCommitReceipt(replica peer.ID, commit Commit) { + + msgID := getMessageID(commit.View, commit.SequenceNumber) + commits, ok := r.commits[msgID] + if !ok { + r.commits[msgID] = newCommitReceipts() + commits = r.commits[msgID] + } + + commits.Lock() + defer commits.Unlock() + + // Have we already seen this commit? + _, exists := commits.m[replica] + if exists { + r.log.Warn().Uint("view", commit.View).Uint("sequence", commit.SequenceNumber).Str("digest", commit.Digest).Msg("ignoring duplicate commit") + return + } + + commits.m[replica] = commit +} diff --git a/consensus/pbft/conditions.go b/consensus/pbft/conditions.go new file mode 100644 index 00000000..5c0243da --- /dev/null +++ b/consensus/pbft/conditions.go @@ -0,0 +1,93 @@ +package pbft + +func (r *Replica) prePrepared(view uint, sequenceNo uint, digest string) bool { + + // NOTE: Digest can be empty (NullRequest). + + // Have we seen this request before? + _, seen := r.requests[digest] + if !seen { + return false + } + + // Do we have a pre-prepare for this request? + preprepare, seen := r.preprepares[getMessageID(view, sequenceNo)] + if !seen { + return false + } + + if preprepare.Digest != digest { + return false + } + + return true +} + +func (r *Replica) prepared(view uint, sequenceNo uint, digest string) bool { + + // Check if we have seen this request before. + // NOTE: This is also checked as part of the pre-prepare check. + _, seen := r.requests[digest] + if !seen { + return false + } + + // Is the pre-prepare condition met for this request? + if !r.prePrepared(view, sequenceNo, digest) { + return false + } + + prepares, ok := r.prepares[getMessageID(view, sequenceNo)] + if !ok { + return false + } + + prepareCount := uint(len(prepares.m)) + haveQuorum := prepareCount >= r.prepareQuorum() + + r.log.Debug().Str("digest", digest).Uint("view", view).Uint("sequence_no", sequenceNo). + Uint("quorum", prepareCount).Bool("have_quorum", haveQuorum). + Msg("number of prepares for a request") + + return haveQuorum +} + +func (r *Replica) committed(view uint, sequenceNo uint, digest string) bool { + + // Is the prepare condition met for this request? + if !r.prepared(view, sequenceNo, digest) { + return false + } + + commits, ok := r.commits[getMessageID(view, sequenceNo)] + if !ok { + return false + } + + commitCount := uint(len(commits.m)) + haveQuorum := commitCount >= r.commitQuorum() + + r.log.Debug().Str("digest", digest).Uint("view", view).Uint("sequence_no", sequenceNo). + Uint("quorum", commitCount).Bool("have_quorum", haveQuorum). + Msg("number of commits for a request") + + return haveQuorum +} + +func (r *Replica) viewChangeReady(view uint) bool { + + vc, ok := r.viewChanges[view] + if !ok { + return false + } + + vc.Lock() + defer vc.Unlock() + + vcCount := uint(len(vc.m)) + haveQuorum := vcCount >= r.commitQuorum() + + r.log.Debug().Uint("view", view).Uint("quorum", vcCount).Bool("have_quorum", haveQuorum).Msg("number of view change messages for a view") + + return haveQuorum +} diff --git a/consensus/pbft/config.go b/consensus/pbft/config.go new file mode 100644 index 00000000..9dcbfeb6 --- /dev/null +++ b/consensus/pbft/config.go @@ -0,0 +1,49 @@ +package pbft + +import ( + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/models/execute" +) + +// Option can be used to set PBFT configuration options. +type Option func(*Config) + +// PostProcessFunc is invoked by the replica after execution is done. +type PostProcessFunc func(requestID string, origin peer.ID, request execute.Request, result execute.Result) + +var DefaultConfig = Config{ + NetworkTimeout: NetworkTimeout, + RequestTimeout: RequestTimeout, +} + +type Config struct { + PostProcessors []PostProcessFunc // Callback functions to be invoked after execution is done. + NetworkTimeout time.Duration + RequestTimeout time.Duration +} + +// WithNetworkTimeout sets how much time we allow for message sending. +func WithNetworkTimeout(d time.Duration) Option { + return func(cfg *Config) { + cfg.NetworkTimeout = d + } +} + +// WithRequestTimeout sets the inactivity period before we trigger a view change. +func WithRequestTimeout(d time.Duration) Option { + return func(cfg *Config) { + cfg.RequestTimeout = d + } +} + +// WithPostProcessors sets the callbacks that will be invoked after execution. +func WithPostProcessors(callbacks ...PostProcessFunc) Option { + return func(cfg *Config) { + var fns []PostProcessFunc + fns = append(fns, callbacks...) + cfg.PostProcessors = fns + } +} diff --git a/consensus/pbft/core.go b/consensus/pbft/core.go new file mode 100644 index 00000000..df946e1a --- /dev/null +++ b/consensus/pbft/core.go @@ -0,0 +1,73 @@ +package pbft + +type pbftCore struct { + // Number of replicas in the cluster. + n uint + + // Number of byzantine replicas we can tolerate. + f uint + + // Sequence number. + sequence uint + + // ViewNumber. + view uint +} + +func newPbftCore(total uint) pbftCore { + + return pbftCore{ + sequence: 0, + view: 0, + n: total, + f: calcByzantineTolerance(total), + } +} + +// given a view number, return the index of the expected primary. +func (c pbftCore) primary(v uint) uint { + return v % c.n +} + +// return the index of the expected primary for the current view. +func (c pbftCore) currentPrimary() uint { + return c.view % c.n +} + +func (c pbftCore) prepareQuorum() uint { + return 2 * c.f +} + +func (c pbftCore) commitQuorum() uint { + return 2*c.f + 1 +} + +// MinClusterResults returns the number of identical results client should expect from the +// cluster before accepting the result as valid. The number is f+1. +func MinClusterResults(n uint) uint { + return calcByzantineTolerance(n) + 1 +} + +// based on the number of replicas, determine how many byzantine replicas we can tolerate. +func calcByzantineTolerance(n uint) uint { + + if n <= 1 { + return 0 + } + + f := (n - 1) / 3 + return f +} + +// messageID is used to identify a specific point in time as view + sequence number combination. +type messageID struct { + view uint + sequence uint +} + +func getMessageID(view uint, sequenceNo uint) messageID { + return messageID{ + view: view, + sequence: sequenceNo, + } +} diff --git a/consensus/pbft/digest.go b/consensus/pbft/digest.go new file mode 100644 index 00000000..7193cfd8 --- /dev/null +++ b/consensus/pbft/digest.go @@ -0,0 +1,14 @@ +package pbft + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +func getDigest(rec any) string { + payload, _ := json.Marshal(rec) + hash := sha256.Sum256(payload) + + return hex.EncodeToString(hash[:]) +} diff --git a/consensus/pbft/execute.go b/consensus/pbft/execute.go new file mode 100644 index 00000000..f01b759c --- /dev/null +++ b/consensus/pbft/execute.go @@ -0,0 +1,119 @@ +package pbft + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/models/blockless" + "github.com/blocklessnetwork/b7s/models/codes" + "github.com/blocklessnetwork/b7s/models/execute" + "github.com/blocklessnetwork/b7s/models/response" +) + +// Execute fullfils the consensus interface by inserting the request into the pipeline. +func (r *Replica) Execute(client peer.ID, requestID string, timestamp time.Time, req execute.Request) (codes.Code, execute.Result, error) { + + request := Request{ + ID: requestID, + Timestamp: timestamp, + Origin: client, + Execute: req, + } + + err := r.processRequest(client, request) + if err != nil { + return codes.Error, execute.Result{}, fmt.Errorf("could not process request: %w", err) + } + + // Nothing to return at this point. + return codes.NoContent, execute.Result{}, nil +} + +// execute executes the request AND sends the result back to origin. +func (r *Replica) execute(view uint, sequence uint, digest string) error { + + // Sanity check, should not happen. + request, ok := r.requests[digest] + if !ok { + return fmt.Errorf("unknown request (digest: %s)", digest) + } + + log := r.log.With().Uint("view", view).Uint("sequence", sequence).Str("digest", digest).Str("request", request.ID).Logger() + + // We don't want to execute a job multiple times. + _, havePending := r.pending[digest] + if !havePending { + log.Warn().Msg("no pending request with matching info - likely already executed") + return nil + } + + // Requests must be executed in order. + if sequence != r.lastExecuted+1 { + log.Error().Msg("requests with lower sequence number have not been executed") + // TODO (pbft): Start execution of earlier requests? + return nil + } + + // Sanity check - should never happen. + if sequence < r.lastExecuted { + log.Error().Uint("last_executed", r.lastExecuted).Msg("requests executed out of order!") + } + + // Remove this request from the list of outstanding requests. + delete(r.pending, digest) + + log.Info().Msg("executing request") + + res, err := r.executor.ExecuteFunction(request.ID, request.Execute) + if err != nil { + log.Error().Err(err).Msg("execution failed") + } + + // Stop the timer since we completed an execution. + r.stopRequestTimer() + + // If we have more pending requests, start a new timer. + if len(r.pending) > 0 { + r.startRequestTimer(true) + } + + log.Info().Msg("executed request") + + r.lastExecuted = sequence + + msg := response.Execute{ + Type: blockless.MessageExecuteResponse, + Code: res.Code, + RequestID: request.ID, + Results: execute.ResultMap{ + r.id: res, + }, + PBFT: response.PBFTResultInfo{ + View: r.view, + RequestTimestamp: request.Timestamp, + Replica: r.id, + }, + } + + // Save this executions in case it's requested again. + r.executions[request.ID] = msg + + // Invoke specified post processor functions. + for _, proc := range r.cfg.PostProcessors { + proc(request.ID, request.Origin, request.Execute, res) + } + + err = msg.Sign(r.host.PrivateKey()) + if err != nil { + return fmt.Errorf("could not sign execution request: %w", err) + } + + err = r.send(request.Origin, msg, blockless.ProtocolID) + if err != nil { + return fmt.Errorf("could not send execution response to node (target: %s, request: %s): %w", request.Origin.String(), request.ID, err) + } + + return nil +} diff --git a/consensus/pbft/message.go b/consensus/pbft/message.go new file mode 100644 index 00000000..ae68fb26 --- /dev/null +++ b/consensus/pbft/message.go @@ -0,0 +1,104 @@ +package pbft + +import ( + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/models/execute" +) + +// JSON encoding related code is in serialization.go +// Signature related code is in message_signature.go + +type MessageType uint + +const ( + MessageRequest MessageType = iota + 1 + MessagePrePrepare + MessagePrepare + MessageCommit + MessageViewChange + MessageNewView +) + +func (m MessageType) String() string { + switch m { + case MessagePrePrepare: + return "MessagePrePrepare" + case MessagePrepare: + return "MessagePrepare" + case MessageCommit: + return "MessageCommit" + case MessageViewChange: + return "MessageViewChange" + case MessageNewView: + return "MessageNewView" + default: + return fmt.Sprintf("unknown: %d", m) + } +} + +type Request struct { + ID string `json:"id"` + Timestamp time.Time `json:"timestamp"` + Origin peer.ID `json:"origin"` + Execute execute.Request `json:"execute"` +} + +type PrePrepare struct { + View uint `json:"view"` + SequenceNumber uint `json:"sequence_number"` + Digest string `json:"digest"` + Request Request `json:"request"` + + // Signed digest of the pre-prepare message. + Signature string `json:"signature,omitempty"` +} + +type Prepare struct { + View uint `json:"view"` + SequenceNumber uint `json:"sequence_number"` + Digest string `json:"digest"` + + // Signed digest of the prepare message. + Signature string `json:"signature,omitempty"` +} + +type Commit struct { + View uint `json:"view"` + SequenceNumber uint `json:"sequence_number"` + Digest string `json:"digest"` + + // Signed digest of the commit message. + Signature string `json:"signature,omitempty"` +} + +type ViewChange struct { + View uint `json:"view"` + Prepares []PrepareInfo `json:"prepares"` + + // Signed digest of the view change message. + Signature string `json:"signature,omitempty"` + + // Technically, view change message also includes: + // - n - sequence number of the last stable checkpoint => not needed here since we don't support checkpoints + // - C - 2f+1 checkpoint messages proving the correctness of s => see above +} + +type PrepareInfo struct { + View uint `json:"view"` + SequenceNumber uint `json:"sequence_number"` + Digest string `json:"digest"` + PrePrepare PrePrepare `json:"preprepare"` + Prepares map[peer.ID]Prepare `json:"prepares"` +} +type NewView struct { + View uint `json:"view"` + Messages map[peer.ID]ViewChange `json:"messages"` + PrePrepares []PrePrepare `json:"preprepares"` + + // Signed digest of the new view message. + Signature string `json:"signature,omitempty"` +} diff --git a/consensus/pbft/message_signature.go b/consensus/pbft/message_signature.go new file mode 100644 index 00000000..ca0480ac --- /dev/null +++ b/consensus/pbft/message_signature.go @@ -0,0 +1,82 @@ +package pbft + +type signable interface { + signableRecord() any + setSignature(string) + getSignature() string +} + +// Returns the payload that is eligible to be signed. This means basically the PrePrepare struct, excluding the signature field. +func (p *PrePrepare) signableRecord() any { + cp := *p + cp.setSignature("") + return cp +} + +func (p *PrePrepare) setSignature(signature string) { + p.Signature = signature +} + +func (p PrePrepare) getSignature() string { + return p.Signature +} + +// Returns the payload that is eligible to be signed. This means basically the Prepare struct, excluding the signature field. +func (p *Prepare) signableRecord() any { + cp := *p + cp.setSignature("") + return cp +} + +func (p *Prepare) setSignature(signature string) { + p.Signature = signature +} + +func (p Prepare) getSignature() string { + return p.Signature +} + +// Returns the payload that is eligible to be signed. This means basically the Commit struct, excluding the signature field. +func (c *Commit) signableRecord() any { + cp := *c + cp.setSignature("") + return cp +} + +func (c *Commit) setSignature(signature string) { + c.Signature = signature +} + +func (c Commit) getSignature() string { + return c.Signature +} + +// Returns the payload that is eligible to be signed. This means basically the ViewChange struct, excluding the signature field. +func (v *ViewChange) signableRecord() any { + cp := *v + cp.setSignature("") + return cp +} + +func (v *ViewChange) setSignature(signature string) { + v.Signature = signature +} + +func (v ViewChange) getSignature() string { + return v.Signature +} + +// Returns the payload that is eligible to be signed. This means basically the NewView struct, excluding the signature field. +func (v *NewView) signableRecord() any { + cp := *v + cp.setSignature("") + return cp +} + +func (v *NewView) setSignature(signature string) { + v.Signature = signature +} + +func (v NewView) getSignature() string { + return v.Signature +} diff --git a/consensus/pbft/messaging.go b/consensus/pbft/messaging.go new file mode 100644 index 00000000..31350bfd --- /dev/null +++ b/consensus/pbft/messaging.go @@ -0,0 +1,95 @@ +package pbft + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/hashicorp/go-multierror" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" +) + +func (r *Replica) send(to peer.ID, msg interface{}, protocol protocol.ID) error { + + // Serialize the message. + payload, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("could not encode record: %w", err) + } + + // We don't want to wait indefinitely. + ctx, cancel := context.WithTimeout(context.Background(), r.cfg.NetworkTimeout) + defer cancel() + + // Send message. + err = r.host.SendMessageOnProtocol(ctx, to, payload, protocol) + if err != nil { + return fmt.Errorf("could not send message: %w", err) + } + + return nil +} + +// broadcast sends message to all peers in the replica set. +func (r *Replica) broadcast(msg interface{}) error { + + // Serialize the message. + payload, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("could not encode record: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), r.cfg.NetworkTimeout) + defer cancel() + + var ( + wg sync.WaitGroup + multierr *multierror.Error + lock sync.Mutex + ) + + for _, target := range r.peers { + // Skip self. + if target == r.id { + continue + } + + wg.Add(1) + + // Send concurrently to everyone. + go func(peer peer.ID) { + defer wg.Done() + + // NOTE: We could potentially retry sending if we fail once. On the other hand, somewhat unlikely they're + // back online split second later. + + err := r.host.SendMessageOnProtocol(ctx, peer, payload, r.protocolID) + if err != nil { + + lock.Lock() + defer lock.Unlock() + + multierr = multierror.Append(multierr, err) + } + }(target) + } + + wg.Wait() + + // If all went well, just return. + sendErr := multierr.ErrorOrNil() + if sendErr == nil { + return nil + } + + // Warn if we had more send errors than we bargained for. + errCount := uint(multierr.Len()) + if errCount > r.f { + r.log.Warn().Uint("f", r.f).Uint("errors", errCount).Msg("broadcast error count higher than pBFT f value") + } + + return fmt.Errorf("could not broadcast message: %w", sendErr) +} diff --git a/consensus/pbft/new_view.go b/consensus/pbft/new_view.go new file mode 100644 index 00000000..e205332c --- /dev/null +++ b/consensus/pbft/new_view.go @@ -0,0 +1,302 @@ +package pbft + +import ( + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +func (r *Replica) startNewView(view uint) error { + + log := r.log.With().Uint("view", view).Logger() + + log.Info().Msg("starting a new view") + + projectedPrimary := r.peers[r.primary(view)] + if projectedPrimary != r.id { + return fmt.Errorf("am not the expected primary for the specified view (view: %v, primary: %v)", view, projectedPrimary.String()) + } + + vcs, ok := r.viewChanges[view] + if !ok { + return fmt.Errorf("no view change messages for the specified view (view: %v)", view) + } + + // If we don't have our own view change message added yet - do it now. + // Don't defer unlock because we invoke viewChangeReady, which locks the same view change slot. + vcs.Lock() + _, ok = vcs.m[r.id] + if !ok { + + vc := ViewChange{ + View: view, + Prepares: r.getPrepareSet(), + } + + vcs.m[r.id] = vc + } + vcs.Unlock() + + // Recheck that we have a valid view change state (quorum). + if !r.viewChangeReady(view) { + return fmt.Errorf("new view sequence started but not enough view change messages present (view: %v)", view) + } + + log.Info().Msg("view change ready, broadcasting new view message") + + preprepares := r.generatePreprepares(view, vcs.m) + + for i := 0; i < len(preprepares); i++ { + err := r.sign(&preprepares[i]) + if err != nil { + return fmt.Errorf("new-view - could not sign preprepare message: %w", err) + } + } + + newView := NewView{ + View: view, + Messages: vcs.m, + PrePrepares: preprepares, + } + + err := r.sign(&newView) + if err != nil { + return fmt.Errorf("could not sign the new view message: %w", err) + } + + err = r.broadcast(newView) + if err != nil { + return fmt.Errorf("could not broadcast new-view message (view: %v): %w", view, err) + } + + log.Info().Interface("new_view", newView).Msg("new view message successfully broadcast") + + r.cleanupState(view) + + // Now, save any information we did not have previously (e.g. new pre-prepares, requests), change the current view for the replica and enter the view (set as active). + for _, preprepare := range preprepares { + + r.preprepares[getMessageID(preprepare.View, preprepare.SequenceNumber)] = preprepare + + _, found := r.requests[preprepare.Digest] + if !found { + r.requests[preprepare.Digest] = preprepare.Request + } + + _, found = r.pending[preprepare.Digest] + if !found { + r.pending[preprepare.Digest] = preprepare.Request + } + } + + r.view = view + r.activeView = true + + log.Info().Msg("new view started") + + // See any pending requests you've seen and add them to the pipeline. + outstandingRequests := r.outstandingRequests() + for _, request := range outstandingRequests { + + _, pending := r.pending[getDigest(request)] + if pending { + r.log.Debug().Str("request", request.ID).Msg("outstanding request but already pending as part of a new-view payload, skipping") + continue + } + + err = r.processRequest(request.Origin, request) + if err != nil { + r.log.Error().Err(err).Str("request", request.ID).Msg("could not process request") + // Log but continue. + } + } + + return nil +} + +func (r *Replica) generatePreprepares(view uint, vcs map[peer.ID]ViewChange) []PrePrepare { + + log := r.log.With().Uint("view", view).Logger() + + // Phase 1. We don't have checkpoints, so our lower sequence number bound is 0. + // Determine the upper higher sequence bound by going through the view change messages + // and examining the prepare certificates. + max := getHighestSequenceNumber(vcs) + + log.Info().Uint("max", max).Msg("generating preprepares for new view, determined max sequence number") + + // Phase 2. Go through all sequence numbers from 1 to max. If there is a prepare certificate + // for a sequence number in the view change messages - create a pre-prepare message for m,v+1,n. + // If there are multiple prepare certificates with different view numbers - use the highest view number. + preprepares := make([]PrePrepare, 0, max) + for sequenceNo := uint(1); sequenceNo <= max; sequenceNo++ { + + log := log.With().Uint("sequence", sequenceNo).Logger() + + prepare, exists := getPrepare(vcs, sequenceNo) + // If we have a prepare certificate for this sequence number, add it. + if exists { + + log.Info().Str("digest", prepare.Digest).Str("request", prepare.PrePrepare.Request.ID).Msg("generating preprepares for new view, found prepare certificate") + + preprepare := PrePrepare{ + View: view, + SequenceNumber: sequenceNo, + Digest: prepare.Digest, + Request: prepare.PrePrepare.Request, + } + + preprepares = append(preprepares, preprepare) + continue + } + + log.Info().Msg("generating preprepares for new view, no prepare certificate found, using a null request") + + // We don't have a prepare certificate for this sequence number - create a preprepare for a null request. + preprepare := PrePrepare{ + View: view, + SequenceNumber: sequenceNo, + Digest: "", + Request: NullRequest, + } + + preprepares = append(preprepares, preprepare) + } + + return preprepares +} + +func getHighestSequenceNumber(vcs map[peer.ID]ViewChange) uint { + + var max uint + + // For each view change message (from a replica). + for _, vc := range vcs { + // Go through all prepares. + for _, prepare := range vc.Prepares { + // Update the max sequence number seen if current one is higher. + if prepare.SequenceNumber > max { + max = prepare.SequenceNumber + } + } + } + + return max +} + +func getPrepare(vcs map[peer.ID]ViewChange, sequenceNo uint) (PrepareInfo, bool) { + + var ( + out PrepareInfo + found bool + ) + + // For each view change message (from a replica). + for _, vc := range vcs { + + // Go through prepares. + for _, prepare := range vc.Prepares { + + // Only observe the prepares with this sequence number. + if prepare.SequenceNumber != sequenceNo { + continue + } + + // In case we have multiple prepares for the same sequence number, + // keep the one from the highest view. + if !found || (prepare.View > out.View) { + out = prepare + // Could also compare with an empty PrepareInfo tbh. + found = true + } + } + } + + return out, found +} + +func (r *Replica) processNewView(replica peer.ID, newView NewView) error { + + log := r.log.With().Str("replica", replica.String()).Uint("new_view", newView.View).Logger() + + log.Info().Interface("new_view", newView).Msg("processing new view message") + + if newView.View < r.view { + log.Warn().Uint("current_view", r.view).Msg("received new view message for a view lower than ours, discarding") + return nil + } + + // Make sure that the replica sending this is the replica that will be the primary for the view in question. + projectedPrimary := r.peers[r.primary(newView.View)] + if projectedPrimary != replica { + return fmt.Errorf("sender of the new-view message is not the projected primary for the view (sender: %v, projected: %v)", + replica.String(), + projectedPrimary.String()) + } + + err := r.verifySignature(&newView, projectedPrimary) + if err != nil { + return fmt.Errorf("could not verify signature of the new view message: %w", err) + } + + // Verify number of messages included. + count := uint(len(newView.Messages)) + haveQuorum := count >= r.commitQuorum() + + if !haveQuorum { + return fmt.Errorf("new-view message does not have a quorum of view-change messages (sender: %v, count: %v)", replica.String(), count) + } + + for replica, vc := range newView.Messages { + + err := r.validViewChange(vc) + if err != nil { + return fmt.Errorf("new-view - included view change message is invalid: %w", err) + } + + err = r.verifySignature(&vc, replica) + if err != nil { + return fmt.Errorf("new-view - included view change message has an invalid signature: %w", err) + } + } + + for i, preprepare := range newView.PrePrepares { + if preprepare.View != newView.View { + return fmt.Errorf("new view preprepare message for a wrong view (preprepare_view: %v, new_view: %v)", preprepare.View, newView.View) + } + + // Verify sequence numbers are all there, though offset by one. + if uint(i+1) != preprepare.SequenceNumber { + log.Warn().Interface("preprepares", newView.PrePrepares).Msg("preprepares have unexpected sequence number value (possible gap)") + return fmt.Errorf("unexpected sequence number list") + } + } + + // Update our local view, switch to active view. + r.view = newView.View + r.activeView = true + + log.Info().Msg("processed new view message") + + r.log.Info().Str("primary", r.primaryReplicaID().String()).Uint("view", r.view).Msg("entered new view") + + r.cleanupState(newView.View) + + // Start processing preprepares. + for _, preprepare := range newView.PrePrepares { + err := r.processPrePrepare(replica, preprepare) + if err != nil { + log.Error().Err(err).Uint("view", preprepare.View).Uint("sequence", preprepare.SequenceNumber).Msg("error processing preprepare message") + // Continue despite errors. + } + } + + log.Info().Msg("processed preprepares from the new view") + + if len(r.outstandingRequests()) > 0 { + r.log.Info().Msg("outstanding requests found, starting a new view change timer") + r.startRequestTimer(false) + } + + return nil +} diff --git a/consensus/pbft/outstanding.go b/consensus/pbft/outstanding.go new file mode 100644 index 00000000..93165993 --- /dev/null +++ b/consensus/pbft/outstanding.go @@ -0,0 +1,30 @@ +package pbft + +// outstandingRequests returns the list of requests that have been seen by the replica, but are not already in the pipeline. +// This is called on view change to check if replicas should re-start their view change timers to keep the new primary honest. +// New primary should do the same thing to see if it has seen some requests that the previous replica has not made progress on and, +// if there are any, make actions related to these requests (by issuing preprepares). +func (r *Replica) outstandingRequests() []Request { + + r.log.Debug().Msg("checking if there are any requests not yet in the pipeline") + + var requests []Request + + for digest, request := range r.requests { + + log := r.log.With().Str("digest", digest).Str("request", request.ID).Logger() + + _, executed := r.executions[request.ID] + if executed { + log.Debug().Msg("request already executed, skipping") + continue + } + + log.Info().Msg("request not yet in the pipeline nor executed") + + // This means there's a request we've seen that hasn't been executed and not in the pipeline. + requests = append(requests, request) + } + + return requests +} diff --git a/consensus/pbft/params.go b/consensus/pbft/params.go new file mode 100644 index 00000000..cefe2720 --- /dev/null +++ b/consensus/pbft/params.go @@ -0,0 +1,35 @@ +package pbft + +import ( + "errors" + "time" + + "github.com/libp2p/go-libp2p/core/protocol" +) + +const ( + // Protocol to use for PBFT related communication. + Protocol protocol.ID = "/b7s/consensus/pbft/1.0.0" + + // PBFT offers no resiliency towards Byzantine nodes with less than four nodes. + MinimumReplicaCount = 4 + + // How long do the send/broadcast operation have until we consider it failed. + NetworkTimeout = 5 * time.Second + + // How long is the inactivity period before we trigger a view change. + RequestTimeout = 10 * time.Second + + EnvVarByzantine = "B7S_PBFT_BYZANTINE" +) + +var ( + ErrViewChange = errors.New("view change in progress") + ErrActiveView = errors.New("replica is currently in an active view") + ErrConflictingPreprepare = errors.New("conflicting pre-prepare") + ErrInvalidSignature = errors.New("invalid signature") +) + +var ( + NullRequest = Request{} +) diff --git a/consensus/pbft/pbft.go b/consensus/pbft/pbft.go new file mode 100644 index 00000000..5aa66e4c --- /dev/null +++ b/consensus/pbft/pbft.go @@ -0,0 +1,273 @@ +package pbft + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + + "github.com/blocklessnetwork/b7s/consensus" + "github.com/blocklessnetwork/b7s/host" + "github.com/blocklessnetwork/b7s/models/blockless" +) + +// TODO (pbft): Request timestamp - execution exactly once, prevent multiple/out of order executions. + +// Replica is a single PBFT node. Both Primary and Backup nodes are all replicas. +type Replica struct { + // PBFT related data. + pbftCore + replicaState + + cfg Config + + // Track inactivity period to trigger a view change. + requestTimer *time.Timer + + // Components. + log zerolog.Logger + host *host.Host + executor blockless.Executor + + // Cluster identity. + id peer.ID + peers []peer.ID + clusterID string + protocolID protocol.ID + + // TODO (pbft): This is used for testing ATM, remove later. + byzantine bool +} + +// NewReplica creates a new PBFT replica. +func NewReplica(log zerolog.Logger, host *host.Host, executor blockless.Executor, peers []peer.ID, clusterID string, options ...Option) (*Replica, error) { + + total := uint(len(peers)) + + if total < MinimumReplicaCount { + return nil, fmt.Errorf("too small cluster for a valid PBFT (have: %v, minimum: %v)", total, MinimumReplicaCount) + } + + cfg := DefaultConfig + for _, option := range options { + option(&cfg) + } + + replica := Replica{ + pbftCore: newPbftCore(total), + replicaState: newState(), + + cfg: cfg, + + log: log.With().Str("component", "pbft").Str("cluster", clusterID).Logger(), + host: host, + executor: executor, + clusterID: clusterID, + protocolID: protocol.ID(fmt.Sprintf("%s/cluster/%s", Protocol, clusterID)), + + id: host.ID(), + peers: peers, + + byzantine: isByzantine(), + } + + replica.log.Info().Strs("replicas", peerIDList(peers)).Uint("n", total).Uint("f", replica.f).Bool("byzantine", replica.byzantine).Msg("created PBFT replica") + + // Set the message handlers. + + // Handling messages on the PBFT protocol. + replica.setPBFTMessageHandler() + + return &replica, nil +} + +func (r *Replica) Consensus() consensus.Type { + return consensus.PBFT +} + +func (r *Replica) Shutdown() error { + r.host.RemoveStreamHandler(r.protocolID) + r.stopRequestTimer() + return nil +} + +func (r *Replica) setPBFTMessageHandler() { + + // We want to only accept messages from replicas in our cluster. + // Create a map so we can perform a faster lookup. + pm := make(map[peer.ID]struct{}) + for _, peer := range r.peers { + pm[peer] = struct{}{} + } + + r.host.Host.SetStreamHandler(r.protocolID, func(stream network.Stream) { + defer stream.Close() + + from := stream.Conn().RemotePeer() + + // On this protocol we only allow messages from other replicas in the cluster. + _, known := pm[from] + if !known { + r.log.Info().Str("peer", from.String()).Msg("received message from a peer not in our cluster, discarding") + return + } + + buf := bufio.NewReader(stream) + msg, err := buf.ReadBytes('\n') + if err != nil && !errors.Is(err, io.EOF) { + stream.Reset() + r.log.Error().Err(err).Msg("error receiving direct message") + return + } + + r.log.Debug().Str("peer", from.String()).Msg("received message") + + err = r.processMessage(from, msg) + if err != nil { + r.log.Error().Err(err).Str("peer", from.String()).Msg("message processing failed") + } + }) +} + +func (r *Replica) processMessage(from peer.ID, payload []byte) error { + + // If we're acting as a byzantine replica, just don't do anything. + // At this point we're not trying any elaborate sus behavior. + if r.byzantine { + return errors.New("we're a byzantine replica, ignoring received message") + } + + msg, err := unpackMessage(payload) + if err != nil { + return fmt.Errorf("could not unpack message: %w", err) + } + + // Access to individual segments (pre-prepares, prepares, commits etc) could be managed on an individual level, + // but it's probably not worth it. This way we just do it request by request. + // NOTE: Perhaps lock as early as possible or force serialization. For some things we want to force in-order processing of messages, + // e.g. `new-view` first, THEN any `preprepares` for that view. + r.sl.Lock() + defer r.sl.Unlock() + + err = r.isMessageAllowed(msg) + if err != nil { + return fmt.Errorf("message not allowed (message: %T): %w", msg, err) + } + + switch m := msg.(type) { + + case Request: + return r.processRequest(from, m) + + case PrePrepare: + return r.processPrePrepare(from, m) + + case Prepare: + return r.processPrepare(from, m) + + case Commit: + return r.processCommit(from, m) + + case ViewChange: + return r.processViewChange(from, m) + + case NewView: + return r.processNewView(from, m) + } + + return fmt.Errorf("unexpected message type (from: %s): %T", from, msg) +} + +func (r *Replica) primaryReplicaID() peer.ID { + return r.peers[r.currentPrimary()] +} + +func (r *Replica) isPrimary() bool { + return r.id == r.primaryReplicaID() +} + +// helper function to to convert a slice of multiaddrs to strings. +func peerIDList(ids []peer.ID) []string { + peerIDs := make([]string, 0, len(ids)) + for _, rp := range ids { + peerIDs = append(peerIDs, rp.String()) + } + return peerIDs +} + +func (r *Replica) isMessageAllowed(msg interface{}) error { + + // If we're in an active view, we accept all but new-view messages. + if r.activeView { + + switch msg.(type) { + case NewView: + return ErrActiveView + default: + return nil + } + } + + // We are in a view change. Only accept view-change and new-view messages. + // PBFT also supports checkpoint messages, but we don't use those. + switch msg.(type) { + case ViewChange, NewView: + return nil + default: + return ErrViewChange + } +} + +// cleanupState will discard old preprepares, prepares, commist and pending requests. +// Call this before updating the list of pending requests since for those we don't know +// in which view they were scheduled - we remove all of them. +func (r *Replica) cleanupState(thresholdView uint) { + + r.log.Debug().Uint("threshold_view", thresholdView).Msg("cleaning up replica state") + + // Cleanup pending requests. + for id := range r.pending { + delete(r.pending, id) + } + + // Cleanup old preprepares. + for id := range r.preprepares { + if id.view < thresholdView { + delete(r.preprepares, id) + } + } + + // Cleanup old prepares. + for id := range r.prepares { + if id.view < thresholdView { + delete(r.prepares, id) + } + } + + // Cleanup old commits. + for id := range r.commits { + if id.view < thresholdView { + delete(r.commits, id) + } + } +} + +func isByzantine() bool { + env := strings.ToLower(os.Getenv(EnvVarByzantine)) + + switch env { + case "y", "yes", "true", "1": + return true + default: + return false + } +} diff --git a/consensus/pbft/pbft_test.go b/consensus/pbft/pbft_test.go new file mode 100644 index 00000000..b6118ce5 --- /dev/null +++ b/consensus/pbft/pbft_test.go @@ -0,0 +1,33 @@ +package pbft + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blocklessnetwork/b7s/host" + "github.com/blocklessnetwork/b7s/testing/mocks" +) + +const ( + loopback = "127.0.0.1" +) + +func newDummyReplica(t *testing.T) *Replica { + t.Helper() + + var ( + logger = mocks.NoopLogger + executor = mocks.BaselineExecutor(t) + clusterID = mocks.GenericUUID.String() + peers = mocks.GenericPeerIDs[:4] + ) + + host, err := host.New(logger, loopback, 0) + require.NoError(t, err) + + replica, err := NewReplica(logger, host, executor, peers, clusterID) + require.NoError(t, err) + + return replica +} diff --git a/consensus/pbft/prepare.go b/consensus/pbft/prepare.go new file mode 100644 index 00000000..3f01bb51 --- /dev/null +++ b/consensus/pbft/prepare.go @@ -0,0 +1,87 @@ +package pbft + +import ( + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// Send a prepare message. Naturally this is only sent by the non-primary replicas, +// as a response to a pre-prepare message. +func (r *Replica) sendPrepare(preprepare PrePrepare) error { + + msg := Prepare{ + View: preprepare.View, + SequenceNumber: preprepare.SequenceNumber, + Digest: preprepare.Digest, + } + + log := r.log.With().Str("digest", msg.Digest).Uint("view", msg.View).Uint("sequence_number", msg.SequenceNumber).Logger() + + log.Info().Msg("broadcasting prepare message") + + err := r.sign(&msg) + if err != nil { + return fmt.Errorf("could not sign prepare message: %w", err) + } + + err = r.broadcast(msg) + if err != nil { + return fmt.Errorf("could not broadcast prepare message: %w", err) + } + + log.Info().Msg("prepare message successfully broadcast") + + // Record this prepare message. + r.recordPrepareReceipt(r.id, msg) + + return nil +} + +func (r *Replica) recordPrepareReceipt(replica peer.ID, prepare Prepare) { + + msgID := getMessageID(prepare.View, prepare.SequenceNumber) + prepares, ok := r.prepares[msgID] + if !ok { + r.prepares[msgID] = newPrepareReceipts() + prepares = r.prepares[msgID] + } + + prepares.Lock() + defer prepares.Unlock() + + _, exists := prepares.m[replica] + if exists { + r.log.Warn().Uint("view", prepare.View).Uint("sequence", prepare.SequenceNumber).Str("digest", prepare.Digest).Str("replica", replica.String()).Msg("ignoring duplicate prepare message") + return + } + + prepares.m[replica] = prepare +} + +func (r *Replica) processPrepare(replica peer.ID, prepare Prepare) error { + + log := r.log.With().Str("replica", replica.String()).Uint("view", prepare.View).Uint("sequence_no", prepare.SequenceNumber).Str("digest", prepare.Digest).Logger() + + log.Info().Msg("received prepare message") + + if replica == r.primaryReplicaID() { + log.Warn().Msg("received prepare message from primary, ignoring") + return nil + } + + if prepare.View != r.view { + return fmt.Errorf("prepare has an invalid view value (received: %v, current: %v)", prepare.View, r.view) + } + + err := r.verifySignature(&prepare, replica) + if err != nil { + return fmt.Errorf("could not verify signature for the prepare message: %w", err) + } + + r.recordPrepareReceipt(replica, prepare) + + log.Info().Msg("processed prepare message") + + return r.maybeSendCommit(prepare.View, prepare.SequenceNumber, prepare.Digest) +} diff --git a/consensus/pbft/preprepare.go b/consensus/pbft/preprepare.go new file mode 100644 index 00000000..488a7023 --- /dev/null +++ b/consensus/pbft/preprepare.go @@ -0,0 +1,126 @@ +package pbft + +import ( + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +func (r *Replica) sendPrePrepare(req Request) error { + + // Only primary replica can send pre-prepares. + if !r.isPrimary() { + return nil + } + + r.sequence++ + sequence := r.sequence + + msg := PrePrepare{ + View: r.view, + SequenceNumber: sequence, + Request: req, + Digest: getDigest(req), + } + + log := r.log.With().Uint("view", msg.View).Uint("sequence_number", msg.SequenceNumber).Str("digest", msg.Digest).Logger() + + if r.conflictingPrePrepare(msg) { + return fmt.Errorf("dropping pre-prepare as we have a conflicting one") + } + + err := r.sign(&msg) + if err != nil { + return fmt.Errorf("could not sign pre-prepare message: %w", err) + } + + log.Info().Msg("broadcasting pre-prepare message") + + err = r.broadcast(msg) + if err != nil { + return fmt.Errorf("could not broadcast pre-prepare message: %w", err) + } + + log.Info().Msg("pre-prepare message successfully broadcast") + + // Take a note of this pre-prepare. This will naturally only happen on the primary replica. + r.preprepares[getMessageID(msg.View, msg.SequenceNumber)] = msg + + return nil +} + +// Process a pre-prepare message. This should only happen on backup replicas. +func (r *Replica) processPrePrepare(replica peer.ID, msg PrePrepare) error { + + if r.isPrimary() { + r.log.Warn().Msg("primary replica received a pre-prepare, dropping") + return nil + } + + log := r.log.With().Str("replica", replica.String()).Uint("view", msg.View).Uint("sequence_no", msg.SequenceNumber).Str("digest", msg.Digest).Logger() + + log.Info().Msg("received pre-prepare message") + + if replica != r.primaryReplicaID() { + log.Error().Str("primary", r.primaryReplicaID().String()).Msg("pre-prepare came from a replica that is not the primary, dropping") + return nil + } + + if msg.View != r.view { + return fmt.Errorf("pre-prepare for an invalid view (received: %v, current: %v)", msg.View, r.view) + } + + err := r.verifySignature(&msg, r.primaryReplicaID()) + if err != nil { + return fmt.Errorf("pre-prepare message signature not valid: %w", err) + } + + id := getMessageID(msg.View, msg.SequenceNumber) + + existing, ok := r.preprepares[id] + if ok { + log.Error().Str("existing_digest", existing.Digest).Msg("pre-prepare message already exists for this view and sequence number, dropping") + return ErrConflictingPreprepare + } + + // We don't have this pre-prepare. Save it now. + r.preprepares[id] = msg + + // TODO (pbft): See if this is the same request we saw. If it isn't consider triggering a view change right here and now. + // Save this request. + r.requests[msg.Digest] = msg.Request + r.pending[msg.Digest] = msg.Request + + r.startRequestTimer(false) + + // Just a sanity check at this point, since we've set up the state just now. + if !r.prePrepared(msg.View, msg.SequenceNumber, msg.Digest) { + log.Warn().Msg("request is not pre-prepared, stopping") + return nil + } + + log.Info().Msg("processed pre-prepare") + + // Broadcast prepare message. + err = r.sendPrepare(msg) + if err != nil { + return fmt.Errorf("could not send prepare message: %w", err) + } + + // There's a possibility our prepare was the one that pushes us into the quorum + // and we now have the commit condition achieved. + return r.maybeSendCommit(msg.View, msg.SequenceNumber, msg.Digest) +} + +func (r *Replica) conflictingPrePrepare(preprepare PrePrepare) bool { + + for _, pp := range r.preprepares { + + // If we have a pre-prepare with the same view and same digest but different sequence number - invalid. + if pp.View == preprepare.View && pp.Digest == preprepare.Digest && pp.SequenceNumber != preprepare.SequenceNumber { + return true + } + } + + return false +} diff --git a/consensus/pbft/receipts.go b/consensus/pbft/receipts.go new file mode 100644 index 00000000..c9eb1688 --- /dev/null +++ b/consensus/pbft/receipts.go @@ -0,0 +1,53 @@ +package pbft + +import ( + "sync" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// prepareReceipts maps a peer/replica ID to the prepare message it sent. +type prepareReceipts struct { + m map[peer.ID]Prepare + *sync.Mutex +} + +func newPrepareReceipts() *prepareReceipts { + + pr := prepareReceipts{ + m: make(map[peer.ID]Prepare), + Mutex: &sync.Mutex{}, + } + + return &pr +} + +type commitReceipts struct { + m map[peer.ID]Commit + *sync.Mutex +} + +func newCommitReceipts() *commitReceipts { + + cr := commitReceipts{ + m: make(map[peer.ID]Commit), + Mutex: &sync.Mutex{}, + } + + return &cr +} + +type viewChangeReceipts struct { + m map[peer.ID]ViewChange + *sync.Mutex +} + +func newViewChangeReceipts() *viewChangeReceipts { + + vcr := viewChangeReceipts{ + m: make(map[peer.ID]ViewChange), + Mutex: &sync.Mutex{}, + } + + return &vcr +} diff --git a/consensus/pbft/request.go b/consensus/pbft/request.go new file mode 100644 index 00000000..56a78d98 --- /dev/null +++ b/consensus/pbft/request.go @@ -0,0 +1,77 @@ +package pbft + +import ( + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/models/blockless" +) + +func (r *Replica) processRequest(from peer.ID, req Request) error { + + pub, err := req.Origin.ExtractPublicKey() + if err != nil { + return fmt.Errorf("could not extract public key from client: %w", err) + } + + err = req.Execute.VerifySignature(pub) + if err != nil { + return fmt.Errorf("request is not properly signed by the client: %w", err) + } + + digest := getDigest(req) + + log := r.log.With().Str("client", from.String()).Str("request", req.ID).Str("digest", digest).Logger() + + log.Info().Msg("received a request") + + // Check if we've executed this before. If yes, just return the result. + result, ok := r.executions[req.ID] + if ok { + log.Info().Msg("request already executed, sending result to client") + + err := r.send(req.Origin, result, blockless.ProtocolID) + if err != nil { + return fmt.Errorf("could not send execution result back to client (request: %s, client: %s): %w", req.ID, req.Origin.String(), err) + } + + return nil + } + + // If we're not the primary, we'll drop the request. We do start a request timer though. + if !r.isPrimary() { + r.startRequestTimer(false) + log.Info().Str("primary", r.primaryReplicaID().String()).Msg("we are not the primary replica, dropping the request") + + // Just to be safe, store the request we've seen. + r.requests[digest] = req + return nil + } + + log.Info().Msg("we are the primary, processing the request") + + _, pending := r.pending[digest] + if pending { + return fmt.Errorf("this request is already queued, dropping (request: %v)", req.ID) + } + + _, seen := r.requests[digest] + if seen { + log.Info().Msg("already seen this request, resubmitted") + } + + // Take a note of this request. + r.requests[digest] = req + r.pending[digest] = req + + // Broadcast a pre-prepare message. + err = r.sendPrePrepare(req) + if err != nil { + return fmt.Errorf("could not broadcast pre-prepare message (request: %v): %w", req.ID, err) + } + + log.Info().Msg("processed request") + + return nil +} diff --git a/consensus/pbft/serialization.go b/consensus/pbft/serialization.go new file mode 100644 index 00000000..7145a915 --- /dev/null +++ b/consensus/pbft/serialization.go @@ -0,0 +1,295 @@ +package pbft + +import ( + "encoding/json" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +// messageRecord is used as an interim format to supplement the original type with its type. +// Useful for serialization to automatically include the message type field. +type messageRecord struct { + Type MessageType `json:"type"` + Data any `json:"data"` +} + +// messageEnvelope is used as an interim format to extract the original type from the `messageRecord` format. +type messageEnvelope struct { + Type MessageType `json:"type"` + Data json.RawMessage `json:"data"` +} + +func (r Request) MarshalJSON() ([]byte, error) { + type alias Request + rec := messageRecord{ + Type: MessageRequest, + Data: alias(r), + } + return json.Marshal(rec) +} + +func (r *Request) UnmarshalJSON(data []byte) error { + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + type alias *Request + return json.Unmarshal(rec.Data, alias(r)) +} + +func (p PrePrepare) MarshalJSON() ([]byte, error) { + type alias PrePrepare + rec := messageRecord{ + Type: MessagePrePrepare, + Data: alias(p), + } + return json.Marshal(rec) +} + +func (p *PrePrepare) UnmarshalJSON(data []byte) error { + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + type alias *PrePrepare + return json.Unmarshal(rec.Data, alias(p)) +} + +func (p Prepare) MarshalJSON() ([]byte, error) { + type alias Prepare + rec := messageRecord{ + Type: MessagePrepare, + Data: alias(p), + } + return json.Marshal(rec) +} + +func (p *Prepare) UnmarshalJSON(data []byte) error { + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + type alias *Prepare + return json.Unmarshal(rec.Data, alias(p)) +} + +func (c Commit) MarshalJSON() ([]byte, error) { + type alias Commit + rec := messageRecord{ + Type: MessageCommit, + Data: alias(c), + } + return json.Marshal(rec) +} + +func (c *Commit) UnmarshalJSON(data []byte) error { + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + type alias *Commit + return json.Unmarshal(rec.Data, alias(c)) +} + +type prepareInfoEncoded struct { + View uint `json:"view"` + SequenceNumber uint `json:"sequence_number"` + Digest string `json:"digest"` + PrePrepare PrePrepare `json:"preprepare"` + Prepares map[string]Prepare `json:"prepares"` +} + +func (p PrepareInfo) MarshalJSON() ([]byte, error) { + + encodedPrepareMap := make(map[string]Prepare) + for id, prepare := range p.Prepares { + encodedPrepareMap[id.String()] = prepare + } + + rec := prepareInfoEncoded{ + View: p.View, + SequenceNumber: p.SequenceNumber, + Digest: p.Digest, + PrePrepare: p.PrePrepare, + Prepares: encodedPrepareMap, + } + + return json.Marshal(rec) +} + +func (p *PrepareInfo) UnmarshalJSON(data []byte) error { + + var info prepareInfoEncoded + err := json.Unmarshal(data, &info) + if err != nil { + return err + } + + prepareMap := make(map[peer.ID]Prepare) + for idStr, prepare := range info.Prepares { + id, err := peer.Decode(idStr) + if err != nil { + return fmt.Errorf("could not decode peer.ID (str: %s): %w", idStr, err) + } + prepareMap[id] = prepare + } + + *p = PrepareInfo{ + View: info.View, + SequenceNumber: info.SequenceNumber, + Digest: info.Digest, + PrePrepare: info.PrePrepare, + Prepares: prepareMap, + } + + return nil +} + +func (v ViewChange) MarshalJSON() ([]byte, error) { + type alias ViewChange + rec := messageRecord{ + Type: MessageViewChange, + Data: alias(v), + } + return json.Marshal(rec) +} + +func (v *ViewChange) UnmarshalJSON(data []byte) error { + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + type alias *ViewChange + return json.Unmarshal(rec.Data, alias(v)) +} + +type newViewEncode struct { + View uint `json:"view"` + Messages map[string]ViewChange `json:"messages"` + PrePrepares []PrePrepare `json:"preprepares"` + Signature string `json:"signature"` +} + +func (v NewView) MarshalJSON() ([]byte, error) { + + // To properly handle `peer.ID` serialization, this is a bit more involved. + // See documentation for `ResultMap.MarshalJSON` in `models/execute/response.go`. + messages := make(map[string]ViewChange) + for replica, vc := range v.Messages { + messages[replica.String()] = vc + } + + nv := newViewEncode{ + View: v.View, + Messages: messages, + PrePrepares: v.PrePrepares, + Signature: v.Signature, + } + + rec := messageRecord{ + Type: MessageNewView, + Data: nv, + } + + return json.Marshal(rec) +} + +func (n *NewView) UnmarshalJSON(data []byte) error { + + var rec messageEnvelope + err := json.Unmarshal(data, &rec) + if err != nil { + return err + } + + var nv newViewEncode + err = json.Unmarshal(rec.Data, &nv) + if err != nil { + return err + } + + messages := make(map[peer.ID]ViewChange) + for idStr, vc := range nv.Messages { + id, err := peer.Decode(idStr) + if err != nil { + return fmt.Errorf("could not decode peer.ID (str: %s): %w", idStr, err) + } + messages[id] = vc + } + + *n = NewView{ + View: nv.View, + Messages: messages, + PrePrepares: nv.PrePrepares, + Signature: nv.Signature, + } + + return nil +} + +func unpackMessage(payload []byte) (any, error) { + + var msg messageEnvelope + err := json.Unmarshal(payload, &msg) + if err != nil { + return nil, fmt.Errorf("could not unpack base message: %w", err) + } + + switch msg.Type { + case MessageRequest: + var request Request + err = json.Unmarshal(payload, &request) + if err != nil { + return nil, fmt.Errorf("could not unpack request: %w", err) + } + return request, nil + + case MessagePrePrepare: + var preprepare PrePrepare + err = json.Unmarshal(payload, &preprepare) + if err != nil { + return nil, fmt.Errorf("could not unpack pre-prepare message: %w", err) + } + return preprepare, nil + + case MessagePrepare: + var prepare Prepare + err = json.Unmarshal(payload, &prepare) + if err != nil { + return nil, fmt.Errorf("could not unpack prepare message: %w", err) + } + return prepare, nil + + case MessageCommit: + var commit Commit + err = json.Unmarshal(payload, &commit) + if err != nil { + return nil, fmt.Errorf("could not unpack commit message: %w", err) + } + return commit, nil + + case MessageViewChange: + var viewChange ViewChange + err = json.Unmarshal(payload, &viewChange) + if err != nil { + return nil, fmt.Errorf("could not unpack view change message: %w", err) + } + return viewChange, nil + + case MessageNewView: + var newView NewView + err = json.Unmarshal(payload, &newView) + if err != nil { + return nil, fmt.Errorf("could not unpack new view message: %w", err) + } + return newView, nil + } + + return nil, fmt.Errorf("unexpected message type (type: %v)", msg.Type) +} diff --git a/consensus/pbft/serialization_test.go b/consensus/pbft/serialization_test.go new file mode 100644 index 00000000..948bfbc0 --- /dev/null +++ b/consensus/pbft/serialization_test.go @@ -0,0 +1,222 @@ +package pbft + +import ( + "encoding/json" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/blocklessnetwork/b7s/testing/mocks" +) + +var ( + genericRequest = Request{ + ID: mocks.GenericUUID.String(), + Timestamp: time.Now().UTC(), + Origin: mocks.GenericPeerID, + Execute: mocks.GenericExecutionRequest, + } + + genericPrepareInfo = PrepareInfo{ + View: 1, + SequenceNumber: 14, + Digest: "abcdef123456789", + PrePrepare: PrePrepare{ + View: 1, + SequenceNumber: 14, + Digest: "abcdef123456789", + Request: Request{ + ID: mocks.GenericUUID.String(), + Timestamp: time.Now().UTC(), + Origin: mocks.GenericPeerID, + Execute: mocks.GenericExecutionRequest, + }, + }, + // These are all different but it's test data so it's fine. + Prepares: map[peer.ID]Prepare{ + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x91}): { + View: 45, + SequenceNumber: 19, + Digest: "abcdef123456789", + }, + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x92}): { + View: 98, + SequenceNumber: 12, + Digest: "987654321", + }, + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x93}): { + View: 100, + SequenceNumber: 91, + Digest: "abc123def456", + }, + }, + } +) + +func TestRequest_Serialization(t *testing.T) { + + orig := genericRequest + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked Request + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + + require.Equal(t, orig, unpacked) +} + +func TestPrePrepare_Serialization(t *testing.T) { + + orig := PrePrepare{ + View: 1, + SequenceNumber: 2, + Digest: "123456789", + Request: genericRequest, + } + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked PrePrepare + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + + require.Equal(t, orig, unpacked) +} + +func TestPrepare_Serialization(t *testing.T) { + + orig := Prepare{ + View: 14, + SequenceNumber: 45, + Digest: "abc123def", + } + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked Prepare + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + require.Equal(t, orig, unpacked) +} + +func TestCommit_Serialization(t *testing.T) { + + orig := Commit{ + View: 23, + SequenceNumber: 51, + Digest: "987xyz", + } + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked Commit + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + require.Equal(t, orig, unpacked) +} + +func TestPrepareInfo_Serialization(t *testing.T) { + + orig := genericPrepareInfo + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked PrepareInfo + err = json.Unmarshal(encoded, &unpacked) + + require.NoError(t, err) + require.Equal(t, orig, unpacked) +} + +func TestViewChange_Serialization(t *testing.T) { + + // Use the generic type as the baseline but change individual fields. + info1 := genericPrepareInfo + info1.View = 973 + + info2 := genericPrepareInfo + info2.SequenceNumber = 175 + + info3 := genericPrepareInfo + info3.PrePrepare.Digest = "xyz" + + orig := ViewChange{ + View: 15, + Prepares: []PrepareInfo{info1, info2, info3}, + } + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked ViewChange + + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + require.Equal(t, orig, unpacked) +} + +func TestNewView_Serialization(t *testing.T) { + + var ( + genericPeerID1 = peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x91}) + genericPeerID2 = peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x92}) + genericPeerID3 = peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x56, 0x77, 0x86, 0x82, 0x76, 0xa, 0xc5, 0x9, 0x63, 0xde, 0xe4, 0x31, 0xfc, 0x44, 0x75, 0xdd, 0x5a, 0x27, 0xee, 0x6b, 0x94, 0x13, 0xed, 0xe2, 0xa3, 0x6d, 0x8a, 0x1d, 0x57, 0xb6, 0xb8, 0x93}) + + orig = NewView{ + View: 654, + PrePrepares: []PrePrepare{ + { + View: 12, + SequenceNumber: 32, + Request: genericRequest, + }, + { + View: 32, + SequenceNumber: 45, + Request: genericRequest, + }, + { + View: 78, + SequenceNumber: 32, + Request: genericRequest, + }, + }, + Messages: map[peer.ID]ViewChange{ + genericPeerID1: { + View: 13, + Prepares: []PrepareInfo{ + genericPrepareInfo, + }, + }, + genericPeerID2: { + View: 13, + Prepares: []PrepareInfo{ + genericPrepareInfo, + }, + }, + genericPeerID3: { + View: 13, + Prepares: []PrepareInfo{ + genericPrepareInfo, + }, + }, + }, + } + ) + + encoded, err := json.Marshal(orig) + require.NoError(t, err) + + var unpacked NewView + err = json.Unmarshal(encoded, &unpacked) + require.NoError(t, err) + require.Equal(t, orig, unpacked) +} diff --git a/consensus/pbft/signature.go b/consensus/pbft/signature.go new file mode 100644 index 00000000..a0db994f --- /dev/null +++ b/consensus/pbft/signature.go @@ -0,0 +1,48 @@ +package pbft + +import ( + "encoding/hex" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +func (r *Replica) sign(rec signable) error { + + digest := getDigest(rec.signableRecord()) + sig, err := r.host.PrivateKey().Sign([]byte(digest)) + if err != nil { + return fmt.Errorf("could not sign preprepare: %w", err) + } + + rec.setSignature(hex.EncodeToString(sig)) + + return nil +} + +func (r *Replica) verifySignature(rec signable, signer peer.ID) error { + + // Get the digest of the message, excluding the signature. + digest := getDigest(rec.signableRecord()) + + pub, err := signer.ExtractPublicKey() + if err != nil { + return fmt.Errorf("could not extract public key from peer ID (id: %s): %w", signer.String(), err) + } + + sig, err := hex.DecodeString(rec.getSignature()) + if err != nil { + return fmt.Errorf("could not decode signature from hex: %w", err) + } + + ok, err := pub.Verify([]byte(digest), sig) + if err != nil { + return fmt.Errorf("could not verify signature: %w", err) + } + + if !ok { + return ErrInvalidSignature + } + + return nil +} diff --git a/consensus/pbft/signature_test.go b/consensus/pbft/signature_test.go new file mode 100644 index 00000000..8decfc3f --- /dev/null +++ b/consensus/pbft/signature_test.go @@ -0,0 +1,169 @@ +package pbft + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/blocklessnetwork/b7s/testing/mocks" +) + +func TestSign_PrePrepare(t *testing.T) { + + var ( + samplePrePrepare = PrePrepare{ + View: 14, + SequenceNumber: 45, + Digest: "abcdef123456789", + Request: Request{ + ID: mocks.GenericUUID.String(), + Timestamp: time.Now().UTC(), + Origin: mocks.GenericPeerID, + Execute: mocks.GenericExecutionRequest, + }, + } + ) + + t.Run("nominal case", func(t *testing.T) { + + prePrepare := samplePrePrepare + require.Empty(t, prePrepare.Signature) + + replica := newDummyReplica(t) + err := replica.sign(&prePrepare) + require.NoError(t, err) + require.NotEmpty(t, prePrepare.Signature) + + verifier := newDummyReplica(t) + + err = verifier.verifySignature(&prePrepare, replica.host.ID()) + require.NoError(t, err) + }) + t.Run("catch tampering", func(t *testing.T) { + + prePrepare := samplePrePrepare + + replica := newDummyReplica(t) + err := replica.sign(&prePrepare) + require.NoError(t, err) + require.NotEmpty(t, prePrepare.Signature) + + verifier := newDummyReplica(t) + + prePrepare.View++ + + err = verifier.verifySignature(&prePrepare, replica.host.ID()) + require.Error(t, err) + }) + t.Run("validating empty signature fails", func(t *testing.T) { + + prePrepare := samplePrePrepare + replica := newDummyReplica(t) + + err := replica.verifySignature(&prePrepare, replica.host.ID()) + require.Error(t, err) + }) +} + +func TestSign_Prepare(t *testing.T) { + + var ( + samplePrepare = Prepare{ + View: 78, + SequenceNumber: 15, + Digest: "987654321abcdef", + } + ) + + t.Run("nominal case", func(t *testing.T) { + + prepare := samplePrepare + require.Empty(t, prepare.Signature) + + replica := newDummyReplica(t) + err := replica.sign(&prepare) + require.NoError(t, err) + require.NotEmpty(t, prepare.Signature) + + verifier := newDummyReplica(t) + + err = verifier.verifySignature(&prepare, replica.host.ID()) + require.NoError(t, err) + }) + t.Run("catch tampering", func(t *testing.T) { + + prepare := samplePrepare + + replica := newDummyReplica(t) + err := replica.sign(&prepare) + require.NoError(t, err) + require.NotEmpty(t, prepare.Signature) + + verifier := newDummyReplica(t) + + prepare.Digest += " " + + err = verifier.verifySignature(&prepare, replica.host.ID()) + require.Error(t, err) + }) + t.Run("validating empty signature fails", func(t *testing.T) { + + prepare := samplePrepare + replica := newDummyReplica(t) + + err := replica.verifySignature(&prepare, replica.host.ID()) + require.Error(t, err) + }) +} + +func TestSign_Commit(t *testing.T) { + + var ( + sampleCommit = Commit{ + View: 32, + SequenceNumber: 41, + Digest: "123456789pqrs", + } + ) + + t.Run("nominal case", func(t *testing.T) { + + commit := sampleCommit + require.Empty(t, commit.Signature) + + replica := newDummyReplica(t) + err := replica.sign(&commit) + require.NoError(t, err) + require.NotEmpty(t, commit.Signature) + + verifier := newDummyReplica(t) + + err = verifier.verifySignature(&commit, replica.host.ID()) + require.NoError(t, err) + }) + t.Run("catch tampering", func(t *testing.T) { + + commit := sampleCommit + + replica := newDummyReplica(t) + err := replica.sign(&commit) + require.NoError(t, err) + require.NotEmpty(t, commit.Signature) + + verifier := newDummyReplica(t) + + commit.SequenceNumber = 0 + + err = verifier.verifySignature(&commit, replica.host.ID()) + require.Error(t, err) + }) + t.Run("validating empty signature fails", func(t *testing.T) { + + commit := sampleCommit + replica := newDummyReplica(t) + + err := replica.verifySignature(&commit, replica.host.ID()) + require.Error(t, err) + }) +} diff --git a/consensus/pbft/state.go b/consensus/pbft/state.go new file mode 100644 index 00000000..91ec56fc --- /dev/null +++ b/consensus/pbft/state.go @@ -0,0 +1,55 @@ +package pbft + +import ( + "sync" + + "github.com/blocklessnetwork/b7s/models/response" +) + +type replicaState struct { + // State lock. This is a global lock for all state maps (pre-prepares, prepares, commits etc). + // Even though access to those could be managed on a more granular level, at the moment I'm not + // sure it's worth it. + sl *sync.Mutex + + // False if view change is in progress. + activeView bool + + // Sequence number of last execution. + lastExecuted uint + + // Keep track of seen requests. Map request to the digest. + requests map[string]Request + // Keep track of requests queued for execution. Could also be tracked via a single map. + pending map[string]Request + + // Keep track of seen pre-prepare messages. + preprepares map[messageID]PrePrepare + // Keep track of seen prepare messages. + prepares map[messageID]*prepareReceipts + // Keep track of seen commit messages. + commits map[messageID]*commitReceipts + // Keep track of view change messages. + viewChanges map[uint]*viewChangeReceipts + + // Keep track of past executions. Results are mapped to request IDs, not digests. + executions map[string]response.Execute +} + +func newState() replicaState { + + state := replicaState{ + sl: &sync.Mutex{}, + + activeView: true, + requests: make(map[string]Request), + pending: make(map[string]Request), + preprepares: make(map[messageID]PrePrepare), + prepares: make(map[messageID]*prepareReceipts), + commits: make(map[messageID]*commitReceipts), + viewChanges: make(map[uint]*viewChangeReceipts), + executions: make(map[string]response.Execute), + } + + return state +} diff --git a/consensus/pbft/timer.go b/consensus/pbft/timer.go new file mode 100644 index 00000000..7ab515b0 --- /dev/null +++ b/consensus/pbft/timer.go @@ -0,0 +1,50 @@ +package pbft + +import ( + "time" +) + +func (r *Replica) startRequestTimer(overrideExisting bool) { + + r.log.Debug().Msg("starting view change timer") + + if r.requestTimer != nil && !overrideExisting { + r.log.Debug().Msg("view change timer running, not overriding") + return + } + + if r.requestTimer != nil { + r.requestTimer.Stop() + } + + // Evaluate the view number now. Potentially, we could've already advanced + // to the next view before our inactivity timer fires. + targetView := r.view + 1 + + r.requestTimer = time.AfterFunc(r.cfg.RequestTimeout, func() { + r.sl.Lock() + defer r.sl.Unlock() + + err := r.startViewChange(targetView) + if err != nil { + r.log.Error().Err(err).Msg("could not start view change") + } + }) + + r.log.Debug().Msg("view change timer started") +} + +func (r *Replica) stopRequestTimer() { + + r.log.Debug().Msg("stopping view change timer") + + if r.requestTimer == nil { + r.log.Debug().Msg("no active view change timer") + return + } + + r.requestTimer.Stop() + r.requestTimer = nil + + r.log.Debug().Msg("view change timer stopped") +} diff --git a/consensus/pbft/view_change.go b/consensus/pbft/view_change.go new file mode 100644 index 00000000..c3078ae2 --- /dev/null +++ b/consensus/pbft/view_change.go @@ -0,0 +1,267 @@ +package pbft + +import ( + "errors" + "fmt" + + "github.com/libp2p/go-libp2p/core/peer" +) + +func (r *Replica) startViewChange(view uint) error { + + if view <= r.view { + r.log.Debug().Uint("view", view).Uint("current_view", r.view).Msg("ignoring view change start for an old view") + return nil + } + + r.log.Info().Uint("current_view", r.view).Msg("starting view change") + + r.stopRequestTimer() + + r.view = view + r.activeView = false + + vc := ViewChange{ + View: r.view, + Prepares: r.getPrepareSet(), + } + + err := r.sign(&vc) + if err != nil { + return fmt.Errorf("could not sign view change message: %w", err) + } + + err = r.broadcast(vc) + if err != nil { + return fmt.Errorf("could not broadcast view change: %w", err) + } + + r.recordViewChangeReceipt(r.id, vc) + + r.log.Info().Uint("pending_view", r.view).Msg("view change successfully broadcast") + + return nil +} + +func (r *Replica) processViewChange(replica peer.ID, msg ViewChange) error { + + log := r.log.With().Str("replica", replica.String()).Uint("received_view", msg.View).Logger() + log.Info().Msg("processing view change message") + + if msg.View < r.view { + log.Warn().Uint("current_view", r.view).Msg("received view change for an old view") + return nil + } + + err := r.verifySignature(&msg, replica) + if err != nil { + return fmt.Errorf("could not verify signature for the view change message: %w", err) + } + + // Check if the view change message is valid. + err = r.validViewChange(msg) + if err != nil { + return fmt.Errorf("view change message is not valid (replica: %s): %w", replica.String(), err) + } + + r.recordViewChangeReceipt(replica, msg) + + log.Info().Msg("processed view change message") + + // View change for the current view, but we've already transitioned to it. + if msg.View == r.view && r.activeView { + log.Info().Msg("received view change for this view, but we're already transitioned to it") + return nil + } + + nextView, should := r.shouldSendViewChange() + if should { + + log.Info().Uint("next_view", nextView).Msg("we have received enough view change messages, joining view change") + + err = r.startViewChange(nextView) + if err != nil { + log.Error().Err(err).Uint("next_view", nextView).Msg("unable to send view change") + } + } + + projectedPrimary := r.peers[r.primary(msg.View)] + log.Info().Str("primary", projectedPrimary.String()).Msg("expected primary for the view") + + // If `I` am not the expected primary for this view - I've done all I should. + if projectedPrimary != r.id { + log.Info().Msg("I am not the expected primary for this view - done") + return nil + } + + // I am the primary for the view in question. + if !r.viewChangeReady(msg.View) { + log.Info().Msg("I am the expected primary for the view, but not enough view change messages yet") + return nil + } + + log.Info().Msg("I am the expected primary for the new view, have enough view change messages") + + return r.startNewView(msg.View) +} + +func (r *Replica) recordViewChangeReceipt(replica peer.ID, vc ViewChange) { + + vcs, ok := r.viewChanges[vc.View] + if !ok { + r.viewChanges[vc.View] = newViewChangeReceipts() + vcs = r.viewChanges[vc.View] + } + + vcs.Lock() + defer vcs.Unlock() + + _, exists := vcs.m[replica] + if exists { + r.log.Warn().Uint("view", vc.View).Str("replica", replica.String()).Msg("ignoring duplicate view change message") + return + } + + vcs.m[replica] = vc +} + +// Required for a view change, getPrepareSet returns the set of all requests prepared on this replica. +// It includes a valid pre-prepare message and 2f matching, valid prepare messages signed by other backups - same view, sequence number and digest. +func (r *Replica) getPrepareSet() []PrepareInfo { + + r.log.Info().Msg("determining prepare set") + + var out []PrepareInfo + + for msgID, prepare := range r.prepares { + + log := r.log.With().Uint("view", msgID.view).Uint("sequence", msgID.sequence).Logger() + + for digest := range r.requests { + + log = log.With().Str("digest", digest).Logger() + log.Info().Msg("checking if request is suitable for prepare set") + + if !r.prepared(msgID.view, msgID.sequence, digest) { + log.Info().Msg("request not prepared - skipping") + continue + } + + log.Info().Msg("request prepared - including") + + prepareInfo := PrepareInfo{ + View: msgID.view, + SequenceNumber: msgID.sequence, + Digest: digest, + PrePrepare: r.preprepares[msgID], + Prepares: prepare.m, + } + + out = append(out, prepareInfo) + } + } + + r.log.Debug().Interface("prepare_set", out).Msg("prepare set for the replica") + + return out +} + +func (r *Replica) validViewChange(vc ViewChange) error { + + if vc.View == 0 { + return errors.New("invalid view number") + } + + for _, prepare := range vc.Prepares { + + if prepare.View >= vc.View || prepare.SequenceNumber == 0 { + return fmt.Errorf("view change - prepare has an invalid view/sequence number (view: %v, prepare view: %v, sequence: %v)", vc.View, prepare.View, prepare.SequenceNumber) + } + + if prepare.View != prepare.PrePrepare.View || prepare.SequenceNumber != prepare.PrePrepare.SequenceNumber { + return fmt.Errorf("view change - prepare has an unmatching pre-prepare message (view/sequence number)") + } + + // Verify signature of the pre-prepare message. + err := r.verifySignature(&prepare.PrePrepare, r.peers[r.primary(prepare.View)]) + if err != nil { + return fmt.Errorf("view change - preprepare is not signed by the expected primary for the view: %w", err) + } + + if prepare.Digest == "" { + return fmt.Errorf("view change - prepare has an empty digest") + } + + if prepare.Digest != prepare.PrePrepare.Digest { + return fmt.Errorf("view change - prepare has an unmatching pre-prepare message (digest)") + } + + if uint(len(prepare.Prepares)) < r.prepareQuorum() { + return fmt.Errorf("view change - prepare has an insufficient number of prepare messages (have: %v)", len(prepare.Prepares)) + } + + for replica, pp := range prepare.Prepares { + if pp.View != prepare.View || pp.SequenceNumber != prepare.SequenceNumber || pp.Digest != prepare.Digest { + return fmt.Errorf("view change - included prepare message for wrong request") + } + + err = r.verifySignature(&pp, replica) + if err != nil { + return fmt.Errorf("view change - included prepare message signature invalid: %w", err) + } + } + } + + return nil +} + +// Liveness condition - if we received f+1 valid view change messages from other replicas, +// (greater than our current view), send a view change message for the smallest view in the set. Do so +// even if our timer has not expired. +func (r *Replica) shouldSendViewChange() (uint, bool) { + + // If we're already participating in a view change, we're good. + if !r.activeView { + return 0, false + } + + var newView uint + + // Go through view change messages we have received. + for view, vcs := range r.viewChanges { + // Only consider views higher than our current one. + if view <= r.view { + continue + } + + vcs.Lock() + + // See how many view change messages we received. Don't count our own. + count := 0 + for replica := range vcs.m { + if replica != r.id { + count++ + } + + // NOTE: We already check if the view change is valid on receiving it. + } + + vcs.Unlock() + + // If we have more than f+1 view change messages, consider sending one too. + if uint(count) >= r.f+1 { + + if newView == 0 { // Set new view if it was uninitialized. + newView = view + } else if view < newView { // We have multiple sets of f+1 view change messages. Use the lowest one. + newView = view + } + } + } + + if newView != 0 { + return newView, true + } + + return newView, false +} diff --git a/consensus/raft/bootstrap.go b/consensus/raft/bootstrap.go new file mode 100644 index 00000000..75f75b65 --- /dev/null +++ b/consensus/raft/bootstrap.go @@ -0,0 +1,37 @@ +package raft + +import ( + "errors" + "fmt" + + "github.com/hashicorp/raft" +) + +func (r *Replica) bootstrapCluster() error { + + servers := make([]raft.Server, 0, len(r.peers)) + for _, id := range r.peers { + + s := raft.Server{ + Suffrage: raft.Voter, + ID: raft.ServerID(id.String()), + Address: raft.ServerAddress(id), + } + + servers = append(servers, s) + } + + cfg := raft.Configuration{ + Servers: servers, + } + + // Bootstrapping will only succeed for the first node to start it. + // Other attempts will fail with an error that can be ignored. + ret := r.BootstrapCluster(cfg) + err := ret.Error() + if err != nil && !errors.Is(err, raft.ErrCantBootstrap) { + return fmt.Errorf("could not bootstrap cluster: %w", err) + } + + return nil +} diff --git a/consensus/raft/config.go b/consensus/raft/config.go new file mode 100644 index 00000000..5ecb4274 --- /dev/null +++ b/consensus/raft/config.go @@ -0,0 +1,74 @@ +package raft + +import ( + "path/filepath" + "time" + + "github.com/hashicorp/raft" + "github.com/rs/zerolog" + + "github.com/blocklessnetwork/b7s/log/hclog" +) + +// Option can be used to set Raft configuration options. +type Option func(*Config) + +// DefaultConfig represents the default settings for the raft handler. +var DefaultConfig = Config{ + HeartbeatTimeout: DefaultHeartbeatTimeout, + ElectionTimeout: DefaultElectionTimeout, + LeaderLease: DefaultLeaderLease, +} + +type Config struct { + Callbacks []FSMProcessFunc // Callback functions to be invoked by the FSM after execution is done. + + HeartbeatTimeout time.Duration // How often a consensus cluster leader should ping its followers. + ElectionTimeout time.Duration // How long does a consensus cluster node wait for a leader before it triggers an election. + LeaderLease time.Duration // How long does a leader remain a leader if it cannot contact a quorum of cluster nodes. +} + +// WithHeartbeatTimeout sets the heartbeat timeout for the consensus cluster. +func WithHeartbeatTimeout(d time.Duration) Option { + return func(cfg *Config) { + cfg.HeartbeatTimeout = d + } +} + +// WithElectionTimeout sets the election timeout for the consensus cluster. +func WithElectionTimeout(d time.Duration) Option { + return func(cfg *Config) { + cfg.ElectionTimeout = d + } +} + +// WithLeaderLease sets the leader lease for the consensus cluster leader. +func WithLeaderLease(d time.Duration) Option { + return func(cfg *Config) { + cfg.LeaderLease = d + } +} + +func WithCallbacks(callbacks ...FSMProcessFunc) Option { + return func(cfg *Config) { + var fns []FSMProcessFunc + fns = append(fns, callbacks...) + cfg.Callbacks = fns + } +} + +func getRaftConfig(cfg Config, log zerolog.Logger, nodeID string) raft.Config { + + rcfg := raft.DefaultConfig() + rcfg.LocalID = raft.ServerID(nodeID) + rcfg.Logger = hclog.New(log).Named("raft") + rcfg.HeartbeatTimeout = cfg.HeartbeatTimeout + rcfg.ElectionTimeout = cfg.ElectionTimeout + rcfg.LeaderLeaseTimeout = cfg.LeaderLease + + return *rcfg +} + +func consensusDir(workspace string, requestID string) string { + return filepath.Join(workspace, defaultConsensusDirName, requestID) +} diff --git a/consensus/raft/execute.go b/consensus/raft/execute.go new file mode 100644 index 00000000..d7a54666 --- /dev/null +++ b/consensus/raft/execute.go @@ -0,0 +1,62 @@ +package raft + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/models/codes" + "github.com/blocklessnetwork/b7s/models/execute" +) + +func (r *Replica) Execute(from peer.ID, requestID string, timestamp time.Time, req execute.Request) (codes.Code, execute.Result, error) { + + r.log.Info().Time("timestamp", timestamp).Msg("received an execution request") + + if !r.isLeader() { + _, id := r.LeaderWithID() + + r.log.Info().Str("leader", string(id)).Msg("we are not the cluster leader - dropping the request") + return codes.NoContent, execute.Result{}, nil + } + + r.log.Info().Msg("we are the cluster leader, executing the request") + + fsmReq := FSMLogEntry{ + RequestID: requestID, + Origin: from, + Execute: req, + } + + payload, err := json.Marshal(fsmReq) + if err != nil { + return codes.Error, execute.Result{}, fmt.Errorf("could not serialize request for FSM: %w", err) + } + + // Apply Raft log. + future := r.Apply(payload, defaultApplyTimeout) + err = future.Error() + if err != nil { + return codes.Error, execute.Result{}, fmt.Errorf("could not apply raft log: %w", err) + } + + r.log.Info().Msg("node applied raft log") + + // Get execution result. + response := future.Response() + value, ok := response.(execute.Result) + if !ok { + fsmErr, ok := response.(error) + if ok { + return codes.Error, execute.Result{}, fmt.Errorf("execution encountered an error: %w", fsmErr) + } + + return codes.Error, execute.Result{}, fmt.Errorf("unexpected FSM response format: %T", response) + } + + r.log.Info().Msg("cluster leader executed the request") + + return codes.OK, value, nil +} diff --git a/node/fsm.go b/consensus/raft/fsm.go similarity index 77% rename from node/fsm.go rename to consensus/raft/fsm.go index 8b93d10c..99b3b075 100644 --- a/node/fsm.go +++ b/consensus/raft/fsm.go @@ -1,4 +1,4 @@ -package node +package raft import ( "encoding/json" @@ -9,30 +9,31 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/rs/zerolog" + "github.com/blocklessnetwork/b7s/models/blockless" "github.com/blocklessnetwork/b7s/models/execute" ) -type fsmLogEntry struct { +type FSMLogEntry struct { RequestID string `json:"request_id,omitempty"` Origin peer.ID `json:"origin,omitempty"` Execute execute.Request `json:"execute,omitempty"` } -type fsmProcessFunc func(req fsmLogEntry, res execute.Result) +type FSMProcessFunc func(req FSMLogEntry, res execute.Result) type fsmExecutor struct { log zerolog.Logger - executor Executor - processors []fsmProcessFunc + executor blockless.Executor + processors []FSMProcessFunc } -func newFsmExecutor(log zerolog.Logger, executor Executor, processors ...fsmProcessFunc) *fsmExecutor { +func newFsmExecutor(log zerolog.Logger, executor blockless.Executor, processors ...FSMProcessFunc) *fsmExecutor { - ps := make([]fsmProcessFunc, 0, len(processors)) + ps := make([]FSMProcessFunc, 0, len(processors)) ps = append(ps, processors...) fsm := fsmExecutor{ - log: log.With().Str("component", "fsm").Logger(), + log: log.With().Str("module", "fsm").Logger(), executor: executor, processors: ps, } @@ -47,7 +48,7 @@ func (f fsmExecutor) Apply(log *raft.Log) interface{} { // Unpack the execution request. payload := log.Data - var logEntry fsmLogEntry + var logEntry FSMLogEntry err := json.Unmarshal(payload, &logEntry) if err != nil { return fmt.Errorf("could not unmarshal request: %w", err) diff --git a/consensus/raft/params.go b/consensus/raft/params.go new file mode 100644 index 00000000..083597ae --- /dev/null +++ b/consensus/raft/params.go @@ -0,0 +1,19 @@ +package raft + +import ( + "time" +) + +// Raft and consensus related parameters. +const ( + defaultConsensusDirName = "consensus" + defaultLogStoreName = "logs.dat" + defaultStableStoreName = "stable.dat" + + defaultApplyTimeout = 0 // No timeout. + DefaultHeartbeatTimeout = 300 * time.Millisecond + DefaultElectionTimeout = 300 * time.Millisecond + DefaultLeaderLease = 200 * time.Millisecond + + consensusTransportTimeout = 1 * time.Minute +) diff --git a/consensus/raft/raft.go b/consensus/raft/raft.go new file mode 100644 index 00000000..11dc17eb --- /dev/null +++ b/consensus/raft/raft.go @@ -0,0 +1,195 @@ +package raft + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/raft" + boltdb "github.com/hashicorp/raft-boltdb/v2" + "github.com/rs/zerolog" + + libp2praft "github.com/libp2p/go-libp2p-raft" + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/consensus" + "github.com/blocklessnetwork/b7s/host" + "github.com/blocklessnetwork/b7s/models/blockless" +) + +type Replica struct { + *raft.Raft + logStore *boltdb.BoltStore + stable *boltdb.BoltStore + + cfg Config + log zerolog.Logger + + rootDir string + peers []peer.ID +} + +// New creates a new raft replica, bootstraps the cluster and waits until a first leader is elected. We do this because +// only after the election the cluster is really operational and ready to process requests. +func New(log zerolog.Logger, host *host.Host, workspace string, requestID string, executor blockless.Executor, peers []peer.ID, options ...Option) (*Replica, error) { + + // Step 1: Create a new raft replica. + replica, err := newReplica(log, host, workspace, requestID, executor, peers, options...) + if err != nil { + return nil, fmt.Errorf("could not create raft handler: %w", err) + } + + // Step 2: Register an observer to monitor leadership changes. More precisely, + // wait on the first leader election, so we know when the cluster is operational. + + obsCh := make(chan raft.Observation, 1) + observer := raft.NewObserver(obsCh, false, func(obs *raft.Observation) bool { + _, ok := obs.Data.(raft.LeaderObservation) + return ok + }) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + + // Wait on leadership observation. + obs := <-obsCh + leaderObs, ok := obs.Data.(raft.LeaderObservation) + if !ok { + replica.log.Error().Type("type", obs.Data).Msg("invalid observation type received") + return + } + + // We don't need the observer anymore. + replica.DeregisterObserver(observer) + + replica.log.Info().Str("request", requestID).Str("leader", string(leaderObs.LeaderID)).Msg("observed a leadership event - ready") + }() + + replica.RegisterObserver(observer) + + // Step 3: Bootstrap the cluster. + err = replica.bootstrapCluster() + if err != nil { + return nil, fmt.Errorf("could not bootstrap cluster: %w", err) + } + + wg.Wait() + + return replica, nil +} + +func (r *Replica) Consensus() consensus.Type { + return consensus.Raft +} + +func newReplica(log zerolog.Logger, host *host.Host, workspace string, requestID string, executor blockless.Executor, peers []peer.ID, options ...Option) (*Replica, error) { + + if len(peers) == 0 { + return nil, errors.New("empty peer list") + } + + cfg := DefaultConfig + for _, option := range options { + option(&cfg) + } + + // Determine directory that should be used for consensus for this request. + rootDir := consensusDir(workspace, requestID) + err := os.MkdirAll(rootDir, os.ModePerm) + if err != nil { + return nil, fmt.Errorf("could not create consensus work directory: %w", err) + } + + // Transport layer for raft communication. + transport, err := libp2praft.NewLibp2pTransport(host, consensusTransportTimeout) + if err != nil { + return nil, fmt.Errorf("could not create libp2p transport: %w", err) + } + + // Create log store. + logDB := filepath.Join(rootDir, defaultLogStoreName) + logStore, err := boltdb.NewBoltStore(logDB) + if err != nil { + return nil, fmt.Errorf("could not create log store (path: %s): %w", logDB, err) + } + + // Create stable store. + stableDB := filepath.Join(rootDir, defaultStableStoreName) + stableStore, err := boltdb.NewBoltStore(stableDB) + if err != nil { + return nil, fmt.Errorf("could not create stable store (path: %s): %w", stableDB, err) + } + + // Create snapshot store. We never really expect we'll need snapshots + // since our clusters are short lived, so this should be fine. + snapshot := raft.NewDiscardSnapshotStore() + + fsm := newFsmExecutor(log, executor, cfg.Callbacks...) + + raftCfg := getRaftConfig(cfg, log, host.ID().String()) + + // Tag the logger with the cluster ID (request ID). + raftCfg.Logger = raftCfg.Logger.With("cluster", requestID) + + raftNode, err := raft.NewRaft(&raftCfg, fsm, logStore, stableStore, snapshot, transport) + if err != nil { + return nil, fmt.Errorf("could not create a raft node: %w", err) + } + + rh := Replica{ + Raft: raftNode, + logStore: logStore, + stable: stableStore, + + log: log.With().Str("module", "raft").Str("cluster", requestID).Logger(), + cfg: cfg, + rootDir: rootDir, + peers: peers, + } + + rh.log.Info().Strs("peers", blockless.PeerIDsToStr(peers)).Msg("created new raft handler") + + return &rh, nil +} + +func (r *Replica) Shutdown() error { + + r.log.Info().Msg("shuttting down cluster") + + future := r.Raft.Shutdown() + err := future.Error() + if err != nil { + return fmt.Errorf("could not shutdown raft cluster: %w", err) + } + + // We'll log the actual error but return an "umbrella" one if we fail to close any of the two stores. + var multierr *multierror.Error + + err = r.logStore.Close() + if err != nil { + multierr = multierror.Append(multierr, fmt.Errorf("could not close log store: %w", err)) + } + + err = r.stable.Close() + if err != nil { + multierr = multierror.Append(multierr, fmt.Errorf("could not close stable store: %w", err)) + } + + // Delete residual files. This may fail if we failed to close the databases above. + err = os.RemoveAll(r.rootDir) + if err != nil { + multierr = multierror.Append(multierr, fmt.Errorf("could not delete consensus dir: %w", err)) + } + + return multierr.ErrorOrNil() +} + +func (r *Replica) isLeader() bool { + return r.State() == raft.Leader +} diff --git a/host/host.go b/host/host.go index 153989c5..d395a4e2 100644 --- a/host/host.go +++ b/host/host.go @@ -120,6 +120,16 @@ func New(log zerolog.Logger, address string, port uint, options ...func(*Config) return &host, nil } +// PrivateKey returns the private key of the libp2p host. +func (h *Host) PrivateKey() crypto.PrivKey { + return h.Peerstore().PrivKey(h.ID()) +} + +// PublicKey returns the public key of the libp2p host. +func (h *Host) PublicKey() crypto.PubKey { + return h.Peerstore().PubKey(h.ID()) +} + // Addresses returns the list of p2p addresses of the host. func (h *Host) Addresses() []string { diff --git a/host/send.go b/host/send.go index d2298ed1..5325d360 100644 --- a/host/send.go +++ b/host/send.go @@ -5,14 +5,20 @@ import ( "fmt" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" "github.com/blocklessnetwork/b7s/models/blockless" ) -// SendMessage sends a message directly to the specified peer. +// SendMessage sends a message directly to the specified peer, on the standard blockless protocol. func (h *Host) SendMessage(ctx context.Context, to peer.ID, payload []byte) error { + return h.SendMessageOnProtocol(ctx, to, payload, blockless.ProtocolID) +} + +// SendMessageOnProtocol sends a message directly to the specified peer, using the specified protocol. +func (h *Host) SendMessageOnProtocol(ctx context.Context, to peer.ID, payload []byte, protocol protocol.ID) error { - stream, err := h.Host.NewStream(ctx, to, blockless.ProtocolID) + stream, err := h.Host.NewStream(ctx, to, protocol) if err != nil { return fmt.Errorf("could not create stream: %w", err) } diff --git a/models/blockless/executor.go b/models/blockless/executor.go new file mode 100644 index 00000000..ff9737f3 --- /dev/null +++ b/models/blockless/executor.go @@ -0,0 +1,9 @@ +package blockless + +import ( + "github.com/blocklessnetwork/b7s/models/execute" +) + +type Executor interface { + ExecuteFunction(requestID string, request execute.Request) (execute.Result, error) +} diff --git a/models/blockless/peer.go b/models/blockless/peer.go index b4a1d441..a5346c93 100644 --- a/models/blockless/peer.go +++ b/models/blockless/peer.go @@ -10,3 +10,14 @@ type Peer struct { MultiAddr string `json:"multiaddress,omitempty"` AddrInfo peer.AddrInfo `json:"addrinfo,omitempty"` } + +// PeerIDsToStr will convert a list of peer.IDs to strings. +func PeerIDsToStr(ids []peer.ID) []string { + + out := make([]string, 0, len(ids)) + for _, id := range ids { + out = append(out, id.String()) + } + + return out +} diff --git a/models/blockless/role.go b/models/blockless/role.go index 7d6e3630..3635308d 100644 --- a/models/blockless/role.go +++ b/models/blockless/role.go @@ -1,8 +1,5 @@ package blockless -// TODO: Reconsider the package name - typically I'd use the name of the project - `b7s`. -// Package `blockless` might be too wide. - // NodeRole is a representation of the node's role. type NodeRole uint8 diff --git a/models/execute/request.go b/models/execute/request.go index 251a84e7..5272dcb1 100644 --- a/models/execute/request.go +++ b/models/execute/request.go @@ -6,6 +6,9 @@ type Request struct { Method string `json:"method"` Parameters []Parameter `json:"parameters,omitempty"` Config Config `json:"config"` + + // Optional signature of the request. + Signature string `json:"signature,omitempty"` } // Parameter represents an execution parameter, modeled as a key-value pair. @@ -24,6 +27,8 @@ type Config struct { // NodeCount specifies how many nodes should execute this request. NodeCount int `json:"number_of_nodes,omitempty"` + // Consensus algorithm to use. Raft and PBFT are supported at this moment. + ConsensusAlgorithm string `json:"consensus_algorithm,omitempty"` // Threshold (percentage) defines how many nodes should respond with a result to consider this execution successful. Threshold float64 `json:"threshold,omitempty"` diff --git a/models/execute/request_signature.go b/models/execute/request_signature.go new file mode 100644 index 00000000..4f8f4888 --- /dev/null +++ b/models/execute/request_signature.go @@ -0,0 +1,57 @@ +package execute + +import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + "github.com/libp2p/go-libp2p/core/crypto" +) + +func (e *Request) Sign(key crypto.PrivKey) error { + + cp := *e + e.Signature = "" + + payload, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("could not get byte representation of the record: %w", err) + } + + sig, err := key.Sign(payload) + if err != nil { + return fmt.Errorf("could not sign digest: %w", err) + } + + e.Signature = hex.EncodeToString(sig) + return nil +} + +func (e Request) VerifySignature(key crypto.PubKey) error { + + // Exclude signature and the `from` field from the signature. + cp := e + cp.Signature = "" + + payload, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("could not get byte representation of the record: %w", err) + } + + sig, err := hex.DecodeString(e.Signature) + if err != nil { + return fmt.Errorf("could not decode signature from hex: %w", err) + } + + ok, err := key.Verify(payload, sig) + if err != nil { + return fmt.Errorf("could not verify signature: %w", err) + } + + if !ok { + return errors.New("invalid signature") + } + + return nil +} diff --git a/models/execute/request_signature_test.go b/models/execute/request_signature_test.go new file mode 100644 index 00000000..35ea180c --- /dev/null +++ b/models/execute/request_signature_test.go @@ -0,0 +1,66 @@ +package execute + +import ( + "testing" + + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/stretchr/testify/require" +) + +func TestExecute_Signing(t *testing.T) { + + sampleReq := Request{ + FunctionID: "function-di", + Method: "method-value", + Parameters: []Parameter{ + { + Name: "parameter-name", + Value: "parameter-value", + }, + }, + Config: Config{}, + } + + t.Run("nominal case", func(t *testing.T) { + + req := sampleReq + priv, pub := newKey(t) + + err := req.Sign(priv) + require.NoError(t, err) + + err = req.VerifySignature(pub) + require.NoError(t, err) + }) + t.Run("empty signature verification fails", func(t *testing.T) { + + req := sampleReq + req.Signature = "" + + _, pub := newKey(t) + + err := req.VerifySignature(pub) + require.Error(t, err) + }) + t.Run("tampered data signature verification fails", func(t *testing.T) { + + req := sampleReq + priv, pub := newKey(t) + + err := req.Sign(priv) + require.NoError(t, err) + + req.FunctionID += " " + + err = req.VerifySignature(pub) + require.Error(t, err) + }) +} + +func newKey(t *testing.T) (crypto.PrivKey, crypto.PubKey) { + t.Helper() + priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, 0) + require.NoError(t, err) + + return priv, pub +} diff --git a/models/execute/response.go b/models/execute/response.go index d615d408..3210729c 100644 --- a/models/execute/response.go +++ b/models/execute/response.go @@ -28,7 +28,7 @@ type RuntimeOutput struct { Stdout string `json:"stdout"` Stderr string `json:"stderr"` ExitCode int `json:"exit_code"` - Log string `json:"-"` // TODO: Check do we want to send this over the wire too? + Log string `json:"-"` } // Usage represents the resource usage information for a particular execution. diff --git a/models/request/execute.go b/models/request/execute.go index 58e449b2..b401e14d 100644 --- a/models/request/execute.go +++ b/models/request/execute.go @@ -1,6 +1,8 @@ package request import ( + "time" + "github.com/libp2p/go-libp2p/core/peer" "github.com/blocklessnetwork/b7s/models/execute" @@ -8,14 +10,15 @@ import ( // Execute describes the `MessageExecute` request payload. type Execute struct { - Type string `json:"type,omitempty"` - From peer.ID `json:"from,omitempty"` - Code string `json:"code,omitempty"` - FunctionID string `json:"function_id,omitempty"` - Method string `json:"method,omitempty"` - Parameters []execute.Parameter `json:"parameters,omitempty"` - Config execute.Config `json:"config,omitempty"` + Type string `json:"type,omitempty"` + From peer.ID `json:"from,omitempty"` + Code string `json:"code,omitempty"` + + execute.Request // execute request is embedded. // RequestID may be set initially, if the execution request is relayed via roll-call. RequestID string `json:"request_id,omitempty"` + + // Execution request timestamp is a factor for PBFT. + Timestamp time.Time `json:"timestamp,omitempty"` } diff --git a/models/request/form_cluster.go b/models/request/form_cluster.go index 834ed9e4..60b83f59 100644 --- a/models/request/form_cluster.go +++ b/models/request/form_cluster.go @@ -2,13 +2,16 @@ package request import ( "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/consensus" ) // FormCluster describes the `MessageFormCluster` request payload. // It is sent on clustered execution of a request. type FormCluster struct { - Type string `json:"type,omitempty"` - From peer.ID `json:"from,omitempty"` - RequestID string `json:"request_id,omitempty"` - Peers []peer.ID `json:"peers,omitempty"` + Type string `json:"type,omitempty"` + From peer.ID `json:"from,omitempty"` + RequestID string `json:"request_id,omitempty"` + Peers []peer.ID `json:"peers,omitempty"` + Consensus consensus.Type `json:"consensus,omitempty"` } diff --git a/models/request/roll_call.go b/models/request/roll_call.go index 14247029..c4e006ea 100644 --- a/models/request/roll_call.go +++ b/models/request/roll_call.go @@ -2,14 +2,16 @@ package request import ( "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/consensus" ) // RollCall describes the `MessageRollCall` message payload. type RollCall struct { - From peer.ID `json:"from,omitempty"` - Type string `json:"type,omitempty"` - Origin peer.ID `json:"origin,omitempty"` // Origin is the peer that initiated the roll call. - FunctionID string `json:"function_id,omitempty"` - RequestID string `json:"request_id,omitempty"` - ConsensusNeeded bool `json:"consensus_needed"` + From peer.ID `json:"from,omitempty"` + Type string `json:"type,omitempty"` + Origin peer.ID `json:"origin,omitempty"` // Origin is the peer that initiated the roll call. + FunctionID string `json:"function_id,omitempty"` + RequestID string `json:"request_id,omitempty"` + Consensus consensus.Type `json:"consensus"` } diff --git a/models/response/execute.go b/models/response/execute.go index d6a0734c..3e0898f7 100644 --- a/models/response/execute.go +++ b/models/response/execute.go @@ -1,6 +1,13 @@ package response import ( + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/blocklessnetwork/b7s/models/codes" @@ -16,6 +23,66 @@ type Execute struct { Results execute.ResultMap `json:"results,omitempty"` Cluster execute.Cluster `json:"cluster,omitempty"` + PBFT PBFTResultInfo `json:"pbft,omitempty"` + // Signed digest of the response. + Signature string `json:"signature,omitempty"` + // Used to communicate the reason for failure to the user. Message string `json:"message,omitempty"` } + +type PBFTResultInfo struct { + View uint `json:"view"` + RequestTimestamp time.Time `json:"request_timestamp,omitempty"` + Replica peer.ID `json:"replica,omitempty"` +} + +func (e *Execute) Sign(key crypto.PrivKey) error { + + // Exclude signature and the `from` field from the signature. + cp := *e + cp.Signature = "" + cp.From = "" + + payload, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("could not get byte representation of the record: %w", err) + } + + sig, err := key.Sign(payload) + if err != nil { + return fmt.Errorf("could not sign digest: %w", err) + } + + e.Signature = hex.EncodeToString(sig) + return nil +} + +func (e Execute) VerifySignature(key crypto.PubKey) error { + + // Exclude signature and the `from` field from the signature. + cp := e + cp.Signature = "" + cp.From = "" + + payload, err := json.Marshal(cp) + if err != nil { + return fmt.Errorf("could not get byte representation of the record: %w", err) + } + + sig, err := hex.DecodeString(e.Signature) + if err != nil { + return fmt.Errorf("could not decode signature from hex: %w", err) + } + + ok, err := key.Verify(payload, sig) + if err != nil { + return fmt.Errorf("could not verify signature: %w", err) + } + + if !ok { + return errors.New("invalid signature") + } + + return nil +} diff --git a/models/response/execute_test.go b/models/response/execute_test.go new file mode 100644 index 00000000..13ad3d4a --- /dev/null +++ b/models/response/execute_test.go @@ -0,0 +1,72 @@ +package response + +import ( + "testing" + + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/stretchr/testify/require" + + "github.com/blocklessnetwork/b7s/models/blockless" + "github.com/blocklessnetwork/b7s/models/codes" + "github.com/blocklessnetwork/b7s/models/execute" + "github.com/blocklessnetwork/b7s/testing/mocks" +) + +func TestExecute_Signing(t *testing.T) { + + sampleRes := Execute{ + Type: blockless.MessageExecuteResponse, + RequestID: mocks.GenericUUID.String(), + From: mocks.GenericPeerID, + Code: codes.OK, + Results: execute.ResultMap{ + mocks.GenericPeerID: mocks.GenericExecutionResult, + }, + Cluster: execute.Cluster{ + Peers: mocks.GenericPeerIDs[:4], + }, + } + + t.Run("nominal case", func(t *testing.T) { + + res := sampleRes + priv, pub := newKey(t) + + err := res.Sign(priv) + require.NoError(t, err) + + err = res.VerifySignature(pub) + require.NoError(t, err) + }) + t.Run("empty signature verification fails", func(t *testing.T) { + + res := sampleRes + res.Signature = "" + + _, pub := newKey(t) + + err := res.VerifySignature(pub) + require.Error(t, err) + }) + t.Run("tampered data signature verification fails", func(t *testing.T) { + + res := sampleRes + priv, pub := newKey(t) + + err := res.Sign(priv) + require.NoError(t, err) + + res.RequestID += " " + + err = res.VerifySignature(pub) + require.Error(t, err) + }) +} + +func newKey(t *testing.T) (crypto.PrivKey, crypto.PubKey) { + t.Helper() + priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, 0) + require.NoError(t, err) + + return priv, pub +} diff --git a/models/response/form_cluster.go b/models/response/form_cluster.go index 2223f5fe..53004980 100644 --- a/models/response/form_cluster.go +++ b/models/response/form_cluster.go @@ -3,13 +3,15 @@ package response import ( "github.com/libp2p/go-libp2p/core/peer" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/codes" ) // FormCluster describes the `MessageFormClusteRr` response. type FormCluster struct { - Type string `json:"type,omitempty"` - RequestID string `json:"request_id,omitempty"` - From peer.ID `json:"from,omitempty"` - Code codes.Code `json:"code,omitempty"` + Type string `json:"type,omitempty"` + RequestID string `json:"request_id,omitempty"` + From peer.ID `json:"from,omitempty"` + Code codes.Code `json:"code,omitempty"` + Consensus consensus.Type `json:"consensus,omitempty"` } diff --git a/node/cluster.go b/node/cluster.go index 3a243871..5fa0a609 100644 --- a/node/cluster.go +++ b/node/cluster.go @@ -4,10 +4,12 @@ import ( "context" "encoding/json" "fmt" + "sync" - "github.com/hashicorp/raft" "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog/log" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/blockless" "github.com/blocklessnetwork/b7s/models/codes" "github.com/blocklessnetwork/b7s/models/request" @@ -16,6 +18,12 @@ import ( func (n *Node) processFormCluster(ctx context.Context, from peer.ID, payload []byte) error { + // Should never happen. + if !n.isWorker() { + n.log.Warn().Str("peer", from.String()).Msg("only worker nodes participate in consensus clusters") + return nil + } + // Unpack the request. var req request.FormCluster err := json.Unmarshal(payload, &req) @@ -24,61 +32,17 @@ func (n *Node) processFormCluster(ctx context.Context, from peer.ID, payload []b } req.From = from - n.log.Info().Str("request", req.RequestID).Strs("peers", peerIDList(req.Peers)).Msg("received request to form consensus cluster") + n.log.Info().Str("request", req.RequestID).Strs("peers", blockless.PeerIDsToStr(req.Peers)).Str("consensus", req.Consensus.String()).Msg("received request to form consensus cluster") - raftHandler, err := n.newRaftHandler(req.RequestID) - if err != nil { - return fmt.Errorf("could not create raft node: %w", err) - } + switch req.Consensus { + case consensus.Raft: + return n.createRaftCluster(ctx, from, req) - // Register an observer to monitor leadership changes. More precisely, - // wait on the first leader election, so we know when the cluster is operational. - - obsCh := make(chan raft.Observation, 1) - observer := raft.NewObserver(obsCh, false, func(obs *raft.Observation) bool { - _, ok := obs.Data.(raft.LeaderObservation) - return ok - }) - - go func() { - // Wait on leadership observation. - obs := <-obsCh - leaderObs, ok := obs.Data.(raft.LeaderObservation) - if !ok { - n.log.Error().Type("type", obs.Data).Msg("invalid observation type received") - return - } - - // We don't need the observer anymore. - raftHandler.DeregisterObserver(observer) - - n.log.Info().Str("peer", from.String()).Str("leader", string(leaderObs.LeaderID)).Msg("observed a leadership event - sending response") - - res := response.FormCluster{ - Type: blockless.MessageFormClusterResponse, - RequestID: req.RequestID, - Code: codes.OK, - } - - err = n.send(ctx, from, res) - if err != nil { - n.log.Error().Err(err).Msg("could not send cluster confirmation message") - return - } - }() - - raftHandler.RegisterObserver(observer) - - err = bootstrapCluster(raftHandler, req.Peers) - if err != nil { - return fmt.Errorf("could not bootstrap cluster: %w", err) + case consensus.PBFT: + return n.createPBFTCluster(ctx, from, req) } - n.clusterLock.Lock() - n.clusters[req.RequestID] = raftHandler - n.clusterLock.Unlock() - - return nil + return fmt.Errorf("invalid consensus specified (%v %s)", req.Consensus, req.Consensus.String()) } // processFormClusterResponse will record the cluster formation response. @@ -103,6 +67,12 @@ func (n *Node) processFormClusterResponse(ctx context.Context, from peer.ID, pay // processDisbandCluster will start cluster shutdown command. func (n *Node) processDisbandCluster(ctx context.Context, from peer.ID, payload []byte) error { + // Should never happen. + if !n.isWorker() { + n.log.Warn().Str("peer", from.String()).Msg("only worker nodes participate in consensus clusters") + return nil + } + // Unpack the request. var req request.DisbandCluster err := json.Unmarshal(payload, &req) @@ -111,31 +81,101 @@ func (n *Node) processDisbandCluster(ctx context.Context, from peer.ID, payload } req.From = from - n.log.Info().Str("request", req.RequestID).Msg("received request to disband consensus cluster") + n.log.Info().Str("peer", from.String()).Str("request", req.RequestID).Msg("received request to disband consensus cluster") err = n.leaveCluster(req.RequestID) if err != nil { return fmt.Errorf("could not disband cluster (request: %s): %w", req.RequestID, err) } + n.log.Info().Str("peer", from.String()).Str("request", req.RequestID).Msg("left consensus cluster") + return nil } -func (n *Node) leaveCluster(requestID string) error { +func (n *Node) formCluster(ctx context.Context, requestID string, replicas []peer.ID, consensus consensus.Type) error { - ctx, cancel := context.WithTimeout(context.Background(), raftClusterDisbandTimeout) - defer cancel() + // Create cluster formation request. + reqCluster := request.FormCluster{ + Type: blockless.MessageFormCluster, + RequestID: requestID, + Peers: replicas, + Consensus: consensus, + } - // We know that the request is done executing when we have a result for it. - _, ok := n.executeResponses.WaitFor(ctx, requestID) + // Request execution from peers. + err := n.sendToMany(ctx, replicas, reqCluster) + if err != nil { + return fmt.Errorf("could not send cluster formation request to peers: %w", err) + } - n.log.Info().Bool("executed_work", ok).Str("request", requestID).Msg("waiting for execution done, leaving raft cluster") + // Wait for cluster confirmation messages. + n.log.Debug().Str("request", requestID).Msg("waiting for cluster to be formed") + + // We're willing to wait for a limited amount of time. + clusterCtx, exCancel := context.WithTimeout(ctx, n.cfg.ExecutionTimeout) + defer exCancel() + + // Wait for confirmations for cluster forming. + bootstrapped := make(map[string]struct{}) + var rlock sync.Mutex + var rw sync.WaitGroup + rw.Add(len(replicas)) + + // Wait on peers asynchronously. + for _, rp := range replicas { + rp := rp + + go func() { + defer rw.Done() + key := consensusResponseKey(requestID, rp) + res, ok := n.consensusResponses.WaitFor(clusterCtx, key) + if !ok { + return + } + + n.log.Info().Str("request", requestID).Str("peer", rp.String()).Msg("accounted consensus cluster response from roll called peer") + + fc := res.(response.FormCluster) + if fc.Code != codes.OK { + log.Warn().Str("peer", rp.String()).Msg("peer failed to join consensus cluster") + return + } + + rlock.Lock() + defer rlock.Unlock() + bootstrapped[rp.String()] = struct{}{} + }() + } + + // Wait for results, whatever they may be. + rw.Wait() + + // Err if not all peers joined the cluster successfully. + if len(bootstrapped) != len(replicas) { + return fmt.Errorf("some peers failed to join consensus cluster (have: %d, want: %d)", len(bootstrapped), len(replicas)) + } + + return nil +} - err := n.shutdownCluster(requestID) +func (n *Node) disbandCluster(requestID string, replicas []peer.ID) error { + + msgDisband := request.DisbandCluster{ + Type: blockless.MessageDisbandCluster, + RequestID: requestID, + } + + ctx, cancel := context.WithTimeout(context.Background(), consensusClusterSendTimeout) + defer cancel() + + err := n.sendToMany(ctx, replicas, msgDisband) if err != nil { - return fmt.Errorf("could not leave raft cluster (request: %v): %w", requestID, err) + return fmt.Errorf("could not send cluster disband request (request: %s): %w", requestID, err) } + log.Info().Err(err).Strs("peers", blockless.PeerIDsToStr(replicas)).Msg("sent cluster disband request") + return nil } diff --git a/node/config.go b/node/config.go index 59716501..afb31372 100644 --- a/node/config.go +++ b/node/config.go @@ -2,12 +2,10 @@ package node import ( "errors" - "fmt" "path/filepath" "time" - "github.com/hashicorp/raft" - + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/blockless" ) @@ -16,32 +14,28 @@ type Option func(*Config) // DefaultConfig represents the default settings for the node. var DefaultConfig = Config{ - Role: blockless.WorkerNode, - Topic: DefaultTopic, - HealthInterval: DefaultHealthInterval, - RollCallTimeout: DefaultRollCallTimeout, - Concurrency: DefaultConcurrency, - ExecutionTimeout: DefaultExecutionTimeout, - ClusterFormationTimeout: DefaultClusterFormationTimeout, - ConsensusHeartbeatTimeout: DefaultRaftHeartbeatTimeout, - ConsensusElectionTimeout: DefaultRaftElectionTimeout, - ConsensusLeaderLease: DefaultRaftLeaderLease, + Role: blockless.WorkerNode, + Topic: DefaultTopic, + HealthInterval: DefaultHealthInterval, + RollCallTimeout: DefaultRollCallTimeout, + Concurrency: DefaultConcurrency, + ExecutionTimeout: DefaultExecutionTimeout, + ClusterFormationTimeout: DefaultClusterFormationTimeout, + DefaultConsensus: DefaultConsensusAlgorithm, } // Config represents the Node configuration. type Config struct { - Role blockless.NodeRole // Node role. - Topic string // Topic to subscribe to. - Execute Executor // Executor to use for running functions. - HealthInterval time.Duration // How often should we emit the health ping. - RollCallTimeout time.Duration // How long do we wait for roll call responses. - Concurrency uint // How many requests should the node process in parallel. - ExecutionTimeout time.Duration // How long does the head node wait for worker nodes to send their execution results. - ClusterFormationTimeout time.Duration // How long do we wait for the nodes to form a cluster for an execution. - Workspace string // Directory where we can store files needed for execution. - ConsensusHeartbeatTimeout time.Duration // How often a consensus cluster leader should ping its followers. - ConsensusElectionTimeout time.Duration // How long does a consensus cluster node wait for a leader before it triggers an election. - ConsensusLeaderLease time.Duration // How long does a leader remain a leader if it cannot contact a quorum of cluster nodes. + Role blockless.NodeRole // Node role. + Topic string // Topic to subscribe to. + Execute blockless.Executor // Executor to use for running functions. + HealthInterval time.Duration // How often should we emit the health ping. + RollCallTimeout time.Duration // How long do we wait for roll call responses. + Concurrency uint // How many requests should the node process in parallel. + ExecutionTimeout time.Duration // How long does the head node wait for worker nodes to send their execution results. + ClusterFormationTimeout time.Duration // How long do we wait for the nodes to form a cluster for an execution. + Workspace string // Directory where we can store files needed for execution. + DefaultConsensus consensus.Type // Default consensus algorithm to use. } // Validate checks if the given configuration is correct. @@ -66,13 +60,6 @@ func (n *Node) ValidateConfig() error { if n.cfg.Execute == nil { return errors.New("execution component is required") } - - // Make sure we have a valid consensus configuration. - rcfg := n.getRaftConfig(n.host.ID().String()) - err := raft.ValidateConfig(&rcfg) - if err != nil { - return fmt.Errorf("consensus configuration is not valid: %w", err) - } } // Head node specific validation. @@ -102,7 +89,7 @@ func WithTopic(topic string) Option { } // WithExecutor specifies the executor to be used for running Blockless functions -func WithExecutor(execute Executor) Option { +func WithExecutor(execute blockless.Executor) Option { return func(cfg *Config) { cfg.Execute = execute } @@ -150,24 +137,10 @@ func WithWorkspace(path string) Option { } } -// WithConsensusHeartbeatTimeout sets the heartbeat timeout for the consensus cluster. -func WithConsensusHeartbeatTimeout(d time.Duration) Option { - return func(cfg *Config) { - cfg.ConsensusHeartbeatTimeout = d - } -} - -// WithConsensusElectionTimeout sets the election timeout for the consensus cluster. -func WithConsensusElectionTimeout(d time.Duration) Option { - return func(cfg *Config) { - cfg.ConsensusElectionTimeout = d - } -} - -// WithConsensusLeaderLease sets the leader lease for the consensus cluster leader. -func WithConsensusLeaderLease(d time.Duration) Option { +// WithDefaultConsensus specifies the consensus algorithm to use, if not specified in the request. +func WithDefaultConsensus(c consensus.Type) Option { return func(cfg *Config) { - cfg.ConsensusLeaderLease = d + cfg.DefaultConsensus = c } } diff --git a/node/consensus.go b/node/consensus.go new file mode 100644 index 00000000..eced6f9f --- /dev/null +++ b/node/consensus.go @@ -0,0 +1,164 @@ +package node + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + + "github.com/blocklessnetwork/b7s/consensus" + "github.com/blocklessnetwork/b7s/consensus/pbft" + "github.com/blocklessnetwork/b7s/consensus/raft" + "github.com/blocklessnetwork/b7s/models/blockless" + "github.com/blocklessnetwork/b7s/models/codes" + "github.com/blocklessnetwork/b7s/models/execute" + "github.com/blocklessnetwork/b7s/models/request" + "github.com/blocklessnetwork/b7s/models/response" +) + +// consensusExecutor defines the interface we have for managing clustered execution. +// Execute often does not mean a direct execution but instead just pipelining the request, where execution is done asynchronously. +type consensusExecutor interface { + Consensus() consensus.Type + Execute(from peer.ID, id string, timestamp time.Time, request execute.Request) (codes.Code, execute.Result, error) + Shutdown() error +} + +func (n *Node) createRaftCluster(ctx context.Context, from peer.ID, fc request.FormCluster) error { + + // Add a callback function to cache the execution result + cacheFn := func(req raft.FSMLogEntry, res execute.Result) { + n.executeResponses.Set(req.RequestID, res) + } + + // Add a callback function to send the execution result to origin. + sendFn := func(req raft.FSMLogEntry, res execute.Result) { + + ctx, cancel := context.WithTimeout(context.Background(), consensusClusterSendTimeout) + defer cancel() + + msg := response.Execute{ + Type: blockless.MessageExecuteResponse, + Code: res.Code, + RequestID: req.RequestID, + Results: execute.ResultMap{ + n.host.ID(): res, + }, + } + + err := n.send(ctx, req.Origin, msg) + if err != nil { + n.log.Error().Err(err).Str("peer", req.Origin.String()).Msg("could not send execution result to node") + } + } + + rh, err := raft.New( + n.log, + n.host, + n.cfg.Workspace, + fc.RequestID, + n.executor, + fc.Peers, + raft.WithCallbacks(cacheFn, sendFn), + ) + if err != nil { + return fmt.Errorf("could not create raft node: %w", err) + } + + n.clusterLock.Lock() + n.clusters[fc.RequestID] = rh + n.clusterLock.Unlock() + + res := response.FormCluster{ + Type: blockless.MessageFormClusterResponse, + RequestID: fc.RequestID, + Code: codes.OK, + Consensus: fc.Consensus, + } + + err = n.send(ctx, from, res) + if err != nil { + return fmt.Errorf("could not send cluster confirmation message: %w", err) + } + + return nil +} + +func (n *Node) createPBFTCluster(ctx context.Context, from peer.ID, fc request.FormCluster) error { + + cacheFn := func(requestID string, origin peer.ID, request execute.Request, result execute.Result) { + n.executeResponses.Set(requestID, result) + } + + ph, err := pbft.NewReplica( + n.log, + n.host, + n.executor, + fc.Peers, + fc.RequestID, + pbft.WithPostProcessors(cacheFn), + ) + if err != nil { + return fmt.Errorf("could not create PBFT node: %w", err) + } + + n.clusterLock.Lock() + n.clusters[fc.RequestID] = ph + n.clusterLock.Unlock() + + res := response.FormCluster{ + Type: blockless.MessageFormClusterResponse, + RequestID: fc.RequestID, + Code: codes.OK, + Consensus: fc.Consensus, + } + + err = n.send(ctx, from, res) + if err != nil { + return fmt.Errorf("could not send cluster confirmation message: %w", err) + } + + return nil +} + +func (n *Node) leaveCluster(requestID string) error { + + // Shutdown can take a while so use short locking intervals. + n.clusterLock.RLock() + cluster, ok := n.clusters[requestID] + n.clusterLock.RUnlock() + + if !ok { + return errors.New("no cluster with that ID") + } + + n.log.Info().Str("consensus", cluster.Consensus().String()).Str("request", requestID).Msg("leaving consensus cluster") + + ctx, cancel := context.WithTimeout(context.Background(), consensusClusterDisbandTimeout) + defer cancel() + + // We know that the request is done executing when we have a result for it. + _, ok = n.executeResponses.WaitFor(ctx, requestID) + + log := n.log.With().Str("request", requestID).Logger() + log.Info().Bool("executed_work", ok).Msg("waiting for execution done, leaving cluster") + + err := cluster.Shutdown() + if err != nil { + // Not much we can do at this point. + return fmt.Errorf("could not leave cluster (request: %v): %w", requestID, err) + } + + n.clusterLock.Lock() + delete(n.clusters, requestID) + n.clusterLock.Unlock() + + return nil +} + +// helper function just for the sake of readibility. +func consensusRequired(c consensus.Type) bool { + return c != 0 +} diff --git a/node/execute.go b/node/execute.go index 17957649..21da16ed 100644 --- a/node/execute.go +++ b/node/execute.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/libp2p/go-libp2p/core/peer" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/codes" "github.com/blocklessnetwork/b7s/models/execute" "github.com/blocklessnetwork/b7s/models/response" @@ -26,7 +28,7 @@ func (n *Node) processExecuteResponse(ctx context.Context, from peer.ID, payload var res response.Execute err := json.Unmarshal(payload, &res) if err != nil { - return fmt.Errorf("could not not unpack execute response: %w", err) + return fmt.Errorf("could not unpack execute response: %w", err) } res.From = from @@ -69,11 +71,20 @@ func determineOverallCode(results map[string]execute.Result) codes.Code { return codes.Error } -// helper function to to convert a slice of multiaddrs to strings -func peerIDList(ids []peer.ID) []string { - peerIDs := make([]string, 0, len(ids)) - for _, rp := range ids { - peerIDs = append(peerIDs, rp.String()) +func parseConsensusAlgorithm(value string) (consensus.Type, error) { + + if value == "" { + return 0, nil } - return peerIDs + + lv := strings.ToLower(value) + switch lv { + case "raft": + return consensus.Raft, nil + + case "pbft": + return consensus.PBFT, nil + } + + return 0, fmt.Errorf("unknown consensus value (%s)", value) } diff --git a/node/execute_internal_test.go b/node/execute_internal_test.go index 5995b27a..9ccf215e 100644 --- a/node/execute_internal_test.go +++ b/node/execute_internal_test.go @@ -29,12 +29,14 @@ func TestNode_WorkerExecute(t *testing.T) { ) executionRequest := request.Execute{ - Type: blockless.MessageExecute, - RequestID: requestID, - FunctionID: functionID, - Method: functionMethod, - Parameters: []execute.Parameter{}, - Config: execute.Config{}, + Type: blockless.MessageExecute, + RequestID: requestID, + Request: execute.Request{ + FunctionID: functionID, + Method: functionMethod, + Parameters: []execute.Parameter{}, + Config: execute.Config{}, + }, } payload := serialize(t, executionRequest) @@ -258,11 +260,13 @@ func TestNode_HeadExecute(t *testing.T) { ) executionRequest := request.Execute{ - Type: blockless.MessageExecute, - FunctionID: functionID, - Method: functionMethod, - Parameters: []execute.Parameter{}, - Config: execute.Config{}, + Type: blockless.MessageExecute, + Request: execute.Request{ + FunctionID: functionID, + Method: functionMethod, + Parameters: []execute.Parameter{}, + Config: execute.Config{}, + }, } payload := serialize(t, executionRequest) diff --git a/node/execution_results.go b/node/execution_results.go new file mode 100644 index 00000000..f6d5699d --- /dev/null +++ b/node/execution_results.go @@ -0,0 +1,147 @@ +package node + +import ( + "context" + "fmt" + "sync" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/rs/zerolog/log" + + "github.com/blocklessnetwork/b7s/consensus/pbft" + "github.com/blocklessnetwork/b7s/models/execute" + "github.com/blocklessnetwork/b7s/models/response" +) + +// gatherExecutionResultsPBFT collects execution results from a PBFT cluster. This means f+1 identical results. +func (n *Node) gatherExecutionResultsPBFT(ctx context.Context, requestID string, peers []peer.ID) execute.ResultMap { + + exctx, exCancel := context.WithTimeout(ctx, n.cfg.ExecutionTimeout) + defer exCancel() + + type aggregatedResult struct { + result execute.Result + peers []peer.ID + } + + var ( + count = pbft.MinClusterResults(uint(len(peers))) + lock sync.Mutex + wg sync.WaitGroup + + results = make(map[string]aggregatedResult) + out execute.ResultMap = make(map[peer.ID]execute.Result) + ) + + wg.Add(len(peers)) + + for _, rp := range peers { + go func(sender peer.ID) { + defer wg.Done() + + key := executionResultKey(requestID, sender) + res, ok := n.executeResponses.WaitFor(exctx, key) + if !ok { + return + } + + n.log.Info().Str("peer", sender.String()).Str("request", requestID).Msg("accounted execution response from peer") + + er := res.(response.Execute) + + pub, err := sender.ExtractPublicKey() + if err != nil { + log.Error().Err(err).Msg("could not derive public key from peer ID") + return + } + + err = er.VerifySignature(pub) + if err != nil { + log.Error().Err(err).Msg("could not verify signature of an execution response") + return + } + + exres, ok := er.Results[sender] + if !ok { + return + } + + lock.Lock() + defer lock.Unlock() + + // Equality means same result and same timestamp. + reskey := fmt.Sprintf("%+#v-%s", exres.Result, er.PBFT.RequestTimestamp.String()) + result, ok := results[reskey] + if !ok { + results[reskey] = aggregatedResult{ + result: exres, + peers: []peer.ID{ + sender, + }, + } + return + } + + result.peers = append(result.peers, sender) + if uint(len(result.peers)) >= count { + n.log.Info().Str("request", requestID).Int("peers", len(peers)).Uint("matching_results", count).Msg("have enough maching results") + exCancel() + + for _, peer := range result.peers { + out[peer] = result.result + } + } + }(rp) + } + + wg.Wait() + + return out +} + +// gatherExecutionResults collects execution results from direct executions or raft clusters. +func (n *Node) gatherExecutionResults(ctx context.Context, requestID string, peers []peer.ID) execute.ResultMap { + + // We're willing to wait for a limited amount of time. + exctx, exCancel := context.WithTimeout(ctx, n.cfg.ExecutionTimeout) + defer exCancel() + + var ( + results execute.ResultMap = make(map[peer.ID]execute.Result) + reslock sync.Mutex + wg sync.WaitGroup + ) + + wg.Add(len(peers)) + + // Wait on peers asynchronously. + for _, rp := range peers { + rp := rp + + go func() { + defer wg.Done() + key := executionResultKey(requestID, rp) + res, ok := n.executeResponses.WaitFor(exctx, key) + if !ok { + return + } + + log.Info().Str("peer", rp.String()).Msg("accounted execution response from peer") + + er := res.(response.Execute) + + exres, ok := er.Results[rp] + if !ok { + return + } + + reslock.Lock() + defer reslock.Unlock() + results[rp] = exres + }() + } + + wg.Wait() + + return results +} diff --git a/node/executor.go b/node/executor.go deleted file mode 100644 index 9bcc550d..00000000 --- a/node/executor.go +++ /dev/null @@ -1,9 +0,0 @@ -package node - -import ( - "github.com/blocklessnetwork/b7s/models/execute" -) - -type Executor interface { - ExecuteFunction(requestID string, req execute.Request) (execute.Result, error) -} diff --git a/node/head_execute.go b/node/head_execute.go index 4e1b488b..cfc6f7f2 100644 --- a/node/head_execute.go +++ b/node/head_execute.go @@ -5,10 +5,11 @@ import ( "encoding/json" "errors" "fmt" - "sync" + "time" "github.com/libp2p/go-libp2p/core/peer" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/blockless" "github.com/blocklessnetwork/b7s/models/codes" "github.com/blocklessnetwork/b7s/models/execute" @@ -33,15 +34,13 @@ func (n *Node) headProcessExecute(ctx context.Context, from peer.ID, payload []b log := n.log.With().Str("request", req.RequestID).Str("peer", from.String()).Str("function", req.FunctionID).Logger() - code, results, cluster, err := n.headExecute(ctx, requestID, createExecuteRequest(req)) + code, results, cluster, err := n.headExecute(ctx, requestID, req.Request) if err != nil { log.Error().Err(err).Msg("execution failed") } log.Info().Str("code", code.String()).Msg("execution complete") - // NOTE: Head node no longer caches execution results because it doesn't have one of its own. - // Create the execution response from the execution result. res := response.Execute{ Type: blockless.MessageExecuteResponse, @@ -69,177 +68,77 @@ func (n *Node) headProcessExecute(ctx context.Context, from peer.ID, payload []b // The returned map contains execution results, mapped to the peer IDs of peers who reported them. func (n *Node) headExecute(ctx context.Context, requestID string, req execute.Request) (codes.Code, execute.ResultMap, execute.Cluster, error) { - // TODO: (raft) if no cluster/consensus is required - request direct execution. - quorum := 1 + nodeCount := 1 if req.Config.NodeCount > 1 { - quorum = req.Config.NodeCount + nodeCount = req.Config.NodeCount } // Create a logger with relevant context. - log := n.log.With().Str("request", requestID).Str("function", req.FunctionID).Int("quorum", quorum).Logger() + log := n.log.With().Str("request", requestID).Str("function", req.FunctionID).Int("node_count", nodeCount).Logger() - log.Info().Msg("processing execution request") + consensusAlgo, err := parseConsensusAlgorithm(req.Config.ConsensusAlgorithm) + if err != nil { + log.Error().Str("value", req.Config.ConsensusAlgorithm).Str("default", n.cfg.DefaultConsensus.String()).Err(err).Msg("could not parse consensus algorithm from the user request, using default") + consensusAlgo = n.cfg.DefaultConsensus + } - // Phase 1. - Issue roll call to nodes. + if consensusRequired(consensusAlgo) { + log = log.With().Str("consensus", consensusAlgo.String()).Logger() + } - // Create the queue to record roll call respones. - n.rollCall.create(requestID) - defer n.rollCall.remove(requestID) + log.Info().Msg("processing execution request") - consensusNeeded := quorum > 1 - err := n.issueRollCall(ctx, requestID, req.FunctionID, consensusNeeded) + // Phase 1. - Issue roll call to nodes. + reportingPeers, err := n.executeRollCall(ctx, requestID, req.FunctionID, nodeCount, consensusAlgo) if err != nil { - return codes.Error, nil, execute.Cluster{}, fmt.Errorf("could not issue roll call: %w", err) - } - - log.Info().Msg("roll call published") - - // Limit for how long we wait for responses. - tctx, exCancel := context.WithTimeout(ctx, n.cfg.RollCallTimeout) - defer exCancel() - - // Peers that have reported on roll call. - var reportingPeers []peer.ID -rollCallResponseLoop: - for { - // Wait for responses from nodes who want to work on the request. - select { - // Request timed out. - case <-tctx.Done(): - - log.Warn().Msg("roll call timed out") - return codes.Timeout, nil, execute.Cluster{}, blockless.ErrRollCallTimeout - - case reply := <-n.rollCall.responses(requestID): - - // Check if this is the reply we want - shouldn't really happen. - if reply.FunctionID != req.FunctionID { - log.Info().Str("peer", reply.From.String()).Str("function_got", reply.FunctionID).Msg("skipping inadequate roll call response - wrong function") - continue - } - - // Check if we are connected to this peer. - // Since we receive responses to roll call via direct messages - should not happen. - if !n.haveConnection(reply.From) { - n.log.Info().Str("peer", reply.From.String()).Msg("skipping roll call response from unconnected peer") - continue - } - - log.Info().Str("peer", reply.From.String()).Msg("roll called peer chosen for execution") - - reportingPeers = append(reportingPeers, reply.From) - if len(reportingPeers) >= quorum { - log.Info().Msg("enough peers reported for roll call") - break rollCallResponseLoop - } + code := codes.Error + if errors.Is(err, blockless.ErrRollCallTimeout) { + code = codes.Timeout } - } - log.Info().Strs("peers", peerIDList(reportingPeers)).Msg("requesting cluster formation from peers who reported for roll call") + return code, nil, execute.Cluster{}, fmt.Errorf("could not roll call peers (request: %s): %w", requestID, err) + } cluster := execute.Cluster{ Peers: reportingPeers, } - // Phase 2. - Request cluster formation. + // Phase 2. - Request cluster formation, if we need consensus. + if consensusRequired(consensusAlgo) { - // Create cluster formation request. - reqCluster := request.FormCluster{ - Type: blockless.MessageFormCluster, - RequestID: requestID, - Peers: reportingPeers, - } + log.Info().Strs("peers", blockless.PeerIDsToStr(reportingPeers)).Msg("requesting cluster formation from peers who reported for roll call") - // Request execution from peers. - err = n.sendToMany(ctx, reportingPeers, reqCluster) - if err != nil { - return codes.Error, nil, cluster, fmt.Errorf("could not send cluster formation request to peers (function: %s, request: %s): %w", req.FunctionID, requestID, err) - } - - // Wait for cluster confirmation messages. - log.Debug().Msg("waiting for cluster to be formed") - - // When we're done, send a message to disband the cluster. - // NOTE: We could schedule this on the worker nodes when receiving the execution request. - // One variant I tried is waiting on the execution to be done on the leader (using a timed wait on the execution response) and starting raft shutdown after. - // However, this can happen too fast and the execution request might not have been propagated to all of the nodes in the cluster, but "only" to a majority. - // Doing this here allows for more wiggle room and ~probably~ all nodes will have seen the request so far. - defer func() { - go func() { - - msgDisband := request.DisbandCluster{ - Type: blockless.MessageDisbandCluster, - RequestID: requestID, - } - - ctx, cancel := context.WithTimeout(context.Background(), raftClusterSendTimeout) - defer cancel() - - err = n.sendToMany(ctx, reportingPeers, msgDisband) - if err != nil { - log.Error().Err(err).Strs("peers", peerIDList(reportingPeers)).Msg("could not send cluster disband request") - return - } - - log.Error().Err(err).Strs("peers", peerIDList(reportingPeers)).Msg("sent cluster disband request") - }() - }() - - // We're willing to wait for a limited amount of time. - clusterCtx, exCancel := context.WithTimeout(ctx, n.cfg.ExecutionTimeout) - defer exCancel() - - // Wait for confirmations for cluster forming. - bootstrapped := make(map[string]struct{}) - var rlock sync.Mutex - var rw sync.WaitGroup - rw.Add(len(reportingPeers)) - - // Wait on peers asynchronously. - for _, rp := range reportingPeers { - rp := rp - - go func() { - defer rw.Done() - key := consensusResponseKey(requestID, rp) - res, ok := n.consensusResponses.WaitFor(clusterCtx, key) - if !ok { - return - } - - log.Info().Str("peer", rp.String()).Msg("accounted consensus response from roll called peer") - - fc := res.(response.FormCluster) - if fc.Code != codes.OK { - log.Warn().Str("peer", rp.String()).Msg("peer failed to join consensus cluster") - return - } - - rlock.Lock() - defer rlock.Unlock() - bootstrapped[rp.String()] = struct{}{} - }() - } - - // Wait for results, whatever they may be. - rw.Wait() + err := n.formCluster(ctx, requestID, reportingPeers, consensusAlgo) + if err != nil { + return codes.Error, nil, execute.Cluster{}, fmt.Errorf("could not form cluster (request: %s): %w", requestID, err) + } - // Bail if not all peers joined the cluster successfully. - if len(bootstrapped) != quorum { - return codes.NotAvailable, nil, cluster, fmt.Errorf("some peers failed to join consensus cluster (have: %d, want: %d)", len(bootstrapped), quorum) + // When we're done, send a message to disband the cluster. + // NOTE: We could schedule this on the worker nodes when receiving the execution request. + // One variant I tried is waiting on the execution to be done on the leader (using a timed wait on the execution response) and starting raft shutdown after. + // However, this can happen too fast and the execution request might not have been propagated to all of the nodes in the cluster, but "only" to a majority. + // Doing this here allows for more wiggle room and ~probably~ all nodes will have seen the request so far. + defer n.disbandCluster(requestID, reportingPeers) } // Phase 3. - Request execution. // Send the execution request to peers in the cluster. Non-leaders will drop the request. reqExecute := request.Execute{ - Type: blockless.MessageExecute, - FunctionID: req.FunctionID, - Method: req.Method, - Parameters: req.Parameters, - Config: req.Config, - RequestID: requestID, + Type: blockless.MessageExecute, + Request: req, + RequestID: requestID, + Timestamp: time.Now().UTC(), + } + + // If we're working with PBFT, sign the request. + if consensusAlgo == consensus.PBFT { + err := reqExecute.Request.Sign(n.host.PrivateKey()) + if err != nil { + return codes.Error, nil, cluster, fmt.Errorf("could not sign execution request (function: %s, request: %s): %w", req.FunctionID, requestID, err) + } } + err = n.sendToMany(ctx, reportingPeers, reqExecute) if err != nil { return codes.Error, nil, cluster, fmt.Errorf("could not send execution request to peers (function: %s, request: %s): %w", req.FunctionID, requestID, err) @@ -247,47 +146,23 @@ rollCallResponseLoop: log.Debug().Msg("waiting for execution responses") - // We're willing to wait for a limited amount of time. - exctx, exCancel := context.WithTimeout(ctx, n.cfg.ExecutionTimeout) - defer exCancel() - - var ( - // We're waiting for a single execution result now, as only the cluster leader will return a result. - results execute.ResultMap = make(map[peer.ID]execute.Result) - reslock sync.Mutex - wg sync.WaitGroup - ) - - wg.Add(len(reportingPeers)) - - // Wait on peers asynchronously. - for _, rp := range reportingPeers { - rp := rp + var results execute.ResultMap + if consensusAlgo == consensus.PBFT { + results = n.gatherExecutionResultsPBFT(ctx, requestID, reportingPeers) - go func() { - defer wg.Done() - key := executionResultKey(requestID, rp) - res, ok := n.executeResponses.WaitFor(exctx, key) - if !ok { - return - } + log.Info().Msg("received PBFT execution responses") - log.Info().Str("peer", rp.String()).Msg("accounted execution response from peer") - - er := res.(response.Execute) - - exres, ok := er.Results[rp] - if !ok { - return - } + retcode := codes.OK + // Use the return code from the execution as the return code. + for _, res := range results { + retcode = res.Code + break + } - reslock.Lock() - defer reslock.Unlock() - results[rp] = exres - }() + return retcode, results, cluster, nil } - wg.Wait() + results = n.gatherExecutionResults(ctx, requestID, reportingPeers) log.Info().Int("cluster_size", len(reportingPeers)).Int("responded", len(results)).Msg("received execution responses") @@ -296,7 +171,9 @@ rollCallResponseLoop: threshold := determineThreshold(req) retcode := codes.OK - if respondRatio < threshold { + if respondRatio == 0 { + retcode = codes.NoContent + } else if respondRatio < threshold { log.Warn().Float64("expected", threshold).Float64("have", respondRatio).Msg("threshold condition not met") retcode = codes.PartialContent } diff --git a/node/internal/waitmap/waitmap.go b/node/internal/waitmap/waitmap.go index 847bf529..0bd73a28 100644 --- a/node/internal/waitmap/waitmap.go +++ b/node/internal/waitmap/waitmap.go @@ -5,8 +5,6 @@ import ( "sync" ) -// NOTE: Perhaps enable an option to say how long to wait for? - // WaitMap is a key-value store that enables not only setting and getting // values from a map, but also waiting until value for a key becomes available. // Important: Since this implementation is tied pretty closely to how it will be used, diff --git a/node/models.go b/node/models.go deleted file mode 100644 index cfbe9d18..00000000 --- a/node/models.go +++ /dev/null @@ -1,19 +0,0 @@ -package node - -import ( - "github.com/blocklessnetwork/b7s/models/execute" - "github.com/blocklessnetwork/b7s/models/request" -) - -// convert the incoming message format to an execution request. -func createExecuteRequest(req request.Execute) execute.Request { - - er := execute.Request{ - FunctionID: req.FunctionID, - Method: req.Method, - Parameters: req.Parameters, - Config: req.Config, - } - - return er -} diff --git a/node/node.go b/node/node.go index bd4bd296..369e5453 100644 --- a/node/node.go +++ b/node/node.go @@ -27,7 +27,7 @@ type Node struct { log zerolog.Logger host *host.Host - executor Executor + executor blockless.Executor fstore FStore topic *pubsub.Topic @@ -36,8 +36,8 @@ type Node struct { rollCall *rollCallQueue - // clusters maps request ID to the raft cluster the node belongs to. - clusters map[string]*raftHandler + // clusters maps request ID to the cluster the node belongs to. + clusters map[string]consensusExecutor // clusterLock is used to synchronize access to the `clusters` map. clusterLock sync.RWMutex @@ -67,7 +67,7 @@ func New(log zerolog.Logger, host *host.Host, peerStore PeerStore, fstore FStore sema: make(chan struct{}, cfg.Concurrency), rollCall: newQueue(rollCallQueueBufferSize), - clusters: make(map[string]*raftHandler), + clusters: make(map[string]consensusExecutor), executeResponses: waitmap.New(), consensusResponses: waitmap.New(), } diff --git a/node/params.go b/node/params.go index 3789c026..02d74be1 100644 --- a/node/params.go +++ b/node/params.go @@ -3,16 +3,20 @@ package node import ( "errors" "time" + + "github.com/blocklessnetwork/b7s/consensus" ) const ( DefaultTopic = "blockless/b7s/general" DefaultHealthInterval = 1 * time.Minute DefaultRollCallTimeout = 5 * time.Second - DefaultExecutionTimeout = 10 * time.Second + DefaultExecutionTimeout = 20 * time.Second DefaultClusterFormationTimeout = 10 * time.Second DefaultConcurrency = 10 + DefaultConsensusAlgorithm = consensus.Raft + rollCallQueueBufferSize = 1000 defaultExecutionThreshold = 0.6 @@ -22,20 +26,10 @@ const ( // Raft and consensus related parameters. const ( - defaultConsensusDirName = "consensus" - defaultLogStoreName = "logs.dat" - defaultStableStoreName = "stable.dat" - - raftClusterDisbandTimeout = 5 * time.Minute + // When disbanding a cluster, how long do we wait until a potential execution is done. + consensusClusterDisbandTimeout = 5 * time.Minute // Timeout for the context used for sending disband request to cluster nodes. - raftClusterSendTimeout = 10 * time.Second - - defaultRaftApplyTimeout = 0 // No timeout. - DefaultRaftHeartbeatTimeout = 300 * time.Millisecond - DefaultRaftElectionTimeout = 300 * time.Millisecond - DefaultRaftLeaderLease = 200 * time.Millisecond - - consensusTransportTimeout = 1 * time.Minute + consensusClusterSendTimeout = 10 * time.Second ) var ( diff --git a/node/raft.go b/node/raft.go deleted file mode 100644 index 2fa268ff..00000000 --- a/node/raft.go +++ /dev/null @@ -1,204 +0,0 @@ -package node - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/hashicorp/raft" - boltdb "github.com/hashicorp/raft-boltdb/v2" - - libp2praft "github.com/libp2p/go-libp2p-raft" - "github.com/libp2p/go-libp2p/core/peer" - - "github.com/blocklessnetwork/b7s/log/hclog" - "github.com/blocklessnetwork/b7s/models/blockless" - "github.com/blocklessnetwork/b7s/models/execute" - "github.com/blocklessnetwork/b7s/models/response" -) - -type raftHandler struct { - *raft.Raft - - log *boltdb.BoltStore - stable *boltdb.BoltStore -} - -func (n *Node) newRaftHandler(requestID string) (*raftHandler, error) { - - // Determine directory that should be used for consensus for this request. - dirPath := n.consensusDir(requestID) - err := os.MkdirAll(dirPath, os.ModePerm) - if err != nil { - return nil, fmt.Errorf("could not create consensus work directory: %w", err) - } - - // Transport layer for raft communication. - transport, err := libp2praft.NewLibp2pTransport(n.host, consensusTransportTimeout) - if err != nil { - return nil, fmt.Errorf("could not create libp2p transport: %w", err) - } - - // Create log store. - logDB := filepath.Join(dirPath, defaultLogStoreName) - logStore, err := boltdb.NewBoltStore(logDB) - if err != nil { - return nil, fmt.Errorf("could not create log store (path: %s): %w", logDB, err) - } - - // Create stable store. - stableDB := filepath.Join(dirPath, defaultStableStoreName) - stableStore, err := boltdb.NewBoltStore(stableDB) - if err != nil { - return nil, fmt.Errorf("could not create stable store (path: %s): %w", stableDB, err) - } - - // Create snapshot store. We never really expect we'll need snapshots - // since our clusters are short lived, so this should be fine. - snapshot := raft.NewDiscardSnapshotStore() - - // Add a callback function to cache the execution result - cacheFn := func(req fsmLogEntry, res execute.Result) { - n.executeResponses.Set(req.RequestID, res) - } - - // Add a callback function to send the execution result to origin. - sendFn := func(req fsmLogEntry, res execute.Result) { - - ctx, cancel := context.WithTimeout(context.Background(), raftClusterSendTimeout) - defer cancel() - - msg := response.Execute{ - Type: blockless.MessageExecuteResponse, - Code: res.Code, - RequestID: req.RequestID, - Results: execute.ResultMap{ - n.host.ID(): res, - }, - } - - err := n.send(ctx, req.Origin, msg) - if err != nil { - n.log.Error().Err(err).Str("peer", req.Origin.String()).Msg("could not send execution result to node") - } - } - - fsm := newFsmExecutor(n.log, n.executor, cacheFn, sendFn) - - cfg := n.getRaftConfig(n.host.ID().String()) - - // Tag the logger with the cluster ID (request ID). - cfg.Logger = cfg.Logger.With("cluster", requestID) - - raftNode, err := raft.NewRaft(&cfg, fsm, logStore, stableStore, snapshot, transport) - if err != nil { - return nil, fmt.Errorf("could not create a raft node: %w", err) - } - - rh := raftHandler{ - Raft: raftNode, - log: logStore, - stable: stableStore, - } - - return &rh, nil -} - -func (n *Node) getRaftConfig(nodeID string) raft.Config { - - cfg := raft.DefaultConfig() - cfg.LocalID = raft.ServerID(nodeID) - cfg.Logger = hclog.New(n.log).Named("raft") - cfg.HeartbeatTimeout = n.cfg.ConsensusHeartbeatTimeout - cfg.ElectionTimeout = n.cfg.ConsensusElectionTimeout - cfg.LeaderLeaseTimeout = n.cfg.ConsensusLeaderLease - - return *cfg -} - -func bootstrapCluster(raftHandler *raftHandler, peerIDs []peer.ID) error { - - if len(peerIDs) == 0 { - return errors.New("empty peer list") - } - - servers := make([]raft.Server, 0, len(peerIDs)) - for _, id := range peerIDs { - - s := raft.Server{ - Suffrage: raft.Voter, - ID: raft.ServerID(id.String()), - Address: raft.ServerAddress(id), - } - - servers = append(servers, s) - } - - cfg := raft.Configuration{ - Servers: servers, - } - - // Bootstrapping will only succeed for the first node to start it. - // Other attempts will fail with an error that can be ignored. - ret := raftHandler.BootstrapCluster(cfg) - err := ret.Error() - if err != nil && !errors.Is(err, raft.ErrCantBootstrap) { - return fmt.Errorf("could not bootstrap cluster: %w", err) - } - - return nil -} - -func (n *Node) shutdownCluster(requestID string) error { - - log := n.log.With().Str("request", requestID).Logger() - log.Info().Msg("shutting down cluster") - - n.clusterLock.RLock() - raftHandler, ok := n.clusters[requestID] - n.clusterLock.RUnlock() - - if !ok { - return nil - } - - future := raftHandler.Shutdown() - err := future.Error() - if err != nil { - return fmt.Errorf("could not shutdown raft cluster: %w", err) - } - - // We'll log the actual error but return an "umbrella" one if we fail to close any of the two stores. - var retErr error - err = raftHandler.log.Close() - if err != nil { - log.Error().Err(err).Msg("could not close log store") - retErr = fmt.Errorf("could not close raft database") - } - - err = raftHandler.stable.Close() - if err != nil { - log.Error().Err(err).Msg("could not close stable store") - retErr = fmt.Errorf("could not close raft database") - } - - // Delete residual files. This may fail if we failed to close the databases above. - dir := n.consensusDir(requestID) - err = os.RemoveAll(dir) - if err != nil { - log.Error().Err(err).Str("path", dir).Msg("could not delete consensus dir") - retErr = fmt.Errorf("could not delete consensus directory") - } - - n.clusterLock.Lock() - delete(n.clusters, requestID) - n.clusterLock.Unlock() - - return retErr -} - -func (n *Node) consensusDir(requestID string) string { - return filepath.Join(n.cfg.Workspace, defaultConsensusDirName, requestID) -} diff --git a/node/roll_call.go b/node/roll_call.go index f9d24961..3ab4f701 100644 --- a/node/roll_call.go +++ b/node/roll_call.go @@ -7,6 +7,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/models/blockless" "github.com/blocklessnetwork/b7s/models/codes" "github.com/blocklessnetwork/b7s/models/request" @@ -33,7 +34,7 @@ func (n *Node) processRollCall(ctx context.Context, from peer.ID, payload []byte log.Debug().Msg("received roll call request") // TODO: (raft) temporary measure - at the moment we don't support multiple raft clusters on the same node at the same time. - if req.ConsensusNeeded && len(n.clusters) > 0 { + if req.Consensus == consensus.Raft && n.haveRaftClusters() { log.Warn().Msg("cannot respond to a roll call as we're already participating in one raft cluster") return nil } @@ -86,17 +87,78 @@ func (n *Node) processRollCall(ctx context.Context, from peer.ID, payload []byte return nil } -// issueRollCall will create a roll call request for executing the given function. +func (n *Node) executeRollCall(ctx context.Context, requestID string, functionID string, nodeCount int, consensus consensus.Type) ([]peer.ID, error) { + + // Create a logger with relevant context. + log := n.log.With().Str("request", requestID).Str("function", functionID).Int("node_count", nodeCount).Logger() + + log.Info().Msg("performing roll call for request") + + n.rollCall.create(requestID) + defer n.rollCall.remove(requestID) + + err := n.publishRollCall(ctx, requestID, functionID, consensus) + if err != nil { + return nil, fmt.Errorf("could not publish roll call: %w", err) + } + + log.Info().Msg("roll call published") + + // Limit for how long we wait for responses. + tctx, exCancel := context.WithTimeout(ctx, n.cfg.RollCallTimeout) + defer exCancel() + + // Peers that have reported on roll call. + var reportingPeers []peer.ID +rollCallResponseLoop: + for { + // Wait for responses from nodes who want to work on the request. + select { + // Request timed out. + case <-tctx.Done(): + + log.Warn().Msg("roll call timed out") + return nil, blockless.ErrRollCallTimeout + + case reply := <-n.rollCall.responses(requestID): + + // Check if this is the reply we want - shouldn't really happen. + if reply.FunctionID != functionID { + log.Info().Str("peer", reply.From.String()).Str("function_got", reply.FunctionID).Msg("skipping inadequate roll call response - wrong function") + continue + } + + // Check if we are connected to this peer. + // Since we receive responses to roll call via direct messages - should not happen. + if !n.haveConnection(reply.From) { + n.log.Info().Str("peer", reply.From.String()).Msg("skipping roll call response from unconnected peer") + continue + } + + log.Info().Str("peer", reply.From.String()).Msg("roll called peer chosen for execution") + + reportingPeers = append(reportingPeers, reply.From) + if len(reportingPeers) >= nodeCount { + log.Info().Msg("enough peers reported for roll call") + break rollCallResponseLoop + } + } + } + + return reportingPeers, nil +} + +// publishRollCall will create a roll call request for executing the given function. // On successful issuance of the roll call request, we return the ID of the issued request. -func (n *Node) issueRollCall(ctx context.Context, requestID string, functionID string, consensusNeeded bool) error { +func (n *Node) publishRollCall(ctx context.Context, requestID string, functionID string, consensus consensus.Type) error { // Create a roll call request. rollCall := request.RollCall{ - Type: blockless.MessageRollCall, - Origin: n.host.ID(), - FunctionID: functionID, - RequestID: requestID, - ConsensusNeeded: consensusNeeded, + Type: blockless.MessageRollCall, + Origin: n.host.ID(), + FunctionID: functionID, + RequestID: requestID, + Consensus: consensus, } // Publish the mssage. @@ -107,3 +169,18 @@ func (n *Node) issueRollCall(ctx context.Context, requestID string, functionID s return nil } + +// Temporary measure - we can't have multiple Raft clusters at this point. Remove when we remove this limitation. +func (n *Node) haveRaftClusters() bool { + + n.clusterLock.RLock() + defer n.clusterLock.RUnlock() + + for _, cluster := range n.clusters { + if cluster.Consensus() == consensus.Raft { + return true + } + } + + return false +} diff --git a/node/roll_call_internal_test.go b/node/roll_call_internal_test.go index 32a2eedd..e25e2b09 100644 --- a/node/roll_call_internal_test.go +++ b/node/roll_call_internal_test.go @@ -10,6 +10,7 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/stretchr/testify/require" + "github.com/blocklessnetwork/b7s/consensus" "github.com/blocklessnetwork/b7s/host" "github.com/blocklessnetwork/b7s/models/blockless" "github.com/blocklessnetwork/b7s/models/codes" @@ -262,7 +263,7 @@ func TestNode_RollCall(t *testing.T) { requestID, err := newRequestID() require.NoError(t, err) - err = node.issueRollCall(ctx, requestID, functionID, false) + err = node.publishRollCall(ctx, requestID, functionID, consensus.Type(0)) require.NoError(t, err) deadlineCtx, cancel := context.WithTimeout(ctx, publishTimeout) diff --git a/node/worker_execute.go b/node/worker_execute.go index fb9ec7d6..60af4fcf 100644 --- a/node/worker_execute.go +++ b/node/worker_execute.go @@ -4,8 +4,8 @@ import ( "context" "encoding/json" "fmt" + "time" - "github.com/hashicorp/raft" "github.com/libp2p/go-libp2p/core/peer" "github.com/blocklessnetwork/b7s/models/blockless" @@ -34,7 +34,7 @@ func (n *Node) workerProcessExecute(ctx context.Context, from peer.ID, payload [ // NOTE: In case of an error, we do not return early from this function. // Instead, we send the response back to the caller, whatever it may be. - code, result, err := n.workerExecute(ctx, requestID, createExecuteRequest(req), req.From) + code, result, err := n.workerExecute(ctx, requestID, req.Timestamp, req.Request, req.From) if err != nil { log.Error().Err(err).Str("peer", from.String()).Msg("execution failed") } @@ -70,7 +70,7 @@ func (n *Node) workerProcessExecute(ctx context.Context, from peer.ID, payload [ } // workerExecute is called on the worker node to use its executor component to invoke the function. -func (n *Node) workerExecute(ctx context.Context, requestID string, req execute.Request, from peer.ID) (codes.Code, execute.Result, error) { +func (n *Node) workerExecute(ctx context.Context, requestID string, timestamp time.Time, req execute.Request, from peer.ID) (codes.Code, execute.Result, error) { // Check if we have function in store. functionInstalled, err := n.fstore.Installed(req.FunctionID) @@ -83,12 +83,17 @@ func (n *Node) workerExecute(ctx context.Context, requestID string, req execute. } // Determine if we should just execute this function, or are we part of the cluster. - n.clusterLock.RLock() - raftNode, ok := n.clusters[requestID] - n.clusterLock.RUnlock() + + // Here we actually have a bit of a conceptual problem with having the same models for head and worker node. + // Head node receives client requests so it can expect _some_ type of inaccuracy there. Worker node receives + // execution requests from the head node, so it shouldn't really tolerate errors/ambiguities. + consensus, err := parseConsensusAlgorithm(req.Config.ConsensusAlgorithm) + if err != nil { + return codes.Error, execute.Result{}, fmt.Errorf("could not parse consensus algorithm from the head node request, aborting (value: %s): %w", req.Config.ConsensusAlgorithm, err) + } // We are not part of a cluster - just execute the request. - if !ok { + if !consensusRequired(consensus) { res, err := n.executor.ExecuteFunction(requestID, req) if err != nil { return res.Code, res, fmt.Errorf("execution failed: %w", err) @@ -97,52 +102,26 @@ func (n *Node) workerExecute(ctx context.Context, requestID string, req execute. return res.Code, res, nil } - log := n.log.With().Str("request", requestID).Str("function", req.FunctionID).Logger() + // Now we KNOW we need a consensus. A cluster must already exist. - log.Info().Msg("execution request to be executed as part of a cluster") - - if raftNode.State() != raft.Leader { - _, id := raftNode.LeaderWithID() + n.clusterLock.RLock() + cluster, ok := n.clusters[requestID] + n.clusterLock.RUnlock() - log.Info().Str("leader", string(id)).Msg("we are not the cluster leader - dropping the request") - return codes.NoContent, execute.Result{}, nil + if !ok { + return codes.Error, execute.Result{}, fmt.Errorf("consensus required but no cluster found; omitted cluster formation message or error forming cluster (request: %s)", requestID) } - log.Info().Msg("we are the cluster leader, executing the request") + log := n.log.With().Str("request", requestID).Str("function", req.FunctionID).Str("consensus", consensus.String()).Logger() - fsmReq := fsmLogEntry{ - RequestID: requestID, - Origin: from, - Execute: req, - } - - payload, err := json.Marshal(fsmReq) - if err != nil { - return codes.Error, execute.Result{}, fmt.Errorf("could not serialize request for FSM") - } + log.Info().Msg("execution request to be executed as part of a cluster") - // Apply Raft log. - future := raftNode.Apply(payload, defaultRaftApplyTimeout) - err = future.Error() + code, value, err := cluster.Execute(from, requestID, timestamp, req) if err != nil { - return codes.Error, execute.Result{}, fmt.Errorf("could not apply raft log: %w", err) - } - - log.Info().Msg("node applied raft log") - - // Get execution result. - response := future.Response() - value, ok := response.(execute.Result) - if !ok { - fsmErr, ok := response.(error) - if ok { - return codes.Error, execute.Result{}, fmt.Errorf("execution encountered an error: %w", fsmErr) - } - - return codes.Error, execute.Result{}, fmt.Errorf("unexpected FSM response format: %T", response) + return codes.Error, execute.Result{}, fmt.Errorf("execution failed: %w", err) } - log.Info().Msg("cluster leader executed the request") + log.Info().Str("code", string(code)).Msg("node processed the execution request") - return codes.OK, value, nil + return code, value, nil } diff --git a/testing/mocks/generic.go b/testing/mocks/generic.go index 42fe11d2..9cf551de 100644 --- a/testing/mocks/generic.go +++ b/testing/mocks/generic.go @@ -65,4 +65,23 @@ var ( FSRootPath: "/var/tmp/blockless/", Entry: "/var/tmp/blockless/app.wasm", } + + // List of a few peer IDs in case multiple are required. + GenericPeerIDs = []peer.ID{ + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0xe5, 0xc, 0xbd, 0xb8, 0xff, 0xed, 0x5a, 0x74, 0x48, 0x4, 0x2, 0x33, 0x4e, 0x42, 0xc, 0x40, 0xab, 0x28, 0x3a, 0x28, 0xdb, 0x5, 0x7e, 0xc6, 0xf5, 0x0, 0x6f, 0x36, 0xa2, 0x8d, 0x82, 0x48}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x19, 0x2a, 0x38, 0x8d, 0xf9, 0x66, 0xa1, 0x14, 0xea, 0x6c, 0xce, 0x1f, 0xf2, 0x3b, 0xee, 0x3b, 0x56, 0xe9, 0x56, 0x27, 0x7e, 0x70, 0x1b, 0x49, 0x9c, 0x25, 0x5, 0x8e, 0xb, 0xda, 0xa3, 0x87}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x68, 0xbc, 0x89, 0x26, 0x87, 0xd5, 0x10, 0x62, 0xa8, 0x6, 0x83, 0xb4, 0xae, 0x62, 0xe1, 0x87, 0xe7, 0xcd, 0x4e, 0x2b, 0x58, 0xa5, 0x82, 0x3b, 0x6a, 0xf6, 0x57, 0x83, 0x38, 0x80, 0x5b, 0xc2}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x26, 0xa8, 0xd0, 0xc6, 0x7c, 0x0, 0xcb, 0xf, 0x7e, 0x23, 0xd9, 0x1c, 0x88, 0xef, 0xc6, 0x2a, 0xd5, 0xa3, 0x18, 0xb5, 0xde, 0xa7, 0x21, 0x44, 0xb0, 0x38, 0xa7, 0xc9, 0x18, 0x6f, 0xd1, 0x25}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x5e, 0x5d, 0x44, 0xfe, 0xde, 0xbc, 0xd7, 0xbd, 0x82, 0xcd, 0x49, 0x23, 0xe8, 0x48, 0x46, 0x56, 0xca, 0x61, 0x13, 0x9d, 0xf4, 0x14, 0x5, 0x8a, 0x87, 0xdd, 0xd2, 0xd9, 0xd6, 0x1c, 0xc4, 0xc2}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x37, 0xf2, 0xf0, 0x1a, 0x4f, 0x1a, 0xaf, 0xd3, 0x48, 0xf4, 0xe8, 0xa7, 0x4b, 0xb, 0xfe, 0x5, 0xbb, 0x18, 0x1, 0xcb, 0x44, 0x1d, 0xe4, 0x4, 0x31, 0x5c, 0x55, 0xf, 0xbd, 0xae, 0x77, 0x95}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x9, 0xb1, 0x4c, 0x68, 0xd5, 0x17, 0x29, 0x76, 0xbc, 0xca, 0xe8, 0xa8, 0x76, 0x7c, 0x2b, 0x82, 0x68, 0xa1, 0xae, 0xe8, 0x35, 0x4b, 0x42, 0xff, 0x5f, 0xaa, 0xe9, 0x6, 0x2d, 0x46, 0xa1, 0x9c}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0xc3, 0x36, 0xd9, 0xea, 0x3, 0x29, 0x85, 0x17, 0x8e, 0x60, 0xa, 0xc5, 0xf, 0x5d, 0xe3, 0xe8, 0xd, 0x1c, 0x53, 0x7b, 0x31, 0x82, 0x58, 0xc9, 0x4e, 0x80, 0x97, 0x8d, 0x6d, 0x1c, 0x97, 0x86}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x49, 0xbd, 0x82, 0x34, 0x7d, 0x97, 0xd, 0xb5, 0x52, 0xf5, 0x82, 0x47, 0x8b, 0xc, 0x42, 0x16, 0x22, 0xa5, 0x24, 0x32, 0xf2, 0x24, 0xfb, 0xc7, 0x44, 0xd1, 0xfc, 0xe1, 0x2e, 0xd3, 0x70, 0xa7}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x24, 0x49, 0x1b, 0x67, 0x6, 0x91, 0x6, 0x94, 0xa, 0xcb, 0x5d, 0x1c, 0xe5, 0x69, 0x37, 0xbe, 0x7c, 0x6b, 0x7e, 0x97, 0x4b, 0x44, 0xd7, 0xbe, 0x94, 0x22, 0x9f, 0xfa, 0x1e, 0x7e, 0x2d, 0xcf}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x15, 0xf8, 0xc7, 0x4b, 0x28, 0x5e, 0x1e, 0xf9, 0x96, 0xd8, 0xbd, 0x15, 0xf9, 0xde, 0x46, 0x39, 0x30, 0xe1, 0xa1, 0x2e, 0xa4, 0x17, 0x5f, 0xef, 0xbd, 0x2d, 0xe4, 0xd1, 0x43, 0xe8, 0xcb, 0x53}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0xd0, 0xca, 0xd3, 0x5c, 0x95, 0xf5, 0xdd, 0xb7, 0x73, 0x4e, 0xe3, 0x6a, 0x6b, 0xfc, 0x73, 0xe5, 0x55, 0x8d, 0xbf, 0x78, 0x2f, 0xa8, 0x42, 0xc9, 0x1d, 0x70, 0xd6, 0xce, 0x2b, 0x9e, 0x4f, 0xf7}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0x9a, 0xc4, 0x96, 0xd, 0x16, 0x58, 0x7d, 0x93, 0x40, 0xa, 0x7a, 0xf, 0xdf, 0x48, 0x12, 0x19, 0x46, 0xc5, 0x4d, 0x2d, 0x8e, 0x11, 0x96, 0xb2, 0xf6, 0xb7, 0x4e, 0x51, 0xff, 0xee, 0x0, 0xb5}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0xaa, 0xed, 0x63, 0x5d, 0xa1, 0xdf, 0x41, 0x2b, 0xe8, 0x9c, 0x49, 0xed, 0xe8, 0x0, 0x5c, 0xa8, 0x64, 0x58, 0x1d, 0x3, 0xf3, 0x59, 0x41, 0x74, 0xff, 0x2b, 0xcd, 0xde, 0x37, 0xfe, 0x15, 0xc6}), + peer.ID([]byte{0x0, 0x24, 0x8, 0x1, 0x12, 0x20, 0xc6, 0x8f, 0x95, 0xd3, 0x98, 0x66, 0x40, 0x6b, 0xc4, 0x6c, 0x19, 0x5e, 0x80, 0xe0, 0x8c, 0x9b, 0x15, 0x4f, 0x8c, 0x6b, 0xd0, 0x1d, 0x5b, 0x83, 0x23, 0x7b, 0x9a, 0x97, 0xc0, 0x9b, 0x9d, 0x9b}), + } )