Fuzz test convert and twins scenario

.gitignore

@@ -35,3 +35,5 @@ measurements.json
 *.pdf
 
 twins.json
+fuzz/previous_messages.b64
+fuzz/previous_messages.seed

.vscode/dict.txt

@@ -16,14 +16,17 @@ Debugf
 durationpb
 emptypb
 Erevik
+eventloop
 Fangyu
 fasthotstuff
 felixge
 Feng
 ferr
 fgprof
 fgprofprofile
+Fuzzer
 gocyclo
+gofuzz
 golangci
 golint
 gomock
@@ -41,13 +44,16 @@ Infof
 Jalalzai
 Jehl
 Jianyu
+keygen
 kilic
 latencygen
 Malkhi
 Mathieu
 Meling
 mitchellh
 nolint
+Oneof
+Oneofs
 orchestrationpb
 partitioner
 Paulo
@@ -59,6 +65,7 @@ propsed
 proto
 protobuf
 protoc
+protoreflect
 protostream
 ptypes
 QC's
@@ -71,6 +78,7 @@ simplehotstuff
 Sonnino
 SSWU
 structs
+subconfiguration
 throughputvslatency
 timestamppb
 TMPDIR

.vscode/settings.json

@@ -18,3 +18,4 @@
 	],
 	"cSpell.enabled": true
 }
+

Makefile

@@ -3,7 +3,9 @@ proto_src := internal/proto/clientpb/client.proto          \
 		internal/proto/hotstuffpb/hotstuff.proto           \
 		internal/proto/orchestrationpb/orchestration.proto \
 		internal/proto/handelpb/handel.proto               \
-		metrics/types/types.proto
+		metrics/types/types.proto                          \
+		fuzz/fuzz.proto
+
 proto_go := $(proto_src:%.proto=%.pb.go)
 gorums_go := internal/proto/clientpb/client_gorums.pb.go \
 		internal/proto/hotstuffpb/hotstuff_gorums.pb.go  \

fuzz/doc.go

@@ -0,0 +1,2 @@
+// Package fuzz contains fuzzing functions for the fuzzing protobuf/hotstuff messages.
+package fuzz

fuzz/error.go

@@ -0,0 +1,192 @@
+package fuzz
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/meling/proto2"
+)
+
+type typeCount map[reflect.Type]int
+
+func (tc typeCount) Add(typ reflect.Type) {
+	tc[typ]++
+}
+
+func (tc typeCount) String(typeTotalCount typeCount) string {
+	keys := make([]reflect.Type, 0)
+	for key := range tc {
+		keys = append(keys, key)
+	}
+
+	sort.Slice(keys, func(i, j int) bool { return keys[i].String() < keys[j].String() })
+
+	str := ""
+	for _, key := range keys {
+		str += key.String() + ": " + strconv.Itoa(tc[key]) + " / " + strconv.Itoa(typeTotalCount[key]) + "\n"
+	}
+	return str
+}
+
+type panicInfo struct {
+	Err        any
+	StackTrace string
+	FuzzMsg    string
+	FuzzMsgB64 string
+	Seed       *int64
+	LineNum    int
+	TypeCount  typeCount
+}
+
+type errorInfo struct {
+	currentFuzzMsg     *FuzzMsg
+	currentFuzzMsgSeed *int64
+	errorCount         int
+	panics             map[string]panicInfo
+	totalScenarios     int
+	failedScenarios    int
+	totalMessages      int
+	failedMessages     int
+	TypePanicCount     typeCount
+	TypeTotalCount     typeCount
+}
+
+func (ei *errorInfo) init() {
+	ei.panics = make(map[string]panicInfo)
+	ei.TypeTotalCount = make(typeCount)
+	ei.TypePanicCount = make(typeCount)
+}
+
+func (ei *errorInfo) outputErrors(t *testing.T) {
+	ei.outputInfo(t)
+	for _, panicInfo := range ei.panics {
+		t.Error(panicInfo.Err)
+	}
+}
+
+func (ei *errorInfo) outputInfo(t *testing.T) {
+	t.Helper()
+	b64s := ""
+	seeds := ""
+
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+
+	fmt.Println("ERROR INFO")
+
+	keys := make([]string, 0)
+	for key := range ei.panics {
+		keys = append(keys, key)
+	}
+
+	// sorting the keys of the
+	sort.Strings(keys)
+
+	for i, key := range keys {
+		panicInfo := ei.panics[key]
+		b64s += panicInfo.FuzzMsgB64 + "\n"
+
+		if panicInfo.Seed != nil {
+			seeds += strconv.FormatInt(*panicInfo.Seed, 10) + "\n"
+		}
+
+		fmt.Println()
+		fmt.Printf("ERROR NUMBER %d\n", i+1)
+		// contains error location, err text and recover point
+		fmt.Println(key)
+		fmt.Println()
+		fmt.Println("crash amounts grouped by type:")
+		fmt.Println(panicInfo.TypeCount.String(ei.TypeTotalCount))
+		fmt.Println("- STACK TRACE BEGIN")
+		fmt.Print(panicInfo.StackTrace)
+		fmt.Println("- STACK TRACE END")
+		fmt.Println()
+		fmt.Println("- FUZZ MESSAGE BEGIN")
+		fmt.Println(panicInfo.FuzzMsg)
+		fmt.Println("- FUZZ MESSAGE END")
+		fmt.Println()
+	}
+
+	err := saveStringToFile("previous_messages.b64", b64s)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if seeds != "" {
+		err := saveStringToFile("previous_messages.seed", seeds)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+
+	fmt.Println()
+	fmt.Println("SUMMARY")
+	fmt.Printf("unique errors found: %d\n", len(ei.panics))
+	fmt.Printf("%d runs were errors\n", ei.errorCount)
+	fmt.Printf("%d of %d scenarios failed\n", ei.failedScenarios, ei.totalScenarios)
+	fmt.Printf("%d of %d messages failed\n", ei.failedMessages, ei.totalMessages)
+	fmt.Println()
+	fmt.Println("crash amounts grouped by type:")
+	fmt.Println(ei.TypePanicCount.String(ei.TypeTotalCount))
+}
+
+func (ei *errorInfo) addTotal(fuzzMessage *FuzzMsg, seed *int64) {
+	ei.totalMessages++
+	ei.currentFuzzMsg = fuzzMessage
+	ei.currentFuzzMsgSeed = seed
+	protoMsg := extractProtoMsg(ei.currentFuzzMsg)
+	typ := reflect.TypeOf(protoMsg)
+	ei.TypeTotalCount.Add(typ)
+}
+
+func (ei *errorInfo) addPanic(fullStack string, err2 any, info string) {
+	simpleStack := simplifyStack(fullStack)
+	identifier := "error location:\t" + simpleStack + "\nerror info:\t" + fmt.Sprint(err2) + "\nrecovered from:\t" + info
+
+	ei.errorCount++
+
+	oldPanic, okPanic := ei.panics[identifier]
+
+	b64, err := fuzzMsgToB64(ei.currentFuzzMsg)
+	if err != nil {
+		panic(err)
+	}
+
+	protoMsg := extractProtoMsg(ei.currentFuzzMsg)
+	fuzzMsgString := proto2.GoStruct(protoMsg)
+	newLines := strings.Count(fuzzMsgString, "\n")
+
+	newPanic := panicInfo{
+		Err:        err2,
+		StackTrace: fullStack,
+		FuzzMsg:    fuzzMsgString,
+		FuzzMsgB64: b64,
+		Seed:       ei.currentFuzzMsgSeed,
+		LineNum:    newLines,
+	}
+
+	if okPanic {
+		newPanic.TypeCount = oldPanic.TypeCount
+	} else {
+		newPanic.TypeCount = make(typeCount)
+	}
+
+	oldLines := oldPanic.LineNum
+	if !okPanic || newLines < oldLines {
+		ei.panics[identifier] = newPanic
+	}
+	typ := reflect.TypeOf(protoMsg)
+	ei.panics[identifier].TypeCount.Add(typ)
+	ei.TypePanicCount.Add(typ)
+}
+
+func simplifyStack(stack string) string {
+	stackLines := strings.Split(strings.ReplaceAll(stack, "\r\n", "\n"), "\n")
+	// line 9 tells us where the panic happened, found through testing
+	return stackLines[8][1:]
+}

fuzz/extra_test.go

@@ -0,0 +1,59 @@
+package fuzz
+
+// this file includes a couple of extra unnecessary tests
+// used for probability tests, profiling and other things
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+)
+
+func TestFrequencyErrorFuzz(t *testing.T) {
+	if os.Getenv("FUZZ") == "" {
+		t.Skip("Skipping slow test; run with FUZZ=1 to enable")
+	}
+	frequency := make(map[string]int, 0)
+
+	f := initFuzz()
+	for j := 0; j < 1000; j++ {
+		errInfo := new(errorInfo)
+		errInfo.init()
+
+		iterations := 1
+		for i := 0; i < iterations; i++ {
+			fuzzMessage := createFuzzMessage(f, nil)
+			useFuzzMessage(errInfo, fuzzMessage, nil)
+		}
+
+		for key := range errInfo.panics {
+			frequency[key]++
+		}
+	}
+
+	sum := 0
+	for key, val := range frequency {
+		sum += val
+		fmt.Println(key)
+		fmt.Println(val)
+		fmt.Println()
+	}
+	fmt.Println(sum)
+}
+
+func BenchmarkFuzz(b *testing.B) {
+	errInfo := new(errorInfo)
+	errInfo.init()
+
+	f := initFuzz()
+
+	for i := 0; i < b.N; i++ {
+		seed := rand.Int63()
+		fuzzMessage := createFuzzMessage(f, &seed)
+		useFuzzMessage(errInfo, fuzzMessage, &seed)
+	}
+
+	fmt.Println(b.Elapsed())
+	fmt.Println()
+}