Introduce fuzz testing to separate unit tests from repetition tests

.github/workflows/ci.yml

@@ -38,10 +38,6 @@ jobs:
         with:
           go-version: '1.21'
       - name: Test
-        env:
-          # Override test repetition parallelism, since the default runner seem to have a single core.
-          # This shaves ~30 seconds off the CI test duration. 
-          F3_TEST_REPETITION_PARALLELISM: 2
         run: make test/cover
       - name: Upload coverage to Codecov
         # Commit SHA corresponds to version v4.4.0
@@ -57,6 +53,21 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
+  fuzz:
+    name: Fuzz
+    runs-on: ubuntu-latest
+    needs:
+      - test # Do not bother running the fuzz tests if tests fail
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Fuzz
+        env:
+          FUZZTIME: 30s
+        run: make fuzz
+
   generate:
     name: Generate
     runs-on: ubuntu-latest

Makefile

@@ -1,6 +1,9 @@
-all: generate test lint
+SHELL := /usr/bin/env bash
+
+GOGC ?= 1000 # Reduce GC frequency during testing, default to 1000 if unset.
+
+all: generate test fuzz lint
 
-test: GOGC ?= 1000 # Reduce the GC frequency, default to 1000 if unset.
 test:
 	GOGC=$(GOGC) go test $(GOTEST_ARGS) ./...
 .PHONY: test
@@ -9,8 +12,18 @@ test/cover: test
 test/cover: GOTEST_ARGS=-coverprofile=coverage.txt -covermode=atomic -coverpkg=./...
 .PHONY: test/cover
 
+fuzz: FUZZTIME ?= 10s # The duration to run fuzz testing, default to 10s if unset.
+fuzz: # List all fuzz tests across the repo, and run them one at a time with the configured fuzztime.
+	@go list ./... | while read -r package; do \
+		go test -list '^Fuzz' "$$package" | grep '^Fuzz' | while read -r func; do \
+			echo "Running $$package $$func for $(FUZZTIME)..."; \
+			GOGC=$(GOGC) go test "$$package" -run '^$$' -fuzz="$$func" -fuzztime=$(FUZZTIME) || exit 1; \
+		done; \
+	done;
+.PHONY: fuzz
 
 lint:
+	go mod tidy
 	golangci-lint run ./...
 .PHONY: lint
 

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/whyrusleeping/cbor-gen v0.1.0
 	go.uber.org/multierr v1.11.0
 	golang.org/x/crypto v0.21.0
-	golang.org/x/sync v0.3.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
 )
 

sim/cmd/f3sim/f3sim.go

@@ -30,13 +30,8 @@ func main() {
 		seed := *latencySeed + int64(i)
 		fmt.Printf("Iteration %d: seed=%d, mean=%f\n", i, seed, *latencyMean)
 
-		latencyModel, err := latency.NewLogNormal(*latencySeed, time.Duration(*latencyMean*float64(time.Second)))
-		if err != nil {
-			log.Panicf("failed to instantiate log normal latency model: %c\n", err)
-		}
-
 		options := []sim.Option{
-			sim.WithLatencyModel(latencyModel),
+			sim.WithLatencyModel(latency.NewLogNormal(*latencySeed, time.Duration(*latencyMean*float64(time.Second)))),
 			sim.WithECEpochDuration(30 * time.Second),
 			sim.WithECStabilisationDelay(0),
 			sim.AddHonestParticipants(

sim/latency/log_normal.go

@@ -1,7 +1,6 @@
 package latency
 
 import (
-	"errors"
 	"math"
 	"math/rand"
 	"sync"
@@ -25,16 +24,14 @@ type LogNormal struct {
 }
 
 // NewLogNormal instantiates a new latency model of log normal latency
-// distribution with the given mean.
-func NewLogNormal(seed int64, mean time.Duration) (*LogNormal, error) {
-	if mean < 0 {
-		return nil, errors.New("mean duration cannot be negative")
-	}
+// distribution with the given mean. This model will always return zero if mean
+// latency duration is less than or equal to zero.
+func NewLogNormal(seed int64, mean time.Duration) *LogNormal {
 	return &LogNormal{
 		rng:           rand.New(rand.NewSource(seed)),
 		mean:          mean,
 		latencyFromTo: make(map[gpbft.ActorID]map[gpbft.ActorID]time.Duration),
-	}, nil
+	}
 }
 
 // Sample returns latency samples that correspond to the log normal distribution
@@ -43,9 +40,10 @@ func NewLogNormal(seed int64, mean time.Duration) (*LogNormal, error) {
 // distribution. Latency from one participant to another may be asymmetric and
 // once generated remains constant for the lifetime of a simulation.
 //
-// Note, here from and to are the same the latency sample will always be zero.
+// Note, where from and to are the same or mean configured latency is not larger
+// than zero the latency sample will always be zero.
 func (l *LogNormal) Sample(_ time.Time, from gpbft.ActorID, to gpbft.ActorID) time.Duration {
-	if from == to {
+	if from == to || l.mean <= 0 {
 		return 0
 	}
 

test/absent_test.go

@@ -8,19 +8,25 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestAbsent(t *testing.T) {
-	t.Parallel()
+func FuzzAbsentAdversary(f *testing.F) {
+	f.Add(98465230)
+	f.Add(651)
+	f.Add(-56)
+	f.Add(22)
+	f.Add(0)
+	f.Fuzz(absentAdversaryTest)
+}
 
-	repeatInParallel(t, 1, func(t *testing.T, repetition int) {
-		// Total network size of 3 + 1, where the adversary has 1/4 of power.
-		sm, err := sim.NewSimulation(asyncOptions(t, repetition,
+func absentAdversaryTest(t *testing.T, latencySeed int) {
+	sm, err := sim.NewSimulation(
+		asyncOptions(latencySeed,
+			// Total network size of 3 + 1, where the adversary has 1/4 of power.
 			sim.AddHonestParticipants(
 				3,
 				sim.NewUniformECChainGenerator(tipSetGeneratorSeed, 1, 10),
 				uniformOneStoragePower),
 			sim.WithAdversary(adversary.NewAbsentGenerator(oneStoragePower)),
 		)...)
-		require.NoError(t, err)
-		require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe())
-	})
+	require.NoError(t, err)
+	require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe())
 }

test/constants_test.go

@@ -1,13 +1,11 @@
 package test
 
 import (
-	"testing"
 	"time"
 
 	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-f3/sim"
 	"github.com/filecoin-project/go-f3/sim/latency"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -16,7 +14,6 @@ const (
 
 	latencyAsync         = 100 * time.Millisecond
 	maxRounds            = 10
-	asyncIterations      = 5000
 	EcEpochDuration      = 30 * time.Second
 	EcStabilisationDelay = 3 * time.Second
 )
@@ -44,11 +41,9 @@ func syncOptions(o ...sim.Option) []sim.Option {
 	)
 }
 
-func asyncOptions(t *testing.T, latencySeed int, o ...sim.Option) []sim.Option {
-	lm, err := latency.NewLogNormal(int64(latencySeed), latencyAsync)
-	require.NoError(t, err)
+func asyncOptions(latencySeed int, o ...sim.Option) []sim.Option {
 	return append(o,
-		sim.WithLatencyModel(lm),
+		sim.WithLatencyModel(latency.NewLogNormal(int64(latencySeed), latencyAsync)),
 		sim.WithECEpochDuration(EcEpochDuration),
 		sim.WitECStabilisationDelay(EcStabilisationDelay),
 		sim.WithGpbftOptions(testGpbftOptions...),

test/decide_test.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/filecoin-project/go-f3/gpbft"
@@ -10,19 +11,30 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestImmediateDecide(t *testing.T) {
-	tsg := sim.NewTipSetGenerator(984615)
+func FuzzImmediateDecideAdversary(f *testing.F) {
+	f.Add(98562314)
+	f.Add(8)
+	f.Add(-9554)
+	f.Add(95)
+	f.Add(65)
+	f.Fuzz(immediateDecideAdversaryTest)
+}
+
+func immediateDecideAdversaryTest(t *testing.T, seed int) {
+	rng := rand.New(rand.NewSource(int64(seed)))
+	tsg := sim.NewTipSetGenerator(tipSetGeneratorSeed)
 	baseChain := generateECChain(t, tsg)
 	adversaryValue := baseChain.Extend(tsg.Sample())
-	sm, err := sim.NewSimulation(asyncOptions(t, 1413,
-		sim.AddHonestParticipants(
-			1,
-			sim.NewUniformECChainGenerator(tipSetGeneratorSeed, 1, 10),
-			uniformOneStoragePower),
-		sim.WithBaseChain(&baseChain),
-		// Add the adversary to the simulation with 3/4 of total power.
-		sim.WithAdversary(adversary.NewImmediateDecideGenerator(adversaryValue, gpbft.NewStoragePower(3))),
-	)...)
+	sm, err := sim.NewSimulation(
+		asyncOptions(rng.Int(),
+			sim.AddHonestParticipants(
+				1,
+				sim.NewUniformECChainGenerator(rng.Uint64(), 1, 10),
+				uniformOneStoragePower),
+			sim.WithBaseChain(&baseChain),
+			// Add the adversary to the simulation with 3/4 of total power.
+			sim.WithAdversary(adversary.NewImmediateDecideGenerator(adversaryValue, gpbft.NewStoragePower(3))),
+		)...)
 	require.NoError(t, err)
 
 	err = sm.Run(1, maxRounds)