Introduce fuzz testing to separate unit tests from repetition tests

Prior to changes introduced here the majority of time during testing was
spent on repeating the same test while changing the seed for generators
or latency models.

Instead of repeating the same test over and over use the fuzz testing
mechanism provided by Go SDK. This change allows the unit tests to run
much faster while offering a configurable "fuzztime" that dictates for
how long the test cases should be fuzzed.

The changes here introduce the following fuzz tests:
* `FuzzAbsentAdversary`
* `FuzzImmediateDecideAdversary`
* `FuzzHonest_AsyncRequireStrongQuorumToProgress`
* `FuzzHonest_SyncMajorityCommonPrefix`
* `FuzzHonest_AsyncMajorityCommonPrefix`
* `FuzzHonestMultiInstance_AsyncDisagreement`
* `FuzzHonestMultiInstance_SyncAgreement`
* `FuzzHonestMultiInstance_AsyncAgreement`
* `FuzzStoragePower_SyncIncreaseMidSimulation`
* `FuzzStoragePower_AsyncIncreaseMidSimulation`
* `FuzzStoragePower_SyncDecreaseRevertsToBase`
* `FuzzStoragePower_AsyncDecreaseRevertsToBase`
* `FuzzRepeatAdversary`

The CI workflow is updated to run each of the fuzz tests for 30 seconds
in a dedicated job.

Author: masih

.github/workflows/ci.yml

@@ -38,10 +38,6 @@ jobs:
         with:
           go-version: '1.21'
       - name: Test
-        env:
-          # Override test repetition parallelism, since the default runner seem to have a single core.
-          # This shaves ~30 seconds off the CI test duration. 
-          F3_TEST_REPETITION_PARALLELISM: 2
         run: make test/cover
       - name: Upload coverage to Codecov
         # Commit SHA corresponds to version v4.4.0
@@ -57,6 +53,21 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
+  fuzz:
+    name: Fuzz
+    runs-on: ubuntu-latest
+    needs:
+      - test # Do not bother running the fuzz tests if tests fail
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Fuzz
+        env:
+          FUZZTIME: 30s
+        run: make fuzz
+
   generate:
     name: Generate
     runs-on: ubuntu-latest

Makefile

@@ -1,6 +1,9 @@
-all: generate test lint
+SHELL := /usr/bin/env bash
+
+GOGC ?= 1000 # Reduce GC frequency during testing, default to 1000 if unset.
+
+all: generate test fuzz lint
 
-test: GOGC ?= 1000 # Reduce the GC frequency, default to 1000 if unset.
 test:
 	GOGC=$(GOGC) go test $(GOTEST_ARGS) ./...
 .PHONY: test
@@ -9,8 +12,18 @@ test/cover: test
 test/cover: GOTEST_ARGS=-coverprofile=coverage.txt -covermode=atomic -coverpkg=./...
 .PHONY: test/cover
 
+fuzz: FUZZTIME ?= 10s # The duration to run fuzz testing, default to 10s if unset.
+fuzz: # List all fuzz tests across the repo, and run them one at a time with the configured fuzztime.
+	@go list ./... | while read -r package; do \
+		go test -list '^Fuzz' "$$package" | grep '^Fuzz' | while read -r func; do \
+			echo "Running $$package $$func for $(FUZZTIME)..."; \
+			GOGC=$(GOGC) go test "$$package" -run '^$$' -fuzz="$$func" -fuzztime=$(FUZZTIME) || exit 1; \
+		done; \
+	done;
+.PHONY: fuzz
 
 lint:
+	go mod tidy
 	golangci-lint run ./...
 .PHONY: lint
 

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/whyrusleeping/cbor-gen v0.1.0
 	go.uber.org/multierr v1.11.0
-	golang.org/x/sync v0.3.0
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028
 )
 

sim/cmd/f3sim/f3sim.go

@@ -30,13 +30,8 @@ func main() {
 		seed := *latencySeed + int64(i)
 		fmt.Printf("Iteration %d: seed=%d, mean=%f\n", i, seed, *latencyMean)
 
-		latencyModel, err := latency.NewLogNormal(*latencySeed, time.Duration(*latencyMean*float64(time.Second)))
-		if err != nil {
-			log.Panicf("failed to instantiate log normal latency model: %c\n", err)
-		}
-
 		options := []sim.Option{
-			sim.WithLatencyModel(latencyModel),
+			sim.WithLatencyModel(latency.NewLogNormal(*latencySeed, time.Duration(*latencyMean*float64(time.Second)))),
 			sim.WithECEpochDuration(30 * time.Second),
 			sim.WithECStabilisationDelay(0),
 			sim.AddHonestParticipants(

sim/latency/log_normal.go

@@ -1,7 +1,6 @@
 package latency
 
 import (
-	"errors"
 	"math"
 	"math/rand"
 	"sync"
@@ -25,16 +24,14 @@ type LogNormal struct {
 }
 
 // NewLogNormal instantiates a new latency model of log normal latency
-// distribution with the given mean.
-func NewLogNormal(seed int64, mean time.Duration) (*LogNormal, error) {
-	if mean < 0 {
-		return nil, errors.New("mean duration cannot be negative")
-	}
+// distribution with the given mean. This model will always return zero if mean
+// latency duration is less than or equal to zero.
+func NewLogNormal(seed int64, mean time.Duration) *LogNormal {
 	return &LogNormal{
 		rng:           rand.New(rand.NewSource(seed)),
 		mean:          mean,
 		latencyFromTo: make(map[gpbft.ActorID]map[gpbft.ActorID]time.Duration),
-	}, nil
+	}
 }
 
 // Sample returns latency samples that correspond to the log normal distribution
@@ -43,9 +40,10 @@ func NewLogNormal(seed int64, mean time.Duration) (*LogNormal, error) {
 // distribution. Latency from one participant to another may be asymmetric and
 // once generated remains constant for the lifetime of a simulation.
 //
-// Note, here from and to are the same the latency sample will always be zero.
+// Note, where from and to are the same or mean configured latency is not larger
+// than zero the latency sample will always be zero.
 func (l *LogNormal) Sample(_ time.Time, from gpbft.ActorID, to gpbft.ActorID) time.Duration {
-	if from == to {
+	if from == to || l.mean <= 0 {
 		return 0
 	}
 

test/absent_test.go

@@ -8,19 +8,25 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestAbsent(t *testing.T) {
-	t.Parallel()
+func FuzzAbsentAdversary(f *testing.F) {
+	f.Add(98465230)
+	f.Add(651)
+	f.Add(-56)
+	f.Add(22)
+	f.Add(0)
+	f.Fuzz(absentAdversaryTest)
+}
 
-	repeatInParallel(t, 1, func(t *testing.T, repetition int) {
-		// Total network size of 3 + 1, where the adversary has 1/4 of power.
-		sm, err := sim.NewSimulation(asyncOptions(t, repetition,
+func absentAdversaryTest(t *testing.T, latencySeed int) {
+	sm, err := sim.NewSimulation(
+		asyncOptions(latencySeed,
+			// Total network size of 3 + 1, where the adversary has 1/4 of power.
 			sim.AddHonestParticipants(
 				3,
 				sim.NewUniformECChainGenerator(tipSetGeneratorSeed, 1, 10),
 				uniformOneStoragePower),
 			sim.WithAdversary(adversary.NewAbsentGenerator(oneStoragePower)),
 		)...)
-		require.NoError(t, err)
-		require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe())
-	})
+	require.NoError(t, err)
+	require.NoErrorf(t, sm.Run(1, maxRounds), "%s", sm.Describe())
 }

test/constants_test.go

@@ -1,13 +1,11 @@
 package test
 
 import (
-	"testing"
 	"time"
 
 	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-f3/sim"
 	"github.com/filecoin-project/go-f3/sim/latency"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -16,7 +14,6 @@ const (
 
 	latencyAsync         = 100 * time.Millisecond
 	maxRounds            = 10
-	asyncIterations      = 5000
 	EcEpochDuration      = 30 * time.Second
 	EcStabilisationDelay = 3 * time.Second
 )
@@ -44,11 +41,9 @@ func syncOptions(o ...sim.Option) []sim.Option {
 	)
 }
 
-func asyncOptions(t *testing.T, latencySeed int, o ...sim.Option) []sim.Option {
-	lm, err := latency.NewLogNormal(int64(latencySeed), latencyAsync)
-	require.NoError(t, err)
+func asyncOptions(latencySeed int, o ...sim.Option) []sim.Option {
 	return append(o,
-		sim.WithLatencyModel(lm),
+		sim.WithLatencyModel(latency.NewLogNormal(int64(latencySeed), latencyAsync)),
 		sim.WithECEpochDuration(EcEpochDuration),
 		sim.WitECStabilisationDelay(EcStabilisationDelay),
 		sim.WithGpbftOptions(testGpbftOptions...),

test/decide_test.go

@@ -2,6 +2,7 @@ package test
 
 import (
 	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/filecoin-project/go-f3/gpbft"
@@ -10,19 +11,30 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestImmediateDecide(t *testing.T) {
-	tsg := sim.NewTipSetGenerator(984615)
+func FuzzImmediateDecideAdversary(f *testing.F) {
+	f.Add(98562314)
+	f.Add(8)
+	f.Add(-9554)
+	f.Add(95)
+	f.Add(65)
+	f.Fuzz(immediateDecideAdversaryTest)
+}
+
+func immediateDecideAdversaryTest(t *testing.T, seed int) {
+	rng := rand.New(rand.NewSource(int64(seed)))
+	tsg := sim.NewTipSetGenerator(tipSetGeneratorSeed)
 	baseChain := generateECChain(t, tsg)
 	adversaryValue := baseChain.Extend(tsg.Sample())
-	sm, err := sim.NewSimulation(asyncOptions(t, 1413,
-		sim.AddHonestParticipants(
-			1,
-			sim.NewUniformECChainGenerator(tipSetGeneratorSeed, 1, 10),
-			uniformOneStoragePower),
-		sim.WithBaseChain(&baseChain),
-		// Add the adversary to the simulation with 3/4 of total power.
-		sim.WithAdversary(adversary.NewImmediateDecideGenerator(adversaryValue, gpbft.NewStoragePower(3))),
-	)...)
+	sm, err := sim.NewSimulation(
+		asyncOptions(rng.Int(),
+			sim.AddHonestParticipants(
+				1,
+				sim.NewUniformECChainGenerator(rng.Uint64(), 1, 10),
+				uniformOneStoragePower),
+			sim.WithBaseChain(&baseChain),
+			// Add the adversary to the simulation with 3/4 of total power.
+			sim.WithAdversary(adversary.NewImmediateDecideGenerator(adversaryValue, gpbft.NewStoragePower(3))),
+		)...)
 	require.NoError(t, err)
 
 	err = sm.Run(1, maxRounds)