Skip to content

Commit d1f0179

Browse files
committed
test setup and reorg tests
1 parent 147e153 commit d1f0179

File tree

14 files changed

+2046
-35
lines changed

14 files changed

+2046
-35
lines changed

.dockerignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
.env
2+
.mockery.yaml
23

34
configs/secrets*

.github/workflows/format.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ jobs:
99
runs-on: ubuntu-latest
1010
steps:
1111
- name: Checkout code
12-
uses: actions/checkout@v3
12+
uses: actions/checkout@v4
1313

1414
- name: Set up Go
1515
uses: actions/setup-go@v4

.github/workflows/test.yml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: Go Unit Tests
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
types: [opened, synchronize]
8+
workflow_dispatch:
9+
10+
jobs:
11+
test:
12+
name: Test
13+
timeout-minutes: 15
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- name: Set up Go
19+
uses: actions/setup-go@v4
20+
with:
21+
go-version: '1.23'
22+
23+
- name: Install dependencies
24+
run: go mod download
25+
26+
- name: Run Unit Tests
27+
run: go test ./... -v
28+
env:
29+
GO_ENV: "test"

.mockery.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
with-expecter: true
2+
mockname: "Mock{{.InterfaceName}}"
3+
filename: "{{.MockName}}.go"
4+
outpkg: mocks
5+
dir: test/mocks
6+
mock-build-tags: "!production"
7+
packages:
8+
github.com/thirdweb-dev/indexer/internal/rpc:
9+
interfaces:
10+
IRPCClient:
11+
github.com/thirdweb-dev/indexer/internal/storage:
12+
interfaces:
13+
IStorage:
14+
IMainStorage:
15+
IStagingStorage:
16+
IOrchestratorStorage:

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ RUN go mod download
99

1010
COPY . .
1111

12-
RUN go build -o /bin/app main.go
12+
RUN go build -o /bin/app -tags=production main.go
1313

1414
FROM alpine:3.18
1515

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ git clone https://github.com/thirdweb-dev/insight.git
3636
4. Create `secrets.yml` from `secrects.example.yml` and set the needed credentials
3737
5. Build an instance
3838
```
39-
go build -o main
39+
go build -o main -tags=production
4040
```
4141
6. Run insight
4242
```

go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ require (
1313
github.com/rs/zerolog v1.33.0
1414
github.com/spf13/cobra v1.8.1
1515
github.com/spf13/viper v1.18.0
16+
github.com/stretchr/testify v1.9.0
1617
github.com/swaggo/files v1.0.1
1718
github.com/swaggo/gin-swagger v1.6.0
1819
github.com/swaggo/swag v1.16.3
@@ -35,6 +36,7 @@ require (
3536
github.com/consensys/gnark-crypto v0.12.1 // indirect
3637
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
3738
github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
39+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
3840
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
3941
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
4042
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -76,6 +78,7 @@ require (
7678
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
7779
github.com/pierrec/lz4/v4 v4.1.21 // indirect
7880
github.com/pkg/errors v0.9.1 // indirect
81+
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
7982
github.com/prometheus/client_model v0.6.1 // indirect
8083
github.com/prometheus/common v0.55.0 // indirect
8184
github.com/prometheus/procfs v0.15.1 // indirect
@@ -88,6 +91,7 @@ require (
8891
github.com/spf13/afero v1.11.0 // indirect
8992
github.com/spf13/cast v1.6.0 // indirect
9093
github.com/spf13/pflag v1.0.5 // indirect
94+
github.com/stretchr/objx v0.5.2 // indirect
9195
github.com/subosito/gotenv v1.6.0 // indirect
9296
github.com/supranational/blst v0.3.11 // indirect
9397
github.com/tklauser/go-sysconf v0.3.12 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,8 @@ github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9
276276
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
277277
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
278278
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
279+
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
280+
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
279281
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
280282
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
281283
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/orchestrator/reorg_handler.go

Lines changed: 44 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -71,45 +71,56 @@ func (rh *ReorgHandler) Start() {
7171
log.Debug().Msgf("Reorg handler running")
7272
go func() {
7373
for range ticker.C {
74-
lookbackFrom := new(big.Int).Add(rh.lastCheckedBlock, big.NewInt(int64(rh.blocksPerScan)))
75-
blockHeaders, err := rh.storage.MainStorage.LookbackBlockHeaders(rh.rpc.GetChainID(), rh.blocksPerScan, lookbackFrom)
74+
// need to include lastCheckedBlock to check if next block's parent matches
75+
lookbackFrom := new(big.Int).Add(rh.lastCheckedBlock, big.NewInt(int64(rh.blocksPerScan-1)))
76+
mostRecentBlockChecked, err := rh.RunFromBlock(lookbackFrom)
7677
if err != nil {
77-
log.Error().Err(err).Msg("Error getting recent block headers")
78+
log.Error().Err(err).Msg("Error during reorg handling")
7879
continue
7980
}
80-
if len(blockHeaders) == 0 {
81-
log.Warn().Msg("No block headers found")
81+
if mostRecentBlockChecked == nil {
8282
continue
8383
}
84-
mostRecentBlockHeader := blockHeaders[0]
85-
reorgEndIndex := findReorgEndIndex(blockHeaders)
86-
if reorgEndIndex == -1 {
87-
rh.lastCheckedBlock = mostRecentBlockHeader.Number
88-
rh.storage.OrchestratorStorage.SetLastReorgCheckedBlockNumber(rh.rpc.GetChainID(), mostRecentBlockHeader.Number)
89-
metrics.ReorgHandlerLastCheckedBlock.Set(float64(mostRecentBlockHeader.Number.Int64()))
90-
continue
91-
}
92-
metrics.ReorgCounter.Inc()
93-
forkPoint, err := rh.findForkPoint(blockHeaders[reorgEndIndex:])
94-
if err != nil {
95-
log.Error().Err(err).Msg("Error while finding fork point")
96-
continue
97-
}
98-
err = rh.handleReorg(forkPoint, lookbackFrom)
99-
if err != nil {
100-
log.Error().Err(err).Msg("Error while handling reorg")
101-
continue
102-
}
103-
rh.lastCheckedBlock = mostRecentBlockHeader.Number
104-
rh.storage.OrchestratorStorage.SetLastReorgCheckedBlockNumber(rh.rpc.GetChainID(), mostRecentBlockHeader.Number)
105-
metrics.ReorgHandlerLastCheckedBlock.Set(float64(mostRecentBlockHeader.Number.Int64()))
84+
85+
rh.lastCheckedBlock = mostRecentBlockChecked
86+
rh.storage.OrchestratorStorage.SetLastReorgCheckedBlockNumber(rh.rpc.GetChainID(), mostRecentBlockChecked)
87+
metrics.ReorgHandlerLastCheckedBlock.Set(float64(mostRecentBlockChecked.Int64()))
10688
}
10789
}()
10890

10991
// Keep the program running (otherwise it will exit)
11092
select {}
11193
}
11294

95+
func (rh *ReorgHandler) RunFromBlock(lookbackFrom *big.Int) (lastCheckedBlock *big.Int, err error) {
96+
blockHeaders, err := rh.storage.MainStorage.LookbackBlockHeaders(rh.rpc.GetChainID(), rh.blocksPerScan, lookbackFrom)
97+
if err != nil {
98+
return nil, fmt.Errorf("error getting recent block headers: %w", err)
99+
}
100+
if len(blockHeaders) == 0 {
101+
log.Warn().Msg("No block headers found during reorg handling")
102+
return nil, nil
103+
}
104+
mostRecentBlockHeader := blockHeaders[0]
105+
reorgEndIndex := findReorgEndIndex(blockHeaders)
106+
log.Debug().Msgf("Reorg end index: %d", reorgEndIndex)
107+
if reorgEndIndex == -1 {
108+
return mostRecentBlockHeader.Number, nil
109+
}
110+
reorgEndBlock := blockHeaders[reorgEndIndex].Number
111+
metrics.ReorgCounter.Inc()
112+
forkPoint, err := rh.findFirstForkedBlockNumber(blockHeaders[reorgEndIndex:])
113+
if err != nil {
114+
return nil, fmt.Errorf("error while finding fork point: %w", err)
115+
}
116+
log.Debug().Msgf("Fork point: %s", forkPoint.String())
117+
err = rh.handleReorg(forkPoint, reorgEndBlock)
118+
if err != nil {
119+
return nil, fmt.Errorf("error while handling reorg: %w", err)
120+
}
121+
return mostRecentBlockHeader.Number, nil
122+
}
123+
113124
func findReorgEndIndex(reversedBlockHeaders []common.BlockHeader) (index int) {
114125
for i := 0; i < len(reversedBlockHeaders)-1; i++ {
115126
currentBlock := reversedBlockHeaders[i]
@@ -129,20 +140,21 @@ func findReorgEndIndex(reversedBlockHeaders []common.BlockHeader) (index int) {
129140
return -1
130141
}
131142

132-
func (rh *ReorgHandler) findForkPoint(reversedBlockHeaders []common.BlockHeader) (forkPoint *big.Int, err error) {
143+
func (rh *ReorgHandler) findFirstForkedBlockNumber(reversedBlockHeaders []common.BlockHeader) (forkPoint *big.Int, err error) {
133144
newBlocksByNumber, err := rh.getNewBlocksByNumber(reversedBlockHeaders)
134145
if err != nil {
135146
return nil, err
136147
}
137148

138-
for i := 0; i < len(reversedBlockHeaders)-1; i++ {
149+
// skip first because that is the reorg end block
150+
for i := 1; i < len(reversedBlockHeaders); i++ {
139151
blockHeader := reversedBlockHeaders[i]
140152
block, ok := (*newBlocksByNumber)[blockHeader.Number.String()]
141153
if !ok {
142154
return nil, fmt.Errorf("block not found: %s", blockHeader.Number.String())
143155
}
144-
if block.Hash == blockHeader.Hash {
145-
previousBlock := reversedBlockHeaders[i+1]
156+
if blockHeader.ParentHash == block.ParentHash && blockHeader.Hash == block.Hash {
157+
previousBlock := reversedBlockHeaders[i-1]
146158
return previousBlock.Number, nil
147159
}
148160
}
@@ -151,7 +163,7 @@ func (rh *ReorgHandler) findForkPoint(reversedBlockHeaders []common.BlockHeader)
151163
if err != nil {
152164
return nil, fmt.Errorf("error getting next headers batch: %w", err)
153165
}
154-
return rh.findForkPoint(nextHeadersBatch)
166+
return rh.findFirstForkedBlockNumber(nextHeadersBatch)
155167
}
156168

157169
func (rh *ReorgHandler) getNewBlocksByNumber(reversedBlockHeaders []common.BlockHeader) (*map[string]common.Block, error) {

0 commit comments

Comments
 (0)