-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
indexes.go
675 lines (569 loc) · 17.9 KB
/
indexes.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
package main
import (
"context"
"database/sql"
"fmt"
"path"
"path/filepath"
"strings"
"time"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/exitcode"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/types/ethtypes"
lcli "github.com/filecoin-project/lotus/cli"
)
const (
// same as in chain/events/index.go
eventExists = `SELECT MAX(id) FROM event WHERE height=? AND tipset_key=? AND tipset_key_cid=? AND emitter_addr=? AND event_index=? AND message_cid=? AND message_index=?`
insertEvent = `INSERT OR IGNORE INTO event(height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted) VALUES(?, ?, ?, ?, ?, ?, ?, ?)`
insertEntry = `INSERT OR IGNORE INTO event_entry(event_id, indexed, flags, key, codec, value) VALUES(?, ?, ?, ?, ?, ?)`
)
func withCategory(cat string, cmd *cli.Command) *cli.Command {
cmd.Category = strings.ToUpper(cat)
return cmd
}
var indexesCmd = &cli.Command{
Name: "indexes",
Usage: "Commands related to managing sqlite indexes",
HideHelpCommand: true,
Subcommands: []*cli.Command{
withCategory("msgindex", backfillMsgIndexCmd),
withCategory("msgindex", pruneMsgIndexCmd),
withCategory("txhash", backfillTxHashCmd),
withCategory("events", backfillEventsCmd),
},
}
var backfillEventsCmd = &cli.Command{
Name: "backfill-events",
Usage: "Backfill the events.db for a number of epochs starting from a specified height",
Flags: []cli.Flag{
&cli.UintFlag{
Name: "from",
Value: 0,
Usage: "the tipset height to start backfilling from (0 is head of chain)",
},
&cli.IntFlag{
Name: "epochs",
Value: 2000,
Usage: "the number of epochs to backfill",
},
&cli.BoolFlag{
Name: "temporary-index",
Value: false,
Usage: "use a temporary index to speed up the backfill process",
},
&cli.BoolFlag{
Name: "vacuum",
Value: false,
Usage: "run VACUUM on the database after backfilling is complete; this will reclaim space from deleted rows, but may take a long time",
},
},
Action: func(cctx *cli.Context) error {
srv, err := lcli.GetFullNodeServices(cctx)
if err != nil {
return err
}
defer srv.Close() //nolint:errcheck
api := srv.FullNodeAPI()
ctx := lcli.ReqContext(cctx)
// currTs will be the tipset where we start backfilling from
currTs, err := api.ChainHead(ctx)
if err != nil {
return err
}
if cctx.IsSet("from") {
// we need to fetch the tipset after the epoch being specified since we will need to advance currTs
currTs, err = api.ChainGetTipSetAfterHeight(ctx, abi.ChainEpoch(cctx.Int("from")+1), currTs.Key())
if err != nil {
return err
}
}
// advance currTs by one epoch and maintain prevTs as the previous tipset (this allows us to easily use the ChainGetParentMessages/Receipt API)
prevTs := currTs
currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
if err != nil {
return fmt.Errorf("failed to load tipset %s: %w", prevTs.Parents(), err)
}
epochs := cctx.Int("epochs")
basePath, err := homedir.Expand(cctx.String("repo"))
if err != nil {
return err
}
log.Infof(
"WARNING: If this command is run against a node that is currently collecting events with DisableHistoricFilterAPI=false, " +
"it may cause the node to fail to record recent events due to the need to obtain an exclusive lock on the database for writes.")
dbPath := path.Join(basePath, "sqlite", "events.db")
db, err := sql.Open("sqlite3", dbPath+"?_txlock=immediate")
if err != nil {
return err
}
defer func() {
err := db.Close()
if err != nil {
fmt.Printf("ERROR: closing db: %s", err)
}
}()
if cctx.Bool("temporary-index") {
log.Info("creating temporary index (tmp_event_backfill_index) on event table to speed up backfill")
_, err := db.Exec("CREATE INDEX IF NOT EXISTS tmp_event_backfill_index ON event (height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted);")
if err != nil {
return err
}
}
addressLookups := make(map[abi.ActorID]address.Address)
// TODO: We don't need this address resolution anymore once https://github.com/filecoin-project/lotus/issues/11594 lands
resolveFn := func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) {
// we only want to match using f4 addresses
idAddr, err := address.NewIDAddress(uint64(emitter))
if err != nil {
return address.Undef, false
}
actor, err := api.StateGetActor(ctx, idAddr, ts.Key())
if err != nil || actor.DelegatedAddress == nil {
return idAddr, true
}
return *actor.DelegatedAddress, true
}
isIndexedValue := func(b uint8) bool {
// currently we mark the full entry as indexed if either the key
// or the value are indexed; in the future we will need finer-grained
// management of indices
return b&(types.EventFlagIndexedKey|types.EventFlagIndexedValue) > 0
}
var totalEventsAffected int64
var totalEntriesAffected int64
stmtEventExists, err := db.Prepare(eventExists)
if err != nil {
return err
}
stmtInsertEvent, err := db.Prepare(insertEvent)
if err != nil {
return err
}
stmtInsertEntry, err := db.Prepare(insertEntry)
if err != nil {
return err
}
processHeight := func(ctx context.Context, cnt int, msgs []lapi.Message, receipts []*types.MessageReceipt) error {
var tx *sql.Tx
for {
var err error
tx, err = db.BeginTx(ctx, nil)
if err != nil {
if err.Error() == "database is locked" {
log.Warnf("database is locked, retrying in 200ms")
time.Sleep(200 * time.Millisecond)
continue
}
return err
}
break
}
defer tx.Rollback() //nolint:errcheck
var eventsAffected int64
var entriesAffected int64
// loop over each message receipt and backfill the events
for idx, receipt := range receipts {
msg := msgs[idx]
if receipt.ExitCode != exitcode.Ok {
continue
}
if receipt.EventsRoot == nil {
continue
}
events, err := api.ChainGetEvents(ctx, *receipt.EventsRoot)
if err != nil {
return fmt.Errorf("failed to load events for tipset %s: %w", currTs, err)
}
for eventIdx, event := range events {
addr, found := addressLookups[event.Emitter]
if !found {
var ok bool
addr, ok = resolveFn(ctx, event.Emitter, currTs)
if !ok {
// not an address we will be able to match against
continue
}
addressLookups[event.Emitter] = addr
}
tsKeyCid, err := currTs.Key().Cid()
if err != nil {
return fmt.Errorf("failed to get tipset key cid: %w", err)
}
// select the highest event id that exists in database, or null if none exists
var entryID sql.NullInt64
err = tx.Stmt(stmtEventExists).QueryRow(
currTs.Height(),
currTs.Key().Bytes(),
tsKeyCid.Bytes(),
addr.Bytes(),
eventIdx,
msg.Cid.Bytes(),
idx,
).Scan(&entryID)
if err != nil {
return fmt.Errorf("error checking if event exists: %w", err)
}
// we already have this event
if entryID.Valid {
continue
}
// event does not exist, lets backfill it
res, err := tx.Stmt(stmtInsertEvent).Exec(
currTs.Height(), // height
currTs.Key().Bytes(), // tipset_key
tsKeyCid.Bytes(), // tipset_key_cid
addr.Bytes(), // emitter_addr
eventIdx, // event_index
msg.Cid.Bytes(), // message_cid
idx, // message_index
false, // reverted
)
if err != nil {
return fmt.Errorf("error inserting event: %w", err)
}
entryID.Int64, err = res.LastInsertId()
if err != nil {
return fmt.Errorf("could not get last insert id: %w", err)
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("could not get rows affected: %w", err)
}
eventsAffected += rowsAffected
// backfill the event entries
for _, entry := range event.Entries {
_, err := tx.Stmt(stmtInsertEntry).Exec(
entryID.Int64, // event_id
isIndexedValue(entry.Flags), // indexed
[]byte{entry.Flags}, // flags
entry.Key, // key
entry.Codec, // codec
entry.Value, // value
)
if err != nil {
return fmt.Errorf("error inserting entry: %w", err)
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("could not get rows affected: %w", err)
}
entriesAffected += rowsAffected
}
}
}
err = tx.Commit()
if err != nil {
return fmt.Errorf("failed to commit transaction: %w", err)
}
log.Infof("[%d] backfilling actor events epoch:%d, eventsAffected:%d, entriesAffected:%d", cnt, currTs.Height(), eventsAffected, entriesAffected)
totalEventsAffected += eventsAffected
totalEntriesAffected += entriesAffected
return nil
}
for i := 0; i < epochs; i++ {
select {
case <-ctx.Done():
return nil
default:
}
blockCid := prevTs.Blocks()[0].Cid()
// get messages for the parent of the previous tipset (which will be currTs)
msgs, err := api.ChainGetParentMessages(ctx, blockCid)
if err != nil {
return fmt.Errorf("failed to get parent messages for block %s: %w", blockCid, err)
}
// get receipts for the parent of the previous tipset (which will be currTs)
receipts, err := api.ChainGetParentReceipts(ctx, blockCid)
if err != nil {
return fmt.Errorf("failed to get parent receipts for block %s: %w", blockCid, err)
}
if len(msgs) != len(receipts) {
return fmt.Errorf("mismatched in message and receipt count: %d != %d", len(msgs), len(receipts))
}
err = processHeight(ctx, i, msgs, receipts)
if err != nil {
return err
}
// advance prevTs and currTs up the chain
prevTs = currTs
currTs, err = api.ChainGetTipSet(ctx, currTs.Parents())
if err != nil {
return fmt.Errorf("failed to load tipset %s: %w", currTs, err)
}
}
log.Infof("backfilling events complete, totalEventsAffected:%d, totalEntriesAffected:%d", totalEventsAffected, totalEntriesAffected)
if cctx.Bool("temporary-index") {
log.Info("dropping temporary index (tmp_event_backfill_index) on event table")
_, err := db.Exec("DROP INDEX IF EXISTS tmp_event_backfill_index;")
if err != nil {
fmt.Printf("ERROR: dropping index: %s", err)
}
}
if cctx.Bool("vacuum") {
log.Info("running VACUUM on the database")
_, err := db.Exec("VACUUM;")
if err != nil {
return fmt.Errorf("failed to run VACUUM on the database: %w", err)
}
}
return nil
},
}
var backfillMsgIndexCmd = &cli.Command{
Name: "backfill-msgindex",
Usage: "Backfill the msgindex.db for a number of epochs starting from a specified height",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "from",
Value: 0,
Usage: "height to start the backfill; uses the current head if omitted",
},
&cli.IntFlag{
Name: "epochs",
Value: 1800,
Usage: "number of epochs to backfill; defaults to 1800 (2 finalities)",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
curTs, err := api.ChainHead(ctx)
if err != nil {
return err
}
startHeight := int64(cctx.Int("from"))
if startHeight == 0 {
startHeight = int64(curTs.Height()) - 1
}
epochs := cctx.Int("epochs")
basePath, err := homedir.Expand(cctx.String("repo"))
if err != nil {
return err
}
dbPath := path.Join(basePath, "sqlite", "msgindex.db")
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return err
}
defer func() {
err := db.Close()
if err != nil {
fmt.Printf("ERROR: closing db: %s", err)
}
}()
insertStmt, err := db.Prepare("INSERT OR IGNORE INTO messages (cid, tipset_cid, epoch) VALUES (?, ?, ?)")
if err != nil {
return err
}
var nrRowsAffected int64
for i := 0; i < epochs; i++ {
epoch := abi.ChainEpoch(startHeight - int64(i))
if i%100 == 0 {
log.Infof("%d/%d processing epoch:%d, nrRowsAffected:%d", i, epochs, epoch, nrRowsAffected)
}
ts, err := api.ChainGetTipSetByHeight(ctx, epoch, curTs.Key())
if err != nil {
return fmt.Errorf("failed to get tipset at epoch %d: %w", epoch, err)
}
tsCid, err := ts.Key().Cid()
if err != nil {
return fmt.Errorf("failed to get tipset cid at epoch %d: %w", epoch, err)
}
msgs, err := api.ChainGetMessagesInTipset(ctx, ts.Key())
if err != nil {
return fmt.Errorf("failed to get messages in tipset at epoch %d: %w", epoch, err)
}
for _, msg := range msgs {
key := msg.Cid.String()
tskey := tsCid.String()
res, err := insertStmt.Exec(key, tskey, int64(epoch))
if err != nil {
return fmt.Errorf("failed to insert message cid %s in tipset %s at epoch %d: %w", key, tskey, epoch, err)
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("failed to get rows affected for message cid %s in tipset %s at epoch %d: %w", key, tskey, epoch, err)
}
nrRowsAffected += rowsAffected
}
}
log.Infof("Done backfilling, nrRowsAffected:%d", nrRowsAffected)
return nil
},
}
var pruneMsgIndexCmd = &cli.Command{
Name: "prune-msgindex",
Usage: "Prune the msgindex.db for messages included before a given epoch",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "from",
Usage: "height to start the prune; if negative it indicates epochs from current head",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
startHeight := int64(cctx.Int("from"))
if startHeight < 0 {
curTs, err := api.ChainHead(ctx)
if err != nil {
return err
}
startHeight += int64(curTs.Height())
if startHeight < 0 {
return xerrors.Errorf("bogus start height %d", startHeight)
}
}
basePath, err := homedir.Expand(cctx.String("repo"))
if err != nil {
return err
}
dbPath := path.Join(basePath, "sqlite", "msgindex.db")
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return err
}
defer func() {
err := db.Close()
if err != nil {
fmt.Printf("ERROR: closing db: %s", err)
}
}()
tx, err := db.Begin()
if err != nil {
return err
}
if _, err := tx.Exec("DELETE FROM messages WHERE epoch < ?", startHeight); err != nil {
if err := tx.Rollback(); err != nil {
fmt.Printf("ERROR: rollback: %s", err)
}
return err
}
if err := tx.Commit(); err != nil {
return err
}
return nil
},
}
var backfillTxHashCmd = &cli.Command{
Name: "backfill-txhash",
Usage: "Backfills the txhash.db for a number of epochs starting from a specified height",
Flags: []cli.Flag{
&cli.UintFlag{
Name: "from",
Value: 0,
Usage: "the tipset height to start backfilling from (0 is head of chain)",
},
&cli.IntFlag{
Name: "epochs",
Value: 2000,
Usage: "the number of epochs to backfill",
},
},
Action: func(cctx *cli.Context) error {
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
curTs, err := api.ChainHead(ctx)
if err != nil {
return err
}
startHeight := int64(cctx.Int("from"))
if startHeight == 0 {
startHeight = int64(curTs.Height()) - 1
}
epochs := cctx.Int("epochs")
basePath, err := homedir.Expand(cctx.String("repo"))
if err != nil {
return err
}
dbPath := filepath.Join(basePath, "sqlite", "txhash.db")
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return err
}
defer func() {
err := db.Close()
if err != nil {
fmt.Printf("ERROR: closing db: %s", err)
}
}()
insertStmt, err := db.Prepare("INSERT OR IGNORE INTO eth_tx_hashes(hash, cid) VALUES(?, ?)")
if err != nil {
return err
}
var totalRowsAffected int64 = 0
for i := 0; i < epochs; i++ {
epoch := abi.ChainEpoch(startHeight - int64(i))
select {
case <-cctx.Done():
fmt.Println("request cancelled")
return nil
default:
}
curTsk := curTs.Parents()
execTs, err := api.ChainGetTipSet(ctx, curTsk)
if err != nil {
return fmt.Errorf("failed to call ChainGetTipSet for %s: %w", curTsk, err)
}
if i%100 == 0 {
log.Infof("%d/%d processing epoch:%d", i, epochs, epoch)
}
for _, blockheader := range execTs.Blocks() {
blkMsgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
if err != nil {
log.Infof("Could not get block messages at epoch: %d, stopping walking up the chain", epoch)
epochs = i
break
}
for _, smsg := range blkMsgs.SecpkMessages {
if smsg.Signature.Type != crypto.SigTypeDelegated {
continue
}
tx, err := ethtypes.EthTransactionFromSignedFilecoinMessage(smsg)
if err != nil {
return fmt.Errorf("failed to convert from signed message: %w at epoch: %d", err, epoch)
}
hash, err := tx.TxHash()
if err != nil {
return fmt.Errorf("failed to calculate hash for ethTx: %w at epoch: %d", err, epoch)
}
res, err := insertStmt.Exec(hash.String(), smsg.Cid().String())
if err != nil {
return fmt.Errorf("error inserting tx mapping to db: %s at epoch: %d", err, epoch)
}
rowsAffected, err := res.RowsAffected()
if err != nil {
return fmt.Errorf("error getting rows affected: %s at epoch: %d", err, epoch)
}
if rowsAffected > 0 {
log.Debugf("Inserted txhash %s, cid: %s at epoch: %d", hash.String(), smsg.Cid().String(), epoch)
}
totalRowsAffected += rowsAffected
}
}
curTs = execTs
}
log.Infof("Done, inserted %d missing txhashes", totalRowsAffected)
return nil
},
}