Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prefetch state claims to change O(n) calls to Lotus State to O(1) #1855

Merged
merged 1 commit into from
Dec 21, 2023
Merged

Prefetch state claims to change O(n) calls to Lotus State to O(1) #1855

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prefetch state claims to change O(n) calls to Lotus State to O(1) 
In my case (140.000 updates) shaves off 1.5 seconds per call (+ the load on the daemon) -> turns a 39 hour job into seconds
  • Loading branch information
@RobQuistNL
RobQuistNL authored Dec 21, 2023
commit 45d88e8548c09b7742bc8136fad82d934b02734e
17 changes: 11 additions & 6 deletions piecedirectory/doctor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/filecoin-project/lotus/chain/types"
"github.com/ipfs/go-cid"
logging "github.com/ipfs/go-log/v2"
verifregtypes "github.com/filecoin-project/go-state-types/builtin/v9/verifreg"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you please check with v12 verifreg? We are trying to use latest actors as much as possible.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure that will work? The v9 one is the one used in the return type of the API call

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahhh! Looks like a Lotus side dependency. Merging.

)

var doclog = logging.Logger("piecedoc")
Expand Down Expand Up @@ -77,8 +78,16 @@ func (d *Doctor) Run(ctx context.Context) {

// Check each piece for problems
doclog.Debugw("piece doctor: checking pieces", "count", len(pcids))

// Prefetch the state claims
doclog.Debugw("prefetching state claims", "address", d.maddr)
claims, err := d.fullnodeApi.StateGetClaims(ctx, d.maddr, types.EmptyTSK)
if err != nil {
return fmt.Errorf("getting claims for the miner %s: %w", d.maddr, err)
}

for _, pcid := range pcids {
err := d.checkPiece(ctx, pcid, lu, head)
err := d.checkPiece(ctx, pcid, lu, head, claims)
if err != nil {
if errors.Is(err, context.Canceled) {
return err
Expand Down Expand Up @@ -107,7 +116,7 @@ func (d *Doctor) Run(ctx context.Context) {
}
}

func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid, lu *sectorstatemgr.SectorStateUpdates, head *types.TipSet) error {
func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid, lu *sectorstatemgr.SectorStateUpdates, head *types.TipSet, claims map[verifregtypes.ClaimId]verifregtypes.Claim) error {
defer func(start time.Time) { log.Debugw("checkPiece processing", "took", time.Since(start)) }(time.Now())

// Check if piece belongs to an active sector
Expand Down Expand Up @@ -161,10 +170,6 @@ func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid, lu *sectorsta
// Check that Deal is actually on-chain for the active sectors
if d.fullnodeApi != nil { // nil in tests
found := false
claims, err := d.fullnodeApi.StateGetClaims(ctx, d.maddr, types.EmptyTSK)
if err != nil {
return fmt.Errorf("getting claims for the miner %s: %w", d.maddr, err)
}
for _, dealId := range chainDeals {
if dealId.IsDirectDeal {
doclog.Debugw("checking state for direct deal", "piece", pieceCid, "allocation", dealId.ChainDealID)
Expand Down
Loading