Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: snapshot recover from exporter error #13935

Merged
merged 6 commits into from
Mar 6, 2023
Merged

fix: snapshot recover from exporter error #13935

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
267c64e
Snapshot create and restore debug logs for real-time monitoring
chillyvee Mar 3, 2023
14d4f35
Merge branch 'main' into v0466_snapshot_recover_from_exporter_error_m…
tac0turtle Mar 4, 2023
dbf802d
Remove encoding/hex dependency
chillyvee Mar 6, 2023
217f71c
Merge branch 'main' into v0466_snapshot_recover_from_exporter_error_m…
odeke-em Mar 6, 2023
40cdf66
Update store/rootmulti/store.go
tac0turtle Mar 6, 2023
b2d811e
Merge branch 'main' into v0466_snapshot_recover_from_exporter_error_m…
tac0turtle Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions store/rootmulti/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package rootmulti

import (
"encoding/hex"
chillyvee marked this conversation as resolved.
Show resolved Hide resolved
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -211,6 +212,7 @@ func (rs *Store) LoadVersion(ver int64) error {
func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
infos := make(map[string]types.StoreInfo)

rs.logger.Debug("loadVersion", "ver", ver)
cInfo := &types.CommitInfo{}

// load old data if we are not version 0
Expand Down Expand Up @@ -248,12 +250,13 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
for _, key := range storesKeys {
storeParams := rs.storesParams[key]
commitID := rs.getCommitID(infos, key.Name())
rs.logger.Debug("loadVersion commitID", "key", key, "ver", ver, "hash", hex.EncodeToString(commitID.Hash))
chillyvee marked this conversation as resolved.
Show resolved Hide resolved

// If it has been added, set the initial version
if upgrades.IsAdded(key.Name()) || upgrades.RenamedFrom(key.Name()) != "" {
storeParams.initialVersion = uint64(ver) + 1
} else if commitID.Version != ver && storeParams.typ == types.StoreTypeIAVL {
return fmt.Errorf("version of store %s mismatch root store's version; expected %d got %d", key.Name(), ver, commitID.Version)
return fmt.Errorf("version of store %s mismatch root store's version; expected %d got %d; New stores should be added using StoreUpgrades", key.Name(), ver, commitID.Version)
tac0turtle marked this conversation as resolved.
Show resolved Hide resolved
}

store, err := rs.loadCommitStoreFromParams(key, commitID, storeParams)
Expand Down Expand Up @@ -605,9 +608,11 @@ func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (
return nil
}

rs.logger.Debug("pruning heights", "heights", pruningHeights)
rs.logger.Debug("pruning store", "heights", pruningHeights)

for key, store := range rs.stores {
rs.logger.Debug("pruning store", "key", key) // Also log store.name (a private variable)?

// If the store is wrapped with an inter-block cache, we must first unwrap
// it to get the underlying IAVL store.
if store.GetStoreType() != types.StoreTypeIAVL {
Expand Down Expand Up @@ -773,8 +778,14 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
// and the following messages contain a SnapshotNode (i.e. an ExportNode). Store changes
// are demarcated by new SnapshotStore items.
for _, store := range stores {
rs.logger.Debug("starting snapshot", "store", store.name, "height", height)
exporter, err := store.Export(int64(height))
if exporter == nil {
rs.logger.Error("snapshot failed; exporter is nil", "store", store.name)
Copy link
Member

@julienrbrt julienrbrt Mar 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at store.Export this will always trigger when there is an error, and the error check below is now not so useful. Was it intended?

cosmos-sdk/store/iavl/store.go

Lines 275 to 288 in 5d559dd

func (st *Store) Export(version int64) (*iavl.Exporter, error) {
istore, err := st.GetImmutable(version)
if err != nil {
return nil, errorsmod.Wrapf(err, "iavl export failed for version %v", version)
}
tree, ok := istore.tree.(*immutableTree)
if !ok || tree == nil {
return nil, fmt.Errorf("iavl export failed: unable to fetch tree for version %v", version)
}
export, err := tree.Export()
if err != nil {
return nil, err
}
return export, nil

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we modified the exporter function after the fact and probably never cleaned this up

return err
}
if err != nil {
rs.logger.Error("snapshot failed; exporter error", "store", store.name, "err", err)
return err
}

Expand All @@ -789,12 +800,16 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
},
})
if err != nil {
rs.logger.Error("snapshot failed; item store write failed", "store", store.name, "err", err)
return err
}

nodeCount := 0
for {
node, err := exporter.Next()
if err == iavltree.ErrorExportDone {
rs.logger.Debug("Snapshot Done", "store", store.name, "nodeCount", nodeCount)
nodeCount = 0
break
} else if err != nil {
return err
Expand All @@ -812,6 +827,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
if err != nil {
return err
}
nodeCount++
}

return nil
Expand Down Expand Up @@ -863,9 +879,12 @@ loop:
return snapshottypes.SnapshotItem{}, errorsmod.Wrap(err, "import failed")
}
defer importer.Close()
// Importer height must reflect the node height (which usually matches the block height, but not always)
rs.logger.Debug("restoring snapshot", "store", item.Store.Name)

case *snapshottypes.SnapshotItem_IAVL:
if importer == nil {
rs.logger.Error("failed to restore; received IAVL node item before store item")
return snapshottypes.SnapshotItem{}, errorsmod.Wrap(types.ErrLogic, "received IAVL node item before store item")
}
if item.IAVL.Height > math.MaxInt8 {
Expand Down