From 2c33d365142e84045d5361ec5adf1cd086704f89 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 30 May 2022 08:36:39 +0200 Subject: [PATCH] third phase: insert sorted files into the tree (#33) --- cmd/geth/converkle.go | 104 ++++++++++++++++++++++++++++++++++++++++++ cmd/geth/snapshot.go | 16 +++++++ go.mod | 6 +-- go.sum | 10 ++++ 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/cmd/geth/converkle.go b/cmd/geth/converkle.go index ff4d0d91f477..c7a314128c23 100644 --- a/cmd/geth/converkle.go +++ b/cmd/geth/converkle.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "fmt" + "io" "math/rand" "net/http" _ "net/http/pprof" @@ -39,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" trieUtils "github.com/ethereum/go-ethereum/trie/utils" + "github.com/gballet/go-verkle" "github.com/golang/snappy" "github.com/holiman/uint256" "gopkg.in/urfave/cli.v1" @@ -440,3 +442,105 @@ func sortFiles() error { } return nil } + +func doInsertion(ctx *cli.Context) error { + num_files := 0 + for ; ; num_files++ { + idxFile := fmt.Sprintf("index-%02d.verkle", num_files) + if _, err := os.Stat(idxFile); err != nil { + break + } + } + + dataFile, err := os.OpenFile(fmt.Sprintf("dump-%02d.verkle", 0), os.O_RDONLY, 0600) + if err != nil { + return err + } + defer dataFile.Close() + + var ( + start = time.Now() + lastReport time.Time + indexFiles = make([]*os.File, num_files) + recordList = make([]Index, num_files) + eofList = make([]bool, num_files) + root = verkle.New() + count = 0 + ) + + // open all the files and read the first record of each + for i := 0; i < num_files; i++ { + f, err := os.OpenFile(fmt.Sprintf("index-%02d.verkle", i), os.O_RDONLY, 0600) + if err != nil { + return err + } + indexFiles[i] = f + err = binary.Read(indexFiles[i], binary.LittleEndian, &recordList[i]) + eofList[i] = err == io.EOF + defer indexFiles[i].Close() + } + + for { + smallest := 0 + done := true + for i := 0; i < num_files; i++ { + if eofList[i] { + continue + } + done = false + + if bytes.Compare(recordList[smallest].Stem[:], recordList[i].Stem[:]) < 0 { + smallest = i + } + } + if done { + break + } + + if time.Since(lastReport) > time.Second*8 { + log.Info("Inserting nodes", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) + lastReport = time.Now() + } + + dataFile.Seek(int64(recordList[smallest].Offset), io.SeekStart) + valuesSerializedCompressed := make([]byte, recordList[smallest].Size) + n, err := dataFile.Read(valuesSerializedCompressed) + if err != nil || uint32(n) != recordList[smallest].Size { + return fmt.Errorf("error reading data: %w size=%d != %d", err, n, recordList[smallest].Size) + } + + rlpLen, err := snappy.DecodedLen(valuesSerializedCompressed) + if err != nil { + return fmt.Errorf("problem getting the size of compressed data for account %d: %w", count, err) + } + valuesSerialized := make([]byte, rlpLen) + snappy.Decode(valuesSerialized, valuesSerializedCompressed) + values := make( [][]byte, 256) + list, _, _ := rlp.SplitList(valuesSerialized) + for i := range values { + values[i], list, _ = rlp.SplitString(list) + } + + var stem [31]byte + copy(stem[:], recordList[smallest].Stem[:]) + leaf := verkle.NewLeafNode(stem[:], values) + if err != nil { + return fmt.Errorf("error deserializing leaf: %w", err) + } + + root.(*verkle.InternalNode).InsertStemOrdered(stem[:], leaf, nil) + + err = binary.Read(indexFiles[smallest], binary.LittleEndian, &recordList[smallest]) + if err != nil && err != io.EOF { + return err + } + eofList[smallest] = err == io.EOF + + count++ + } + + log.Info("Insertion done", "root commitment", fmt.Sprintf("%x", root.ComputeCommitment().Bytes()), "elapsed", common.PrettyDuration(time.Since(start))) + root.ComputeCommitment() + + return nil +} diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index bbd5c9680c3f..b9264c56d3cf 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -191,6 +191,22 @@ block is used. utils.GoerliFlag, }, Description: ` +geth snapshot sort-files + `, + }, + { + Name: "insert-nodes", + Usage: "insert the sorted nodes into the tree", + ArgsUsage: "", + Action: utils.MigrateFlags(doInsertion), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` geth snapshot sort-files `, }, diff --git a/go.mod b/go.mod index 8c72d9ee1b0b..c5cd1a84e902 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.14.0 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f - github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e // indirect github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 github.com/deepmap/oapi-codegen v1.8.2 // indirect @@ -26,7 +26,7 @@ require ( github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1 // indirect + github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c // indirect github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 github.com/golang/protobuf v1.4.3 @@ -66,7 +66,7 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.0 diff --git a/go.sum b/go.sum index 15a6bf3caf94..ea70c919a997 100644 --- a/go.sum +++ b/go.sum @@ -90,6 +90,10 @@ github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800 h1:X6RlKj/5XE5 github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/crate-crypto/go-ipa v0.0.0-20220516134728-b2cd5dda440e h1:gxuOODMooJDWRS+Ov44j/IeFkCPxdY6zYpp0H1vyj5M= github.com/crate-crypto/go-ipa v0.0.0-20220516134728-b2cd5dda440e/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= +github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e h1:lerU+Bwfo7izG4zhMOT+hYFVic+3uN6sgSikUEysL2U= +github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= +github.com/crate-crypto/go-ipa v0.0.0-20220524122216-93013fc5e327 h1:E6A+t+Jx11nqwgQwUX41rM/BYa1fQOjRQqafNSBo0DE= +github.com/crate-crypto/go-ipa v0.0.0-20220524122216-93013fc5e327/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -161,6 +165,10 @@ github.com/gballet/go-verkle v0.0.0-20220517160324-3697942fe9c9 h1:TG3UdDu+msuoi github.com/gballet/go-verkle v0.0.0-20220517160324-3697942fe9c9/go.mod h1:tJlhnY02dv8C/jS2PjNfkazg/Zen6yyFmmzgXyMeb4U= github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1 h1:8RjTDBZwTSfZe1ZVr1xDebkVyuuGD/tPZpmcGeqqR4c= github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1/go.mod h1:tJlhnY02dv8C/jS2PjNfkazg/Zen6yyFmmzgXyMeb4U= +github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c h1:hPcRl8HZES5vM0RpBcBdpDLpfLy9EnBrU1B3dfu7DpY= +github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c/go.mod h1:Q8VN4hcqHTwwrcU72/31sCU30IUQHuH0rcFe/b2SIFc= +github.com/gballet/go-verkle v0.0.0-20220527070744-f7946ac010f2 h1:M2EhCRamsasDrba35nzsD9ifTZlw55Mk5h6jFhTV9q4= +github.com/gballet/go-verkle v0.0.0-20220527070744-f7946ac010f2/go.mod h1:Q8VN4hcqHTwwrcU72/31sCU30IUQHuH0rcFe/b2SIFc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -585,6 +593,8 @@ golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXH golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=