third phase: insert sorted files into the tree (#33)

cmd/geth/converkle.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"io"
 	"math/rand"
 	"net/http"
 	_ "net/http/pprof"
@@ -39,6 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	trieUtils "github.com/ethereum/go-ethereum/trie/utils"
+	"github.com/gballet/go-verkle"
 	"github.com/golang/snappy"
 	"github.com/holiman/uint256"
 	"gopkg.in/urfave/cli.v1"
@@ -440,3 +442,105 @@ func sortFiles() error {
 	}
 	return nil
 }
+
+func doInsertion(ctx *cli.Context) error {
+	num_files := 0
+	for ; ; num_files++ {
+		idxFile := fmt.Sprintf("index-%02d.verkle", num_files)
+		if _, err := os.Stat(idxFile); err != nil {
+			break
+		}
+	}
+
+	dataFile, err := os.OpenFile(fmt.Sprintf("dump-%02d.verkle", 0), os.O_RDONLY, 0600)
+	if err != nil {
+		return err
+	}
+	defer dataFile.Close()
+
+	var (
+		start      = time.Now()
+		lastReport time.Time
+		indexFiles = make([]*os.File, num_files)
+		recordList = make([]Index, num_files)
+		eofList    = make([]bool, num_files)
+		root       = verkle.New()
+		count      = 0
+	)
+
+	// open all the files and read the first record of each
+	for i := 0; i < num_files; i++ {
+		f, err := os.OpenFile(fmt.Sprintf("index-%02d.verkle", i), os.O_RDONLY, 0600)
+		if err != nil {
+			return err
+		}
+		indexFiles[i] = f
+		err = binary.Read(indexFiles[i], binary.LittleEndian, &recordList[i])
+		eofList[i] = err == io.EOF
+		defer indexFiles[i].Close()
+	}
+
+	for {
+		smallest := 0
+		done := true
+		for i := 0; i < num_files; i++ {
+			if eofList[i] {
+				continue
+			}
+			done = false
+
+			if bytes.Compare(recordList[smallest].Stem[:], recordList[i].Stem[:]) < 0 {
+				smallest = i
+			}
+		}
+		if done {
+			break
+		}
+
+		if time.Since(lastReport) > time.Second*8 {
+			log.Info("Inserting nodes", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
+			lastReport = time.Now()
+		}
+
+		dataFile.Seek(int64(recordList[smallest].Offset), io.SeekStart)
+		valuesSerializedCompressed := make([]byte, recordList[smallest].Size)
+		n, err := dataFile.Read(valuesSerializedCompressed)
+		if err != nil || uint32(n) != recordList[smallest].Size {
+			return fmt.Errorf("error reading data: %w size=%d != %d", err, n, recordList[smallest].Size)
+		}
+
+		rlpLen, err := snappy.DecodedLen(valuesSerializedCompressed)
+		if err != nil {
+			return fmt.Errorf("problem getting the size of compressed data for account %d: %w", count, err)
+		}
+		valuesSerialized := make([]byte, rlpLen)
+		snappy.Decode(valuesSerialized, valuesSerializedCompressed)
+		values := make( [][]byte, 256)
+		list, _, _ := rlp.SplitList(valuesSerialized)
+		for i := range values {
+			values[i], list, _ = rlp.SplitString(list)
+		}
+
+		var stem [31]byte
+		copy(stem[:], recordList[smallest].Stem[:])
+		leaf := verkle.NewLeafNode(stem[:], values)
+		if err != nil {
+			return fmt.Errorf("error deserializing leaf: %w", err)
+		}
+
+		root.(*verkle.InternalNode).InsertStemOrdered(stem[:], leaf, nil)
+
+		err = binary.Read(indexFiles[smallest], binary.LittleEndian, &recordList[smallest])
+		if err != nil && err != io.EOF {
+			return err
+		}
+		eofList[smallest] = err == io.EOF
+
+		count++
+	}
+
+	log.Info("Insertion done", "root commitment", fmt.Sprintf("%x", root.ComputeCommitment().Bytes()), "elapsed", common.PrettyDuration(time.Since(start)))
+	root.ComputeCommitment()
+
+	return nil
+}

cmd/geth/snapshot.go

@@ -191,6 +191,22 @@ block is used.
 					utils.GoerliFlag,
 				},
 				Description: `
+geth snapshot sort-files
+ `,
+			},
+			{
+				Name:      "insert-nodes",
+				Usage:     "insert the sorted nodes into the tree",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(doInsertion),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
 geth snapshot sort-files
  `,
 			},

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/cespare/cp v0.1.0
 	github.com/cloudflare/cloudflare-go v0.14.0
 	github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f
-	github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/deckarep/golang-set v1.8.0
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
@@ -26,7 +26,7 @@ require (
 	github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
-	github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1 // indirect
+	github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c // indirect
 	github.com/go-stack/stack v1.8.0
 	github.com/golang-jwt/jwt/v4 v4.3.0
 	github.com/golang/protobuf v1.4.3
@@ -66,7 +66,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 	golang.org/x/text v0.3.7
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	golang.org/x/tools v0.1.0

go.sum

@@ -90,6 +90,10 @@ github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800 h1:X6RlKj/5XE5
 github.com/crate-crypto/go-ipa v0.0.0-20220516125006-b630877e3800/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
 github.com/crate-crypto/go-ipa v0.0.0-20220516134728-b2cd5dda440e h1:gxuOODMooJDWRS+Ov44j/IeFkCPxdY6zYpp0H1vyj5M=
 github.com/crate-crypto/go-ipa v0.0.0-20220516134728-b2cd5dda440e/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
+github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e h1:lerU+Bwfo7izG4zhMOT+hYFVic+3uN6sgSikUEysL2U=
+github.com/crate-crypto/go-ipa v0.0.0-20220518143002-bb5c76ff150e/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
+github.com/crate-crypto/go-ipa v0.0.0-20220524122216-93013fc5e327 h1:E6A+t+Jx11nqwgQwUX41rM/BYa1fQOjRQqafNSBo0DE=
+github.com/crate-crypto/go-ipa v0.0.0-20220524122216-93013fc5e327/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
@@ -161,6 +165,10 @@ github.com/gballet/go-verkle v0.0.0-20220517160324-3697942fe9c9 h1:TG3UdDu+msuoi
 github.com/gballet/go-verkle v0.0.0-20220517160324-3697942fe9c9/go.mod h1:tJlhnY02dv8C/jS2PjNfkazg/Zen6yyFmmzgXyMeb4U=
 github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1 h1:8RjTDBZwTSfZe1ZVr1xDebkVyuuGD/tPZpmcGeqqR4c=
 github.com/gballet/go-verkle v0.0.0-20220518130120-950819b706d1/go.mod h1:tJlhnY02dv8C/jS2PjNfkazg/Zen6yyFmmzgXyMeb4U=
+github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c h1:hPcRl8HZES5vM0RpBcBdpDLpfLy9EnBrU1B3dfu7DpY=
+github.com/gballet/go-verkle v0.0.0-20220526095713-e14979f99d7c/go.mod h1:Q8VN4hcqHTwwrcU72/31sCU30IUQHuH0rcFe/b2SIFc=
+github.com/gballet/go-verkle v0.0.0-20220527070744-f7946ac010f2 h1:M2EhCRamsasDrba35nzsD9ifTZlw55Mk5h6jFhTV9q4=
+github.com/gballet/go-verkle v0.0.0-20220527070744-f7946ac010f2/go.mod h1:Q8VN4hcqHTwwrcU72/31sCU30IUQHuH0rcFe/b2SIFc=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -585,6 +593,8 @@ golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXH
 golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM=
 golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=