pbss implemention

cmd/geth/chaincmd.go

@@ -50,6 +50,7 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.StateSchemeFlag,
 			utils.OverrideMorph203TimeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",

cmd/geth/dbcmd.go

@@ -70,6 +70,7 @@ Remove blockchain and state databases`,
 			dbDumpFreezerIndex,
 			dbImportCmd,
 			dbExportCmd,
+			dbHbss2PbssCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
@@ -254,6 +255,18 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		},
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
+	dbHbss2PbssCmd = cli.Command{
+		Action:    hbss2pbss,
+		Name:      "hbss-to-pbss",
+		ArgsUsage: "<jobnum (optional)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.AncientFlag,
+		},
+		Usage:       "Convert Hash-Base to Path-Base trie node.",
+		Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`,
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -706,3 +719,38 @@ func exportChaindata(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
+
+func hbss2pbss(ctx *cli.Context) error {
+	stack, config := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	h2p, err := trie.NewHbss2Pbss(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal))
+	if err != nil {
+		return err
+	}
+
+	if ctx.NArg() > 0 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+
+	lastStateID := rawdb.ReadPersistentStateID(chaindb)
+	if lastStateID == 0 {
+		h2p.Run()
+	}
+
+	// prune hbss trie node
+	err = rawdb.PruneHashTrieNodeInDataBase(chaindb)
+	if err != nil {
+		log.Error("Prune Hash trie node in database failed", "error", err)
+		return err
+	}
+	err = h2p.Compact()
+	if err != nil {
+		log.Error("Compact trie node failed", "error", err)
+		return err
+	}
+
+	return nil
+}

cmd/geth/main.go

@@ -159,6 +159,8 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
 		utils.CatalystFlag,
+		utils.StateSchemeFlag,
+		utils.PathDBSyncFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/snapshot.go

@@ -223,6 +223,9 @@ func pruneZKState(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
+		log.Crit("Offline pruning is not required for path scheme")
+	}
 	zkPruner, err := pruner.NewZKPruner(chaindb, ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal))
 	if err != nil {
 		return err

cmd/geth/usage.go

@@ -57,6 +57,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.IdentityFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,
+			utils.StateSchemeFlag,
+			utils.PathDBSyncFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -75,6 +75,7 @@ import (
 	"github.com/morph-l2/go-ethereum/params"
 	"github.com/morph-l2/go-ethereum/rollup/tracing"
 	"github.com/morph-l2/go-ethereum/rpc"
+	"github.com/morph-l2/go-ethereum/trie"
 )
 
 func init() {
@@ -855,6 +856,16 @@ var (
 		Name:  "rpc.getlogs.maxrange",
 		Usage: "Limit max fetched block range for `eth_getLogs` method",
 	}
+
+	StateSchemeFlag = &cli.StringFlag{
+		Name:  "state.scheme",
+		Usage: "Scheme to use for storing zktrie state ('hash' or 'path')",
+	}
+
+	PathDBSyncFlag = cli.BoolFlag{
+		Name:  "pathdb.sync",
+		Usage: "Sync flush nodes cache to disk in path schema",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1722,6 +1733,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 			cfg.EthDiscoveryURLs = SplitAndTrim(urls)
 		}
 	}
+	if ctx.GlobalIsSet(PathDBSyncFlag.Name) {
+		cfg.PathSyncFlush = true
+	}
+
+	scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
+	if err != nil {
+		Fatalf("%v", err)
+	}
+	cfg.StateScheme = scheme
+	if cfg.StateScheme == rawdb.PathScheme {
+		trie.GenesisStateInPathZkTrie = true
+	}
+
 	// Override any default configs for hard coded networks.
 	switch {
 	case ctx.GlobalBool(MainnetFlag.Name):
@@ -1769,6 +1793,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// cheked for path zktrie
+		if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() {
+			log.Crit("Must cooperate with zktrie enable to use --state.scheme=path")
+		}
 	case ctx.GlobalBool(MorphHoleskyFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 2810
@@ -1784,6 +1813,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// cheked for path zktrie
+		if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() {
+			log.Crit("Must cooperate with zktrie enable to use --state.scheme=path")
+		}
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 1337
@@ -2151,3 +2185,22 @@ func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error
 		return action(ctx)
 	}
 }
+
+// ParseCLIAndConfigStateScheme parses state scheme in CLI and config.
+func ParseCLIAndConfigStateScheme(cliScheme, cfgScheme string) (string, error) {
+	if cliScheme == "" {
+		if cfgScheme != "" {
+			log.Info("Use config state scheme", "config", cfgScheme)
+		}
+		return cfgScheme, nil
+	}
+
+	if !rawdb.ValidateStateScheme(cliScheme) {
+		return "", fmt.Errorf("invalid state scheme in CLI: %s", cliScheme)
+	}
+	if cfgScheme == "" || cliScheme == cfgScheme {
+		log.Info("Use CLI state scheme", "CLI", cliScheme)
+		return cliScheme, nil
+	}
+	return "", fmt.Errorf("incompatible state scheme, CLI: %s, config: %s", cliScheme, cfgScheme)
+}

common/hexutil/json.go

@@ -23,13 +23,16 @@ import (
 	"math/big"
 	"reflect"
 	"strconv"
+
+	"github.com/holiman/uint256"
 )
 
 var (
 	bytesT  = reflect.TypeOf(Bytes(nil))
 	bigT    = reflect.TypeOf((*Big)(nil))
 	uintT   = reflect.TypeOf(Uint(0))
 	uint64T = reflect.TypeOf(Uint64(0))
+	u256T   = reflect.TypeOf((*uint256.Int)(nil))
 )
 
 // Bytes marshals/unmarshals as a JSON string with 0x prefix.
@@ -225,6 +228,48 @@ func (b *Big) UnmarshalGraphQL(input interface{}) error {
 	return err
 }
 
+// U256 marshals/unmarshals as a JSON string with 0x prefix.
+// The zero value marshals as "0x0".
+type U256 uint256.Int
+
+// MarshalText implements encoding.TextMarshaler
+func (b U256) MarshalText() ([]byte, error) {
+	u256 := (*uint256.Int)(&b)
+	return []byte(u256.Hex()), nil
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (b *U256) UnmarshalJSON(input []byte) error {
+	// The uint256.Int.UnmarshalJSON method accepts "dec", "0xhex"; we must be
+	// more strict, hence we check string and invoke SetFromHex directly.
+	if !isString(input) {
+		return errNonString(u256T)
+	}
+	// The hex decoder needs to accept empty string ("") as '0', which uint256.Int
+	// would reject.
+	if len(input) == 2 {
+		(*uint256.Int)(b).Clear()
+		return nil
+	}
+	err := (*uint256.Int)(b).SetFromHex(string(input[1 : len(input)-1]))
+	if err != nil {
+		return &json.UnmarshalTypeError{Value: err.Error(), Type: u256T}
+	}
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *U256) UnmarshalText(input []byte) error {
+	// The uint256.Int.UnmarshalText method accepts "dec", "0xhex"; we must be
+	// more strict, hence we check string and invoke SetFromHex directly.
+	return (*uint256.Int)(b).SetFromHex(string(input))
+}
+
+// String returns the hex encoding of b.
+func (b *U256) String() string {
+	return (*uint256.Int)(b).Hex()
+}
+
 // Uint64 marshals/unmarshals as a JSON string with 0x prefix.
 // The zero value marshals as "0x0".
 type Uint64 uint64

common/types.go

@@ -226,6 +226,11 @@ func IsHexAddress(s string) bool {
 	return len(s) == 2*AddressLength && isHex(s)
 }
 
+// Cmp compares two addresses.
+func (a Address) Cmp(other Address) int {
+	return bytes.Compare(a[:], other[:])
+}
+
 // Bytes gets the string representation of the underlying address.
 func (a Address) Bytes() []byte { return a[:] }
 
@@ -433,3 +438,62 @@ func (ma *MixedcaseAddress) ValidChecksum() bool {
 func (ma *MixedcaseAddress) Original() string {
 	return ma.original
 }
+
+func ReverseBytes(b []byte) []byte {
+	o := make([]byte, len(b))
+	for i := range b {
+		o[len(b)-1-i] = b[i]
+	}
+	return o
+}
+
+func bitReverseForNibble(b byte) byte {
+	switch b {
+	case 0:
+		return 0
+	case 1:
+		return 8
+	case 2:
+		return 4
+	case 3:
+		return 12
+	case 4:
+		return 2
+	case 5:
+		return 10
+	case 6:
+		return 6
+	case 7:
+		return 14
+	case 8:
+		return 1
+	case 9:
+		return 9
+	case 10:
+		return 5
+	case 11:
+		return 13
+	case 12:
+		return 3
+	case 13:
+		return 11
+	case 14:
+		return 7
+	case 15:
+		return 15
+	default:
+		panic("unexpected input")
+	}
+}
+
+func BitReverse(inp []byte) (out []byte) {
+
+	l := len(inp)
+	out = make([]byte, l)
+
+	for i, b := range inp {
+		out[l-i-1] = bitReverseForNibble(b&15)<<4 + bitReverseForNibble(b>>4)
+	}
+
+	return
+}