From 12735a38495e33957724a03daaa112e30ea032ff Mon Sep 17 00:00:00 2001 From: Qi Zhou Date: Tue, 7 Jun 2022 22:04:30 -0700 Subject: [PATCH] add precompiled contracts for sharded storage (#84) * add precompiled contracts for sharded storage * threadsafe for trie.Database * add sstorage StateDB snapshot support * StateDB/Database Sread/write follows the same API as ShardManager * error handling Co-authored-by: Qi Zhou --- cmd/geth/main.go | 2 + cmd/sstorage/main.go | 10 ++- cmd/utils/flags.go | 39 +++++++++ core/state/journal.go | 19 +++++ core/state/statedb.go | 64 +++++++++++++++ core/vm/contracts.go | 166 +++++++++++++++++++++++++++++++++++++- core/vm/contracts_test.go | 8 +- core/vm/evm.go | 15 +++- core/vm/interface.go | 4 + core/vm/interpreter.go | 1 + eth/ethconfig/config.go | 4 + internal/ethapi/api.go | 2 +- sstorage/data_shard.go | 3 + sstorage/shard_config.go | 4 +- trie/database.go | 67 +++++++++++++++ 15 files changed, 390 insertions(+), 18 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7374123d2f41..08188a2006cb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -131,6 +131,8 @@ var ( utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerifyFlag, + utils.SstorageShardFlag, + utils.SstorageFileFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/sstorage/main.go b/cmd/sstorage/main.go index 6f16ac6f27e9..6a04ca4f39f4 100644 --- a/cmd/sstorage/main.go +++ b/cmd/sstorage/main.go @@ -14,8 +14,8 @@ import ( ) var ( - chunkIdxLen *uint64 - filenames *[]string + chunkLen *uint64 + filenames *[]string verbosity *int @@ -60,7 +60,7 @@ var ShardWriteCmd = &cobra.Command{ } func init() { - chunkIdxLen = CreateCmd.Flags().Uint64("len", 0, "Chunk idx len to create") + chunkLen = CreateCmd.Flags().Uint64("len", 0, "Chunk idx len to create") filenames = rootCmd.PersistentFlags().StringArray("filename", []string{}, "Data filename") verbosity = rootCmd.PersistentFlags().Int("verbosity", 3, "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail") @@ -100,7 +100,9 @@ func runCreate(cmd *cobra.Command, args []string) { log.Crit("must provide single filename") } - _, err := sstorage.Create((*filenames)[0], *chunkIdx, *chunkIdxLen, sstorage.MASK_KECCAK_256) + log.Info("Creating data file", "chunkIdx", *chunkIdx, "chunkLen", *chunkLen) + + _, err := sstorage.Create((*filenames)[0], *chunkIdx, *chunkLen, sstorage.MASK_KECCAK_256) if err != nil { log.Crit("create failed", "error", err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 432abab9f12b..ad2a7adde7c9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -68,6 +68,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/sstorage" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" "gopkg.in/urfave/cli.v1" @@ -554,6 +555,17 @@ var ( Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", Value: ethconfig.Defaults.RPCTxFeeCap, } + // Sharded Storage settings + SstorageShardFlag = cli.StringSliceFlag{ + Name: "sstorage.shard", + Usage: "Add perferred storage shard", + Value: nil, + } + SstorageFileFlag = cli.StringSliceFlag{ + Name: "sstorage.file", + Usage: "Add sharded storage data file", + Value: nil, + } // Logging and debug settings EthStatsURLFlag = cli.StringFlag{ Name: "ethstats", @@ -1106,6 +1118,32 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { } } +func setSstorage(ctx *cli.Context, cfg *ethconfig.Config) { + if ctx.GlobalIsSet(SstorageShardFlag.Name) { + cfg.SstorageShards = ctx.GlobalStringSlice(SstorageShardFlag.Name) + } + if ctx.GlobalIsSet(SstorageFileFlag.Name) { + cfg.SstorageFiles = ctx.GlobalStringSlice(SstorageFileFlag.Name) + } + + sstorage.InitializeConfig() + for _, s := range cfg.SstorageShards { + if err := sstorage.AddDataShardFromConfig(s); err != nil { + Fatalf("Failed to add data shard: %s, %v", s, err) + } + } + + for _, s := range cfg.SstorageFiles { + if err := sstorage.AddDataFileFromConfig(s); err != nil { + Fatalf("Failed to add data file: %s, %v", s, err) + } + } + + if err := sstorage.IsComplete(); err != nil { + Fatalf("Shard is not complete: %v", err) + } +} + // MakeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. func MakeDatabaseHandles() int { @@ -1571,6 +1609,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { setMiner(ctx, &cfg.Miner) setWhitelist(ctx, cfg) setLes(ctx, cfg) + setSstorage(ctx, cfg) // Cap the cache allowance and tune the garbage collector mem, err := gopsutil.VirtualMemory() diff --git a/core/state/journal.go b/core/state/journal.go index 57a692dc7ffa..70113f6125fd 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -138,6 +138,12 @@ type ( address *common.Address slot *common.Hash } + // Sstorage change + sstorageChange struct { + prevBytes []byte // nil means not exist in StateDB (but may exist in underlying DB) + address *common.Address + kvIdx uint64 + } ) func (ch createObjectChange) revert(s *StateDB) { @@ -267,3 +273,16 @@ func (ch accessListAddSlotChange) revert(s *StateDB) { func (ch accessListAddSlotChange) dirtied() *common.Address { return nil } + +func (ch sstorageChange) dirtied() *common.Address { + // the account structure is not touched. + return nil +} + +func (ch sstorageChange) revert(s *StateDB) { + if ch.prevBytes == nil { + delete(s.shardedStorage[*ch.address], ch.kvIdx) + } else { + s.shardedStorage[*ch.address][ch.kvIdx] = ch.prevBytes + } +} diff --git a/core/state/statedb.go b/core/state/statedb.go index 4ff2c3e48da0..bb45eec62a96 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,6 +18,7 @@ package state import ( + "bytes" "errors" "fmt" "math/big" @@ -79,6 +80,9 @@ type StateDB struct { stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution + // This map holds sharded KV puts + shardedStorage map[common.Address]map[uint64][]byte + // DB error. // State objects are used by the consensus core and VM which are // unable to deal with database-level errors. Any error that occurs @@ -143,6 +147,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) journal: newJournal(), accessList: newAccessList(), hasher: crypto.NewKeccakState(), + shardedStorage: make(map[common.Address]map[uint64][]byte), } if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { @@ -154,6 +159,46 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) return sdb, nil } +func (s *StateDB) SstorageMaxKVSize(addr common.Address) uint64 { + return s.db.TrieDB().SstorageMaxKVSize(addr) +} + +func (s *StateDB) SstorageWrite(addr common.Address, kvIdx uint64, data []byte) error { + if len(data) > int(s.SstorageMaxKVSize(addr)) { + return fmt.Errorf("put too large") + } + + if _, ok := s.shardedStorage[addr]; !ok { + s.shardedStorage[addr] = make(map[uint64][]byte) + } + + s.journal.append(sstorageChange{ + address: &addr, + prevBytes: s.shardedStorage[addr][kvIdx], + kvIdx: kvIdx, + }) + // Assume data is immutable + s.shardedStorage[addr][kvIdx] = data + return nil +} + +func (s *StateDB) SstorageRead(addr common.Address, kvIdx uint64, readLen int) ([]byte, bool, error) { + if readLen > int(s.SstorageMaxKVSize(addr)) { + return nil, false, fmt.Errorf("readLen too large") + } + + if m, ok0 := s.shardedStorage[addr]; ok0 { + if b, ok1 := m[kvIdx]; ok1 { + if readLen > len(b) { + return append(b, bytes.Repeat([]byte{0}, readLen-len(b))...), true, nil + } + return b[0:readLen], true, nil + } + } + + return s.db.TrieDB().SstorageRead(addr, kvIdx, readLen) +} + // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. @@ -657,6 +702,7 @@ func (s *StateDB) Copy() *StateDB { preimages: make(map[common.Hash][]byte, len(s.preimages)), journal: newJournal(), hasher: crypto.NewKeccakState(), + shardedStorage: make(map[common.Address]map[uint64][]byte), } // Copy the dirty states, logs, and preimages for addr := range s.journal.dirties { @@ -738,6 +784,12 @@ func (s *StateDB) Copy() *StateDB { state.snapStorage[k] = temp } } + for addr, m := range s.shardedStorage { + state.shardedStorage[addr] = make(map[uint64][]byte) + for k, v := range m { + state.shardedStorage[addr][k] = v + } + } return state } @@ -981,6 +1033,18 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } + // Write the sharded storage changes to the underlying trie db + // This assumes no fork (with instant finality consensus) + triedb := s.db.TrieDB() + for addr, m := range s.shardedStorage { + for k, v := range m { + err := triedb.SstorageWrite(addr, k, v) + if err != nil { + log.Crit("failed to flush sstorage data", "err", err) + } + } + } + s.shardedStorage = make(map[common.Address]map[uint64][]byte) return root, err } diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 9210f5486c57..1215c51035f7 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -17,9 +17,12 @@ package vm import ( + "bytes" "crypto/sha256" "encoding/binary" + "encoding/hex" "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -42,6 +45,15 @@ type PrecompiledContract interface { Run(input []byte) ([]byte, error) // Run runs the precompiled contract } +type PrecompiledContractCallEnv struct { + evm *EVM + caller ContractRef +} + +type PrecompiledContractWithEVM interface { + RunWith(env *PrecompiledContractCallEnv, input []byte) ([]byte, error) +} + // PrecompiledContractsHomestead contains the default set of pre-compiled Ethereum // contracts used in the Frontier and Homestead releases. var PrecompiledContractsHomestead = map[common.Address]PrecompiledContract{ @@ -92,6 +104,22 @@ var PrecompiledContractsBerlin = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{9}): &blake2F{}, } +// PrecompiledContractsPisa contains the default set of pre-compiled Ethereum +// contracts used in the Berlin release. +var PrecompiledContractsPisa = map[common.Address]PrecompiledContract{ + common.BytesToAddress([]byte{1}): &ecrecover{}, + common.BytesToAddress([]byte{2}): &sha256hash{}, + common.BytesToAddress([]byte{3}): &ripemd160hash{}, + common.BytesToAddress([]byte{4}): &dataCopy{}, + common.BytesToAddress([]byte{5}): &bigModExp{eip2565: true}, + common.BytesToAddress([]byte{6}): &bn256AddIstanbul{}, + common.BytesToAddress([]byte{7}): &bn256ScalarMulIstanbul{}, + common.BytesToAddress([]byte{8}): &bn256PairingIstanbul{}, + common.BytesToAddress([]byte{9}): &blake2F{}, + common.BytesToAddress([]byte{3, 0x33, 1}): &systemContractDeployer{}, + common.BytesToAddress([]byte{3, 0x33, 2}): &sstoragePisa{}, +} + // PrecompiledContractsBLS contains the set of pre-compiled Ethereum // contracts specified in EIP-2537. These are exported for testing purposes. var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{ @@ -147,14 +175,18 @@ func ActivePrecompiles(rules params.Rules) []common.Address { // - the returned bytes, // - the _remaining_ gas, // - any error that occurred -func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uint64) (ret []byte, remainingGas uint64, err error) { +func RunPrecompiledContract(env *PrecompiledContractCallEnv, p PrecompiledContract, input []byte, suppliedGas uint64) (ret []byte, remainingGas uint64, err error) { gasCost := p.RequiredGas(input) if suppliedGas < gasCost { return nil, 0, ErrOutOfGas } suppliedGas -= gasCost - output, err := p.Run(input) - return output, suppliedGas, err + if pw, ok := p.(PrecompiledContractWithEVM); ok { + ret, err = pw.RunWith(env, input) + } else { + ret, err = p.Run(input) + } + return ret, suppliedGas, err } // ECRECOVER implemented as a native contract. @@ -619,6 +651,134 @@ func (c *blake2F) Run(input []byte) ([]byte, error) { return output, nil } +type systemContractDeployer struct{} + +var ( + // contract at 0x0000000000000000000000000000000003330001 is complied DecentralizedKV(0x33302, 4096, 1652641001, 1000000000000000, 340282365784068676928457747575078800565)+3b2d31+0.8.13 solc + systemContracts = map[common.Address][]byte{ + common.HexToAddress("0x0000000000000000000000000000000003330001"): common.Hex2Bytes("6080604052600436106100dd5760003560e01c8063749cf2821161007f5780639cf001fe116100595780639cf001fe146102c6578063a097365f14610312578063a4a8435e14610346578063afd5644d1461037a57600080fd5b8063749cf2821461024557806378e979251461027257806395bc2673146102a657600080fd5b8063429dd7ad116100bb578063429dd7ad1461016e57806344e77d99146101a257806349bdd6f5146101b757806373e8b3d4146101d757600080fd5b80631ccbc6da146100e2578063258ae5821461010a5780633cb2fecc1461013a575b600080fd5b3480156100ee57600080fd5b506100f76103dc565b6040519081526020015b60405180910390f35b34801561011657600080fd5b5061012a610125366004610e28565b6103ec565b6040519015158152602001610101565b34801561014657600080fd5b506100f77f00000000000000000000000000000000000000000000000000038d7ea4c6800081565b34801561017a57600080fd5b5060005461018c9064ffffffffff1681565b60405164ffffffffff9091168152602001610101565b6101b56101b0366004610e28565b6104ed565b005b3480156101c357600080fd5b506101b56101d2366004610eb2565b6107f1565b3480156101e357600080fd5b5061012a6101f2366004610eee565b60408051336020808301919091528183019390935281518082038301815260609091018252805190830120600090815260019092529081902054600160401b9004901b67ffffffffffffffff1916151590565b34801561025157600080fd5b50610265610260366004610f07565b610aa1565b6040516101019190610f8f565b34801561027e57600080fd5b506100f77f0000000000000000000000000000000000000000000000000000000062814ce981565b3480156102b257600080fd5b506101b56102c1366004610eee565b610ccf565b3480156102d257600080fd5b506102fa7f000000000000000000000000000000000000000000000000000000000003330281565b6040516001600160a01b039091168152602001610101565b34801561031e57600080fd5b506100f77f000000000000000000000000000000000000000000000000000000000000100081565b34801561035257600080fd5b506100f77f00000000000000000000000000000000fffffff1a6935a84491b53a3b65e4cb581565b34801561038657600080fd5b506100f7610395366004610eee565b6040805133602080830191909152818301939093528151808203830181526060909101825280519083012060009081526001909252902054600160281b900462ffffff1690565b60006103e742610cdc565b905090565b60408051336020820152908101839052600090819060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff81168652600160281b810462ffffff1693860193909352600160401b909204831b67ffffffffffffffff19169284018390529350036104aa5760405162461bcd60e51b815260206004820152600c60248201526b1add881b9bdd08195e1a5cdd60a21b60448201526064015b60405180910390fd5b8351816020015162ffffff16146104c6576000925050506104e7565b8351602085012060409091015167ffffffffffffffff199081169116149150505b92915050565b7f00000000000000000000000000000000000000000000000000000000000010008151111561054f5760405162461bcd60e51b815260206004820152600e60248201526d6461746120746f6f206c6172676560901b60448201526064016104a1565b6040805133602082015290810183905260009060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff81168652600160281b810462ffffff1693860193909352600160401b909204831b67ffffffffffffffff1916928401839052935003610674576105d76103dc565b34101561061b5760405162461bcd60e51b81526020600482015260126024820152711b9bdd08195b9bdd59da081c185e5b595b9d60721b60448201526064016104a1565b6000805464ffffffffff908116808452855162ffffff166020808601919091529083526002905260408220849055905461065791166001610fb8565b6000805464ffffffffff191664ffffffffff929092169190911790555b825160208085019190912067ffffffffffffffff19908116604080850191825260008681526001855281812086518154968801519451841c600160401b0262ffffff909516600160281b029690951664ffffffffff8616179590951767ffffffffffffffff169290921790935591516001600160a01b037f000000000000000000000000000000000000000000000000000000000003330216916304fb033960e41b9161072691908890602401610fe1565b60408051601f198184030181529181526020820180516001600160e01b03166001600160e01b03199094169390931790925290516107649190611009565b6000604051808303816000865af19150503d80600081146107a1576040519150601f19603f3d011682016040523d82523d6000602084013e6107a6565b606091505b50509050806107ea5760405162461bcd60e51b815260206004820152601060248201526f6661696c656420746f2070757452617760801b60448201526064016104a1565b5050505050565b6040805133602082015290810183905260009060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff8116808752600160281b820462ffffff1694870194909452600160401b9004841b67ffffffffffffffff1916938501849052909450909190036108ac5760405162461bcd60e51b815260206004820152600c60248201526b1add881b9bdd08195e1a5cdd60a21b60448201526064016104a1565b6040805160608101825260008082526020808301828152838501838152888452600192839052858420945185549251915190961c600160401b0262ffffff91909116600160281b0267ffffffffffffffff199290921664ffffffffff968716179190911767ffffffffffffffff16179092558054909260029284926109319216611025565b64ffffffffff908116825260208083019390935260409182016000908120548683168083526002808752858420839055828452600196879052948320805464ffffffffff1916909117905581549095509093849261098f9216611025565b64ffffffffff9081168252602082019290925260400160009081209290925590546109bd9160019116611025565b6000805464ffffffffff191664ffffffffff928316908117909155604051633625b3bb60e11b8152600481019190915290831660248201527f00000000000000000000000000000000000000000000000000000000000333026001600160a01b031690636c4b677690604401600060405180830381600087803b158015610a4357600080fd5b505af1158015610a57573d6000803e3d6000fd5b50505050846001600160a01b03166108fc610a706103dc565b6040518115909202916000818181858888f19350505050158015610a98573d6000803e3d6000fd5b50505050505050565b606081600003610ac05750604080516000815260208101909152610cc8565b6040805133602082015290810185905260009060600160408051808303601f1901815282825280516020918201206000818152600183528390206060850184525464ffffffffff81168552600160281b810462ffffff16928501839052600160401b9004831b67ffffffffffffffff1916928401929092529092508510610b595750506040805160008152602081019091529050610cc8565b602081015162ffffff16610b6d868661104b565b1115610b8b5784816020015162ffffff16610b889190611063565b93505b6040818101518251825167ffffffffffffffff19909216602483015264ffffffffff1660448201526064810187905260848082018790528251808303909101815260a490910182526020810180516001600160e01b031663f835367f60e01b179052905160009182917f00000000000000000000000000000000000000000000000000000000000333026001600160a01b031691610c2891611009565b600060405180830381855afa9150503d8060008114610c63576040519150601f19603f3d011682016040523d82523d6000602084013e610c68565b606091505b509150915081610cad5760405162461bcd60e51b815260206004820152601060248201526f6661696c656420746f2067657452617760801b60448201526064016104a1565b80806020019051810190610cc1919061107a565b9450505050505b9392505050565b610cd981336107f1565b50565b60006104e77f00000000000000000000000000000000000000000000000000038d7ea4c68000610d2c7f0000000000000000000000000000000000000000000000000000000062814ce985611063565b60006080610d5a7f00000000000000000000000000000000fffffff1a6935a84491b53a3b65e4cb584610d6d565b610d6490856110f1565b901c9392505050565b6000600160801b5b8215610cc85782600116600103610d97576080610d9285836110f1565b901c90505b6080610da385806110f1565b901c9350610db2600284611110565b9250610d75565b634e487b7160e01b600052604160045260246000fd5b604051601f8201601f1916810167ffffffffffffffff81118282101715610df857610df8610db9565b604052919050565b600067ffffffffffffffff821115610e1a57610e1a610db9565b50601f01601f191660200190565b60008060408385031215610e3b57600080fd5b82359150602083013567ffffffffffffffff811115610e5957600080fd5b8301601f81018513610e6a57600080fd5b8035610e7d610e7882610e00565b610dcf565b818152866020838501011115610e9257600080fd5b816020840160208301376000602083830101528093505050509250929050565b60008060408385031215610ec557600080fd5b8235915060208301356001600160a01b0381168114610ee357600080fd5b809150509250929050565b600060208284031215610f0057600080fd5b5035919050565b600080600060608486031215610f1c57600080fd5b505081359360208301359350604090920135919050565b60005b83811015610f4e578181015183820152602001610f36565b83811115610f5d576000848401525b50505050565b60008151808452610f7b816020860160208601610f33565b601f01601f19169290920160200192915050565b602081526000610cc86020830184610f63565b634e487b7160e01b600052601160045260246000fd5b600064ffffffffff808316818516808303821115610fd857610fd8610fa2565b01949350505050565b64ffffffffff831681526040602082015260006110016040830184610f63565b949350505050565b6000825161101b818460208701610f33565b9190910192915050565b600064ffffffffff8381169083168181101561104357611043610fa2565b039392505050565b6000821982111561105e5761105e610fa2565b500190565b60008282101561107557611075610fa2565b500390565b60006020828403121561108c57600080fd5b815167ffffffffffffffff8111156110a357600080fd5b8201601f810184136110b457600080fd5b80516110c2610e7882610e00565b8181528560208385010111156110d757600080fd5b6110e8826020830160208601610f33565b95945050505050565b600081600019048311821515161561110b5761110b610fa2565b500290565b60008261112d57634e487b7160e01b600052601260045260246000fd5b50049056fea2646970667358221220a681d6af7f28ec6eaae1a368ab315d95af7a70e728eae30bf889df807a79f36d64736f6c634300080d0033"), + // debug code with putRaw() and getRaw() direclty. + // common.HexToAddress("0x0000000000000000000000000000000003330001"): common.Hex2Bytes("6080604052600436106100fe5760003560e01c806373e8b3d4116100955780639cf001fe116100645780639cf001fe14610327578063a097365f14610373578063a4a8435e146103a7578063afd5644d146103db578063bc5c77be1461043d57600080fd5b806373e8b3d414610245578063749cf282146102b357806378e97925146102d357806395bc26731461030757600080fd5b8063429dd7ad116100d1578063429dd7ad146101bc57806344e77d99146101f057806349bdd6f5146102055780634fb033901461022557600080fd5b80631ccbc6da14610103578063258ae5821461012b57806333b073391461015b5780633cb2fecc14610188575b600080fd5b34801561010f57600080fd5b5061011861045d565b6040519081526020015b60405180910390f35b34801561013757600080fd5b5061014b6101463660046111e3565b61046d565b6040519015158152602001610122565b34801561016757600080fd5b5061017b61017636600461122a565b61056e565b60405161012291906112a8565b34801561019457600080fd5b506101187f00000000000000000000000000000000000000000000000000038d7ea4c6800081565b3480156101c857600080fd5b506000546101da9064ffffffffff1681565b60405164ffffffffff9091168152602001610122565b6102036101fe3660046111e3565b61069a565b005b34801561021157600080fd5b506102036102203660046112bb565b610980565b34801561023157600080fd5b506102036102403660046111e3565b610c30565b34801561025157600080fd5b5061014b6102603660046112f7565b60408051336020808301919091528183019390935281518082038301815260609091018252805190830120600090815260019092529081902054600160401b9004901b67ffffffffffffffff1916151590565b3480156102bf57600080fd5b5061017b6102ce366004611310565b610d1d565b3480156102df57600080fd5b506101187f0000000000000000000000000000000000000000000000000000000062814ce981565b34801561031357600080fd5b506102036103223660046112f7565b610f4b565b34801561033357600080fd5b5061035b7f000000000000000000000000000000000000000000000000000000000003330281565b6040516001600160a01b039091168152602001610122565b34801561037f57600080fd5b506101187f000000000000000000000000000000000000000000000000000000000000100081565b3480156103b357600080fd5b506101187f00000000000000000000000000000000fffffff1a6935a84491b53a3b65e4cb581565b3480156103e757600080fd5b506101186103f63660046112f7565b6040805133602080830191909152818301939093528151808203830181526060909101825280519083012060009081526001909252902054600160281b900462ffffff1690565b34801561044957600080fd5b506102036104583660046111e3565b610f58565b600061046842611041565b905090565b60408051336020820152908101839052600090819060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff81168652600160281b810462ffffff1693860193909352600160401b909204831b67ffffffffffffffff191692840183905293500361052b5760405162461bcd60e51b815260206004820152600c60248201526b1add881b9bdd08195e1a5cdd60a21b60448201526064015b60405180910390fd5b8351816020015162ffffff161461054757600092505050610568565b8351602085012060409091015167ffffffffffffffff199081169116149150505b92915050565b60408051600060248201819052604482018590526064820181905260848083018590528351808403909101815260a490920183526020820180516001600160e01b031663f835367f60e01b17905291516060929182917f00000000000000000000000000000000000000000000000000000000000333026001600160a01b0316916105f89161133c565b600060405180830381855afa9150503d8060008114610633576040519150601f19603f3d011682016040523d82523d6000602084013e610638565b606091505b50915091508161067d5760405162461bcd60e51b815260206004820152601060248201526f6661696c656420746f2067657452617760801b6044820152606401610522565b808060200190518101906106919190611358565b95945050505050565b7f0000000000000000000000000000000000000000000000000000000000001000815111156106fc5760405162461bcd60e51b815260206004820152600e60248201526d6461746120746f6f206c6172676560901b6044820152606401610522565b6040805133602082015290810183905260009060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff81168652600160281b810462ffffff1693860193909352600160401b909204831b67ffffffffffffffff19169284018390529350036108215761078461045d565b3410156107c85760405162461bcd60e51b81526020600482015260126024820152711b9bdd08195b9bdd59da081c185e5b595b9d60721b6044820152606401610522565b6000805464ffffffffff908116808452855162ffffff1660208086019190915290835260029052604082208490559054610804911660016113dc565b6000805464ffffffffff191664ffffffffff929092169190911790555b825162ffffff908116602080840191825285518187012067ffffffffffffffff199081166040808701918252600088815260019094528084208751815496519351831c600160401b0293909716600160281b029590931664ffffffffff8716179490941767ffffffffffffffff16179055905190916001600160a01b037f000000000000000000000000000000000000000000000000000000000003330216916304fb033960e41b916108d8918890602401611405565b60408051601f198184030181529181526020820180516001600160e01b03166001600160e01b0319909416939093179092529051610916919061133c565b6000604051808303816000865af19150503d8060008114610953576040519150601f19603f3d011682016040523d82523d6000602084013e610958565b606091505b50509050806109795760405162461bcd60e51b81526004016105229061142d565b5050505050565b6040805133602082015290810183905260009060600160408051808303601f1901815282825280516020918201206000818152600183528381206060860185525464ffffffffff8116808752600160281b820462ffffff1694870194909452600160401b9004841b67ffffffffffffffff191693850184905290945090919003610a3b5760405162461bcd60e51b815260206004820152600c60248201526b1add881b9bdd08195e1a5cdd60a21b6044820152606401610522565b6040805160608101825260008082526020808301828152838501838152888452600192839052858420945185549251915190961c600160401b0262ffffff91909116600160281b0267ffffffffffffffff199290921664ffffffffff968716179190911767ffffffffffffffff1617909255805490926002928492610ac09216611457565b64ffffffffff908116825260208083019390935260409182016000908120548683168083526002808752858420839055828452600196879052948320805464ffffffffff19169091179055815490955090938492610b1e9216611457565b64ffffffffff908116825260208201929092526040016000908120929092559054610b4c9160019116611457565b6000805464ffffffffff191664ffffffffff928316908117909155604051633625b3bb60e11b8152600481019190915290831660248201527f00000000000000000000000000000000000000000000000000000000000333026001600160a01b031690636c4b677690604401600060405180830381600087803b158015610bd257600080fd5b505af1158015610be6573d6000803e3d6000fd5b50505050846001600160a01b03166108fc610bff61045d565b6040518115909202916000818181858888f19350505050158015610c27573d6000803e3d6000fd5b50505050505050565b60007f00000000000000000000000000000000000000000000000000000000000333026001600160a01b0316634fb0339060e01b8484604051602401610c7792919061147d565b60408051601f198184030181529181526020820180516001600160e01b03166001600160e01b0319909416939093179092529051610cb5919061133c565b6000604051808303816000865af19150503d8060008114610cf2576040519150601f19603f3d011682016040523d82523d6000602084013e610cf7565b606091505b5050905080610d185760405162461bcd60e51b81526004016105229061142d565b505050565b606081600003610d3c5750604080516000815260208101909152610f44565b6040805133602082015290810185905260009060600160408051808303601f1901815282825280516020918201206000818152600183528390206060850184525464ffffffffff81168552600160281b810462ffffff16928501839052600160401b9004831b67ffffffffffffffff1916928401929092529092508510610dd55750506040805160008152602081019091529050610f44565b602081015162ffffff16610de98686611496565b1115610e075784816020015162ffffff16610e0491906114ae565b93505b6040818101518251825167ffffffffffffffff19909216602483015264ffffffffff1660448201526064810187905260848082018790528251808303909101815260a490910182526020810180516001600160e01b031663f835367f60e01b179052905160009182917f00000000000000000000000000000000000000000000000000000000000333026001600160a01b031691610ea49161133c565b600060405180830381855afa9150503d8060008114610edf576040519150601f19603f3d011682016040523d82523d6000602084013e610ee4565b606091505b509150915081610f295760405162461bcd60e51b815260206004820152601060248201526f6661696c656420746f2067657452617760801b6044820152606401610522565b80806020019051810190610f3d9190611358565b9450505050505b9392505050565b610f558133610980565b50565b60007f00000000000000000000000000000000000000000000000000000000000333026001600160a01b0316634fb0339060e01b8484604051602401610f9f92919061147d565b60408051601f198184030181529181526020820180516001600160e01b03166001600160e01b0319909416939093179092529051610fdd919061133c565b6000604051808303816000865af19150503d806000811461101a576040519150601f19603f3d011682016040523d82523d6000602084013e61101f565b606091505b505090508015610d185760405162461bcd60e51b81526004016105229061142d565b60006105687f00000000000000000000000000000000000000000000000000038d7ea4c680006110917f0000000000000000000000000000000000000000000000000000000062814ce9856114ae565b600060806110bf7f00000000000000000000000000000000fffffff1a6935a84491b53a3b65e4cb5846110d2565b6110c990856114c5565b901c9392505050565b6000600160801b5b8215610f4457826001166001036110fc5760806110f785836114c5565b901c90505b608061110885806114c5565b901c93506111176002846114e4565b92506110da565b634e487b7160e01b600052604160045260246000fd5b604051601f8201601f1916810167ffffffffffffffff8111828210171561115d5761115d61111e565b604052919050565b600067ffffffffffffffff82111561117f5761117f61111e565b50601f01601f191660200190565b600082601f83011261119e57600080fd5b81356111b16111ac82611165565b611134565b8181528460208386010111156111c657600080fd5b816020850160208301376000918101602001919091529392505050565b600080604083850312156111f657600080fd5b82359150602083013567ffffffffffffffff81111561121457600080fd5b6112208582860161118d565b9150509250929050565b6000806040838503121561123d57600080fd5b50508035926020909101359150565b60005b8381101561126757818101518382015260200161124f565b83811115611276576000848401525b50505050565b6000815180845261129481602086016020860161124c565b601f01601f19169290920160200192915050565b602081526000610f44602083018461127c565b600080604083850312156112ce57600080fd5b8235915060208301356001600160a01b03811681146112ec57600080fd5b809150509250929050565b60006020828403121561130957600080fd5b5035919050565b60008060006060848603121561132557600080fd5b505081359360208301359350604090920135919050565b6000825161134e81846020870161124c565b9190910192915050565b60006020828403121561136a57600080fd5b815167ffffffffffffffff81111561138157600080fd5b8201601f8101841361139257600080fd5b80516113a06111ac82611165565b8181528560208385010111156113b557600080fd5b61069182602083016020860161124c565b634e487b7160e01b600052601160045260246000fd5b600064ffffffffff8083168185168083038211156113fc576113fc6113c6565b01949350505050565b64ffffffffff83168152604060208201526000611425604083018461127c565b949350505050565b60208082526010908201526f6661696c656420746f2070757452617760801b604082015260600190565b600064ffffffffff83811690831681811015611475576114756113c6565b039392505050565b828152604060208201526000611425604083018461127c565b600082198211156114a9576114a96113c6565b500190565b6000828210156114c0576114c06113c6565b500390565b60008160001904831182151516156114df576114df6113c6565b500290565b60008261150157634e487b7160e01b600052601260045260246000fd5b50049056fea26469706673582212200938bd543a1cf6d60f50cbadfe7df73030e2c6803e6091684535c570e4a31c0c64736f6c634300080d0033"), + } +) + +func (l *systemContractDeployer) RequiredGas(input []byte) uint64 { + if len(input) < 32 { + return 0 + } + + addr := common.BytesToAddress(input[0:32]) + if b, ok := systemContracts[addr]; ok { + return uint64(len(b)) / params.CreateDataGas + } else { + return 0 + } +} + +func (l *systemContractDeployer) Run(input []byte) ([]byte, error) { + panic("not supported") +} + +func (l *systemContractDeployer) RunWith(env *PrecompiledContractCallEnv, input []byte) ([]byte, error) { + if len(input) < 32 { + return nil, errors.New("invalid input length") + } + + evm := env.evm + addr := common.BytesToAddress(input[0:32]) + if b, ok := systemContracts[addr]; ok { + if !evm.StateDB.Exist(addr) { + evm.StateDB.CreateAccount(addr) + } + // allow override to upgrade the contract + evm.StateDB.SetCode(addr, b) + return nil, nil + } else { + return nil, errors.New("contract not found") + } +} + +var ( + putRawMethodId, _ = hex.DecodeString("4fb03390") // putRaw(uint256,bytes) + getRawMethodId, _ = hex.DecodeString("f835367f") // getRaw(bytes32,uint256,uint256,uint256) + removeRawMethodId, _ = hex.DecodeString("6c4b6776") // removeRaw(uint256,uint256) +) + +type sstoragePisa struct{} + +func (l *sstoragePisa) RequiredGas(input []byte) uint64 { + if len(input) < 4 { + return 0 + } + + if bytes.Equal(input[0:4], putRawMethodId) { + return params.SstoreResetGasEIP2200 + } else if bytes.Equal(input[0:4], getRawMethodId) { + return params.SloadGasEIP2200 + } else { + // TODO: remove is not supported yet + return 0 + } +} + +func (l *sstoragePisa) Run(input []byte) ([]byte, error) { + panic("not supported") +} + +func (l *sstoragePisa) RunWith(env *PrecompiledContractCallEnv, input []byte) ([]byte, error) { + if len(input) < 4 { + return nil, errors.New("invalid input length") + } + + evm := env.evm + caller := env.caller.Address() + maxKVSize := evm.StateDB.SstorageMaxKVSize(caller) + if maxKVSize == 0 { + return nil, errors.New("invalid caller") + } + + if bytes.Equal(input[0:4], putRawMethodId) { + if evm.interpreter.readOnly { + return nil, ErrWriteProtection + } + kvIdx := new(big.Int).SetBytes(getData(input, 4, 32)).Uint64() + dataPtr := new(big.Int).SetBytes(getData(input, 4+32, 32)).Uint64() + if 4+dataPtr > uint64(len(input)) { + return nil, errors.New("dataptr too large") + } + putLen := new(big.Int).SetBytes(getData(input, 4+dataPtr, 32)).Uint64() + + if putLen > maxKVSize { + return nil, errors.New("put len too large") + } + evm.StateDB.SstorageWrite(caller, kvIdx, getData(input, 4+dataPtr+32, putLen)) + return nil, nil + } else if bytes.Equal(input[0:4], getRawMethodId) { + if !evm.Config.IsJsonRpc { + return nil, errors.New("getRaw() must be called in JSON RPC") + } + // TODO: check hash correctness + // hash := new(big.Int).SetBytes(getData(input, 4, 4+32)) + kvIdx := new(big.Int).SetBytes(getData(input, 4+32, 32)).Uint64() + kvOff := new(big.Int).SetBytes(getData(input, 4+64, 32)).Uint64() + kvLen := new(big.Int).SetBytes(getData(input, 4+96, 32)).Uint64() + fb, ok, err := evm.StateDB.SstorageRead(caller, kvIdx, int(kvLen+kvOff)) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("shard data not found: %s, %d", common.Bytes2Hex(env.caller.Address().Bytes()), kvIdx) + } + b := fb[kvOff:] + pb := make([]byte, 64) + binary.BigEndian.PutUint64(pb[32-8:32], 32) + binary.BigEndian.PutUint64(pb[64-8:64], uint64(len(b))) + return append(pb, b...), nil + } + // TODO: remove is not supported yet + return nil, errors.New("unsupported method") +} + var ( errBLS12381InvalidInputLength = errors.New("invalid input length") errBLS12381InvalidFieldElementTopBytes = errors.New("invalid field element top bytes") diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 30d9b49f719b..e2450fe4644c 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -96,7 +96,7 @@ func testPrecompiled(addr string, test precompiledTest, t *testing.T) { in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - if res, _, err := RunPrecompiledContract(p, in, gas); err != nil { + if res, _, err := RunPrecompiledContract(nil, p, in, gas); err != nil { t.Error(err) } else if common.Bytes2Hex(res) != test.Expected { t.Errorf("Expected %v, got %v", test.Expected, common.Bytes2Hex(res)) @@ -118,7 +118,7 @@ func testPrecompiledOOG(addr string, test precompiledTest, t *testing.T) { gas := p.RequiredGas(in) - 1 t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - _, _, err := RunPrecompiledContract(p, in, gas) + _, _, err := RunPrecompiledContract(nil, p, in, gas) if err.Error() != "out of gas" { t.Errorf("Expected error [out of gas], got [%v]", err) } @@ -135,7 +135,7 @@ func testPrecompiledFailure(addr string, test precompiledFailureTest, t *testing in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(test.Name, func(t *testing.T) { - _, _, err := RunPrecompiledContract(p, in, gas) + _, _, err := RunPrecompiledContract(nil, p, in, gas) if err.Error() != test.ExpectedError { t.Errorf("Expected error [%v], got [%v]", test.ExpectedError, err) } @@ -167,7 +167,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { bench.ResetTimer() for i := 0; i < bench.N; i++ { copy(data, in) - res, _, err = RunPrecompiledContract(p, data, reqGas) + res, _, err = RunPrecompiledContract(nil, p, data, reqGas) } bench.StopTimer() elapsed := uint64(time.Since(start)) diff --git a/core/vm/evm.go b/core/vm/evm.go index 8af230160af5..02336b579cc5 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/sstorage" "github.com/holiman/uint256" ) @@ -44,6 +45,8 @@ type ( func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) { var precompiles map[common.Address]PrecompiledContract switch { + case evm.chainRules.IsPisa: + precompiles = PrecompiledContractsPisa case evm.chainRules.IsBerlin: precompiles = PrecompiledContractsBerlin case evm.chainRules.IsIstanbul: @@ -78,6 +81,10 @@ type BlockContext struct { Random *common.Hash // Provides information for RANDOM } +type SstorageContext struct { + ContractToShardManager map[common.Address]*sstorage.ShardManager +} + // TxContext provides the EVM with information about a transaction. // All fields can change between transactions. type TxContext struct { @@ -212,7 +219,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas } if isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas) + ret, gas, err = RunPrecompiledContract(&PrecompiledContractCallEnv{evm, caller}, p, input, gas) } else { // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. @@ -279,7 +286,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas) + ret, gas, err = RunPrecompiledContract(&PrecompiledContractCallEnv{evm, caller}, p, input, gas) } else { addrCopy := addr // Initialise a new contract and set the code that is to be used by the EVM. @@ -324,7 +331,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas) + ret, gas, err = RunPrecompiledContract(&PrecompiledContractCallEnv{evm, caller}, p, input, gas) } else { addrCopy := addr // Initialise a new contract and make initialise the delegate values @@ -377,7 +384,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte } if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas) + ret, gas, err = RunPrecompiledContract(&PrecompiledContractCallEnv{evm, caller}, p, input, gas) } else { // At this point, we use a copy of address. If we don't, the go compiler will // leak the 'contract' to the outer scope, and make allocation for 'contract' diff --git a/core/vm/interface.go b/core/vm/interface.go index ad9b05d666a8..5ab7dcdbf308 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -74,6 +74,10 @@ type StateDB interface { AddPreimage(common.Hash, []byte) ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error + + SstorageMaxKVSize(common.Address) uint64 // 0 means not exist + SstorageWrite(common.Address, uint64, []byte) error // following the same interface as ShardManager.TryWrite() + SstorageRead(common.Address, uint64, int) ([]byte, bool, error) // following the same interface as ShardManager.TryRead() } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 21e3c914e139..df61980695fc 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -34,6 +34,7 @@ type Config struct { JumpTable *JumpTable // EVM instruction table, automatically populated if unset ExtraEips []int // Additional EIPS that are to be enabled + IsJsonRpc bool // Whether the call is in context of JsonRpc } // ScopeContext contains the things that are per-call, such as stack and memory, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 35152d7cb4ba..2565b3e417ea 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -217,6 +217,10 @@ type Config struct { ValContract string ValChainId uint64 ValidatorChangeEpochId uint64 + + // Sstorage config + SstorageFiles []string `toml:",omitempty"` + SstorageShards []string `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 77131ba0e0a4..1bcc939241be 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -916,7 +916,7 @@ func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash if err != nil { return nil, err } - evm, vmError, err := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}) + evm, vmError, err := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true, IsJsonRpc: true}) if err != nil { return nil, err } diff --git a/sstorage/data_shard.go b/sstorage/data_shard.go index c44df40da12e..1d5d0858552a 100644 --- a/sstorage/data_shard.go +++ b/sstorage/data_shard.go @@ -70,6 +70,9 @@ func (ds *DataShard) ReadUnmasked(kvIdx uint64, readLen int) ([]byte, error) { if !ds.Contains(kvIdx) { return nil, fmt.Errorf("kv not found") } + if readLen > int(ds.kvSize) { + return nil, fmt.Errorf("read len too large") + } var data []byte for i := uint64(0); i < ds.chunksPerKv; i++ { if readLen == 0 { diff --git a/sstorage/shard_config.go b/sstorage/shard_config.go index 2c31869a5580..abac935d0ddc 100644 --- a/sstorage/shard_config.go +++ b/sstorage/shard_config.go @@ -19,7 +19,7 @@ type ShardInfo struct { // TODO: move to chain specific config? var ShardInfos = []*ShardInfo{ - {common.HexToAddress("0x1234"), 4 * 1024, 32 * 1024 * 1024}, + {common.HexToAddress("0x0000000000000000000000000000000003330001"), 4 * 1024, 256 * 1024}, } func InitializeConfig() { @@ -38,7 +38,7 @@ func findShardManaager(kvSize uint64) *ShardManager { } func parseKvSize(s string) (uint64, error) { - if s[len(s)] == 'k' || s[len(s)] == 'K' { + if s[len(s)-1] == 'k' || s[len(s)-1] == 'K' { if v, err := strconv.Atoi(s[0 : len(s)-1]); err != nil { return 0, err } else { diff --git a/trie/database.go b/trie/database.go index 58ca4e6f3caa..4f7f37894424 100644 --- a/trie/database.go +++ b/trie/database.go @@ -17,6 +17,7 @@ package trie import ( + "bytes" "errors" "fmt" "io" @@ -32,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/sstorage" ) var ( @@ -89,6 +91,57 @@ type Database struct { preimagesSize common.StorageSize // Storage size of the preimages cache lock sync.RWMutex + + // This map holds sharded KV puts + shardedStorage map[common.Address]map[uint64][]byte + contractToShardManager map[common.Address]*sstorage.ShardManager +} + +func (s *Database) SstorageMaxKVSize(addr common.Address) uint64 { + if sm, ok := s.contractToShardManager[addr]; !ok { + return 0 + } else { + return sm.MaxKvSize() + } +} + +func (s *Database) SstorageWrite(addr common.Address, kvIdx uint64, data []byte) error { + s.lock.Lock() + defer s.lock.Unlock() + + if len(data) > int(s.SstorageMaxKVSize(addr)) { + return fmt.Errorf("put too large") + } + + if _, ok := s.shardedStorage[addr]; !ok { + s.shardedStorage[addr] = make(map[uint64][]byte) + } + // Assume data is immutable. + s.shardedStorage[addr][kvIdx] = data + return nil +} + +func (s *Database) SstorageRead(addr common.Address, kvIdx uint64, readLen int) ([]byte, bool, error) { + s.lock.RLock() + defer s.lock.RUnlock() + + if readLen > int(s.SstorageMaxKVSize(addr)) { + return nil, false, fmt.Errorf("readLen too large") + } + + if m, ok0 := s.shardedStorage[addr]; ok0 { + if b, ok1 := m[kvIdx]; ok1 { + if readLen > len(b) { + return append(b, bytes.Repeat([]byte{0}, readLen-len(b))...), true, nil + } + return b[0:readLen], true, nil + } + } + + if s, ok0 := s.contractToShardManager[addr]; ok0 { + return s.TryRead(kvIdx, readLen) + } + return nil, false, nil } // rawNode is a simple binary blob used to differentiate between collapsed trie @@ -304,6 +357,8 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, + contractToShardManager: sstorage.ContractToShardManager, + shardedStorage: make(map[common.Address]map[uint64][]byte), } if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future db.preimages = make(map[common.Hash][]byte) @@ -723,6 +778,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H log.Error("Failed to write trie to disk", "err", err) return err } + // Uncache any leftovers in the last batch db.lock.Lock() defer db.lock.Unlock() @@ -730,6 +786,17 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Replay(uncacher) batch.Reset() + for addr, m := range db.shardedStorage { + sm := db.contractToShardManager[addr] + for kvIdx, b := range m { + _, err := sm.TryWrite(kvIdx, b) + if err != nil { + log.Error("Failed to write sstorage", "kvIdx", kvIdx, "err", err) + } + } + } + db.shardedStorage = make(map[common.Address]map[uint64][]byte) + // Reset the storage counters and bumped metrics if db.preimages != nil { db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0