op-program: Record the kv format used. (#11900)

Automatically use the correct format if it has been recorded.

Change the default format to directory. Compatibility with op-challenger is preserved because it now uses the automatic format detection, defaulting to file if not specified (e.g for kona-host).

op-challenger/game/fault/trace/asterisc/provider.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/vm"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
 	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
+	kvtypes "github.com/ethereum-optimism/optimism/op-program/host/types"
 	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -44,8 +45,8 @@ func NewTraceProvider(logger log.Logger, m vm.Metricer, cfg vm.Config, vmCfg vm.
 		prestate:  asteriscPrestate,
 		generator: vm.NewExecutor(logger, m, cfg, vmCfg, asteriscPrestate, localInputs),
 		gameDepth: gameDepth,
-		preimageLoader: utils.NewPreimageLoader(func() utils.PreimageSource {
-			return kvstore.NewFileKV(vm.PreimageDir(dir))
+		preimageLoader: utils.NewPreimageLoader(func() (utils.PreimageSource, error) {
+			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
 		PrestateProvider: prestateProvider,
 		stateConverter:   NewStateConverter(),
@@ -169,8 +170,8 @@ func NewTraceProviderForTest(logger log.Logger, m vm.Metricer, cfg *config.Confi
 		prestate:  cfg.AsteriscAbsolutePreState,
 		generator: vm.NewExecutor(logger, m, cfg.Asterisc, vm.NewOpProgramServerExecutor(), cfg.AsteriscAbsolutePreState, localInputs),
 		gameDepth: gameDepth,
-		preimageLoader: utils.NewPreimageLoader(func() utils.PreimageSource {
-			return kvstore.NewFileKV(vm.PreimageDir(dir))
+		preimageLoader: utils.NewPreimageLoader(func() (utils.PreimageSource, error) {
+			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
 		stateConverter: NewStateConverter(),
 		cfg:            cfg.Asterisc,

op-challenger/game/fault/trace/cannon/provider.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 
+	kvtypes "github.com/ethereum-optimism/optimism/op-program/host/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 
@@ -45,8 +46,8 @@ func NewTraceProvider(logger log.Logger, m vm.Metricer, cfg vm.Config, vmCfg vm.
 		prestate:  prestate,
 		generator: vm.NewExecutor(logger, m, cfg, vmCfg, prestate, localInputs),
 		gameDepth: gameDepth,
-		preimageLoader: utils.NewPreimageLoader(func() utils.PreimageSource {
-			return kvstore.NewFileKV(vm.PreimageDir(dir))
+		preimageLoader: utils.NewPreimageLoader(func() (utils.PreimageSource, error) {
+			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
 		PrestateProvider: prestateProvider,
 		stateConverter:   &StateConverter{},
@@ -168,8 +169,8 @@ func NewTraceProviderForTest(logger log.Logger, m vm.Metricer, cfg *config.Confi
 		prestate:  cfg.CannonAbsolutePreState,
 		generator: vm.NewExecutor(logger, m, cfg.Cannon, vm.NewOpProgramServerExecutor(), cfg.CannonAbsolutePreState, localInputs),
 		gameDepth: gameDepth,
-		preimageLoader: utils.NewPreimageLoader(func() utils.PreimageSource {
-			return kvstore.NewFileKV(vm.PreimageDir(dir))
+		preimageLoader: utils.NewPreimageLoader(func() (utils.PreimageSource, error) {
+			return kvstore.NewDiskKV(logger, vm.PreimageDir(dir), kvtypes.DataFormatFile)
 		}),
 		stateConverter: NewStateConverter(),
 		cfg:            cfg.Cannon,

op-challenger/game/fault/trace/utils/preimage.go

@@ -31,7 +31,7 @@ type PreimageSource interface {
 	Close() error
 }
 
-type PreimageSourceCreator func() PreimageSource
+type PreimageSourceCreator func() (PreimageSource, error)
 
 type PreimageLoader struct {
 	makeSource PreimageSourceCreator
@@ -61,7 +61,10 @@ func (l *PreimageLoader) loadBlobPreimage(proof *ProofData) (*types.PreimageOrac
 	// The key for a blob field element is a keccak hash of commitment++fieldElementIndex.
 	// First retrieve the preimage of the key as a keccak hash so we have the commitment and required field element
 	inputsKey := preimage.Keccak256Key(proof.OracleKey).PreimageKey()
-	source := l.makeSource()
+	source, err := l.makeSource()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open preimage store: %w", err)
+	}
 	defer source.Close()
 	inputs, err := source.Get(inputsKey)
 	if err != nil {
@@ -111,7 +114,10 @@ func (l *PreimageLoader) loadBlobPreimage(proof *ProofData) (*types.PreimageOrac
 
 func (l *PreimageLoader) loadPrecompilePreimage(proof *ProofData) (*types.PreimageOracleData, error) {
 	inputKey := preimage.Keccak256Key(proof.OracleKey).PreimageKey()
-	source := l.makeSource()
+	source, err := l.makeSource()
+	if err != nil {
+		return nil, fmt.Errorf("failed to open preimage store: %w", err)
+	}
 	defer source.Close()
 	input, err := source.Get(inputKey)
 	if err != nil {

op-challenger/game/fault/trace/utils/preimage_test.go

@@ -21,8 +21,8 @@ import (
 
 func TestPreimageLoader_NoPreimage(t *testing.T) {
 	kv := kvstore.NewMemKV()
-	loader := NewPreimageLoader(func() PreimageSource {
-		return kv
+	loader := NewPreimageLoader(func() (PreimageSource, error) {
+		return kv, nil
 	})
 	actual, err := loader.LoadPreimage(&ProofData{})
 	require.NoError(t, err)
@@ -31,8 +31,8 @@ func TestPreimageLoader_NoPreimage(t *testing.T) {
 
 func TestPreimageLoader_LocalPreimage(t *testing.T) {
 	kv := kvstore.NewMemKV()
-	loader := NewPreimageLoader(func() PreimageSource {
-		return kv
+	loader := NewPreimageLoader(func() (PreimageSource, error) {
+		return kv, nil
 	})
 	proof := &ProofData{
 		OracleKey:    common.Hash{byte(preimage.LocalKeyType), 0xaa, 0xbb}.Bytes(),
@@ -55,8 +55,8 @@ func TestPreimageLoader_SimpleTypes(t *testing.T) {
 		keyType := keyType
 		t.Run(fmt.Sprintf("type-%v", keyType), func(t *testing.T) {
 			kv := kvstore.NewMemKV()
-			loader := NewPreimageLoader(func() PreimageSource {
-				return kv
+			loader := NewPreimageLoader(func() (PreimageSource, error) {
+				return kv, nil
 			})
 			proof := &ProofData{
 				OracleKey:    common.Hash{byte(keyType), 0xaa, 0xbb}.Bytes(),
@@ -99,8 +99,8 @@ func TestPreimageLoader_BlobPreimage(t *testing.T) {
 
 	t.Run("NoKeyPreimage", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		proof := &ProofData{
 			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xaf}.Bytes(),
@@ -113,8 +113,8 @@ func TestPreimageLoader_BlobPreimage(t *testing.T) {
 
 	t.Run("InvalidKeyPreimage", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		proof := &ProofData{
 			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xad}.Bytes(),
@@ -128,8 +128,8 @@ func TestPreimageLoader_BlobPreimage(t *testing.T) {
 
 	t.Run("MissingBlobs", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		proof := &ProofData{
 			OracleKey:    common.Hash{byte(preimage.BlobKeyType), 0xae}.Bytes(),
@@ -143,8 +143,8 @@ func TestPreimageLoader_BlobPreimage(t *testing.T) {
 
 	t.Run("Valid", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		storeBlob(t, kv, gokzg4844.KZGCommitment(commitment), gokzg4844.Blob(blob))
 		actual, err := loader.LoadPreimage(proof)
@@ -178,16 +178,16 @@ func TestPreimageLoader_PrecompilePreimage(t *testing.T) {
 
 	t.Run("NoInputPreimage", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		_, err := loader.LoadPreimage(proof)
 		require.ErrorIs(t, err, kvstore.ErrNotFound)
 	})
 	t.Run("Valid", func(t *testing.T) {
 		kv := kvstore.NewMemKV()
-		loader := NewPreimageLoader(func() PreimageSource {
-			return kv
+		loader := NewPreimageLoader(func() (PreimageSource, error) {
+			return kv, nil
 		})
 		require.NoError(t, kv.Put(preimage.Keccak256Key(proof.OracleKey).PreimageKey(), input))
 		actual, err := loader.LoadPreimage(proof)

op-program/host/config/config.go

@@ -138,7 +138,7 @@ func NewConfig(
 		L2ClaimBlockNumber:  l2ClaimBlockNum,
 		L1RPCKind:           sources.RPCKindStandard,
 		IsCustomChainConfig: isCustomConfig,
-		DataFormat:          types.DataFormatFile,
+		DataFormat:          types.DataFormatDirectory,
 	}
 }
 

op-program/host/flags/flags.go

@@ -40,7 +40,7 @@ var (
 		Name:    "data.format",
 		Usage:   fmt.Sprintf("Format to use for preimage data storage. Available formats: %s", openum.EnumString(types.SupportedDataFormats)),
 		EnvVars: prefixEnvVars("DATA_FORMAT"),
-		Value:   string(types.DataFormatFile),
+		Value:   string(types.DataFormatDirectory),
 	}
 	L2NodeAddr = &cli.StringFlag{
 		Name:    "l2",

op-program/host/host.go

@@ -16,7 +16,6 @@ import (
 	"github.com/ethereum-optimism/optimism/op-program/host/flags"
 	"github.com/ethereum-optimism/optimism/op-program/host/kvstore"
 	"github.com/ethereum-optimism/optimism/op-program/host/prefetcher"
-	"github.com/ethereum-optimism/optimism/op-program/host/types"
 	opservice "github.com/ethereum-optimism/optimism/op-service"
 	"github.com/ethereum-optimism/optimism/op-service/client"
 	"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
@@ -175,20 +174,14 @@ func PreimageServer(ctx context.Context, logger log.Logger, cfg *config.Config,
 		logger.Info("Using in-memory storage")
 		kv = kvstore.NewMemKV()
 	} else {
-		logger.Info("Creating disk storage", "datadir", cfg.DataDir, "format", cfg.DataFormat)
 		if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
 			return fmt.Errorf("creating datadir: %w", err)
 		}
-		switch cfg.DataFormat {
-		case types.DataFormatFile:
-			kv = kvstore.NewFileKV(cfg.DataDir)
-		case types.DataFormatDirectory:
-			kv = kvstore.NewDirectoryKV(cfg.DataDir)
-		case types.DataFormatPebble:
-			kv = kvstore.NewPebbleKV(cfg.DataDir)
-		default:
-			return fmt.Errorf("invalid data format: %s", cfg.DataFormat)
+		store, err := kvstore.NewDiskKV(logger, cfg.DataDir, cfg.DataFormat)
+		if err != nil {
+			return fmt.Errorf("creating kvstore: %w", err)
 		}
+		kv = store
 	}
 
 	var (

op-program/host/kvstore/directory.go

@@ -12,30 +12,30 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-// DirectoryKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
-// DirectoryKV is safe for concurrent use with a single DirectoryKV instance.
-// DirectoryKV is safe for concurrent use between different DirectoryKV instances of the same disk directory as long as the
+// directoryKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
+// directoryKV is safe for concurrent use with a single directoryKV instance.
+// directoryKV is safe for concurrent use between different directoryKV instances of the same disk directory as long as the
 // file system supports atomic renames.
-type DirectoryKV struct {
+type directoryKV struct {
 	sync.RWMutex
 	path string
 }
 
-// NewDirectoryKV creates a DirectoryKV that puts/gets pre-images as files in the given directory path.
+// newDirectoryKV creates a directoryKV that puts/gets pre-images as files in the given directory path.
 // The path must exist, or subsequent Put/Get calls will error when it does not.
-func NewDirectoryKV(path string) *DirectoryKV {
-	return &DirectoryKV{path: path}
+func newDirectoryKV(path string) *directoryKV {
+	return &directoryKV{path: path}
 }
 
 // pathKey returns the file path for the given key.
 // This is composed of the first characters of the non-0x-prefixed hex key as a directory, and the rest as the file name.
-func (d *DirectoryKV) pathKey(k common.Hash) string {
+func (d *directoryKV) pathKey(k common.Hash) string {
 	key := k.String()
 	dir, name := key[2:6], key[6:]
 	return path.Join(d.path, dir, name+".txt")
 }
 
-func (d *DirectoryKV) Put(k common.Hash, v []byte) error {
+func (d *directoryKV) Put(k common.Hash, v []byte) error {
 	d.Lock()
 	defer d.Unlock()
 	f, err := openTempFile(d.path, k.String()+".txt.*")
@@ -61,7 +61,7 @@ func (d *DirectoryKV) Put(k common.Hash, v []byte) error {
 	return nil
 }
 
-func (d *DirectoryKV) Get(k common.Hash) ([]byte, error) {
+func (d *directoryKV) Get(k common.Hash) ([]byte, error) {
 	d.RLock()
 	defer d.RUnlock()
 	f, err := os.OpenFile(d.pathKey(k), os.O_RDONLY, filePermission)
@@ -79,8 +79,8 @@ func (d *DirectoryKV) Get(k common.Hash) ([]byte, error) {
 	return hex.DecodeString(string(dat))
 }
 
-func (d *DirectoryKV) Close() error {
+func (d *directoryKV) Close() error {
 	return nil
 }
 
-var _ KV = (*DirectoryKV)(nil)
+var _ KV = (*directoryKV)(nil)

op-program/host/kvstore/directory_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestDirectoryKV(t *testing.T) {
 	tmp := t.TempDir() // automatically removed by testing cleanup
-	kv := NewDirectoryKV(tmp)
+	kv := newDirectoryKV(tmp)
 	t.Cleanup(func() { // Can't use defer because kvTest runs tests in parallel.
 		require.NoError(t, kv.Close())
 	})
@@ -20,7 +20,7 @@ func TestDirectoryKV(t *testing.T) {
 func TestDirectoryKV_CreateMissingDirectory(t *testing.T) {
 	tmp := t.TempDir()
 	dir := filepath.Join(tmp, "data")
-	kv := NewDirectoryKV(dir)
+	kv := newDirectoryKV(dir)
 	defer kv.Close()
 	val := []byte{1, 2, 3, 4}
 	key := crypto.Keccak256Hash(val)

op-program/host/kvstore/file.go

@@ -15,26 +15,26 @@ import (
 // read/write mode for user/group/other, not executable.
 const filePermission = 0666
 
-// FileKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
-// FileKV is safe for concurrent use with a single FileKV instance.
-// FileKV is safe for concurrent use between different FileKV instances of the same disk directory as long as the
+// fileKV is a disk-backed key-value store, every key-value pair is a hex-encoded .txt file, with the value as content.
+// fileKV is safe for concurrent use with a single fileKV instance.
+// fileKV is safe for concurrent use between different fileKV instances of the same disk directory as long as the
 // file system supports atomic renames.
-type FileKV struct {
+type fileKV struct {
 	sync.RWMutex
 	path string
 }
 
-// NewFileKV creates a FileKV that puts/gets pre-images as files in the given directory path.
+// newFileKV creates a fileKV that puts/gets pre-images as files in the given directory path.
 // The path must exist, or subsequent Put/Get calls will error when it does not.
-func NewFileKV(path string) *FileKV {
-	return &FileKV{path: path}
+func newFileKV(path string) *fileKV {
+	return &fileKV{path: path}
 }
 
-func (d *FileKV) pathKey(k common.Hash) string {
+func (d *fileKV) pathKey(k common.Hash) string {
 	return path.Join(d.path, k.String()+".txt")
 }
 
-func (d *FileKV) Put(k common.Hash, v []byte) error {
+func (d *fileKV) Put(k common.Hash, v []byte) error {
 	d.Lock()
 	defer d.Unlock()
 	f, err := openTempFile(d.path, k.String()+".txt.*")
@@ -72,7 +72,7 @@ func openTempFile(dir string, nameTemplate string) (*os.File, error) {
 	return f, nil
 }
 
-func (d *FileKV) Get(k common.Hash) ([]byte, error) {
+func (d *fileKV) Get(k common.Hash) ([]byte, error) {
 	d.RLock()
 	defer d.RUnlock()
 	f, err := os.OpenFile(d.pathKey(k), os.O_RDONLY, filePermission)
@@ -90,8 +90,8 @@ func (d *FileKV) Get(k common.Hash) ([]byte, error) {
 	return hex.DecodeString(string(dat))
 }
 
-func (d *FileKV) Close() error {
+func (d *fileKV) Close() error {
 	return nil
 }
 
-var _ KV = (*FileKV)(nil)
+var _ KV = (*fileKV)(nil)

op-program/host/kvstore/file_test.go

@@ -10,7 +10,7 @@ import (
 
 func TestFileKV(t *testing.T) {
 	tmp := t.TempDir() // automatically removed by testing cleanup
-	kv := NewFileKV(tmp)
+	kv := newFileKV(tmp)
 	t.Cleanup(func() { // Can't use defer because kvTest runs tests in parallel.
 		require.NoError(t, kv.Close())
 	})
@@ -20,7 +20,7 @@ func TestFileKV(t *testing.T) {
 func TestFileKV_CreateMissingDirectory(t *testing.T) {
 	tmp := t.TempDir()
 	dir := filepath.Join(tmp, "data")
-	kv := NewFileKV(dir)
+	kv := newFileKV(dir)
 	defer kv.Close()
 	val := []byte{1, 2, 3, 4}
 	key := crypto.Keccak256Hash(val)