feat: backfill provider scaffolder (#19)

* feat: metafiller scaffolder

* feat: metafiller scaffolder

* update gitignore

* block cutoff config

* fix tests

* breakout get function

* remove block cutooff

* remove block cutooff

* nits

* rename to NewBackFillerProvider

* lowercase backfill

* add tests

* add TODOs

Author: ericvelazquez

.gitignore

@@ -18,7 +18,6 @@
 .env
 
 # Build files
-op-translator
 bin/
 generated/
 openapi/

op-translator/internal/config/config.go

@@ -31,6 +31,7 @@ type Config struct {
 	SettlementChainRPCURL      string `koanf:"settlement_chain_rpc_url"`
 	SequencingChainRPCURL      string `koanf:"sequencing_chain_rpc_url"`
 	MetaBasedChainRPCURL       string `koanf:"meta_based_chain_rpc_url"`
+	MetafillerURL              string `koanf:"metafiller_url"`
 	SequencingContractAddress  string `koanf:"sequencing_contract_address"`
 	BatcherAddress             string `koanf:"batcher_address"`
 	BatchInboxAddress          string `koanf:"batch_inbox_address"`
@@ -51,6 +52,7 @@ func setCLIFlags(f *pflag.FlagSet) {
 	f.String("settlement_chain_rpc_url", "https://sepolia.base.org", "Settlement chain address")
 	f.String("sequencing_chain_rpc_url", "https://sepolia.base.org", "Sequencing chain address")
 	f.String("meta_based_chain_rpc_url", "https://sepolia.base.org", "Meta based chain address")
+	f.String("metafiller_url", "http://localhost:8080", "Metafiller URL")
 	f.String("log_level", constants.Info.String(), "Log level for the app")
 	f.Int("frame_size", defaultFrameSize, "Size of each frame in bytes. Max is 1,000,000")
 	f.Bool("pretty", false, "Pretty print JSON log responses")
@@ -70,6 +72,7 @@ func hydrateFromConfMap(config *Config) {
 	config.SettlementChainRPCURL = k.String("settlement_chain_rpc_url")
 	config.SequencingChainRPCURL = k.String("sequencing_chain_rpc_url")
 	config.MetaBasedChainRPCURL = k.String("meta_based_chain_rpc_url")
+	config.MetafillerURL = k.String("metafiller_url")
 	config.FrameSize = k.Int("frame_size")
 	config.LogLevel = k.String("log_level")
 	config.Pretty = k.Bool("pretty")
@@ -151,6 +154,11 @@ func ValidateConfigValues(config *Config) (result error) {
 		result = multierror.Append(result, fmt.Errorf("invalid URL for meta based chain address: %w", err))
 	}
 
+	_, err = url.ParseRequestURI(config.MetafillerURL)
+	if err != nil {
+		result = multierror.Append(result, fmt.Errorf("invalid URL for metafiller: %w", err))
+	}
+
 	if !constants.IsValidLogLevel(config.LogLevel) {
 		result = multierror.Append(result, errors.New("invalid log level"))
 	}

op-translator/internal/config/config_test.go

@@ -18,6 +18,7 @@ func validConfig() config.Config {
 		SequencingChainRPCURL:      "http://example.com",
 		SettlementChainRPCURL:      "https://example.org",
 		MetaBasedChainRPCURL:       "https://example.io",
+		MetafillerURL:              "https://metafiller.io",
 		LogLevel:                   constants.Info.String(),
 		Pretty:                     false,
 		SequencingContractAddress:  "0x0000000000000000000000000000000000000000",

op-translator/mocks/config_mock.go

@@ -33,6 +33,7 @@ var DefaultTestingConfig = &config.Config{
 	SettlementChainRPCURL:      "http://localhost:8545",
 	SequencingChainRPCURL:      "http://localhost:8545",
 	MetaBasedChainRPCURL:       "http://localhost:8545",
+	MetafillerURL:              "http://localhost:8080",
 	BatchInboxAddress:          TestingBatchInboxAddress,
 	BatcherAddress:             TestingBatcherAddress,
 	SettlementChainID:          TestingSettlementChainID,

op-translator/mocks/http_client_mock.go

@@ -0,0 +1,16 @@
+package mocks
+
+import (
+	"net/http"
+
+	"github.com/stretchr/testify/mock"
+)
+
+type HTTPClientMock struct {
+	mock.Mock
+}
+
+func (m *HTTPClientMock) Do(req *http.Request) (*http.Response, error) {
+	args := m.Called(req)
+	return args.Get(0).(*http.Response), args.Error(1)
+}

op-translator/pkg/backfill/backfill_provider.go

@@ -0,0 +1,76 @@
+package backfill
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/internal/config"
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/types"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// TODO SEQ-141: spike: performant Go HTTP/JSON-RPC lib
+type HTTPClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+type BackfillProvider struct {
+	Client        HTTPClient
+	MetafillerURL string
+}
+
+func NewBackfillerProvider(cfg *config.Config) *BackfillProvider {
+	return &BackfillProvider{
+		MetafillerURL: cfg.MetafillerURL,
+		Client:        &http.Client{},
+	}
+}
+
+type BackfillData struct {
+	Data      string      `json:"data"` // Hex format
+	EpochHash common.Hash `json:"epochHash"`
+}
+
+func (b *BackfillProvider) GetBackfillData(ctx context.Context, epochNumber string) (*BackfillData, error) {
+	fullURL := b.MetafillerURL + "/" + epochNumber
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, http.NoBody)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := b.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	fmt.Println("BODY", body)
+	if err != nil {
+		return nil, err
+	}
+	// TODO SEQ-209: Think most optimal way to send/receive this data
+	var data *BackfillData
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (b *BackfillProvider) GetBackfillFrames(ctx context.Context, epochNumber string) ([]*types.Frame, error) {
+	backfillData, err := b.GetBackfillData(ctx, epochNumber)
+	if err != nil {
+		return nil, err
+	}
+
+	frames, err := types.ToFrames([]byte(backfillData.Data), config.MaxFrameSize, backfillData.EpochHash)
+	if err != nil {
+		return nil, err
+	}
+
+	return frames, nil
+}

op-translator/pkg/backfill/backfill_provider_test.go

@@ -0,0 +1,47 @@
+package backfill_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/mocks"
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/backfill"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+func TestGetBackfillFrames(t *testing.T) {
+	mockHTTPClient := new(mocks.HTTPClientMock)
+	jsonData, err := json.Marshal(backfill.BackfillData{
+		Data:      "data",
+		EpochHash: common.HexToHash("0x123"),
+	})
+	assert.NoError(t, err)
+	mockHTTPClient.On("Do", mock.Anything).Return(&http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(bytes.NewBuffer(jsonData)),
+		Header:     make(http.Header),
+	}, nil)
+
+	ctx := context.Background()
+	backfillProvider := backfill.BackfillProvider{
+		Client:        mockHTTPClient,
+		MetafillerURL: "http://metafiller.io",
+	}
+	epoch := "0x1"
+	frames, err := backfillProvider.GetBackfillFrames(ctx, epoch)
+
+	assert.NoError(t, err)
+	assert.Len(t, frames, 1)
+
+	frame := frames[0]
+	assert.NotNil(t, frame.ID)
+	assert.Equal(t, uint16(0), frame.FrameNumber)
+	assert.Equal(t, []byte("data"), frame.Data)
+	assert.Equal(t, true, frame.IsLast)
+}

op-translator/pkg/translator/translator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/internal/config"
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/backfill"
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/rpc-clients"
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/types"
 	"github.com/ethereum/go-ethereum/common"
@@ -30,6 +31,7 @@ var _ IRPCClient = (*rpc.RPCClient)(nil)
 type OPTranslator struct {
 	SettlementChain     IRPCClient
 	BatchProvider       IBatchProvider
+	BackfillProvider    *backfill.BackfillProvider
 	Signer              Signer
 	BatcherInboxAddress common.Address
 	BatcherAddress      common.Address
@@ -43,12 +45,14 @@ func Init(cfg *config.Config) *OPTranslator {
 
 	metaBasedBatchProvider := InitMetaBasedBatchProvider(cfg)
 	signer := NewSigner(cfg)
+	backfillProvider := backfill.NewBackfillerProvider(cfg)
 
 	return &OPTranslator{
 		SettlementChain:     settlementChain,
 		BatcherInboxAddress: common.HexToAddress(cfg.BatchInboxAddress),
 		BatcherAddress:      common.HexToAddress(cfg.BatcherAddress),
 		BatchProvider:       metaBasedBatchProvider,
+		BackfillProvider:    backfillProvider,
 		Signer:              *signer,
 	}
 }
@@ -63,7 +67,9 @@ func (t *OPTranslator) translateBlock(ctx context.Context, block types.Block) (t
 		return nil, err
 	}
 
-	frames, err := batch.ToFrames(config.MaxFrameSize)
+	// TODO SEQ-209: Write logic for switching between backfill and regular data fetching
+	frames, err := batch.GetFrames(config.MaxFrameSize)
+
 	if err != nil {
 		return nil, err
 	}

op-translator/pkg/translator/translator_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/mocks"
+	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/backfill"
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/translator"
 	"github.com/SyndicateProtocol/metabased-rollup/op-translator/pkg/types"
 
@@ -25,9 +26,10 @@ func TestGetBlockByNumber(t *testing.T) {
 
 	mockClient.On("GetBlockByNumber", ctx, number, true).Return(settlementBlock, nil)
 	translatorMock := &translator.OPTranslator{
-		SettlementChain: mockClient,
-		BatchProvider:   &mocks.MockBatchProvider{},
-		Signer:          *translator.NewSigner(mockConfig),
+		SettlementChain:  mockClient,
+		BatchProvider:    &mocks.MockBatchProvider{},
+		Signer:           *translator.NewSigner(mockConfig),
+		BackfillProvider: backfill.NewBackfillerProvider(mockConfig),
 	}
 
 	block, err := translatorMock.GetBlockByNumber(ctx, number, true)
@@ -50,6 +52,7 @@ func TestGetBlockByNumber(t *testing.T) {
 
 func TestGetBlockByHash(t *testing.T) {
 	mockClient := new(mocks.MockRPCClient)
+	mockConfig := mocks.DefaultTestingConfig
 	number := "0x1"
 	settlementBlock := types.Block{
 		"number":       number,
@@ -61,9 +64,10 @@ func TestGetBlockByHash(t *testing.T) {
 
 	mockClient.On("GetBlockByHash", ctx, hash, true).Return(settlementBlock, nil)
 	translatorMock := &translator.OPTranslator{
-		SettlementChain: mockClient,
-		BatchProvider:   &mocks.MockBatchProvider{},
-		Signer:          *translator.NewSigner(mocks.DefaultTestingConfig),
+		SettlementChain:  mockClient,
+		BatchProvider:    &mocks.MockBatchProvider{},
+		Signer:           *translator.NewSigner(mockConfig),
+		BackfillProvider: backfill.NewBackfillerProvider(mockConfig),
 	}
 
 	block, err := translatorMock.GetBlockByHash(ctx, hash, true)

op-translator/pkg/types/batch.go

@@ -63,7 +63,7 @@ func (b *Batch) Encode() ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func (b *Batch) ToFrames(frameSize int) ([]*Frame, error) {
+func (b *Batch) GetFrames(frameSize int) ([]*Frame, error) {
 	encodedBatch, err := b.Encode()
 	if err != nil {
 		return nil, err