fix: blockhash mismatch

rollup/missing_header_fields/manager.go

@@ -0,0 +1,185 @@
+package missing_header_fields
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/scroll-tech/go-ethereum/log"
+)
+
+const timeoutDownload = 10 * time.Minute
+
+// Manager is responsible for managing the missing header fields file.
+// It lazily downloads the file if it doesn't exist, verifies its checksum and provides the missing header fields.
+type Manager struct {
+	ctx         context.Context
+	filePath    string
+	downloadURL string
+	checksum    [sha256.Size]byte
+
+	reader *Reader
+}
+
+func NewManager(ctx context.Context, filePath string, downloadURL string, checksum [sha256.Size]byte) *Manager {
+	return &Manager{
+		ctx:         ctx,
+		filePath:    filePath,
+		downloadURL: downloadURL,
+		checksum:    checksum,
+	}
+}
+
+func (m *Manager) GetMissingHeaderFields(headerNum uint64) (difficulty uint64, extraData []byte, err error) {
+	// lazy initialization: if the reader is not initialized this is the first time we read from the file
+	if m.reader == nil {
+		if err = m.initialize(); err != nil {
+			return 0, nil, fmt.Errorf("failed to initialize missing header reader: %v", err)
+		}
+	}
+
+	return m.reader.Read(headerNum)
+}
+
+func (m *Manager) initialize() error {
+	// if the file doesn't exist, download it
+	if _, err := os.Stat(m.filePath); errors.Is(err, os.ErrNotExist) {
+		if err = m.downloadFile(); err != nil {
+			return fmt.Errorf("failed to download file: %v", err)
+		}
+	}
+
+	// verify the checksum
+	f, err := os.Open(m.filePath)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %v", err)
+	}
+
+	h := sha256.New()
+	if _, err = io.Copy(h, f); err != nil {
+		return fmt.Errorf("failed to copy file: %v", err)
+	}
+	if err = f.Close(); err != nil {
+		return fmt.Errorf("failed to close file: %v", err)
+	}
+
+	if !bytes.Equal(h.Sum(nil), m.checksum[:]) {
+		return fmt.Errorf("checksum mismatch")
+	}
+
+	// finally initialize the reader
+	reader, err := NewReader(m.filePath)
+	if err != nil {
+		return err
+	}
+
+	m.reader = reader
+	return nil
+}
+func (m *Manager) Close() error {
+	if m.reader != nil {
+		return m.reader.Close()
+	}
+	return nil
+}
+
+func (m *Manager) downloadFile() error {
+	log.Info("Downloading missing header fields. This might take a while...", "url", m.downloadURL)
+
+	downloadCtx, downloadCtxCancel := context.WithTimeout(m.ctx, timeoutDownload)
+	defer downloadCtxCancel()
+
+	req, err := http.NewRequestWithContext(downloadCtx, http.MethodGet, m.downloadURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create download request: %v", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to download file: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("server returned status code %d", resp.StatusCode)
+	}
+
+	// create a temporary file
+	tmpFilePath := m.filePath + ".tmp" // append .tmp to the file path
+	tmpFile, err := os.Create(tmpFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to create temporary file: %v", err)
+	}
+	var ok bool
+	defer func() {
+		if !ok {
+			_ = os.Remove(tmpFilePath)
+		}
+	}()
+
+	// copy the response body to the temporary file and print progress
+	writeCounter := NewWriteCounter(m.ctx, uint64(resp.ContentLength))
+	if _, err = io.Copy(tmpFile, io.TeeReader(resp.Body, writeCounter)); err != nil {
+		return fmt.Errorf("failed to copy response body: %v", err)
+	}
+
+	if err = tmpFile.Close(); err != nil {
+		return fmt.Errorf("failed to close temporary file: %v", err)
+	}
+
+	// rename the temporary file to the final file path
+	if err = os.Rename(tmpFilePath, m.filePath); err != nil {
+		return fmt.Errorf("failed to rename temporary file: %v", err)
+	}
+
+	ok = true
+	return nil
+}
+
+type WriteCounter struct {
+	ctx                 context.Context
+	total               uint64
+	written             uint64
+	lastProgressPrinted time.Time
+}
+
+func NewWriteCounter(ctx context.Context, total uint64) *WriteCounter {
+	return &WriteCounter{
+		ctx:   ctx,
+		total: total,
+	}
+}
+
+func (wc *WriteCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.written += uint64(n)
+
+	// check if the context is done and return early
+	select {
+	case <-wc.ctx.Done():
+		return n, wc.ctx.Err()
+	default:
+	}
+
+	wc.printProgress()
+
+	return n, nil
+}
+
+func (wc *WriteCounter) printProgress() {
+	if time.Since(wc.lastProgressPrinted) < 5*time.Second {
+		return
+	}
+	wc.lastProgressPrinted = time.Now()
+
+	log.Info(fmt.Sprintf("Downloading missing header fields... %d MB / %d MB", toMB(wc.written), toMB(wc.total)))
+}
+
+func toMB(bytes uint64) uint64 {
+	return bytes / 1024 / 1024
+}

rollup/missing_header_fields/manager_test.go

@@ -0,0 +1,59 @@
+package missing_header_fields
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/log"
+)
+
+func TestManagerDownload(t *testing.T) {
+	t.Skip("skipping test due to long runtime/downloading file")
+	log.Root().SetHandler(log.StdoutHandler)
+
+	// TODO: replace with actual sha256 hash and downloadURL
+	sha256 := [32]byte(common.FromHex("0x575858a53b8cdde8d63a2cc1a5b90f1bbf0c2243b292a66a1ab2931d571eb260"))
+	downloadURL := "https://ftp.halifax.rwth-aachen.de/ubuntu-releases/24.04/ubuntu-24.04-netboot-amd64.tar.gz"
+	filePath := filepath.Join(t.TempDir(), "test_file_path")
+	manager := NewManager(context.Background(), filePath, downloadURL, sha256)
+
+	_, _, err := manager.GetMissingHeaderFields(0)
+	require.NoError(t, err)
+
+	// Check if the file was downloaded and tmp file was removed
+	_, err = os.Stat(filePath)
+	require.NoError(t, err)
+	_, err = os.Stat(filePath + ".tmp")
+	require.Error(t, err)
+}
+
+func TestManagerChecksum(t *testing.T) {
+	// Checksum doesn't match
+	{
+		sha256 := [32]byte(common.FromHex("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
+		downloadURL := "" // since the file exists we don't need to download it
+		filePath := "testdata/missing_headers_1.dedup"
+		manager := NewManager(context.Background(), filePath, downloadURL, sha256)
+
+		_, _, err := manager.GetMissingHeaderFields(0)
+		require.ErrorContains(t, err, "checksum mismatch")
+	}
+
+	// Checksum matches
+	{
+		sha256 := [32]byte(common.FromHex("0x5dee238e74c350c7116868bfe6c5218d440be3613f47f8c052bd5cef46f4ae04"))
+		downloadURL := "" // since the file exists we don't need to download it
+		filePath := "testdata/missing_headers_1.dedup"
+		manager := NewManager(context.Background(), filePath, downloadURL, sha256)
+
+		difficulty, extra, err := manager.GetMissingHeaderFields(0)
+		require.NoError(t, err)
+		require.Equal(t, expectedMissingHeaders1[0].difficulty, difficulty)
+		require.Equal(t, expectedMissingHeaders1[0].extra, extra)
+	}
+}

rollup/missing_header_fields/reader.go

@@ -0,0 +1,152 @@
+package missing_header_fields
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+)
+
+type missingHeader struct {
+	headerNum  uint64
+	difficulty uint64
+	extraData  []byte
+}
+
+type Reader struct {
+	file           *os.File
+	reader         *bufio.Reader
+	sortedVanities map[int][32]byte
+	lastReadHeader *missingHeader
+}
+
+func NewReader(filePath string) (*Reader, error) {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+
+	r := &Reader{
+		file:   f,
+		reader: bufio.NewReader(f),
+	}
+
+	// read the count of unique vanities
+	vanityCount, err := r.reader.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+
+	// read the unique vanities
+	r.sortedVanities = make(map[int][32]byte)
+	for i := uint8(0); i < vanityCount; i++ {
+		var vanity [32]byte
+		if _, err = r.reader.Read(vanity[:]); err != nil {
+			return nil, err
+		}
+		r.sortedVanities[int(i)] = vanity
+	}
+
+	return r, nil
+}
+
+func (r *Reader) Read(headerNum uint64) (difficulty uint64, extraData []byte, err error) {
+	if r.lastReadHeader == nil {
+		if _, _, err = r.ReadNext(); err != nil {
+			return 0, nil, err
+		}
+	}
+
+	if headerNum > r.lastReadHeader.headerNum {
+		// skip the headers until the requested header number
+		for i := r.lastReadHeader.headerNum; i < headerNum; i++ {
+			if _, _, err = r.ReadNext(); err != nil {
+				return 0, nil, err
+			}
+		}
+	}
+
+	if headerNum == r.lastReadHeader.headerNum {
+		return r.lastReadHeader.difficulty, r.lastReadHeader.extraData, nil
+	}
+
+	// headerNum < r.lastReadHeader.headerNum is not supported
+	return 0, nil, fmt.Errorf("requested header %d below last read header number %d", headerNum, r.lastReadHeader.headerNum)
+}
+
+func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) {
+	// read the bitmask
+	bitmask, err := r.reader.ReadByte()
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to read bitmask: %v", err)
+	}
+
+	bits := newBitMask(bitmask)
+
+	seal := make([]byte, bits.sealLen())
+	if _, err = io.ReadFull(r.reader, seal); err != nil {
+		return 0, nil, fmt.Errorf("failed to read seal: %v", err)
+	}
+
+	// construct the extraData field
+	vanity := r.sortedVanities[bits.vanityIndex()]
+	var b bytes.Buffer
+	b.Write(vanity[:])
+	b.Write(seal)
+
+	// we don't have the header number, so we'll just increment the last read header number
+	// we assume that the headers are written in order, starting from 0
+	if r.lastReadHeader == nil {
+		r.lastReadHeader = &missingHeader{
+			headerNum:  0,
+			difficulty: uint64(bits.difficulty()),
+			extraData:  b.Bytes(),
+		}
+	} else {
+		r.lastReadHeader.headerNum++
+		r.lastReadHeader.difficulty = uint64(bits.difficulty())
+		r.lastReadHeader.extraData = b.Bytes()
+	}
+
+	return difficulty, b.Bytes(), nil
+}
+
+func (r *Reader) Close() error {
+	return r.file.Close()
+}
+
+// bitMask is a bitmask that encodes the following information:
+//
+//	bit 0-5: index of the vanity in the sorted vanities list
+//	bit 6: 0 if difficulty is 2, 1 if difficulty is 1
+//	bit 7: 0 if seal length is 65, 1 if seal length is 85
+type bitMask struct {
+	b uint8
+}
+
+func newBitMask(b uint8) bitMask {
+	return bitMask{b}
+}
+
+func (b bitMask) vanityIndex() int {
+	return int(b.b & 0b00111111)
+}
+
+func (b bitMask) difficulty() int {
+	val := (b.b >> 6) & 0x01
+	if val == 0 {
+		return 2
+	} else {
+		return 1
+	}
+}
+
+func (b bitMask) sealLen() int {
+	val := (b.b >> 7) & 0x01
+	if val == 0 {
+		return 65
+	} else {
+		return 85
+	}
+}