change byte layout of deduplicated file due to too many vanities (>64) in Sepolia data

Author: jonastheis

rollup/missing_header_fields/export-headers-toolkit/README.md

@@ -29,9 +29,8 @@ Where:
 - unique_vanity_i: unique vanity i
 - header_i: block header i
 - header: 
-    <flags:uint8><seal:[65|85]byte>
+    <flags:uint8><vanity_index:uint8><seal:[65|85]byte>
     - flags: bitmask, lsb first
-	    - bit 0-5: index of the vanity in the sorted vanities list
 	    - bit 6: 0 if difficulty is 2, 1 if difficulty is 1
 	    - bit 7: 0 if seal length is 65, 1 if seal length is 85
 ```

rollup/missing_header_fields/export-headers-toolkit/cmd/dedup.go

@@ -293,4 +293,6 @@ func verifyOutputFile(verifyFile, outputFile string) {
 			log.Fatalf("ExtraData mismatch: headerNum %d: %x != %x", header.Number, header.ExtraData, extraData)
 		}
 	}
+
+	log.Printf("All headers match in %s and %s\n", verifyFile, outputFile)
 }

rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_reader.go

@@ -86,14 +86,20 @@ func (r *Reader) ReadNext() (difficulty uint64, extraData []byte, err error) {
 
 	bits := newBitMaskFromByte(bitmaskByte)
 
+	// read the vanity index
+	vanityIndex, err := r.reader.ReadByte()
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to read vanity index: %v", err)
+	}
+
 	seal := make([]byte, bits.sealLen())
 
 	if _, err = io.ReadFull(r.reader, seal); err != nil {
 		return 0, nil, fmt.Errorf("failed to read seal: %v", err)
 	}
 
 	// construct the extraData field
-	vanity := r.sortedVanities[bits.vanityIndex()]
+	vanity := r.sortedVanities[int(vanityIndex)]
 	var b bytes.Buffer
 	b.Write(vanity[:])
 	b.Write(seal)

rollup/missing_header_fields/export-headers-toolkit/cmd/missing_header_writer.go

@@ -5,14 +5,15 @@ import (
 	"bytes"
 	"io"
 	"log"
+	"math"
 	"os"
 	"sort"
 
 	"github.com/scroll-tech/go-ethereum/export-headers-toolkit/types"
 )
 
-// maxVanityCount is the maximum number of unique vanities that can be represented with 6 bits in the bitmask
-const maxVanityCount = 1 << 6
+// maxVanityCount is the maximum number of unique vanities that can be represented with a single byte.
+const maxVanityCount = math.MaxUint8
 
 type missingHeaderFileWriter struct {
 	file   *os.File
@@ -94,21 +95,27 @@ func (m *missingHeaderWriter) writeVanities() {
 
 func (m *missingHeaderWriter) write(header *types.Header) {
 	// 1. prepare the bitmask
-	bits := newBitMask(m.sortedVanitiesMap[header.Vanity()], int(header.Difficulty), header.SealLen())
+	bits := newBitMask(int(header.Difficulty), header.SealLen())
+	vanityIndex := m.sortedVanitiesMap[header.Vanity()]
 
-	// 2. write the header: bitmask and seal
+	if vanityIndex >= maxVanityCount {
+		log.Fatalf("Vanity index %d exceeds maximum allowed %d", vanityIndex, maxVanityCount-1)
+	}
+
+	// 2. write the header: bitmask, vanity index and seal
 	if _, err := m.writer.Write(bits.Bytes()); err != nil {
 		log.Fatalf("Error writing bitmask: %v", err)
 	}
-
+	if _, err := m.writer.Write([]byte{uint8(vanityIndex)}); err != nil {
+		log.Fatalf("Error writing vanity index: %v", err)
+	}
 	if _, err := m.writer.Write(header.Seal()); err != nil {
 		log.Fatalf("Error writing seal: %v", err)
 	}
 }
 
 // bitMask is a bitmask that encodes the following information:
 //
-//	bit 0-5: index of the vanity in the sorted vanities list
 //	bit 6: 0 if difficulty is 2, 1 if difficulty is 1
 //	bit 7: 0 if seal length is 65, 1 if seal length is 85
 type bitMask struct {
@@ -119,14 +126,9 @@ func newBitMaskFromByte(b uint8) bitMask {
 	return bitMask{b}
 }
 
-func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask {
+func newBitMask(difficulty int, sealLen int) bitMask {
 	b := uint8(0)
 
-	if vanityIndex >= maxVanityCount {
-		log.Fatalf("Vanity index exceeds maximum: %d >= %d", vanityIndex, maxVanityCount)
-	}
-	b |= uint8(vanityIndex) & 0b00111111
-
 	if difficulty == 1 {
 		b |= 1 << 6
 	} else if difficulty != 2 {
@@ -142,10 +144,6 @@ func newBitMask(vanityIndex int, difficulty int, sealLen int) bitMask {
 	return bitMask{b}
 }
 
-func (b bitMask) vanityIndex() int {
-	return int(b.b & 0b00111111)
-}
-
 func (b bitMask) difficulty() int {
 	val := (b.b >> 6) & 0x01
 	if val == 0 {