op-challenger: Add metric to report memory used in the cannon VM (#11350

)

* op-challenger: Add metric to report memory used in the cannon VM

* op-challenger: Remove config override.

cannon/mipsevm/debug.go

@@ -1,7 +1,10 @@
 package mipsevm
 
+import "github.com/ethereum/go-ethereum/common/hexutil"
+
 type DebugInfo struct {
-	Pages               int `json:"pages"`
-	NumPreimageRequests int `json:"num_preimage_requests"`
-	TotalPreimageSize   int `json:"total_preimage_size"`
+	Pages               int            `json:"pages"`
+	MemoryUsed          hexutil.Uint64 `json:"memory_used"`
+	NumPreimageRequests int            `json:"num_preimage_requests"`
+	TotalPreimageSize   int            `json:"total_preimage_size"`
 }

cannon/mipsevm/memory/memory.go

@@ -323,8 +323,12 @@ func (m *Memory) ReadMemoryRange(addr uint32, count uint32) io.Reader {
 	return &memReader{m: m, addr: addr, count: count}
 }
 
+func (m *Memory) UsageRaw() uint64 {
+	return uint64(len(m.pages)) * PageSize
+}
+
 func (m *Memory) Usage() string {
-	total := uint64(len(m.pages)) * PageSize
+	total := m.UsageRaw()
 	const unit = 1024
 	if total < unit {
 		return fmt.Sprintf("%d B", total)

cannon/mipsevm/multithreaded/instrumented.go

@@ -3,6 +3,7 @@ package multithreaded
 import (
 	"io"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/log"
 
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm"
@@ -106,6 +107,7 @@ func (m *InstrumentedState) GetState() mipsevm.FPVMState {
 func (m *InstrumentedState) GetDebugInfo() *mipsevm.DebugInfo {
 	return &mipsevm.DebugInfo{
 		Pages:               m.state.Memory.PageCount(),
+		MemoryUsed:          hexutil.Uint64(m.state.Memory.UsageRaw()),
 		NumPreimageRequests: m.preimageOracle.NumPreimageRequests(),
 		TotalPreimageSize:   m.preimageOracle.TotalPreimageSize(),
 	}

cannon/mipsevm/singlethreaded/instrumented.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/exec"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/program"
 	"github.com/ethereum-optimism/optimism/op-service/jsonutil"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
 type InstrumentedState struct {
@@ -109,6 +110,7 @@ func (m *InstrumentedState) GetState() mipsevm.FPVMState {
 func (m *InstrumentedState) GetDebugInfo() *mipsevm.DebugInfo {
 	return &mipsevm.DebugInfo{
 		Pages:               m.state.Memory.PageCount(),
+		MemoryUsed:          hexutil.Uint64(m.state.Memory.UsageRaw()),
 		NumPreimageRequests: m.preimageOracle.NumPreimageRequests(),
 		TotalPreimageSize:   m.preimageOracle.TotalPreimageSize(),
 	}

op-challenger/config/config.go

@@ -142,6 +142,7 @@ func NewConfig(
 			L2:           l2EthRpc,
 			SnapshotFreq: DefaultCannonSnapshotFreq,
 			InfoFreq:     DefaultCannonInfoFreq,
+			DebugInfo:    true,
 		},
 		Asterisc: vm.Config{
 			VmType:       types.TraceTypeAsterisc,

op-challenger/config/config_test.go

@@ -213,6 +213,11 @@ func TestCannonRequiredArgs(t *testing.T) {
 			cfg.Cannon.Network = "unknown"
 			require.ErrorIs(t, cfg.Check(), ErrCannonNetworkUnknown)
 		})
+
+		t.Run(fmt.Sprintf("TestDebugInfoEnabled-%v", traceType), func(t *testing.T) {
+			cfg := validConfig(traceType)
+			require.True(t, cfg.Cannon.DebugInfo)
+		})
 	}
 }
 
@@ -319,6 +324,11 @@ func TestAsteriscRequiredArgs(t *testing.T) {
 			cfg.Asterisc.Network = "unknown"
 			require.ErrorIs(t, cfg.Check(), ErrAsteriscNetworkUnknown)
 		})
+
+		t.Run(fmt.Sprintf("TestDebugInfoDisabled-%v", traceType), func(t *testing.T) {
+			cfg := validConfig(traceType)
+			require.False(t, cfg.Asterisc.DebugInfo)
+		})
 	}
 }
 

op-challenger/flags/flags.go

@@ -526,6 +526,7 @@ func NewConfigFromCLI(ctx *cli.Context, logger log.Logger) (*config.Config, erro
 			L2GenesisPath:    ctx.String(CannonL2GenesisFlag.Name),
 			SnapshotFreq:     ctx.Uint(CannonSnapshotFreqFlag.Name),
 			InfoFreq:         ctx.Uint(CannonInfoFreqFlag.Name),
+			DebugInfo:        true,
 		},
 		CannonAbsolutePreState:        ctx.String(CannonPreStateFlag.Name),
 		CannonAbsolutePreStateBaseURL: cannonPrestatesURL,

op-challenger/game/fault/trace/vm/executor.go

@@ -12,11 +12,18 @@ import (
 
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/trace/utils"
 	"github.com/ethereum-optimism/optimism/op-challenger/game/fault/types"
+	"github.com/ethereum-optimism/optimism/op-service/jsonutil"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/log"
 )
 
+const (
+	debugFilename = "debug-info.json"
+)
+
 type Metricer interface {
 	RecordVmExecutionTime(vmType string, t time.Duration)
+	RecordVmMemoryUsed(vmType string, memoryUsed uint64)
 }
 
 type Config struct {
@@ -31,6 +38,7 @@ type Config struct {
 	L2GenesisPath    string
 	SnapshotFreq     uint // Frequency of snapshots to create when executing (in VM instructions)
 	InfoFreq         uint // Frequency of progress log messages (in VM instructions)
+	DebugInfo        bool
 }
 
 type Executor struct {
@@ -86,6 +94,9 @@ func (e *Executor) DoGenerateProof(ctx context.Context, dir string, begin uint64
 	if end < math.MaxUint64 {
 		args = append(args, "--stop-at", "="+strconv.FormatUint(end+1, 10))
 	}
+	if e.cfg.DebugInfo {
+		args = append(args, "--debug-info", filepath.Join(dataDir, debugFilename))
+	}
 	args = append(args, extraVmArgs...)
 	args = append(args,
 		"--",
@@ -123,5 +134,16 @@ func (e *Executor) DoGenerateProof(ctx context.Context, dir string, begin uint64
 	execStart := time.Now()
 	err = e.cmdExecutor(ctx, e.logger.New("proof", end), e.cfg.VmBin, args...)
 	e.metrics.RecordVmExecutionTime(e.cfg.VmType.String(), time.Since(execStart))
+	if e.cfg.DebugInfo && err == nil {
+		if info, err := jsonutil.LoadJSON[debugInfo](filepath.Join(dataDir, debugFilename)); err != nil {
+			e.logger.Warn("Failed to load debug metrics", "err", err)
+		} else {
+			e.metrics.RecordVmMemoryUsed(e.cfg.VmType.String(), uint64(info.MemoryUsed))
+		}
+	}
 	return err
 }
+
+type debugInfo struct {
+	MemoryUsed hexutil.Uint64 `json:"memory_used"`
+}

op-challenger/metrics/metrics.go

@@ -39,6 +39,7 @@ type Metricer interface {
 	RecordGameMove()
 	RecordGameL2Challenge()
 	RecordVmExecutionTime(vmType string, t time.Duration)
+	RecordVmMemoryUsed(vmType string, memoryUsed uint64)
 	RecordClaimResolutionTime(t float64)
 	RecordGameActTime(t float64)
 
@@ -94,6 +95,7 @@ type Metrics struct {
 	claimResolutionTime prometheus.Histogram
 	gameActTime         prometheus.Histogram
 	vmExecutionTime     *prometheus.HistogramVec
+	vmMemoryUsed        *prometheus.HistogramVec
 
 	trackedGames  prometheus.GaugeVec
 	inflightGames prometheus.Gauge
@@ -176,6 +178,13 @@ func NewMetrics() *Metrics {
 				[]float64{1.0, 10.0},
 				prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
 		}, []string{"vm"}),
+		vmMemoryUsed: factory.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Name:      "vm_memory_used",
+			Help:      "Memory used (in bytes) to execute the fault proof VM",
+			// 100MiB increments from 0 to 1.5GiB
+			Buckets: prometheus.LinearBuckets(0, 1024*1024*100, 15),
+		}, []string{"vm"}),
 		bondClaimFailures: factory.NewCounter(prometheus.CounterOpts{
 			Namespace: Namespace,
 			Name:      "claim_failures",
@@ -285,6 +294,10 @@ func (m *Metrics) RecordVmExecutionTime(vmType string, dur time.Duration) {
 	m.vmExecutionTime.WithLabelValues(vmType).Observe(dur.Seconds())
 }
 
+func (m *Metrics) RecordVmMemoryUsed(vmType string, memoryUsed uint64) {
+	m.vmMemoryUsed.WithLabelValues(vmType).Observe(float64(memoryUsed))
+}
+
 func (m *Metrics) RecordClaimResolutionTime(t float64) {
 	m.claimResolutionTime.Observe(t)
 }

op-challenger/metrics/noop.go

@@ -40,6 +40,7 @@ func (*NoopMetricsImpl) RecordBondClaimFailed()   {}
 func (*NoopMetricsImpl) RecordBondClaimed(uint64) {}
 
 func (*NoopMetricsImpl) RecordVmExecutionTime(_ string, _ time.Duration) {}
+func (*NoopMetricsImpl) RecordVmMemoryUsed(_ string, _ uint64)           {}
 func (*NoopMetricsImpl) RecordClaimResolutionTime(t float64)             {}
 func (*NoopMetricsImpl) RecordGameActTime(t float64)                     {}
 

op-challenger/runner/metrics.go

@@ -17,10 +17,13 @@ type Metrics struct {
 	factory  opmetrics.Factory
 	*contractMetrics.ContractMetrics
 
-	vmExecutionTime *prometheus.HistogramVec
-	successTotal    *prometheus.CounterVec
-	failuresTotal   *prometheus.CounterVec
-	invalidTotal    *prometheus.CounterVec
+	vmExecutionTime     *prometheus.HistogramVec
+	vmLastExecutionTime *prometheus.GaugeVec
+	vmMemoryUsed        *prometheus.HistogramVec
+	vmLastMemoryUsed    *prometheus.GaugeVec
+	successTotal        *prometheus.CounterVec
+	failuresTotal       *prometheus.CounterVec
+	invalidTotal        *prometheus.CounterVec
 }
 
 var _ Metricer = (*Metrics)(nil)
@@ -47,6 +50,23 @@ func NewMetrics() *Metrics {
 				[]float64{1.0, 10.0},
 				prometheus.ExponentialBuckets(30.0, 2.0, 14)...),
 		}, []string{"vm"}),
+		vmLastExecutionTime: factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "vm_last_execution_time",
+			Help:      "Time (in seconds) taken for the last execution of the fault proof VM",
+		}, []string{"vm"}),
+		vmMemoryUsed: factory.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Name:      "vm_memory_used",
+			Help:      "Memory used (in bytes) to execute the fault proof VM",
+			// 100MiB increments from 0 to 1.5GiB
+			Buckets: prometheus.LinearBuckets(0, 1024*1024*100, 15),
+		}, []string{"vm"}),
+		vmLastMemoryUsed: factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Name:      "vm_last_memory_used",
+			Help:      "Memory used (in bytes) for the last execution of the fault proof VM",
+		}, []string{"vm"}),
 		successTotal: factory.NewCounterVec(prometheus.CounterOpts{
 			Namespace: Namespace,
 			Name:      "success_total",
@@ -70,7 +90,14 @@ func (m *Metrics) Registry() *prometheus.Registry {
 }
 
 func (m *Metrics) RecordVmExecutionTime(vmType string, dur time.Duration) {
-	m.vmExecutionTime.WithLabelValues(vmType).Observe(dur.Seconds())
+	val := dur.Seconds()
+	m.vmExecutionTime.WithLabelValues(vmType).Observe(val)
+	m.vmLastExecutionTime.WithLabelValues(vmType).Set(val)
+}
+
+func (m *Metrics) RecordVmMemoryUsed(vmType string, memoryUsed uint64) {
+	m.vmMemoryUsed.WithLabelValues(vmType).Observe(float64(memoryUsed))
+	m.vmLastMemoryUsed.WithLabelValues(vmType).Set(float64(memoryUsed))
 }
 
 func (m *Metrics) RecordSuccess(vmType types.TraceType) {