tapdb: switch proof cache to size-based limits and add deep size estimation

docs/release-notes/release-notes-0.8.0.md

@@ -8,6 +8,7 @@
     - [Functional Updates](#functional-updates)
     - [RPC Updates](#rpc-updates)
     - [tapcli Updates](#tapcli-updates)
+    - [Config Changes](#config-changes)
     - [Breaking Changes](#breaking-changes)
     - [Performance Improvements](#performance-improvements)
     - [Deprecations](#deprecations)
@@ -56,6 +57,13 @@
   [new `--group_key` flag](https://github.com/lightninglabs/taproot-assets/pull/1812)
   that allows users to burn assets by group key.
 
+## Config Changes
+
+- [PR#1870](https://github.com/lightninglabs/taproot-assets/pull/1870)
+  The `proofs-per-universe` configuration option is removed. New option 
+  `max-proof-cache-size` sets the proof cache limit in bytes and accepts
+  human-readable values such as `32MB`.
+
 ## Code Health
 
 ## Breaking Changes

docs/release-notes/release-notes-template.md

@@ -8,6 +8,7 @@
     - [Functional Updates](#functional-updates)
     - [RPC Updates](#rpc-updates)
     - [tapcli Updates](#tapcli-updates)
+    - [Config Changes](#config-changes)
     - [Breaking Changes](#breaking-changes)
     - [Performance Improvements](#performance-improvements)
     - [Deprecations](#deprecations)
@@ -36,6 +37,8 @@
 
 ## tapcli Updates
 
+## Config Changes
+
 ## Code Health
 
 ## Breaking Changes

fn/memory.go

@@ -1,13 +1,25 @@
 package fn
 
-import "errors"
+import (
+	"errors"
+	"reflect"
+	"unsafe"
+)
 
 var (
 	// ErrNilPointerDeference is returned when a nil pointer is
 	// dereferenced.
 	ErrNilPointerDeference = errors.New("nil pointer dereference")
 )
 
+var (
+	// sliceHeaderSize is the size of a slice header.
+	sliceHeaderSize = uint64(unsafe.Sizeof([]byte(nil)))
+
+	// stringHeaderSize is the size of a string header.
+	stringHeaderSize = uint64(unsafe.Sizeof(""))
+)
+
 // Ptr returns the pointer of the given value. This is useful in instances
 // where a function returns the value, but a pointer is wanted. Without this,
 // then an intermediate variable is needed.
@@ -68,3 +80,146 @@ func DerefPanic[T any](ptr *T) T {
 
 	return *ptr
 }
+
+// LowerBoundByteSize returns a conservative deep-size estimate in bytes.
+//
+// Notes:
+//   - Pointer-recursive and cycle safe; each heap allocation is counted once
+//     using its data pointer.
+//   - Lower bound: ignores allocator overhead, GC metadata, unused slice
+//     capacity, map buckets/overflow, evacuation, rounding, and runtime
+//     internals (chan/func).
+func LowerBoundByteSize(x any) uint64 {
+	// seen is a map of heap object identities which have already been
+	// counted.
+	seen := make(map[uintptr]struct{})
+	return byteSizeVisit(reflect.ValueOf(x), true, seen)
+}
+
+// byteSizeVisit returns a conservative lower-bound byte count for `subject`.
+//
+// Notes:
+//   - addSelf: include subject’s inline bytes when true. Parents pass false.
+//   - seen: set of heap data pointers to avoid double counting and break
+//     cycles.
+//
+// Lower bound: ignores allocator overhead, GC metadata, unused capacity, and
+// runtime internals.
+func byteSizeVisit(subject reflect.Value, addSelf bool,
+	seen map[uintptr]struct{}) uint64 {
+
+	if !subject.IsValid() {
+		return 0
+	}
+
+	subjectType := subject.Type()
+	subjectTypeKind := subjectType.Kind()
+
+	if subjectTypeKind == reflect.Interface {
+		n := uint64(unsafe.Sizeof(subject.Interface()))
+		if !subject.IsNil() {
+			n += byteSizeVisit(subject.Elem(), true, seen)
+		}
+		return n
+	}
+
+	switch subjectTypeKind {
+	case reflect.Ptr:
+		if subject.IsNil() {
+			return 0
+		}
+
+		ptr := subject.Pointer()
+		if markSeen(ptr, seen) {
+			return 0
+		}
+
+		return byteSizeVisit(subject.Elem(), true, seen)
+
+	case reflect.Struct:
+		n := uint64(0)
+		if addSelf {
+			n += uint64(subjectType.Size())
+		}
+
+		for i := 0; i < subject.NumField(); i++ {
+			n += byteSizeVisit(subject.Field(i), false, seen)
+		}
+
+		return n
+
+	case reflect.Array:
+		n := uint64(0)
+		if addSelf {
+			n += uint64(subjectType.Size())
+		}
+
+		for i := 0; i < subject.Len(); i++ {
+			n += byteSizeVisit(subject.Index(i), false, seen)
+		}
+
+		return n
+
+	case reflect.Slice:
+		if subject.IsNil() {
+			return 0
+		}
+
+		n := sliceHeaderSize
+		dataPtr := subject.Pointer()
+		if dataPtr != 0 && !markSeen(dataPtr, seen) {
+			elem := subjectType.Elem()
+			n += uint64(subject.Len()) * uint64(elem.Size())
+		}
+
+		for i := 0; i < subject.Len(); i++ {
+			n += byteSizeVisit(subject.Index(i), false, seen)
+		}
+
+		return n
+
+	case reflect.String:
+		n := stringHeaderSize
+		dataPtr := subject.Pointer()
+		if dataPtr != 0 && markSeen(dataPtr, seen) {
+			return n
+		}
+
+		return n + uint64(subject.Len())
+
+	case reflect.Map:
+		n := uint64(unsafe.Sizeof(subject.Interface()))
+		if subject.IsNil() {
+			return n
+		}
+
+		it := subject.MapRange()
+		for it.Next() {
+			n += byteSizeVisit(it.Key(), false, seen)
+			n += byteSizeVisit(it.Value(), false, seen)
+		}
+
+		return n
+
+	case reflect.Chan, reflect.Func, reflect.UnsafePointer:
+		return uint64(unsafe.Sizeof(subject.Interface()))
+
+	default:
+		if addSelf {
+			return uint64(subjectType.Size())
+		}
+
+		return 0
+	}
+}
+
+// markSeen marks the given pointer as seen and returns true if it was already
+// seen.
+func markSeen(ptr uintptr, seen map[uintptr]struct{}) bool {
+	if _, ok := seen[ptr]; ok {
+		return true
+	}
+
+	seen[ptr] = struct{}{}
+	return false
+}

fn/memory_test.go

@@ -0,0 +1,131 @@
+package fn
+
+import (
+	"testing"
+	"unsafe"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestLowerBoundByteSizeNilAndPrimitives ensures the byte estimator handles nil
+// interfaces and primitive concrete values using their inline sizes.
+func TestLowerBoundByteSizeNilAndPrimitives(t *testing.T) {
+	actualNil := LowerBoundByteSize(nil)
+	require.Zero(
+		t, actualNil, "nil interface bytes: expected 0, actual %d",
+		actualNil,
+	)
+
+	var num int64 = 99
+	expectedInt := uint64(unsafe.Sizeof(num))
+	actualInt := LowerBoundByteSize(num)
+	require.Equal(
+		t, expectedInt, actualInt,
+		"int64 bytes mismatch: expected %d, actual %d",
+		expectedInt, actualInt,
+	)
+
+	const str = "taproot-assets"
+	expectedString := stringHeaderSize + uint64(len(str))
+	actualString := LowerBoundByteSize(str)
+	require.Equal(
+		t, expectedString, actualString,
+		"string bytes mismatch: expected %d, actual %d",
+		expectedString, actualString,
+	)
+}
+
+// TestLowerBoundByteSizeStructsAndSlices covers structs that embed slices
+// and validates shared backing arrays are only counted once via the seen set.
+func TestLowerBoundByteSizeStructsAndSlices(t *testing.T) {
+	type structWithSlice struct {
+		Count uint16
+		Data  []byte
+	}
+
+	t.Run("structWithSlice", func(t *testing.T) {
+		payload := []byte{1, 2, 3, 4}
+		value := structWithSlice{
+			Count: 42,
+			Data:  payload,
+		}
+
+		expected := uint64(unsafe.Sizeof(structWithSlice{}))
+		expected += sliceHeaderSize
+		expected += uint64(len(payload))
+		actual := LowerBoundByteSize(value)
+
+		require.Equal(
+			t, expected, actual,
+			"struct with slice size mismatch: expected %d, "+
+				"actual %d",
+			expected, actual,
+		)
+	})
+
+	t.Run("sharedBackingArrayCountedOnce", func(t *testing.T) {
+		payload := []byte{5, 6, 7}
+
+		type twoSlices struct {
+			Left  []byte
+			Right []byte
+		}
+
+		value := twoSlices{
+			Left:  payload,
+			Right: payload,
+		}
+
+		expected := uint64(unsafe.Sizeof(twoSlices{}))
+		expected += 2 * sliceHeaderSize
+		expected += uint64(len(payload))
+		actual := LowerBoundByteSize(value)
+
+		require.Equal(
+			t, expected, actual,
+			"shared backing array size mismatch: expected %d, "+
+				"actual %d",
+			expected, actual,
+		)
+	})
+}
+
+// TestLowerBoundByteSizePointerCycle confirms pointer cycles do not blow up the
+// traversal and only count the struct once.
+func TestLowerBoundByteSizePointerCycle(t *testing.T) {
+	type node struct {
+		Value uint32
+		Next  *node
+	}
+
+	root := &node{Value: 1}
+	root.Next = root
+
+	expected := uint64(unsafe.Sizeof(node{}))
+	actual := LowerBoundByteSize(root)
+	require.Equal(
+		t, expected, actual,
+		"pointer cycle size mismatch: expected %d, actual %d",
+		expected, actual,
+	)
+}
+
+// TestLowerBoundByteSizeMap verifies map headers and key/value payloads are
+// included in the lower bound calculation.
+func TestLowerBoundByteSizeMap(t *testing.T) {
+	payload := []byte{9, 8, 7}
+	value := map[string][]byte{
+		"alpha": payload,
+	}
+
+	expected := uint64(unsafe.Sizeof(any(value)))
+	expected += stringHeaderSize + uint64(len("alpha"))
+	expected += sliceHeaderSize + uint64(len(payload))
+	actual := LowerBoundByteSize(value)
+
+	require.Equal(
+		t, expected, actual,
+		"map size mismatch: expected %d, actual %d",
+		expected, actual,
+	)
+}

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/caddyserver/certmagic v0.17.2
 	github.com/davecgh/go-spew v1.1.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0
+	github.com/dustin/go-humanize v1.0.1
 	github.com/go-errors/errors v1.0.1
 	github.com/golang-migrate/migrate/v4 v4.17.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -85,7 +86,6 @@ require (
 	github.com/docker/docker v28.1.1+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fergusstrange/embedded-postgres v1.25.0 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

sample-tapd.conf

@@ -388,8 +388,9 @@
 
 [multiverse-caches]
 
-; The number of proofs that are cached per universe. (default: 5)
-; universe.multiverse-caches.proofs-per-universe=5
+; The maximum total size of the cached proofs. Accepts human readable values
+; such as 32MB or 1GB. (default: 32MB)
+; universe.multiverse-caches.max-proof-cache-size=32MB
 
 ; The number of universes that can have a cache of leaf keys. (default: 2000)
 ; universe.multiverse-caches.leaves-num-cached-universes=2000

tapcfg/server.go

@@ -152,11 +152,14 @@ func genServerConfig(cfg *Config, cfgLogger btclog.Logger,
 	cfgLogger.Debugf("multiverse_cache=%v",
 		spew.Sdump(cfg.Universe.MultiverseCaches))
 
-	multiverse := tapdb.NewMultiverseStore(
+	multiverse, err := tapdb.NewMultiverseStore(
 		multiverseDB, &tapdb.MultiverseStoreConfig{
 			Caches: *cfg.Universe.MultiverseCaches,
 		},
 	)
+	if err != nil {
+		return nil, fmt.Errorf("create multiverse store: %w", err)
+	}
 
 	uniStatsDB := tapdb.NewTransactionExecutor(
 		db, func(tx *sql.Tx) tapdb.UniverseStatsStore {

tapdb/cache_logger.go

@@ -46,6 +46,6 @@ func (c *cacheLogger) log() {
 	total := hit + miss
 	ratio := float64(hit) / float64(total) * 100
 
-	log.Infof("db cache %s: %d hits, %d misses, %.2f%% hit ratio",
+	log.Infof("cacheLogger(name=%s, hits=%d, misses=%d, hit_ratio=%.2f%%)",
 		c.name, hit, miss, ratio)
 }