Problem: memiavl changes are not persisted in real time (#946)

* wal: writing to

* wal

* IndexBlockChangesBytes

* add some tests

* fix tests

* pull changeset

* refactor logic to use new changeset logic

* fix wal test

* cleanup comments

* replay wal

* fix conflict

* choosable wal paths for trees

* cleanup

* wip

* replay wal

* refactor wal

* dead code rm

* add functionality to replay everything from wal

* root hash once

* go mod tidy

* chores: remove unused code, better test for ReplayWAL, fix bugs caught with better test

* gomod2nix

* make wal optional, and fix unit tests

* fix lint

* tidy

* use go 1.20 in CI

* fix lint and unit test

* fix build

* fix go-version

* fix lint&test

* fix tests

* replayWAL

* gofumpt

---------

Co-authored-by: HuangYi <huang@crypto.com>

Author: pysel

.github/workflows/build.yml

@@ -66,7 +66,7 @@ jobs:
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - id: changed-files
         uses: tj-actions/changed-files@v34
         with:

.github/workflows/codeql-analysis.yml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.github/workflows/lint.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19.4
+          go-version: '^1.20.0'
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v19
         with:

.github/workflows/sims.yml

@@ -110,7 +110,7 @@ jobs:
         if: github.event_name == 'push' || github.event_name == 'pull_request' 
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Display go version
         run: go version
       - run: make build
@@ -131,7 +131,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Display go version
         run: go version
       - name: Install runsim
@@ -157,7 +157,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Checkout Comment PR Branch
         uses: actions/checkout@v3
         if: github.event_name == 'issue_comment'
@@ -212,7 +212,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Checkout Comment PR Branch
         uses: actions/checkout@v3
         if: github.event_name == 'issue_comment'
@@ -267,7 +267,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Checkout Comment PR Branch
         uses: actions/checkout@v3
         if: github.event_name == 'issue_comment'
@@ -322,7 +322,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.20.0
+          go-version: '^1.20.0'
       - name: Checkout Comment PR Branch
         uses: actions/checkout@v3
         if: github.event_name == 'issue_comment'

go.mod

@@ -174,6 +174,11 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tendermint/go-amino v0.16.0 // indirect
 	github.com/tidwall/btree v1.5.0 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/tinylru v1.1.0 // indirect
+	github.com/tidwall/wal v1.1.7 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect
 	github.com/torquem-ch/mdbx-go v0.27.5 // indirect

go.sum

@@ -1155,11 +1155,19 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l
 github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
 github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ=
 github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM=
+github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
+github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
+github.com/tidwall/wal v1.1.7 h1:emc1TRjIVsdKKSnpwGBAcsAGg0767SvUk8+ygx7Bb+4=
+github.com/tidwall/wal v1.1.7/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
 github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=

gomod2nix.toml

@@ -494,6 +494,21 @@ schema = 3
   [mod."github.com/tidwall/btree"]
     version = "v1.5.0"
     hash = "sha256-iWll4/+ADLVse3VAHxXYLprILugX/+3u0ZIk0YlLv/Q="
+  [mod."github.com/tidwall/gjson"]
+    version = "v1.14.0"
+    hash = "sha256-TiMGdAn840a2tOsi5mQ6fAK62ohvY6zuoVaUHCTyIwA="
+  [mod."github.com/tidwall/match"]
+    version = "v1.1.1"
+    hash = "sha256-M2klhPId3Q3T3VGkSbOkYl/2nLHnsG+yMbXkPkyrRdg="
+  [mod."github.com/tidwall/pretty"]
+    version = "v1.2.0"
+    hash = "sha256-esRQGsn2Ee/CiySlwyuOICSLdqUkH4P7u8qXszos8Yc="
+  [mod."github.com/tidwall/tinylru"]
+    version = "v1.1.0"
+    hash = "sha256-yjhttcm2IcJn6Rf9KicGqxm3LAOD3oheJel3QTi9znE="
+  [mod."github.com/tidwall/wal"]
+    version = "v1.1.7"
+    hash = "sha256-HLLFUEw8hWSSSSPSWZHt2KjoZftbje3Xxd9gROM2I+U="
   [mod."github.com/tklauser/go-sysconf"]
     version = "v0.3.10"
     hash = "sha256-Zf2NsgM9+HeM949vCce4HQtSbfUiFpeiQ716yKcFyx4="

memiavl/benchmark_test.go

@@ -19,24 +19,28 @@ func BenchmarkByteCompare(b *testing.B) {
 }
 
 func BenchmarkRandomGet(b *testing.B) {
+	b.Cleanup(removeDefaultWal)
+
 	amount := 1000000
 	items := genRandItems(amount)
 	targetKey := items[500].key
 	targetValue := items[500].value
 	targetItem := itemT{key: targetKey}
 
-	tree := New()
+	tree, err := New(DefaultPathToWAL)
+	require.NoError(b, err)
 	for _, item := range items {
 		tree.set(item.key, item.value)
 	}
 
 	snapshotDir := b.TempDir()
-	err := tree.WriteSnapshot(snapshotDir, true)
+	err = tree.WriteSnapshot(snapshotDir, true)
 	require.NoError(b, err)
 	snapshot, err := OpenSnapshot(snapshotDir)
 	require.NoError(b, err)
 	defer snapshot.Close()
-	diskTree := NewFromSnapshot(snapshot)
+	diskTree, err := NewFromSnapshot(snapshot, DefaultPathToWAL)
+	require.NoError(b, err)
 
 	require.Equal(b, targetValue, tree.Get(targetKey))
 	require.Equal(b, targetValue, diskTree.Get(targetKey))
@@ -109,22 +113,24 @@ func BenchmarkRandomGet(b *testing.B) {
 		for _, item := range items {
 			m[string(item.key)] = item.value
 		}
-		v, _ := m[string(targetItem.key)]
+		v := m[string(targetItem.key)]
 		require.Equal(b, targetValue, v)
 
 		b.ResetTimer()
 		for i := 0; i < b.N; i++ {
-			_, _ = m[string(targetKey)]
+			_ = m[string(targetKey)]
 		}
 	})
 }
 
 func BenchmarkRandomSet(b *testing.B) {
+	b.Cleanup(removeDefaultWal)
 	items := genRandItems(1000000)
 	b.ResetTimer()
 	b.Run("memiavl", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			tree := New()
+			tree, err := New(DefaultPathToWAL)
+			require.NoError(b, err)
 			for _, item := range items {
 				tree.set(item.key, item.value)
 			}

memiavl/changeset.go

@@ -22,15 +22,15 @@ import (
 //
 // ]
 // ```
-func MarshalChangeSet(cs *iavl.ChangeSet) ([]byte, error) {
+func MarshalChangeSet(cs iavl.ChangeSet) ([]byte, error) {
 	buf := bytes.NewBuffer(nil)
 	if err := MarshalChangeSetTo(cs, buf); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil
 }
 
-func MarshalChangeSetTo(cs *iavl.ChangeSet, w io.Writer) error {
+func MarshalChangeSetTo(cs iavl.ChangeSet, w io.Writer) error {
 	for _, pair := range cs.Pairs {
 		if _, err := w.Write([]byte{bool2byte(pair.Delete)}); err != nil {
 			return err
@@ -49,18 +49,18 @@ func MarshalChangeSetTo(cs *iavl.ChangeSet, w io.Writer) error {
 	return nil
 }
 
-func UnmarshalChangeSet(data []byte) (*iavl.ChangeSet, error) {
+func UnmarshalChangeSet(data []byte) (iavl.ChangeSet, error) {
 	var offset int
 	var cs iavl.ChangeSet
 	for offset < len(data) {
 		pair, n := readKVPair(data[offset:])
 		if n <= 0 {
-			return nil, fmt.Errorf("decode kv pair failed: %d", n)
+			return iavl.ChangeSet{}, fmt.Errorf("decode kv pair failed: %d", n)
 		}
 		offset += n
 		cs.Pairs = append(cs.Pairs, pair)
 	}
-	return &cs, nil
+	return cs, nil
 }
 
 func bool2byte(b bool) byte {

memiavl/changeset_test.go

@@ -8,11 +8,11 @@ import (
 
 func TestChangeSetMarshal(t *testing.T) {
 	for _, changes := range ChangeSets {
-		bz, err := MarshalChangeSet(&changes)
+		bz, err := MarshalChangeSet(changes)
 		require.NoError(t, err)
 
 		cs, err := UnmarshalChangeSet(bz)
 		require.NoError(t, err)
-		require.Equal(t, changes, *cs)
+		require.Equal(t, changes, cs)
 	}
 }