feat: use mmap-go for windows support

* feat: use mmap-go for windows support * test: fix tests for windows * test: fix connection refused tests * test: fix single prop test * style: gofumpt * test: fix old prop tracker test * fix: rm commit log close
TraceMachina · Jul 26, 2023 · ec37dbc · ec37dbc
1 parent e9195aa
commit ec37dbc
Show file tree

Hide file tree

Showing 21 changed files with 232 additions and 68 deletions.
diff --git a/adapters/clients/cluster_schema_test.go b/adapters/clients/cluster_schema_test.go
@@ -187,7 +187,7 @@ func TestOpenTransactionUnhappyPaths(t *testing.T) {
 			shutdownPrematurely: true,
 			handler: func(w http.ResponseWriter, r *http.Request) {
 			},
-			expectedErrContains: "connection refused",
+			expectedErrContains: "refused",
 		},
 		{
 			name: "tx id mismatch",
@@ -301,7 +301,7 @@ func TestAbortTransactionUnhappyPaths(t *testing.T) {
 			shutdownPrematurely: true,
 			handler: func(w http.ResponseWriter, r *http.Request) {
 			},
-			expectedErrContains: "connection refused",
+			expectedErrContains: "refused",
 		},
 	}
 
@@ -399,7 +399,7 @@ func TestCommitTransactionUnhappyPaths(t *testing.T) {
 			shutdownPrematurely: true,
 			handler: func(w http.ResponseWriter, r *http.Request) {
 			},
-			expectedErrContains: "connection refused",
+			expectedErrContains: "refused",
 		},
 	}
 

diff --git a/adapters/repos/db/disk_use_unix.go b/adapters/repos/db/disk_use_unix.go
@@ -0,0 +1,33 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2023 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: hello@weaviate.io
+//go:build !windows
+
+package db
+
+import (
+	"syscall"
+)
+
+func (d *DB) getDiskUse(diskPath string) diskUse {
+	fs := syscall.Statfs_t{}
+
+	err := syscall.Statfs(diskPath, &fs)
+	if err != nil {
+		d.logger.WithField("action", "read_disk_use").
+			WithField("path", diskPath).
+			Errorf("failed to read disk usage: %s", err)
+	}
+
+	return diskUse{
+		total: fs.Blocks * uint64(fs.Bsize),
+		free:  fs.Bfree * uint64(fs.Bsize),
+		avail: fs.Bfree * uint64(fs.Bsize),
+	}
+}
diff --git a/adapters/repos/db/disk_use_windows.go b/adapters/repos/db/disk_use_windows.go
@@ -0,0 +1,42 @@
+//                           _       _
+// __      _____  __ ___   ___  __ _| |_ ___
+// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
+//  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
+//   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
+//
+//  Copyright © 2016 - 2023 Weaviate B.V. All rights reserved.
+//
+//  CONTACT: hello@weaviate.io
+//go:build windows
+
+package db
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/windows"
+)
+
+func (d *DB) getDiskUse(diskPath string) diskUse {
+	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes uint64
+
+	_, _ = syscall.UTF16PtrFromString(diskPath)
+
+	err := windows.GetDiskFreeSpaceEx(
+		syscall.StringToUTF16Ptr(diskPath),
+		&freeBytesAvailable,
+		&totalNumberOfBytes,
+		&totalNumberOfFreeBytes,
+	)
+	if err != nil {
+		d.logger.WithField("action", "read_disk_use").
+			WithField("path", diskPath).
+			Errorf("failed to read disk usage: %s", err)
+	}
+
+	return diskUse{
+		total: totalNumberOfBytes,
+		free:  totalNumberOfFreeBytes,
+		avail: freeBytesAvailable,
+	}
+}
diff --git a/adapters/repos/db/inverted/prop_length_tracker_test.go b/adapters/repos/db/inverted/prop_length_tracker_test.go
@@ -41,29 +41,33 @@ func Test_PropertyLengthTracker(t *testing.T) {
 		tests := []test{
 			{
 				values:       []float32{2, 2, 3, 100, 100, 500, 7},
-				name:         "mixed values",
+				name:         "mixed_values",
 				floatCompare: true,
-			}, {
+			},
+			{
 				values: []float32{
 					1000, 1200, 1000, 1300, 800, 2000, 2050,
 					2070, 900,
 				},
-				name:         "high values",
+				name:         "high_values",
 				floatCompare: true,
-			}, {
+			},
+			{
 				values: []float32{
 					60000, 50000, 65000,
 				},
-				name:         "very high values",
+				name:         "very_high_values",
 				floatCompare: true,
-			}, {
+			},
+			{
 				values: []float32{
 					1, 2, 4, 3, 4, 2, 1, 5, 6, 7, 8, 2, 7, 2, 3, 5,
 					6, 3, 5, 9, 3, 4, 8,
 				},
-				name:         "very low values",
+				name:         "very_low_values",
 				floatCompare: true,
-			}, {
+			},
+			{
 				values:       []float32{0, 0},
 				name:         "zeros",
 				floatCompare: false,
@@ -72,7 +76,7 @@ func Test_PropertyLengthTracker(t *testing.T) {
 
 		for _, test := range tests {
 			t.Run(test.name, func(t *testing.T) {
-				tracker, err := NewJsonPropertyLengthTracker(trackerPath, l)
+				tracker, err := NewJsonPropertyLengthTracker(trackerPath+test.name, l)
 				require.Nil(t, err)
 
 				actualMean := float32(0)
@@ -90,6 +94,7 @@ func Test_PropertyLengthTracker(t *testing.T) {
 				} else {
 					assert.Equal(t, actualMean, res)
 				}
+				require.Nil(t, tracker.Close())
 			})
 		}
 	})
@@ -122,6 +127,8 @@ func Test_PropertyLengthTracker(t *testing.T) {
 		assert.Equal(t, 3, sum)
 		assert.Equal(t, 1, count)
 		assert.InEpsilon(t, 3, mean, 0.1)
+
+		require.Nil(t, tracker.Close())
 	})
 
 	t.Run("multiple properties (can all fit on one page)", func(t *testing.T) {
@@ -170,6 +177,8 @@ func Test_PropertyLengthTracker(t *testing.T) {
 
 			assert.InEpsilon(t, actualMean, res, 0.1)
 		}
+
+		require.Nil(t, tracker.Close())
 	})
 
 	t.Run("with more properties that can fit on one page", func(t *testing.T) {
@@ -178,6 +187,8 @@ func Test_PropertyLengthTracker(t *testing.T) {
 		require.Nil(t, err)
 
 		create20PropsAndVerify(t, tracker)
+
+		require.Nil(t, tracker.Close())
 	})
 }
 
@@ -415,27 +426,27 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 		tests := []test{
 			{
 				values:       []float32{2, 2, 3, 100, 100, 500, 7},
-				name:         "mixed values",
+				name:         "mixed_values",
 				floatCompare: true,
 			}, {
 				values: []float32{
 					1000, 1200, 1000, 1300, 800, 2000, 2050,
 					2070, 900,
 				},
-				name:         "high values",
+				name:         "high_values",
 				floatCompare: true,
 			}, {
 				values: []float32{
 					60000, 50000, 65000,
 				},
-				name:         "very high values",
+				name:         "very_high_values",
 				floatCompare: true,
 			}, {
 				values: []float32{
 					1, 2, 4, 3, 4, 2, 1, 5, 6, 7, 8, 2, 7, 2, 3, 5,
 					6, 3, 5, 9, 3, 4, 8,
 				},
-				name:         "very low values",
+				name:         "very_low_values",
 				floatCompare: true,
 			}, {
 				values:       []float32{0, 0},
@@ -446,7 +457,7 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 
 		for _, test := range tests {
 			t.Run(test.name, func(t *testing.T) {
-				tracker, err := NewPropertyLengthTracker(trackerPath)
+				tracker, err := NewPropertyLengthTracker(trackerPath + test.name)
 				require.Nil(t, err)
 
 				actualMean := float32(0)
@@ -464,6 +475,7 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 				} else {
 					assert.Equal(t, actualMean, res)
 				}
+				require.Nil(t, tracker.Close())
 			})
 		}
 	})
@@ -496,6 +508,8 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 		assert.Equal(t, 3, sum)
 		assert.Equal(t, 1, count)
 		assert.InEpsilon(t, 3, mean, 0.1)
+
+		require.Nil(t, tracker.Close())
 	})
 
 	t.Run("multiple properties (can all fit on one page)", func(t *testing.T) {
@@ -544,6 +558,8 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 
 			assert.InEpsilon(t, actualMean, res, 0.1)
 		}
+
+		require.Nil(t, tracker.Close())
 	})
 
 	t.Run("with more properties that can fit on one page", func(t *testing.T) {
@@ -552,6 +568,8 @@ func TestOldPropertyLengthTracker(t *testing.T) {
 		require.Nil(t, err)
 
 		create20PropsAndVerify_old(t, tracker)
+
+		require.Nil(t, tracker.Close())
 	})
 }
 
@@ -600,6 +618,10 @@ func TestOldPropertyLengthTracker_Persistence(t *testing.T) {
 		require.Nil(t, err)
 		assert.InEpsilon(t, actualMeanForProp20, res, 0.1)
 	})
+
+	t.Run("shut down the second tracker", func(t *testing.T) {
+		require.Nil(t, secondTracker.Close())
+	})
 }
 
 func Test_PropertyLengthTracker_Overflow(t *testing.T) {
@@ -617,4 +639,6 @@ func Test_PropertyLengthTracker_Overflow(t *testing.T) {
 	// Check that property that would cause the internal counter to overflow is not added
 	err = tracker.TrackProperty("OVERFLOW", float32(123))
 	require.NotNil(t, err)
+
+	require.Nil(t, tracker.Close())
 }
diff --git a/adapters/repos/db/lsmkv/bucket_test.go b/adapters/repos/db/lsmkv/bucket_test.go
@@ -28,6 +28,9 @@ func TestBucket_WasDeleted(t *testing.T) {
 	b, err := NewBucket(context.Background(), tmpDir, "", logger, nil,
 		cyclemanager.NewNoop(), cyclemanager.NewNoop())
 	require.Nil(t, err)
+	t.Cleanup(func() {
+		require.Nil(t, b.Shutdown(context.Background()))
+	})
 
 	var (
 		key = []byte("key")

diff --git a/adapters/repos/db/lsmkv/memtable_roaring_set_test.go b/adapters/repos/db/lsmkv/memtable_roaring_set_test.go
@@ -50,6 +50,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.False(t, setKey2.Additions.Contains(2))
 		assert.True(t, setKey2.Additions.Contains(3))
 		assert.True(t, setKey2.Additions.Contains(4))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("inserting lists", func(t *testing.T) {
@@ -75,6 +77,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.False(t, setKey2.Additions.Contains(2))
 		assert.True(t, setKey2.Additions.Contains(3))
 		assert.True(t, setKey2.Additions.Contains(4))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("inserting bitmaps", func(t *testing.T) {
@@ -102,6 +106,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.False(t, setKey2.Additions.Contains(2))
 		assert.True(t, setKey2.Additions.Contains(3))
 		assert.True(t, setKey2.Additions.Contains(4))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("removing individual entries", func(t *testing.T) {
@@ -123,6 +129,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		require.Nil(t, err)
 		assert.False(t, setKey2.Additions.Contains(8))
 		assert.True(t, setKey2.Deletions.Contains(8))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("removing lists", func(t *testing.T) {
@@ -148,6 +156,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
 		assert.True(t, setKey2.Deletions.Contains(9))
 		assert.True(t, setKey2.Deletions.Contains(10))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("removing bitmaps", func(t *testing.T) {
@@ -173,6 +183,8 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
 		assert.True(t, setKey2.Deletions.Contains(9))
 		assert.True(t, setKey2.Deletions.Contains(10))
+
+		require.Nil(t, m.commitlog.close())
 	})
 
 	t.Run("adding/removing bitmaps", func(t *testing.T) {
@@ -204,5 +216,7 @@ func TestMemtableRoaringSet(t *testing.T) {
 		assert.Equal(t, 2, setKey2.Deletions.GetCardinality())
 		assert.True(t, setKey2.Deletions.Contains(9))
 		assert.True(t, setKey2.Deletions.Contains(10))
+
+		require.Nil(t, m.commitlog.close())
 	})
 }
diff --git a/adapters/repos/db/lsmkv/memtable_test.go b/adapters/repos/db/lsmkv/memtable_test.go
@@ -26,6 +26,9 @@ func Test_MemtableSecondaryKeyBug(t *testing.T) {
 	dir := t.TempDir()
 	m, err := newMemtable(path.Join(dir, "will-never-flush"), StrategyReplace, 1, nil)
 	require.Nil(t, err)
+	t.Cleanup(func() {
+		require.Nil(t, m.commitlog.close())
+	})
 
 	t.Run("add initial value", func(t *testing.T) {
 		err = m.put([]byte("my-key"), []byte("my-value"),