Skip to content

Commit fc0f210

Browse files
End-reyEnd-rey
committed
fstree: refactor tests
Make cleaner tests code, remove duplication. Signed-off-by: Andrey Butusov <andrey@nspcc.io>
1 parent d745fca commit fc0f210

File tree

5 files changed

+182
-258
lines changed

5 files changed

+182
-258
lines changed

‎pkg/local_object_storage/blobstor/fstree/bench_test.go

Lines changed: 68 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"io"
55
"testing"
66

7-
"github.com/nspcc-dev/neofs-node/pkg/core/object"
8-
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
97
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
108
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
119
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@@ -15,203 +13,121 @@ import (
1513
func BenchmarkFSTree_Head(b *testing.B) {
1614
for _, size := range payloadSizes {
1715
b.Run(generateSizeLabel(size), func(b *testing.B) {
18-
fsTree := fstree.New(fstree.WithPath(b.TempDir()))
19-
20-
require.NoError(b, fsTree.Open(false))
21-
require.NoError(b, fsTree.Init())
22-
23-
testReadOp(b, fsTree, fsTree.Head, "Head", size)
16+
runReadBenchmark(b, "Head", size)
2417
})
2518
}
2619
}
2720

2821
func BenchmarkFSTree_Get(b *testing.B) {
2922
for _, size := range payloadSizes {
3023
b.Run(generateSizeLabel(size), func(b *testing.B) {
31-
fsTree := fstree.New(fstree.WithPath(b.TempDir()))
32-
33-
require.NoError(b, fsTree.Open(false))
34-
require.NoError(b, fsTree.Init())
35-
36-
testReadOp(b, fsTree, fsTree.Get, "Get", size)
24+
runReadBenchmark(b, "Get", size)
3725
})
3826
}
3927
}
4028

4129
func BenchmarkFSTree_GetStream(b *testing.B) {
4230
for _, size := range payloadSizes {
4331
b.Run(generateSizeLabel(size), func(b *testing.B) {
44-
fsTree := fstree.New(fstree.WithPath(b.TempDir()))
45-
46-
require.NoError(b, fsTree.Open(false))
47-
require.NoError(b, fsTree.Init())
48-
49-
testGetStreamOp(b, fsTree, size)
32+
runReadBenchmark(b, "GetStream", size)
33+
34+
b.Run("GetStream_with_payload_read", func(b *testing.B) {
35+
freshFSTree := setupFSTree(b)
36+
addr := prepareSingleObject(b, freshFSTree, size)
37+
38+
b.ReportAllocs()
39+
b.ResetTimer()
40+
for range b.N {
41+
header, reader, err := freshFSTree.GetStream(addr)
42+
if err != nil {
43+
b.Fatal(err)
44+
}
45+
if header == nil {
46+
b.Fatal("header is nil")
47+
}
48+
if reader != nil {
49+
// Read all payload to simulate real usage
50+
_, err := io.ReadAll(reader)
51+
if err != nil {
52+
b.Fatal(err)
53+
}
54+
require.NoError(b, reader.Close())
55+
}
56+
}
57+
})
5058
})
5159
}
5260
}
5361

54-
func testReadOp(b *testing.B, fsTree *fstree.FSTree, read func(address oid.Address) (*objectSDK.Object, error),
55-
name string, payloadSize int) {
56-
b.Run(name+"_regular", func(b *testing.B) {
57-
obj := generateTestObject(payloadSize)
58-
addr := object.AddressOf(obj)
59-
60-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
61-
b.ReportAllocs()
62-
b.ResetTimer()
63-
for range b.N {
64-
_, err := read(addr)
65-
if err != nil {
66-
b.Fatal(err)
62+
func runReadBenchmark(b *testing.B, methodName string, payloadSize int) {
63+
testRead := func(fsTree *fstree.FSTree, addr oid.Address) {
64+
var err error
65+
switch methodName {
66+
case "Head":
67+
_, err = fsTree.Head(addr)
68+
case "Get":
69+
_, err = fsTree.Get(addr)
70+
case "GetStream":
71+
var (
72+
header *objectSDK.Object
73+
reader io.ReadCloser
74+
)
75+
header, reader, err = fsTree.GetStream(addr)
76+
if header == nil {
77+
b.Fatal("header is nil")
6778
}
68-
}
69-
})
70-
71-
b.Run(name+"_combined", func(b *testing.B) {
72-
const numObjects = 10
73-
74-
objMap := make(map[oid.Address][]byte, numObjects)
75-
addrs := make([]oid.Address, numObjects)
76-
for i := range numObjects {
77-
o := generateTestObject(payloadSize)
78-
objMap[object.AddressOf(o)] = o.Marshal()
79-
addrs[i] = object.AddressOf(o)
80-
}
81-
require.NoError(b, fsTree.PutBatch(objMap))
82-
83-
b.ReportAllocs()
84-
b.ResetTimer()
85-
for k := range b.N {
86-
_, err := read(addrs[k%numObjects])
87-
if err != nil {
88-
b.Fatal(err)
79+
if reader != nil {
80+
require.NoError(b, reader.Close())
8981
}
9082
}
91-
})
92-
93-
b.Run(name+"_compressed", func(b *testing.B) {
94-
obj := generateTestObject(payloadSize)
95-
addr := object.AddressOf(obj)
96-
97-
compressConfig := &compression.Config{
98-
Enabled: true,
99-
}
100-
require.NoError(b, compressConfig.Init())
101-
fsTree.SetCompressor(compressConfig)
102-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
103-
104-
b.ReportAllocs()
105-
b.ResetTimer()
106-
for range b.N {
107-
_, err := read(addr)
108-
if err != nil {
109-
b.Fatal(err)
110-
}
83+
if err != nil {
84+
b.Fatal(err)
11185
}
112-
})
113-
}
86+
}
11487

115-
func testGetStreamOp(b *testing.B, fsTree *fstree.FSTree, payloadSize int) {
116-
b.Run("GetStream_regular", func(b *testing.B) {
117-
obj := generateTestObject(payloadSize)
118-
addr := object.AddressOf(obj)
88+
b.Run(methodName+"_regular", func(b *testing.B) {
89+
fsTree := setupFSTree(b)
90+
addr := prepareSingleObject(b, fsTree, payloadSize)
11991

120-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
12192
b.ReportAllocs()
12293
b.ResetTimer()
12394
for range b.N {
124-
header, reader, err := fsTree.GetStream(addr)
125-
if err != nil {
126-
b.Fatal(err)
127-
}
128-
if header == nil {
129-
b.Fatal("header is nil")
130-
}
131-
if reader != nil {
132-
reader.Close()
133-
}
95+
testRead(fsTree, addr)
13496
}
13597
})
13698

137-
b.Run("GetStream_combined", func(b *testing.B) {
138-
const numObjects = 10
139-
140-
objMap := make(map[oid.Address][]byte, numObjects)
141-
addrs := make([]oid.Address, numObjects)
142-
for i := range numObjects {
143-
o := generateTestObject(payloadSize)
144-
objMap[object.AddressOf(o)] = o.Marshal()
145-
addrs[i] = object.AddressOf(o)
146-
}
147-
require.NoError(b, fsTree.PutBatch(objMap))
99+
b.Run(methodName+"_combined", func(b *testing.B) {
100+
fsTree := setupFSTree(b)
101+
addrs := prepareMultipleObjects(b, fsTree, payloadSize)
148102

149103
b.ReportAllocs()
150104
b.ResetTimer()
151105
for k := range b.N {
152-
header, reader, err := fsTree.GetStream(addrs[k%numObjects])
153-
if err != nil {
154-
b.Fatal(err)
155-
}
156-
if header == nil {
157-
b.Fatal("header is nil")
158-
}
159-
if reader != nil {
160-
reader.Close()
161-
}
106+
testRead(fsTree, addrs[k%len(addrs)])
162107
}
163108
})
164109

165-
b.Run("GetStream_compressed", func(b *testing.B) {
166-
obj := generateTestObject(payloadSize)
167-
addr := object.AddressOf(obj)
168-
169-
compressConfig := &compression.Config{
170-
Enabled: true,
171-
}
172-
require.NoError(b, compressConfig.Init())
173-
fsTree.SetCompressor(compressConfig)
174-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
110+
b.Run(methodName+"_compressed", func(b *testing.B) {
111+
fsTree := setupFSTree(b)
112+
setupCompressor(b, fsTree)
113+
addr := prepareSingleObject(b, fsTree, payloadSize)
175114

176115
b.ReportAllocs()
177116
b.ResetTimer()
178117
for range b.N {
179-
header, reader, err := fsTree.GetStream(addr)
180-
if err != nil {
181-
b.Fatal(err)
182-
}
183-
if header == nil {
184-
b.Fatal("header is nil")
185-
}
186-
if reader != nil {
187-
reader.Close()
188-
}
118+
testRead(fsTree, addr)
189119
}
190120
})
191121

192-
b.Run("GetStream_with_payload_read", func(b *testing.B) {
193-
obj := generateTestObject(payloadSize)
194-
addr := object.AddressOf(obj)
122+
b.Run(methodName+"_compressed_combined", func(b *testing.B) {
123+
fsTree := setupFSTree(b)
124+
setupCompressor(b, fsTree)
125+
addrs := prepareMultipleObjects(b, fsTree, payloadSize)
195126

196-
require.NoError(b, fsTree.Put(addr, obj.Marshal()))
197127
b.ReportAllocs()
198128
b.ResetTimer()
199-
for range b.N {
200-
header, reader, err := fsTree.GetStream(addr)
201-
if err != nil {
202-
b.Fatal(err)
203-
}
204-
if header == nil {
205-
b.Fatal("header is nil")
206-
}
207-
if reader != nil {
208-
// Read all payload to simulate real usage
209-
_, err := io.ReadAll(reader)
210-
if err != nil {
211-
b.Fatal(err)
212-
}
213-
reader.Close()
214-
}
129+
for k := range b.N {
130+
testRead(fsTree, addrs[k%len(addrs)])
215131
}
216132
})
217133
}

‎pkg/local_object_storage/blobstor/fstree/common_test.go

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package fstree_test
2+
3+
import (
4+
"crypto/rand"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/nspcc-dev/neofs-node/pkg/core/object"
9+
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
10+
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
11+
objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
12+
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
13+
objecttest "github.com/nspcc-dev/neofs-sdk-go/object/test"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
var payloadSizes = []int{
18+
0, // Empty payload
19+
100, // 100 bytes
20+
4 * 1024, // 4 KB
21+
16 * 1024, // 16 KB
22+
32 * 1024, // 32 KB
23+
100 * 1024, // 100 KB
24+
1024 * 1024, // 1 MB
25+
}
26+
27+
func setupFSTree(tb testing.TB) *fstree.FSTree {
28+
fsTree := fstree.New(fstree.WithPath(tb.TempDir()))
29+
require.NoError(tb, fsTree.Open(false))
30+
require.NoError(tb, fsTree.Init())
31+
return fsTree
32+
}
33+
34+
func setupCompressor(tb testing.TB, fsTree *fstree.FSTree) {
35+
compressConfig := &compression.Config{
36+
Enabled: true,
37+
}
38+
require.NoError(tb, compressConfig.Init())
39+
fsTree.SetCompressor(compressConfig)
40+
}
41+
42+
func prepareSingleObject(tb testing.TB, fsTree *fstree.FSTree, payloadSize int) oid.Address {
43+
obj := generateTestObject(payloadSize)
44+
addr := object.AddressOf(obj)
45+
require.NoError(tb, fsTree.Put(addr, obj.Marshal()))
46+
return addr
47+
}
48+
49+
func addAttribute(obj *objectSDK.Object, key, value string) {
50+
var attr objectSDK.Attribute
51+
attr.SetKey(key)
52+
attr.SetValue(value)
53+
54+
attrs := obj.Attributes()
55+
attrs = append(attrs, attr)
56+
obj.SetAttributes(attrs...)
57+
}
58+
59+
func generateTestObject(payloadSize int) *objectSDK.Object {
60+
obj := objecttest.Object()
61+
if payloadSize > 0 {
62+
payload := make([]byte, payloadSize)
63+
_, _ = rand.Read(payload)
64+
obj.SetPayload(payload)
65+
} else {
66+
obj.SetPayload(nil)
67+
}
68+
obj.SetPayloadSize(uint64(payloadSize))
69+
70+
return &obj
71+
}
72+
73+
func generateSizeLabel(size int) string {
74+
switch {
75+
case size == 0:
76+
return "Empty"
77+
case size < 1024:
78+
return fmt.Sprintf("%dB", size)
79+
case size < 1024*1024:
80+
return fmt.Sprintf("%dKB", size/1024)
81+
default:
82+
return fmt.Sprintf("%dMB", size/(1024*1024))
83+
}
84+
}
85+
86+
func prepareMultipleObjects(tb testing.TB, fsTree *fstree.FSTree, payloadSize int) []oid.Address {
87+
const numObjects = 10
88+
objMap := make(map[oid.Address][]byte, numObjects)
89+
addrs := make([]oid.Address, numObjects)
90+
91+
for i := range numObjects {
92+
obj := generateTestObject(payloadSize)
93+
addr := object.AddressOf(obj)
94+
objMap[addr] = obj.Marshal()
95+
addrs[i] = addr
96+
}
97+
98+
require.NoError(tb, fsTree.PutBatch(objMap))
99+
return addrs
100+
}

0 commit comments

Comments
 (0)