benchmark: simple benchmark to register shards

dagstore_benchmark_test.go

@@ -0,0 +1,106 @@
+package dagstore
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	ds "github.com/ipfs/go-datastore"
+	levelds "github.com/ipfs/go-ds-leveldb"
+	measure "github.com/ipfs/go-ds-measure"
+	ldbopts "github.com/syndtr/goleveldb/leveldb/opt"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/dagstore/index"
+	"github.com/filecoin-project/dagstore/mount"
+	"github.com/filecoin-project/dagstore/rand"
+	"github.com/filecoin-project/dagstore/shard"
+	"github.com/filecoin-project/dagstore/testdata"
+)
+
+func BenchmarkDagstore(b *testing.B) {
+	transientsDir, err := ioutil.TempDir("", "markets-repo-dir-transients")
+	if err != nil {
+		panic(err)
+	}
+
+	dsDir, err := ioutil.TempDir("", "markets-ds-dir")
+	if err != nil {
+		panic(err)
+	}
+
+	indexDir, err := ioutil.TempDir("", "markets-index-dir")
+	if err != nil {
+		panic(err)
+	}
+
+	ds, err := newDatastore(dsDir)
+	if err != nil {
+		panic(err)
+	}
+
+	r := mount.NewRegistry()
+	err = r.Register("fs", &mount.FSMount{FS: testdata.FS})
+	if err != nil {
+		panic(err)
+	}
+	err = r.Register("counting", new(mount.Counting))
+	if err != nil {
+		panic(err)
+	}
+
+	irepo, err := index.NewFSRepo(indexDir)
+	if err != nil {
+		panic(err)
+	}
+
+	dagst, err := NewDAGStore(Config{
+		IndexRepo:     irepo,
+		MountRegistry: r,
+		TransientsDir: transientsDir,
+		Datastore:     ds,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	err = dagst.Start(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	for n := 0; n < b.N; n++ {
+		ch := make(chan ShardResult, 1)
+		k := shard.KeyFromString(rand.String(10))
+
+		// even though the fs mount has an empty path, the existing transient will get us through registration.
+		err = dagst.RegisterShard(context.Background(), k, &mount.FSMount{FS: testdata.FS, Path: ""}, ch, RegisterOpts{ExistingTransient: testdata.RootPathCarV2})
+		if err != nil {
+			panic(err)
+		}
+
+		_ = <-ch
+	}
+}
+
+func newDatastore(dir string) (ds.Batching, error) {
+	// Create the datastore directory if it doesn't exist yet.
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return nil, xerrors.Errorf("failed to create directory %s for DAG store datastore: %w", dir, err)
+	}
+
+	// Create a new LevelDB datastore
+	dstore, err := levelds.NewDatastore(dir, &levelds.Options{
+		Compression: ldbopts.NoCompression,
+		NoSync:      false,
+		Strict:      ldbopts.StrictAll,
+		ReadOnly:    false,
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("failed to open datastore for DAG store: %w", err)
+	}
+	// Keep statistics about the datastore
+	mds := measure.New("measure.", dstore)
+	return mds, nil
+}

dagstore_test.go

@@ -29,7 +29,7 @@ var (
 )
 
 func init() {
-	_ = logging.SetLogLevel("dagstore", "DEBUG")
+	_ = logging.SetLogLevel("dagstore", "INFO")
 }
 
 func TestRegisterUsingExistingTransient(t *testing.T) {

go.mod

@@ -3,16 +3,24 @@ module github.com/filecoin-project/dagstore
 go 1.16
 
 require (
+	github.com/filecoin-project/lotus v0.0.0-00010101000000-000000000000
 	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.1.0
 	github.com/ipfs/go-datastore v0.4.5
-	github.com/ipfs/go-log/v2 v2.1.3
-	github.com/ipld/go-car/v2 v2.0.2
+	github.com/ipfs/go-ds-leveldb v0.4.2 // indirect
+	github.com/ipfs/go-ds-measure v0.1.0 // indirect
+	github.com/ipfs/go-log/v2 v2.3.0
+	github.com/ipld/go-car/v2 v2.0.3-0.20210811121346-c514a30114d7
 	github.com/mr-tron/base58 v1.2.0
 	github.com/multiformats/go-multicodec v0.3.0
 	github.com/stretchr/testify v1.7.0
-	github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158
-	golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	github.com/syndtr/goleveldb v1.0.0 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8
+	golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 )
+
+replace github.com/filecoin-project/lotus => ../lotus
+
+replace github.com/filecoin-project/filecoin-ffi => ../lotus/extern/filecoin-ffi