Merge pull request ipfs/go-ipfs-blockstore#38 from ipfs/feat/mh-backe…

…d-datastore

feat: switch to raw multihashes for blocks

This commit was moved from ipfs/go-ipfs-blockstore@451980b

blockstore/arc_cache.go

@@ -59,7 +59,7 @@ func (b *arccache) hasCached(k cid.Cid) (has bool, size int, ok bool) {
 		return false, -1, false
 	}
 
-	h, ok := b.arc.Get(k.KeyString())
+	h, ok := b.arc.Get(string(k.Hash()))
 	if ok {
 		b.hits.Inc()
 		switch h := h.(type) {
@@ -160,11 +160,11 @@ func (b *arccache) HashOnRead(enabled bool) {
 }
 
 func (b *arccache) cacheHave(c cid.Cid, have bool) {
-	b.arc.Add(c.KeyString(), cacheHave(have))
+	b.arc.Add(string(c.Hash()), cacheHave(have))
 }
 
 func (b *arccache) cacheSize(c cid.Cid, blockSize int) {
-	b.arc.Add(c.KeyString(), cacheSize(blockSize))
+	b.arc.Add(string(c.Hash()), cacheSize(blockSize))
 }
 
 func (b *arccache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {

blockstore/blockstore.go

@@ -119,8 +119,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) {
 		log.Error("undefined cid in blockstore")
 		return nil, ErrNotFound
 	}
-
-	bdata, err := bs.datastore.Get(dshelp.CidToDsKey(k))
+	bdata, err := bs.datastore.Get(dshelp.MultihashToDsKey(k.Hash()))
 	if err == ds.ErrNotFound {
 		return nil, ErrNotFound
 	}
@@ -143,7 +142,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) {
 }
 
 func (bs *blockstore) Put(block blocks.Block) error {
-	k := dshelp.CidToDsKey(block.Cid())
+	k := dshelp.MultihashToDsKey(block.Cid().Hash())
 
 	// Has is cheaper than Put, so see if we already have it
 	exists, err := bs.datastore.Has(k)
@@ -159,7 +158,7 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
 		return err
 	}
 	for _, b := range blocks {
-		k := dshelp.CidToDsKey(b.Cid())
+		k := dshelp.MultihashToDsKey(b.Cid().Hash())
 		exists, err := bs.datastore.Has(k)
 		if err == nil && exists {
 			continue
@@ -174,19 +173,19 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
 }
 
 func (bs *blockstore) Has(k cid.Cid) (bool, error) {
-	return bs.datastore.Has(dshelp.CidToDsKey(k))
+	return bs.datastore.Has(dshelp.MultihashToDsKey(k.Hash()))
 }
 
 func (bs *blockstore) GetSize(k cid.Cid) (int, error) {
-	size, err := bs.datastore.GetSize(dshelp.CidToDsKey(k))
+	size, err := bs.datastore.GetSize(dshelp.MultihashToDsKey(k.Hash()))
 	if err == ds.ErrNotFound {
 		return -1, ErrNotFound
 	}
 	return size, err
 }
 
 func (bs *blockstore) DeleteBlock(k cid.Cid) error {
-	return bs.datastore.Delete(dshelp.CidToDsKey(k))
+	return bs.datastore.Delete(dshelp.MultihashToDsKey(k.Hash()))
 }
 
 // AllKeysChan runs a query for keys from the blockstore.
@@ -220,12 +219,12 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
 			}
 
 			// need to convert to key.Key using key.KeyFromDsKey.
-			k, err := dshelp.DsKeyToCid(ds.RawKey(e.Key))
+			bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key))
 			if err != nil {
-				log.Warningf("error parsing key from DsKey: %s", err)
+				log.Warningf("error parsing key from binary: %s", err)
 				continue
 			}
-
+			k := cid.NewCidV1(cid.Raw, bk)
 			select {
 			case <-ctx.Done():
 				return

blockstore/blockstore_test.go

@@ -53,6 +53,24 @@ func TestPutThenGetBlock(t *testing.T) {
 	}
 }
 
+func TestCidv0v1(t *testing.T) {
+	bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+	block := blocks.NewBlock([]byte("some data"))
+
+	err := bs.Put(block)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	blockFromBlockstore, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, block.Cid().Hash()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) {
+		t.Fail()
+	}
+}
+
 func TestPutThenGetSizeBlock(t *testing.T) {
 	bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
 	block := blocks.NewBlock([]byte("some data"))
@@ -218,18 +236,19 @@ func TestAllKeysRespectsContext(t *testing.T) {
 }
 
 func expectMatches(t *testing.T, expect, actual []cid.Cid) {
+	t.Helper()
 
 	if len(expect) != len(actual) {
 		t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
 	}
+
+	actualSet := make(map[string]bool, len(actual))
+	for _, k := range actual {
+		actualSet[string(k.Hash())] = true
+	}
+
 	for _, ek := range expect {
-		found := false
-		for _, ak := range actual {
-			if ek.Equals(ak) {
-				found = true
-			}
-		}
-		if !found {
+		if !actualSet[string(ek.Hash())] {
 			t.Error("expected key not found: ", ek)
 		}
 	}
@@ -269,6 +288,10 @@ func (c *queryTestDS) Query(q dsq.Query) (dsq.Results, error) {
 	return c.ds.Query(q)
 }
 
+func (c *queryTestDS) Sync(key ds.Key) error {
+	return c.ds.Sync(key)
+}
+
 func (c *queryTestDS) Batch() (ds.Batch, error) {
 	return ds.NewBasicBatch(c), nil
 }

blockstore/bloom_cache.go

@@ -103,7 +103,7 @@ func (b *bloomcache) build(ctx context.Context) error {
 				atomic.StoreInt32(&b.active, 1)
 				return nil
 			}
-			b.bloom.AddTS(key.Bytes()) // Use binary key, the more compact the better
+			b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better
 		case <-ctx.Done():
 			b.buildErr = ctx.Err()
 			return b.buildErr
@@ -130,7 +130,7 @@ func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) {
 		return false, false
 	}
 	if b.BloomActive() {
-		blr := b.bloom.HasTS(k.Bytes())
+		blr := b.bloom.HasTS(k.Hash())
 		if !blr { // not contained in bloom is only conclusive answer bloom gives
 			b.hits.Inc()
 			return false, true
@@ -163,7 +163,7 @@ func (b *bloomcache) Put(bl blocks.Block) error {
 	// See comment in PutMany
 	err := b.blockstore.Put(bl)
 	if err == nil {
-		b.bloom.AddTS(bl.Cid().Bytes())
+		b.bloom.AddTS(bl.Cid().Hash())
 	}
 	return err
 }
@@ -178,7 +178,7 @@ func (b *bloomcache) PutMany(bs []blocks.Block) error {
 		return err
 	}
 	for _, bl := range bs {
-		b.bloom.AddTS(bl.Cid().Bytes())
+		b.bloom.AddTS(bl.Cid().Hash())
 	}
 	return nil
 }

blockstore/bloom_cache_test.go

@@ -202,6 +202,11 @@ func (c *callbackDatastore) Query(q dsq.Query) (dsq.Results, error) {
 	return c.ds.Query(q)
 }
 
+func (c *callbackDatastore) Sync(key ds.Key) error {
+	c.CallF()
+	return c.ds.Sync(key)
+}
+
 func (c *callbackDatastore) Batch() (ds.Batch, error) {
 	return ds.NewBasicBatch(c), nil
 }