Skip to content
This repository was archived by the owner on Jun 19, 2023. It is now read-only.

Commit 451980b

Browse files
StebalienStebalien
authored
Merge pull request #38 from ipfs/feat/mh-backed-datastore
feat: switch to raw multihashes for blocks
2 parents e3929f1 + 8f266ca commit 451980b

File tree

8 files changed

+59
-42
lines changed

8 files changed

+59
-42
lines changed

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ os:
44
language: go
55

66
go:
7-
- 1.11.x
7+
- 1.13.x
88

99
env:
1010
global:

‎arc_cache.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (b *arccache) hasCached(k cid.Cid) (has bool, size int, ok bool) {
5959
return false, -1, false
6060
}
6161

62-
h, ok := b.arc.Get(k.KeyString())
62+
h, ok := b.arc.Get(string(k.Hash()))
6363
if ok {
6464
b.hits.Inc()
6565
switch h := h.(type) {
@@ -160,11 +160,11 @@ func (b *arccache) HashOnRead(enabled bool) {
160160
}
161161

162162
func (b *arccache) cacheHave(c cid.Cid, have bool) {
163-
b.arc.Add(c.KeyString(), cacheHave(have))
163+
b.arc.Add(string(c.Hash()), cacheHave(have))
164164
}
165165

166166
func (b *arccache) cacheSize(c cid.Cid, blockSize int) {
167-
b.arc.Add(c.KeyString(), cacheSize(blockSize))
167+
b.arc.Add(string(c.Hash()), cacheSize(blockSize))
168168
}
169169

170170
func (b *arccache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {

‎blockstore.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) {
119119
log.Error("undefined cid in blockstore")
120120
return nil, ErrNotFound
121121
}
122-
123-
bdata, err := bs.datastore.Get(dshelp.CidToDsKey(k))
122+
bdata, err := bs.datastore.Get(dshelp.MultihashToDsKey(k.Hash()))
124123
if err == ds.ErrNotFound {
125124
return nil, ErrNotFound
126125
}
@@ -143,7 +142,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) {
143142
}
144143

145144
func (bs *blockstore) Put(block blocks.Block) error {
146-
k := dshelp.CidToDsKey(block.Cid())
145+
k := dshelp.MultihashToDsKey(block.Cid().Hash())
147146

148147
// Has is cheaper than Put, so see if we already have it
149148
exists, err := bs.datastore.Has(k)
@@ -159,7 +158,7 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
159158
return err
160159
}
161160
for _, b := range blocks {
162-
k := dshelp.CidToDsKey(b.Cid())
161+
k := dshelp.MultihashToDsKey(b.Cid().Hash())
163162
exists, err := bs.datastore.Has(k)
164163
if err == nil && exists {
165164
continue
@@ -174,19 +173,19 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
174173
}
175174

176175
func (bs *blockstore) Has(k cid.Cid) (bool, error) {
177-
return bs.datastore.Has(dshelp.CidToDsKey(k))
176+
return bs.datastore.Has(dshelp.MultihashToDsKey(k.Hash()))
178177
}
179178

180179
func (bs *blockstore) GetSize(k cid.Cid) (int, error) {
181-
size, err := bs.datastore.GetSize(dshelp.CidToDsKey(k))
180+
size, err := bs.datastore.GetSize(dshelp.MultihashToDsKey(k.Hash()))
182181
if err == ds.ErrNotFound {
183182
return -1, ErrNotFound
184183
}
185184
return size, err
186185
}
187186

188187
func (bs *blockstore) DeleteBlock(k cid.Cid) error {
189-
return bs.datastore.Delete(dshelp.CidToDsKey(k))
188+
return bs.datastore.Delete(dshelp.MultihashToDsKey(k.Hash()))
190189
}
191190

192191
// AllKeysChan runs a query for keys from the blockstore.
@@ -220,12 +219,12 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
220219
}
221220

222221
// need to convert to key.Key using key.KeyFromDsKey.
223-
k, err := dshelp.DsKeyToCid(ds.RawKey(e.Key))
222+
bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key))
224223
if err != nil {
225-
log.Warningf("error parsing key from DsKey: %s", err)
224+
log.Warningf("error parsing key from binary: %s", err)
226225
continue
227226
}
228-
227+
k := cid.NewCidV1(cid.Raw, bk)
229228
select {
230229
case <-ctx.Done():
231230
return

‎blockstore_test.go‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,24 @@ func TestPutThenGetBlock(t *testing.T) {
5353
}
5454
}
5555

56+
func TestCidv0v1(t *testing.T) {
57+
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
58+
block := blocks.NewBlock([]byte("some data"))
59+
60+
err := bs.Put(block)
61+
if err != nil {
62+
t.Fatal(err)
63+
}
64+
65+
blockFromBlockstore, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, block.Cid().Hash()))
66+
if err != nil {
67+
t.Fatal(err)
68+
}
69+
if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) {
70+
t.Fail()
71+
}
72+
}
73+
5674
func TestPutThenGetSizeBlock(t *testing.T) {
5775
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
5876
block := blocks.NewBlock([]byte("some data"))
@@ -218,18 +236,19 @@ func TestAllKeysRespectsContext(t *testing.T) {
218236
}
219237

220238
func expectMatches(t *testing.T, expect, actual []cid.Cid) {
239+
t.Helper()
221240

222241
if len(expect) != len(actual) {
223242
t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
224243
}
244+
245+
actualSet := make(map[string]bool, len(actual))
246+
for _, k := range actual {
247+
actualSet[string(k.Hash())] = true
248+
}
249+
225250
for _, ek := range expect {
226-
found := false
227-
for _, ak := range actual {
228-
if ek.Equals(ak) {
229-
found = true
230-
}
231-
}
232-
if !found {
251+
if !actualSet[string(ek.Hash())] {
233252
t.Error("expected key not found: ", ek)
234253
}
235254
}
@@ -269,6 +288,10 @@ func (c *queryTestDS) Query(q dsq.Query) (dsq.Results, error) {
269288
return c.ds.Query(q)
270289
}
271290

291+
func (c *queryTestDS) Sync(key ds.Key) error {
292+
return c.ds.Sync(key)
293+
}
294+
272295
func (c *queryTestDS) Batch() (ds.Batch, error) {
273296
return ds.NewBasicBatch(c), nil
274297
}

‎bloom_cache.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (b *bloomcache) build(ctx context.Context) error {
103103
atomic.StoreInt32(&b.active, 1)
104104
return nil
105105
}
106-
b.bloom.AddTS(key.Bytes()) // Use binary key, the more compact the better
106+
b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better
107107
case <-ctx.Done():
108108
b.buildErr = ctx.Err()
109109
return b.buildErr
@@ -130,7 +130,7 @@ func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) {
130130
return false, false
131131
}
132132
if b.BloomActive() {
133-
blr := b.bloom.HasTS(k.Bytes())
133+
blr := b.bloom.HasTS(k.Hash())
134134
if !blr { // not contained in bloom is only conclusive answer bloom gives
135135
b.hits.Inc()
136136
return false, true
@@ -163,7 +163,7 @@ func (b *bloomcache) Put(bl blocks.Block) error {
163163
// See comment in PutMany
164164
err := b.blockstore.Put(bl)
165165
if err == nil {
166-
b.bloom.AddTS(bl.Cid().Bytes())
166+
b.bloom.AddTS(bl.Cid().Hash())
167167
}
168168
return err
169169
}
@@ -178,7 +178,7 @@ func (b *bloomcache) PutMany(bs []blocks.Block) error {
178178
return err
179179
}
180180
for _, bl := range bs {
181-
b.bloom.AddTS(bl.Cid().Bytes())
181+
b.bloom.AddTS(bl.Cid().Hash())
182182
}
183183
return nil
184184
}

‎bloom_cache_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,11 @@ func (c *callbackDatastore) Query(q dsq.Query) (dsq.Results, error) {
202202
return c.ds.Query(q)
203203
}
204204

205+
func (c *callbackDatastore) Sync(key ds.Key) error {
206+
c.CallF()
207+
return c.ds.Sync(key)
208+
}
209+
205210
func (c *callbackDatastore) Batch() (ds.Batch, error) {
206211
return ds.NewBasicBatch(c), nil
207212
}

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ require (
55
github.com/ipfs/bbloom v0.0.4
66
github.com/ipfs/go-block-format v0.0.2
77
github.com/ipfs/go-cid v0.0.4
8-
github.com/ipfs/go-datastore v0.1.1
9-
github.com/ipfs/go-ipfs-ds-help v0.0.1
8+
github.com/ipfs/go-datastore v0.3.1
9+
github.com/ipfs/go-ipfs-ds-help v0.1.0
1010
github.com/ipfs/go-ipfs-util v0.0.1
1111
github.com/ipfs/go-log v0.0.1
1212
github.com/ipfs/go-metrics-interface v0.0.1
1313
github.com/multiformats/go-multihash v0.0.10
1414
)
1515

16-
go 1.12
16+
go 1.13

‎go.sum‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyF
1010
github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
1111
github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc=
1212
github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
13-
github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
14-
github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
1513
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
1614
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
1715
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
@@ -20,19 +18,15 @@ github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqI
2018
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
2119
github.com/ipfs/go-cid v0.0.1 h1:GBjWPktLnNyX0JiQCNFpUuUSoMw5KMyqrsejHYlILBE=
2220
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
23-
github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
24-
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
2521
github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
2622
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
27-
github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrUms=
28-
github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
29-
github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI=
30-
github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
3123
github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI=
3224
github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
25+
github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU=
26+
github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
3327
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
34-
github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU=
35-
github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
28+
github.com/ipfs/go-ipfs-ds-help v0.1.0 h1:/MfUnPgrtRCJvLdOiAgaODJD/BtV1CpfqQ53eqhWXqM=
29+
github.com/ipfs/go-ipfs-ds-help v0.1.0/go.mod h1:zjN0aB3d6VyzJTSvcylSYxo+LSR/ELEfNnQM8p/vwc8=
3630
github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50=
3731
github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
3832
github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
@@ -68,10 +62,6 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr
6862
github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
6963
github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ=
7064
github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
71-
github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg=
72-
github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
73-
github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4=
74-
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
7565
github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
7666
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
7767
github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=

0 commit comments

Comments
 (0)