From 7a4b99e54ea86b4c4cc6d8c04fb617b9187c34cb Mon Sep 17 00:00:00 2001 From: Alex Trottier Date: Sat, 1 Feb 2020 15:20:52 -0800 Subject: [PATCH 1/4] chore: update dependencies, adjust use of removed functions, go 1.13 --- .travis.yml | 2 +- blockstore.go | 22 ++++++++++++---------- blockstore_test.go | 4 ++++ bloom_cache_test.go | 5 +++++ go.mod | 6 +++--- go.sum | 18 ++++-------------- 6 files changed, 29 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5163d69..a156d3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ os: language: go go: - - 1.11.x + - 1.13.x env: global: diff --git a/blockstore.go b/blockstore.go index 03004b5..a2eb65c 100644 --- a/blockstore.go +++ b/blockstore.go @@ -119,8 +119,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) { log.Error("undefined cid in blockstore") return nil, ErrNotFound } - - bdata, err := bs.datastore.Get(dshelp.CidToDsKey(k)) + bdata, err := bs.datastore.Get(dshelp.MultihashToDsKey(k.Hash())) if err == ds.ErrNotFound { return nil, ErrNotFound } @@ -143,7 +142,7 @@ func (bs *blockstore) Get(k cid.Cid) (blocks.Block, error) { } func (bs *blockstore) Put(block blocks.Block) error { - k := dshelp.CidToDsKey(block.Cid()) + k := dshelp.MultihashToDsKey(block.Cid().Hash()) // Has is cheaper than Put, so see if we already have it exists, err := bs.datastore.Has(k) @@ -159,7 +158,7 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error { return err } for _, b := range blocks { - k := dshelp.CidToDsKey(b.Cid()) + k := dshelp.MultihashToDsKey(b.Cid().Hash()) exists, err := bs.datastore.Has(k) if err == nil && exists { continue @@ -174,11 +173,11 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error { } func (bs *blockstore) Has(k cid.Cid) (bool, error) { - return bs.datastore.Has(dshelp.CidToDsKey(k)) + return bs.datastore.Has(dshelp.MultihashToDsKey(k.Hash())) } func (bs *blockstore) GetSize(k cid.Cid) (int, error) { - size, err := bs.datastore.GetSize(dshelp.CidToDsKey(k)) + size, err := bs.datastore.GetSize(dshelp.MultihashToDsKey(k.Hash())) if err == ds.ErrNotFound { return -1, ErrNotFound } @@ -186,7 +185,7 @@ func (bs *blockstore) GetSize(k cid.Cid) (int, error) { } func (bs *blockstore) DeleteBlock(k cid.Cid) error { - return bs.datastore.Delete(dshelp.CidToDsKey(k)) + return bs.datastore.Delete(dshelp.MultihashToDsKey(k.Hash())) } // AllKeysChan runs a query for keys from the blockstore. @@ -220,12 +219,15 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { } // need to convert to key.Key using key.KeyFromDsKey. - k, err := dshelp.DsKeyToCid(ds.RawKey(e.Key)) + bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key)) if err != nil { - log.Warningf("error parsing key from DsKey: %s", err) + log.Warning("error pasring key from binary: %s", err) continue } - + k, err := cid.Cast(bk) + if err != nil { + log.Warningf("failed to cast cid: %s", err) + } select { case <-ctx.Done(): return diff --git a/blockstore_test.go b/blockstore_test.go index b01574a..4422559 100644 --- a/blockstore_test.go +++ b/blockstore_test.go @@ -269,6 +269,10 @@ func (c *queryTestDS) Query(q dsq.Query) (dsq.Results, error) { return c.ds.Query(q) } +func (c *queryTestDS) Sync(key ds.Key) error { + return c.ds.Sync(key) +} + func (c *queryTestDS) Batch() (ds.Batch, error) { return ds.NewBasicBatch(c), nil } diff --git a/bloom_cache_test.go b/bloom_cache_test.go index fd65c0b..3b290a0 100644 --- a/bloom_cache_test.go +++ b/bloom_cache_test.go @@ -202,6 +202,11 @@ func (c *callbackDatastore) Query(q dsq.Query) (dsq.Results, error) { return c.ds.Query(q) } +func (c *callbackDatastore) Sync(key ds.Key) error { + c.CallF() + return c.ds.Sync(key) +} + func (c *callbackDatastore) Batch() (ds.Batch, error) { return ds.NewBasicBatch(c), nil } diff --git a/go.mod b/go.mod index 3d4b324..0749b04 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,12 @@ require ( github.com/ipfs/bbloom v0.0.4 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-cid v0.0.4 - github.com/ipfs/go-datastore v0.1.1 - github.com/ipfs/go-ipfs-ds-help v0.0.1 + github.com/ipfs/go-datastore v0.3.1 + github.com/ipfs/go-ipfs-ds-help v0.1.0 github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-log v0.0.1 github.com/ipfs/go-metrics-interface v0.0.1 github.com/multiformats/go-multihash v0.0.10 ) -go 1.12 +go 1.13 diff --git a/go.sum b/go.sum index 24781c2..f1cbeaf 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyF github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= @@ -20,19 +18,15 @@ github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqI github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-cid v0.0.1 h1:GBjWPktLnNyX0JiQCNFpUuUSoMw5KMyqrsejHYlILBE= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= -github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= -github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrUms= -github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= -github.com/ipfs/go-datastore v0.1.0 h1:TOxI04l8CmO4zGtesENhzm4PwkFwJXY3rKiYaaMf9fI= -github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= -github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= -github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.0 h1:/MfUnPgrtRCJvLdOiAgaODJD/BtV1CpfqQ53eqhWXqM= +github.com/ipfs/go-ipfs-ds-help v0.1.0/go.mod h1:zjN0aB3d6VyzJTSvcylSYxo+LSR/ELEfNnQM8p/vwc8= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= @@ -68,10 +62,6 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1 h1:HHwN1K12I+XllBCrqKnhX949Orn4oawPkegHMu2vDqQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= -github.com/multiformats/go-multihash v0.0.8 h1:wrYcW5yxSi3dU07n5jnuS5PrNwyHy0zRHGVoUugWvXg= -github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= -github.com/multiformats/go-multihash v0.0.9 h1:aoijQXYYl7Xtb2pUUP68R+ys1TlnlR3eX6wmozr0Hp4= -github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= From e62b875f6ff6a9f4230fa569eeef89bf97e7d2b7 Mon Sep 17 00:00:00 2001 From: Alex Trottier Date: Sat, 1 Feb 2020 15:24:20 -0800 Subject: [PATCH 2/4] fix typo in blockstore.go --- blockstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockstore.go b/blockstore.go index a2eb65c..a7e8b5d 100644 --- a/blockstore.go +++ b/blockstore.go @@ -221,7 +221,7 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { // need to convert to key.Key using key.KeyFromDsKey. bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key)) if err != nil { - log.Warning("error pasring key from binary: %s", err) + log.Warningf("error parsing key from binary: %s", err) continue } k, err := cid.Cast(bk) From 5f9214c8db55d92b36a16971d47854e7123f78e7 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Sat, 1 Feb 2020 17:01:48 -0800 Subject: [PATCH 3/4] fix: return raw block CIDs instead of V0 CIDs from AllKeysChan V0 CIDs are always "dag-pb". Using the "raw" codec indicates that these blocks are just raw data. --- blockstore.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/blockstore.go b/blockstore.go index a7e8b5d..e815642 100644 --- a/blockstore.go +++ b/blockstore.go @@ -224,10 +224,7 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { log.Warningf("error parsing key from binary: %s", err) continue } - k, err := cid.Cast(bk) - if err != nil { - log.Warningf("failed to cast cid: %s", err) - } + k := cid.NewCidV1(cid.Raw, bk) select { case <-ctx.Done(): return From 8f266cac109a60c25dd62959cc03127329fc2976 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Tue, 7 Jan 2020 09:34:11 -0800 Subject: [PATCH 4/4] feat: switch to raw multihashes for blocks Part of: https://github.com/ipfs/go-ipfs/issues/6815 --- arc_cache.go | 6 +++--- blockstore_test.go | 33 ++++++++++++++++++++++++++------- bloom_cache.go | 8 ++++---- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/arc_cache.go b/arc_cache.go index 8e88fa8..e2b930d 100644 --- a/arc_cache.go +++ b/arc_cache.go @@ -59,7 +59,7 @@ func (b *arccache) hasCached(k cid.Cid) (has bool, size int, ok bool) { return false, -1, false } - h, ok := b.arc.Get(k.KeyString()) + h, ok := b.arc.Get(string(k.Hash())) if ok { b.hits.Inc() switch h := h.(type) { @@ -160,11 +160,11 @@ func (b *arccache) HashOnRead(enabled bool) { } func (b *arccache) cacheHave(c cid.Cid, have bool) { - b.arc.Add(c.KeyString(), cacheHave(have)) + b.arc.Add(string(c.Hash()), cacheHave(have)) } func (b *arccache) cacheSize(c cid.Cid, blockSize int) { - b.arc.Add(c.KeyString(), cacheSize(blockSize)) + b.arc.Add(string(c.Hash()), cacheSize(blockSize)) } func (b *arccache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { diff --git a/blockstore_test.go b/blockstore_test.go index 4422559..a165652 100644 --- a/blockstore_test.go +++ b/blockstore_test.go @@ -53,6 +53,24 @@ func TestPutThenGetBlock(t *testing.T) { } } +func TestCidv0v1(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + block := blocks.NewBlock([]byte("some data")) + + err := bs.Put(block) + if err != nil { + t.Fatal(err) + } + + blockFromBlockstore, err := bs.Get(cid.NewCidV1(cid.DagProtobuf, block.Cid().Hash())) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) { + t.Fail() + } +} + func TestPutThenGetSizeBlock(t *testing.T) { bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) block := blocks.NewBlock([]byte("some data")) @@ -218,18 +236,19 @@ func TestAllKeysRespectsContext(t *testing.T) { } func expectMatches(t *testing.T, expect, actual []cid.Cid) { + t.Helper() if len(expect) != len(actual) { t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual)) } + + actualSet := make(map[string]bool, len(actual)) + for _, k := range actual { + actualSet[string(k.Hash())] = true + } + for _, ek := range expect { - found := false - for _, ak := range actual { - if ek.Equals(ak) { - found = true - } - } - if !found { + if !actualSet[string(ek.Hash())] { t.Error("expected key not found: ", ek) } } diff --git a/bloom_cache.go b/bloom_cache.go index 6e28ece..b4fadc2 100644 --- a/bloom_cache.go +++ b/bloom_cache.go @@ -103,7 +103,7 @@ func (b *bloomcache) build(ctx context.Context) error { atomic.StoreInt32(&b.active, 1) return nil } - b.bloom.AddTS(key.Bytes()) // Use binary key, the more compact the better + b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better case <-ctx.Done(): b.buildErr = ctx.Err() return b.buildErr @@ -130,7 +130,7 @@ func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) { return false, false } if b.BloomActive() { - blr := b.bloom.HasTS(k.Bytes()) + blr := b.bloom.HasTS(k.Hash()) if !blr { // not contained in bloom is only conclusive answer bloom gives b.hits.Inc() return false, true @@ -163,7 +163,7 @@ func (b *bloomcache) Put(bl blocks.Block) error { // See comment in PutMany err := b.blockstore.Put(bl) if err == nil { - b.bloom.AddTS(bl.Cid().Bytes()) + b.bloom.AddTS(bl.Cid().Hash()) } return err } @@ -178,7 +178,7 @@ func (b *bloomcache) PutMany(bs []blocks.Block) error { return err } for _, bl := range bs { - b.bloom.AddTS(bl.Cid().Bytes()) + b.bloom.AddTS(bl.Cid().Hash()) } return nil }