Skip to content

Commit 1a040b0

Browse files
authored
Merge pull request #6816 from ipfs/feat/mh-backed-datastore
feat: switch to raw multihashes for blocks
2 parents deb79a2 + 7fe5650 commit 1a040b0

File tree

15 files changed

+101
-141
lines changed

15 files changed

+101
-141
lines changed

core/commands/refs.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ var RefsLocalCmd = &cmds.Command{
136136
Helptext: cmds.HelpText{
137137
Tagline: "List all local references.",
138138
ShortDescription: `
139-
Displays the hashes of all local objects.
139+
Displays the hashes of all local objects. NOTE: This treats all local objects as "raw blocks" and returns CIDv1-Raw CIDs.
140140
`,
141141
},
142142

core/node/storage.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"github.com/ipfs/go-filestore"
1010
"github.com/ipfs/go-ipfs/core/node/helpers"
1111
"github.com/ipfs/go-ipfs/repo"
12-
"github.com/ipfs/go-ipfs/thirdparty/cidv0v1"
1312
"github.com/ipfs/go-ipfs/thirdparty/verifbs"
1413
)
1514

@@ -41,7 +40,6 @@ func BaseBlockstoreCtor(cacheOpts blockstore.CacheOpts, nilRepo bool, hashOnRead
4140
}
4241

4342
bs = blockstore.NewIdStore(bs)
44-
bs = cidv0v1.NewBlockstore(bs)
4543

4644
if hashOnRead { // TODO: review: this is how it was done originally, is there a reason we can't just pass this directly?
4745
bs.HashOnRead(true)

gc/gc.go

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ type Result struct {
2828
Error error
2929
}
3030

31+
// converts a set of CIDs with different codecs to a set of CIDs with the raw codec.
32+
func toRawCids(set *cid.Set) (*cid.Set, error) {
33+
newSet := cid.NewSet()
34+
err := set.ForEach(func(c cid.Cid) error {
35+
newSet.Add(cid.NewCidV1(cid.Raw, c.Hash()))
36+
return nil
37+
})
38+
return newSet, err
39+
}
40+
3141
// GC performs a mark and sweep garbage collection of the blocks in the blockstore
3242
// first, it creates a 'marked' set and adds to it the following:
3343
// - all recursively pinned blocks, plus all of their descendants (recursively)
@@ -60,6 +70,17 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
6070
}
6171
return
6272
}
73+
74+
// The blockstore reports raw blocks. We need to remove the codecs from the CIDs.
75+
gcs, err = toRawCids(gcs)
76+
if err != nil {
77+
select {
78+
case output <- Result{Error: err}:
79+
case <-ctx.Done():
80+
}
81+
return
82+
}
83+
6384
keychan, err := bs.AllKeysChan(ctx)
6485
if err != nil {
6586
select {
@@ -79,6 +100,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
79100
if !ok {
80101
break loop
81102
}
103+
// NOTE: assumes that all CIDs returned by the keychan are _raw_ CIDv1 CIDs.
104+
// This means we keep the block as long as we want it somewhere (CIDv1, CIDv0, Raw, other...).
82105
if !gcs.Has(k) {
83106
err := bs.DeleteBlock(ctx, k)
84107
removed++
@@ -154,7 +177,9 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
154177

155178
for _, c := range roots {
156179
// Walk recursively walks the dag and adds the keys to the given set
157-
err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent())
180+
err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool {
181+
return set.Visit(toCidV1(k))
182+
}, dag.Concurrent())
158183

159184
if err != nil {
160185
err = verboseCidError(err)
@@ -165,6 +190,14 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
165190
return nil
166191
}
167192

193+
// toCidV1 converts any CIDv0s to CIDv1s.
194+
func toCidV1(c cid.Cid) cid.Cid {
195+
if c.Version() == 0 {
196+
return cid.NewCidV1(c.Type(), c.Hash())
197+
}
198+
return c
199+
}
200+
168201
// ColoredSet computes the set of nodes in the graph that are pinned by the
169202
// pins in the given pinner.
170203
func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffortRoots []cid.Cid, output chan<- Result) (*cid.Set, error) {
@@ -225,7 +258,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
225258
return nil, err
226259
}
227260
for _, k := range dkeys {
228-
gcs.Add(k)
261+
gcs.Add(toCidV1(k))
229262
}
230263

231264
ikeys, err := pn.InternalPins(ctx)

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ require (
2525
github.com/ipfs/go-ds-leveldb v0.5.0
2626
github.com/ipfs/go-ds-measure v0.2.0
2727
github.com/ipfs/go-fetcher v1.6.1
28-
github.com/ipfs/go-filestore v0.1.0
28+
github.com/ipfs/go-filestore v1.1.0
2929
github.com/ipfs/go-fs-lock v0.0.7
3030
github.com/ipfs/go-graphsync v0.11.0
31-
github.com/ipfs/go-ipfs-blockstore v0.2.1
31+
github.com/ipfs/go-ipfs-blockstore v1.1.2
3232
github.com/ipfs/go-ipfs-chunker v0.0.5
3333
github.com/ipfs/go-ipfs-cmds v0.6.0
3434
github.com/ipfs/go-ipfs-config v0.18.0

go.sum

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -454,8 +454,8 @@ github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9
454454
github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE=
455455
github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8=
456456
github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo=
457-
github.com/ipfs/go-filestore v0.1.0 h1:qxvDVTzGrbQElddMmkwsJwJn+fDwWb3pHQHtKc1H0a8=
458-
github.com/ipfs/go-filestore v0.1.0/go.mod h1:0KTrzoJnJ3sJDEDM09Vq8nz8H475rRyeq4i0n/bpF00=
457+
github.com/ipfs/go-filestore v1.1.0 h1:Pu4tLBi1bucu6/HU9llaOmb9yLFk/sgP+pW764zNDoE=
458+
github.com/ipfs/go-filestore v1.1.0/go.mod h1:6e1/5Y6NvLuCRdmda/KA4GUhXJQ3Uat6vcWm2DJfxc8=
459459
github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
460460
github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
461461
github.com/ipfs/go-graphsync v0.11.0 h1:PiiD5CnoC3xEHMW8d6uBGqGcoTwiMB5d9CORIEyF6iA=
@@ -464,8 +464,10 @@ github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma
464464
github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
465465
github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
466466
github.com/ipfs/go-ipfs-blockstore v0.1.6/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
467-
github.com/ipfs/go-ipfs-blockstore v0.2.1 h1:624eIDnkZWNdWbp/N8aDBOUtSY0YW75aJu+vbxnNlkA=
468467
github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE=
468+
github.com/ipfs/go-ipfs-blockstore v1.1.1/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
469+
github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw=
470+
github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY=
469471
github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
470472
github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
471473
github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
@@ -479,8 +481,9 @@ github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1Y
479481
github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
480482
github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
481483
github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo=
482-
github.com/ipfs/go-ipfs-ds-help v0.1.1 h1:IW/bXGeaAZV2VH0Kuok+Ohva/zHkHmeLFBxC1k7mNPc=
483484
github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
485+
github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q=
486+
github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU=
484487
github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
485488
github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo=
486489
github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI=

repo/fsrepo/fsrepo.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ const LockFile = "repo.lock"
3636
var log = logging.Logger("fsrepo")
3737

3838
// version number that we are currently expecting to see
39-
var RepoVersion = 11
39+
var RepoVersion = 12
4040

4141
var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
4242
Sorry for the inconvenience. In the future, these will run automatically.`

repo/fsrepo/migrations/fetcher.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010

1111
const (
1212
// Current distribution to fetch migrations from
13-
CurrentIpfsDist = "/ipfs/QmP7tLxzhLU1KauTRX3jkVkF93pCv4skcceyUYMhf4AKJR" // fs-repo-migrations v2.0.2
13+
CurrentIpfsDist = "/ipfs/QmPweMoUxWFt1MSpYwLWsHB1GBcyYYDKPnANdERMY4U6hK" // fs-repo-11-to-12 v1.0.1
1414
// Latest distribution path. Default for fetchers.
1515
LatestIpfsDist = "/ipns/dist.ipfs.io"
1616

test/sharness/t0080-repo.sh

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,17 +109,25 @@ test_expect_success "remove direct pin" '
109109
'
110110

111111
test_expect_success "'ipfs repo gc' removes file" '
112-
ipfs repo gc >actual7 &&
113-
grep "removed $HASH" actual7
112+
ipfs block stat $HASH &&
113+
ipfs repo gc &&
114+
test_must_fail ipfs block stat $HASH
114115
'
115116

117+
# Convert all to a base32-multihash as refs local outputs cidv1 raw
118+
# Technically converting refs local output would suffice, but this is more
119+
# future proof if we ever switch to adding the files with cid-version 1.
116120
test_expect_success "'ipfs refs local' no longer shows file" '
117121
EMPTY_DIR=QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn &&
118-
ipfs refs local >actual8 &&
119-
grep "QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y" actual8 &&
120-
grep "$EMPTY_DIR" actual8 &&
121-
grep "$HASH_WELCOME_DOCS" actual8 &&
122-
test_must_fail grep "$HASH" actual8
122+
HASH_MH=`cid-fmt -b base32 "%M" "$HASH"` &&
123+
HARDCODED_HASH_MH=`cid-fmt -b base32 "%M" "QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y"` &&
124+
EMPTY_DIR_MH=`cid-fmt -b base32 "%M" "$EMPTY_DIR"` &&
125+
HASH_WELCOME_DOCS_MH=`cid-fmt -b base32 "%M" "$HASH_WELCOME_DOCS"` &&
126+
ipfs refs local | cid-fmt -b base32 --filter "%M" >actual8 &&
127+
grep "$HARDCODED_HASH_MH" actual8 &&
128+
grep "$EMPTY_DIR_MH" actual8 &&
129+
grep "$HASH_WELCOME_DOCS_MH" actual8 &&
130+
test_must_fail grep "$HASH_MH" actual8
123131
'
124132

125133
test_expect_success "adding multiblock random file succeeds" '

test/sharness/t0081-repo-pinning.sh

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -207,11 +207,10 @@ test_expect_success "pin lists look good" '
207207
'
208208

209209
test_expect_success "'ipfs repo gc' succeeds" '
210-
ipfs repo gc >gc_out_actual2 &&
211-
echo "removed $HASH_FILE3" > gc_out_exp2 &&
212-
echo "removed $HASH_FILE5" >> gc_out_exp2 &&
213-
echo "removed $HASH_DIR3" >> gc_out_exp2 &&
214-
test_includes_lines gc_out_exp2 gc_out_actual2
210+
ipfs repo gc &&
211+
test_must_fail ipfs block stat $HASH_FILE3 &&
212+
test_must_fail ipfs block stat $HASH_FILE5 &&
213+
test_must_fail ipfs block stat $HASH_DIR3
215214
'
216215

217216
# use object links for HASH_DIR1 here because its children

test/sharness/t0084-repo-read-rehash.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,9 @@ test_check_bad_blocks() {
3636
'
3737

3838
test_expect_success "block shows up in repo verify" '
39-
test_expect_code 1 ipfs repo verify > verify_out &&
40-
grep "$H_BLOCK2" verify_out
39+
test_expect_code 1 ipfs repo verify | cid-fmt --filter -b base32 "%M" > verify_out &&
40+
H_BLOCK2_MH=`cid-fmt -b base32 "%M" $H_BLOCK2` &&
41+
grep "$H_BLOCK2_MH" verify_out
4142
'
4243
}
4344

0 commit comments

Comments
 (0)