chore: upgrade to new multiformats module

BREAKING CHANGE: Uses new CID class

Author: achingbrain

.aegir.js

@@ -27,6 +27,7 @@ module.exports = {
     }
   },
   build: {
+    bundlesizeMax: '45kB',
     config: esbuild
   }
 }

migrations/migration-8/index.js

@@ -1,12 +1,13 @@
 'use strict'
 
-const CID = require('cids')
+const { CID } = require('multiformats')
 const Key = require('interface-datastore').Key
-const mb = require('multibase')
 const log = require('debug')('ipfs:repo:migrator:migration-8')
-const uint8ArrayToString = require('uint8arrays/to-string')
 const { createStore } = require('../../src/utils')
 const length = require('it-length')
+const { base32 } = require('multiformats/bases/base32')
+const raw = require('multiformats/codecs/raw')
+const mhd = require('multiformats/hashes/digest')
 
 /**
  * @typedef {import('../../src/types').Migration} Migration
@@ -16,16 +17,14 @@ const length = require('it-length')
  * @param {Key} key
  */
 function keyToMultihash (key) {
-  const buf = mb.decode(`b${key.toString().slice(1)}`)
+  const buf = base32.decode(`b${key.toString().toLowerCase().slice(1)}`)
 
   // Extract multihash from CID
-  let multihash = new CID(buf).multihash
+  let multihash = CID.decode(buf).multihash.bytes
 
   // Encode and slice off multibase codec
-  multihash = mb.encode('base32', multihash).slice(1)
-
   // Should be uppercase for interop with go
-  const multihashStr = uint8ArrayToString(multihash).toUpperCase()
+  const multihashStr = base32.encode(multihash).slice(1).toUpperCase()
 
   return new Key(`/${multihashStr}`, false)
 }
@@ -34,12 +33,13 @@ function keyToMultihash (key) {
  * @param {Key} key
  */
 function keyToCid (key) {
-  const buf = mb.decode(`b${key.toString().slice(1)}`)
+  const buf = base32.decode(`b${key.toString().toLowerCase().slice(1)}`)
+  const digest = mhd.decode(buf)
 
   // CID to Key
-  const multihash = mb.encode('base32', new CID(1, 'raw', buf).bytes).slice(1)
+  const multihash = base32.encode(CID.createV1(raw.code, digest).bytes).slice(1)
 
-  return new Key(`/${uint8ArrayToString(multihash)}`.toUpperCase(), false)
+  return new Key(`/${multihash.toUpperCase()}`, false)
 }
 
 /**

migrations/migration-9/index.js

@@ -1,20 +1,20 @@
 'use strict'
 
-const CID = require('cids')
-const dagpb = require('ipld-dag-pb')
+const { CID } = require('multiformats')
+const dagPb = require('@ipld/dag-pb')
 const cbor = require('cborg')
-const multicodec = require('multicodec')
-const multibase = require('multibase')
 const pinset = require('./pin-set')
 const { createStore } = require('../../src/utils')
 const { cidToKey, PIN_DS_KEY, PinTypes } = require('./utils')
 const length = require('it-length')
+const { sha256 } = require('multiformats/hashes/sha2')
+const mhd = require('multiformats/hashes/digest')
+const { base32 } = require('multiformats/bases/base32')
 
 /**
  * @typedef {import('../../src/types').Migration} Migration
  * @typedef {import('../../src/types').MigrationProgressCallback} MigrationProgressCallback
  * @typedef {import('interface-datastore').Datastore} Datastore
- * @typedef {import('multicodec').CodecCode} CodecCode
  */
 
  /**
@@ -29,9 +29,9 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
   }
 
   const mh = await datastore.get(PIN_DS_KEY)
-  const cid = new CID(mh)
+  const cid = CID.decode(mh)
   const pinRootBuf = await blockstore.get(cidToKey(cid))
-  const pinRoot = dagpb.util.deserialize(pinRootBuf)
+  const pinRoot = dagPb.decode(pinRootBuf)
   let counter = 0
   let pinCount
 
@@ -40,7 +40,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
   for await (const cid of pinset.loadSet(blockstore, pinRoot, PinTypes.recursive)) {
     counter++
 
-    /** @type {{ depth: number, version?: CID.CIDVersion, codec?: CodecCode }} */
+    /** @type {{ depth: number, version?: 0 | 1, codec?: number }} */
     const pin = {
       depth: Infinity
     }
@@ -49,8 +49,8 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
       pin.version = cid.version
     }
 
-    if (cid.codec !== 'dag-pb') {
-      pin.codec = multicodec.getNumber(cid.codec)
+    if (cid.code !== dagPb.code) {
+      pin.codec = cid.code
     }
 
     await pinstore.put(cidToKey(cid), cbor.encode(pin))
@@ -61,7 +61,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
   for await (const cid of pinset.loadSet(blockstore, pinRoot, PinTypes.direct)) {
     counter++
 
-    /** @type {{ depth: number, version?: CID.CIDVersion, codec?: CodecCode }} */
+    /** @type {{ depth: number, version?: 0 | 1, codec?: number }} */
     const pin = {
       depth: 0
     }
@@ -70,8 +70,8 @@ async function pinsToDatastore (blockstore, datastore, pinstore, onProgress) {
       pin.version = cid.version
     }
 
-    if (cid.codec !== 'dag-pb') {
-      pin.codec = multicodec.getNumber(cid.codec)
+    if (cid.code !== dagPb.code) {
+      pin.codec = cid.code
     }
 
     await pinstore.put(cidToKey(cid), cbor.encode(pin))
@@ -98,7 +98,11 @@ async function pinsToDAG (blockstore, datastore, pinstore, onProgress) {
   for await (const { key, value } of pinstore.query({})) {
     counter++
     const pin = cbor.decode(value)
-    const cid = new CID(pin.version || 0, pin.codec && multicodec.getName(pin.codec) || 'dag-pb', multibase.decode('b' + key.toString().split('/').pop()))
+    const cid = CID.create(
+      pin.version || 0,
+      pin.codec || dagPb.code,
+      mhd.decode(base32.decode('b' + key.toString().toLowerCase().split('/').pop()))
+    )
 
     if (pin.depth === 0) {
       onProgress((counter / pinCount) * 100, `Reverted direct pin ${cid}`)
@@ -112,16 +116,18 @@ async function pinsToDAG (blockstore, datastore, pinstore, onProgress) {
   }
 
   onProgress(100, 'Updating pin root')
-  const pinRoot = new dagpb.DAGNode(new Uint8Array(), [
-    await pinset.storeSet(blockstore, PinTypes.recursive, recursivePins),
-    await pinset.storeSet(blockstore, PinTypes.direct, directPins)
-  ])
-  const buf = pinRoot.serialize()
-  const cid = await dagpb.util.cid(buf, {
-    cidVersion: 0
-  })
+  const pinRoot = {
+    Links: [
+      await pinset.storeSet(blockstore, PinTypes.direct, directPins),
+      await pinset.storeSet(blockstore, PinTypes.recursive, recursivePins)
+    ]
+  }
+  const buf = dagPb.encode(pinRoot)
+  const digest = await sha256.digest(buf)
+  const cid = CID.createV0(digest)
+
   await blockstore.put(cidToKey(cid), buf)
-  await datastore.put(PIN_DS_KEY, cid.multihash)
+  await datastore.put(PIN_DS_KEY, cid.bytes)
 }
 
 /**

migrations/migration-9/pin-set.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const CID = require('cids')
+const { CID } = require('multiformats')
 const {
   ipfs: {
     pin: {
@@ -12,39 +12,35 @@ const {
 // @ts-ignore
 const fnv1a = require('fnv1a')
 const varint = require('varint')
-const dagpb = require('ipld-dag-pb')
-const DAGNode = require('ipld-dag-pb/src/dag-node/dagNode')
-const DAGLink = require('ipld-dag-pb/src/dag-link/dagLink')
-const multihash = require('multihashing-async').multihash
+const dagPb = require('@ipld/dag-pb')
 const { cidToKey, DEFAULT_FANOUT, MAX_ITEMS, EMPTY_KEY } = require('./utils')
 const uint8ArrayConcat = require('uint8arrays/concat')
 const uint8ArrayCompare = require('uint8arrays/compare')
 const uint8ArrayToString = require('uint8arrays/to-string')
 const uint8ArrayFromString = require('uint8arrays/from-string')
-const uint8ArrayEquals = require('uint8arrays/equals')
+const { sha256 } = require('multiformats/hashes/sha2')
 
 /**
  * @typedef {import('interface-datastore').Datastore} Datastore
+ * @typedef {import('@ipld/dag-pb').PBNode} PBNode
  *
  * @typedef {object} Pin
  * @property {CID} key
  * @property {Uint8Array} [data]
  */
 
 /**
- * @param {Uint8Array | CID} hash
- */
-function toB58String (hash) {
-  return new CID(hash).toBaseEncodedString()
-}
-
-/**
- * @param {DAGNode} rootNode
+ * @param {PBNode} rootNode
  */
 function readHeader (rootNode) {
   // rootNode.data should be a buffer of the format:
   // < varint(headerLength) | header | itemData... >
   const rootData = rootNode.Data
+
+  if (!rootData) {
+    throw new Error('No data present')
+  }
+
   const hdrLength = varint.decode(rootData)
   const vBytes = varint.decode.bytes
 
@@ -86,15 +82,15 @@ function hash (seed, key) {
   const buffer = new Uint8Array(4)
   const dataView = new DataView(buffer.buffer)
   dataView.setUint32(0, seed, true)
-  const encodedKey = uint8ArrayFromString(toB58String(key))
+  const encodedKey = uint8ArrayFromString(key.toString())
   const data = uint8ArrayConcat([buffer, encodedKey], buffer.byteLength + encodedKey.byteLength)
 
   return fnv1a(uint8ArrayToString(data))
 }
 
 /**
  * @param {Datastore} blockstore
- * @param {DAGNode} node
+ * @param {PBNode} node
  * @returns {AsyncGenerator<CID, void, undefined>}
  */
 async function * walkItems (blockstore, node) {
@@ -107,10 +103,10 @@ async function * walkItems (blockstore, node) {
       // if a fanout bin is not 'empty', dig into and walk its DAGLinks
       const linkHash = link.Hash
 
-      if (!uint8ArrayEquals(EMPTY_KEY, linkHash.bytes)) {
+      if (!EMPTY_KEY.equals(linkHash)) {
         // walk the links of this fanout bin
         const buf = await blockstore.get(cidToKey(linkHash))
-        const node = dagpb.util.deserialize(buf)
+        const node = dagPb.decode(buf)
 
         yield * walkItems(blockstore, node)
       }
@@ -125,7 +121,7 @@ async function * walkItems (blockstore, node) {
 
 /**
  * @param {Datastore} blockstore
- * @param {DAGNode} rootNode
+ * @param {PBNode} rootNode
  * @param {string} name
  */
 async function * loadSet (blockstore, rootNode, name) {
@@ -136,7 +132,7 @@ async function * loadSet (blockstore, rootNode, name) {
   }
 
   const buf = await blockstore.get(cidToKey(link.Hash))
-  const node = dagpb.util.deserialize(buf)
+  const node = dagPb.decode(buf)
 
   yield * walkItems(blockstore, node)
 }
@@ -164,26 +160,38 @@ function storeItems (blockstore, items) {
     const fanoutLinks = []
 
     for (let i = 0; i < DEFAULT_FANOUT; i++) {
-      fanoutLinks.push(new DAGLink('', 1, EMPTY_KEY))
+      fanoutLinks.push({
+        Name: '',
+        Tsize: 1,
+        Hash: EMPTY_KEY
+      })
     }
 
     if (pins.length <= MAX_ITEMS) {
       const nodes = pins
         .map(item => {
           return ({
-            link: new DAGLink('', 1, item.key),
+            link: {
+              Name: '',
+              Tsize: 1,
+              Hash: item.key
+            },
             data: item.data || new Uint8Array()
           })
         })
         // sorting makes any ordering of `pins` produce the same DAGNode
         .sort((a, b) => {
+          //return a.link.Name.localeCompare(b.link.Name)
           return uint8ArrayCompare(a.link.Hash.bytes, b.link.Hash.bytes)
         })
 
       const rootLinks = fanoutLinks.concat(nodes.map(item => item.link))
       const rootData = uint8ArrayConcat([headerBuf, ...nodes.map(item => item.data)])
 
-      return new DAGNode(rootData, rootLinks)
+      return {
+        Data: rootData,
+        Links: rootLinks
+      }
     } else {
       // If the array of pins is > MAX_ITEMS, we:
       //  - distribute the pins among `DEFAULT_FANOUT` bins
@@ -212,22 +220,30 @@ function storeItems (blockstore, items) {
         idx++
       }
 
-      return new DAGNode(headerBuf, fanoutLinks)
+      return {
+        Data: headerBuf,
+        Links: fanoutLinks
+      }
     }
 
     /**
-     * @param {DAGNode} child
+     * @param {PBNode} child
      * @param {number} binIdx
      */
     async function storeChild (child, binIdx) {
-      const buf = dagpb.util.serialize(child)
-      const cid = await dagpb.util.cid(buf, {
-        cidVersion: 0,
-        hashAlg: multihash.names['sha2-256']
-      })
+      const buf = dagPb.encode(child)
+      const digest = await sha256.digest(buf)
+      const cid = CID.createV0(digest)
+
       await blockstore.put(cidToKey(cid), buf)
 
-      fanoutLinks[binIdx] = new DAGLink('', child.size, cid)
+      let size = child.Links.reduce((acc, curr) => acc + (curr?.Tsize || 0), 0) + buf.length
+
+      fanoutLinks[binIdx] = {
+        Name: '',
+        Tsize: size,
+        Hash: cid
+      }
     }
   }
 }
@@ -243,15 +259,19 @@ async function storeSet (blockstore, type, cids) {
       key: cid
     }
   }))
-  const buf = rootNode.serialize()
-  const cid = await dagpb.util.cid(buf, {
-    cidVersion: 0,
-    hashAlg: multihash.names['sha2-256']
-  })
+  const buf = dagPb.encode(rootNode)
+  const digest = await sha256.digest(buf)
+  const cid = CID.createV0(digest)
 
   await blockstore.put(cidToKey(cid), buf)
 
-  return new DAGLink(type, rootNode.size, cid)
+  let size = rootNode.Links.reduce((acc, curr) => acc + curr.Tsize, 0) + buf.length
+
+  return {
+    Name: type,
+    Tsize: size,
+    Hash: cid
+  }
 }
 
 module.exports = {