feat: Add --cid-version option to ipfs files add (#974)

Author: alanshaw

src/cli/commands/files/add.js

@@ -112,6 +112,15 @@ module.exports = {
     'shard-split-threshold': {
       type: 'integer',
       default: 1000
+    },
+    'raw-leaves': {
+      type: 'boolean',
+      default: undefined,
+      describe: 'Use raw blocks for leaf nodes. (experimental)'
+    },
+    'cid-version': {
+      type: 'integer',
+      describe: 'Cid version. Non-zero value will change default of \'raw-leaves\' to true. (experimental)'
     }
   },
 
@@ -120,7 +129,25 @@ module.exports = {
     const index = inPath.lastIndexOf('/') + 1
     const options = {
       strategy: argv.trickle ? 'trickle' : 'balanced',
-      shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity
+      shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity,
+      'cid-version': argv['cid-version'],
+      'raw-leaves': argv['raw-leaves']
+    }
+
+    // Temporary restriction on raw-leaves:
+    // When cid-version=1 then raw-leaves MUST be present and false.
+    //
+    // This is because raw-leaves is not yet implemented in js-ipfs,
+    // and go-ipfs changes the value of raw-leaves to true when
+    // cid-version > 0 unless explicitly set to false.
+    //
+    // This retains feature parity without having to implement raw-leaves.
+    if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) {
+      throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0')
+    }
+
+    if (argv['raw-leaves']) {
+      throw new Error('Not implemented: raw-leaves')
     }
 
     if (argv.enableShardingExperiment && utils.isDaemonOn()) {

src/core/components/files.js

@@ -4,7 +4,6 @@ const unixfsEngine = require('ipfs-unixfs-engine')
 const importer = unixfsEngine.importer
 const exporter = unixfsEngine.exporter
 const promisify = require('promisify-es6')
-const multihashes = require('multihashes')
 const pull = require('pull-stream')
 const sort = require('pull-sort')
 const pushable = require('pull-pushable')
@@ -13,6 +12,7 @@ const toPull = require('stream-to-pull-stream')
 const waterfall = require('async/waterfall')
 const isStream = require('is-stream')
 const Duplex = require('stream').Duplex
+const CID = require('cids')
 
 module.exports = function files (self) {
   const createAddPullStream = (options) => {
@@ -24,7 +24,7 @@ module.exports = function files (self) {
       pull.map(normalizeContent),
       pull.flatten(),
       importer(self._ipldResolver, opts),
-      pull.asyncMap(prepareFile.bind(null, self))
+      pull.asyncMap(prepareFile.bind(null, self, opts))
     )
   }
 
@@ -68,7 +68,7 @@ module.exports = function files (self) {
       pull(
         pull.values(normalizeContent(data)),
         importer(self._ipldResolver, options),
-        pull.asyncMap(prepareFile.bind(null, self)),
+        pull.asyncMap(prepareFile.bind(null, self, options)),
         sort((a, b) => {
           if (a.path < b.path) return 1
           if (a.path > b.path) return -1
@@ -114,15 +114,23 @@ module.exports = function files (self) {
   }
 }
 
-function prepareFile (self, file, callback) {
-  const bs58mh = multihashes.toB58String(file.multihash)
+function prepareFile (self, opts, file, callback) {
+  opts = opts || {}
 
   waterfall([
     (cb) => self.object.get(file.multihash, cb),
     (node, cb) => {
+      let cid = new CID(node.multihash)
+
+      if (opts['cid-version'] === 1) {
+        cid = cid.toV1()
+      }
+
+      const b58Hash = cid.toBaseEncodedString()
+
       cb(null, {
-        path: file.path || bs58mh,
-        hash: bs58mh,
+        path: file.path || b58Hash,
+        hash: b58Hash,
         size: node.size
       })
     }

src/http-api/resources/files.js

@@ -11,6 +11,7 @@ const toPull = require('stream-to-pull-stream')
 const pushable = require('pull-pushable')
 const EOL = require('os').EOL
 const toStream = require('pull-stream-to-stream')
+const Joi = require('joi')
 
 exports = module.exports
 
@@ -140,6 +141,28 @@ exports.get = {
 }
 
 exports.add = {
+  validate: {
+    query: Joi.object()
+      .keys({
+        'cid-version': Joi.number().integer().min(0).max(1),
+        // Temporary restriction on raw-leaves:
+        // When cid-version=1 then raw-leaves MUST be present and false.
+        //
+        // This is because raw-leaves is not yet implemented in js-ipfs,
+        // and go-ipfs changes the value of raw-leaves to true when
+        // cid-version > 0 unless explicitly set to false.
+        //
+        // This retains feature parity without having to implement raw-leaves.
+        'raw-leaves': Joi.any().when('cid-version', {
+          is: 1,
+          then: Joi.boolean().valid(false).required(),
+          otherwise: Joi.boolean().valid(false)
+        })
+      })
+      // TODO: Necessary until validate "recursive", "stream-channels" etc.
+      .options({ allowUnknown: true })
+  },
+
   handler: (request, reply) => {
     if (!request.payload) {
       return reply({
@@ -181,9 +204,14 @@ exports.add = {
       fileAdder.end()
     })
 
+    const options = {
+      'cid-version': request.query['cid-version'],
+      'raw-leaves': request.query['raw-leaves']
+    }
+
     pull(
       fileAdder,
-      ipfs.files.createAddPullStream(),
+      ipfs.files.createAddPullStream(options),
       pull.map((file) => {
         return {
           Name: file.path ? file.path : file.hash,

src/http-api/routes/files.js

@@ -38,7 +38,8 @@ module.exports = (server) => {
         parse: false,
         output: 'stream'
       },
-      handler: resources.files.add.handler
+      handler: resources.files.add.handler,
+      validate: resources.files.add.validate
     }
   })
 }

test/cli/files.js

@@ -136,6 +136,60 @@ describe('files', () => runOnAndOff((thing) => {
     })
   })
 
+  it('add with cid-version=0', () => {
+    return ipfs('add src/init-files/init-docs/readme --cid-version=0').then((out) => {
+      expect(out)
+        .to.eql('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme\n')
+    })
+  })
+
+  // Temporarily expect to fail as raw-leaves not yet implemented.
+  //
+  // When cid-version=1 then raw-leaves MUST be present and false.
+  //
+  // This is because raw-leaves is not yet implemented in js-ipfs,
+  // and go-ipfs changes the value of raw-leaves to true when
+  // cid-version > 0 unless explicitly set to false.
+  //
+  // This retains feature parity without having to implement raw-leaves.
+  it('add with cid-version=1', () => {
+    return new Promise((resolve, reject) => {
+      ipfs('add src/init-files/init-docs/readme --cid-version=1')
+        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
+        .catch((err) => {
+          expect(err).to.exist()
+          resolve()
+        })
+    })
+  })
+
+  it('add with cid-version=1 and raw-leaves=false', () => {
+    return ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=false').then((out) => {
+      expect(out)
+        .to.eql('added zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 readme\n')
+    })
+  })
+
+  // Temporarily expect to fail as raw-leaves not yet implemented
+  //
+  // When cid-version=1 then raw-leaves MUST be present and false.
+  //
+  // This is because raw-leaves is not yet implemented in js-ipfs,
+  // and go-ipfs changes the value of raw-leaves to true when
+  // cid-version > 0 unless explicitly set to false.
+  //
+  // This retains feature parity without having to implement raw-leaves.
+  it('add with cid-version=1 and raw-leaves=true', () => {
+    return new Promise((resolve, reject) => {
+      ipfs('add src/init-files/init-docs/readme --cid-version=1 --raw-leaves=true')
+        .then(() => reject(new Error('Raw leaves not expected to be implemented')))
+        .catch((err) => {
+          expect(err).to.exist()
+          resolve()
+        })
+    })
+  })
+
   it('cat', () => {
     return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB')
       .then((out) => {