refactor: fetch add

package.json

@@ -17,7 +17,9 @@
   "browser": {
     "glob": false,
     "fs": false,
-    "stream": "readable-stream"
+    "stream": "readable-stream",
+    "./src/add/form-data.js": "./src/add/form-data.browser.js",
+    "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js"
   },
   "repository": "github:ipfs/js-ipfs-http-client",
   "scripts": {
@@ -35,6 +37,7 @@
   "dependencies": {
     "abort-controller": "^3.0.0",
     "async": "^2.6.1",
+    "async-iterator-to-pull-stream": "^1.3.0",
     "bignumber.js": "^9.0.0",
     "bl": "^3.0.0",
     "bs58": "^4.0.1",
@@ -58,6 +61,8 @@
     "is-stream": "^2.0.0",
     "iso-stream-http": "~0.1.2",
     "iso-url": "~0.4.6",
+    "it-pushable": "^1.2.1",
+    "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "just-kebab-case": "^1.1.0",
     "just-map-keys": "^1.1.0",
@@ -77,6 +82,7 @@
     "promisify-es6": "^1.0.3",
     "pull-defer": "~0.2.3",
     "pull-stream": "^3.6.9",
+    "pull-stream-to-async-iterator": "^1.0.2",
     "pull-to-stream": "~0.1.1",
     "pump": "^3.0.0",
     "qs": "^6.5.2",

src/add-from-fs/glob-source.js

@@ -0,0 +1,94 @@
+'use strict'
+
+const Fs = require('fs')
+const Path = require('path')
+const glob = require('glob')
+const pushable = require('it-pushable')
+const errCode = require('err-code')
+
+/**
+* Create an AsyncIterable that can be passed to ipfs.add for the
+* provided file paths.
+*
+* @param {String} ...paths File system path(s) to glob from
+* @param {Object} [options] Optional options
+* @param {Boolean} [options.recursive] Recursively glob all paths in directories
+* @param {Boolean} [options.hidden] Include .dot files in matched paths
+* @param {Array<String>} [options.ignore] Glob paths to ignore
+* @param {Boolean} [options.followSymlinks] follow symlinks
+* @returns {AsyncIterable}
+*/
+module.exports = (...args) => (async function * () {
+  const options = typeof args[args.length - 1] === 'string' ? {} : args.pop()
+  const paths = args
+
+  const globSourceOptions = {
+    recursive: options.recursive,
+    glob: {
+      dot: Boolean(options.hidden),
+      ignore: Array.isArray(options.ignore) ? options.ignore : [],
+      follow: options.followSymlinks != null ? options.followSymlinks : true
+    }
+  }
+
+  // Check the input paths comply with options.recursive and convert to glob sources
+  const results = await Promise.all(paths.map(pathAndType))
+  const globSources = results.map(r => toGlobSource(r, globSourceOptions))
+
+  for (const globSource of globSources) {
+    for await (const { path, contentPath } of globSource) {
+      yield { path, content: Fs.createReadStream(contentPath) }
+    }
+  }
+})()
+
+function toGlobSource ({ path, type }, options) {
+  return (async function * () {
+    options = options || {}
+
+    const baseName = Path.basename(path)
+
+    if (type === 'file') {
+      yield { path: baseName, contentPath: path }
+      return
+    }
+
+    if (type === 'dir' && !options.recursive) {
+      throw errCode(
+        new Error(`'${path}' is a directory and recursive option not set`),
+        'ERR_DIR_NON_RECURSIVE',
+        { path }
+      )
+    }
+
+    const globOptions = Object.assign({}, options.glob, {
+      cwd: path,
+      nodir: true,
+      realpath: false,
+      absolute: false
+    })
+
+    // TODO: want to use pull-glob but it doesn't have the features...
+    const pusher = pushable()
+
+    glob('**/*', globOptions)
+      .on('match', m => pusher.push(m))
+      .on('end', () => pusher.end())
+      .on('abort', () => pusher.end())
+      .on('error', err => pusher.end(err))
+
+    for await (const p of pusher) {
+      yield {
+        path: `${baseName}/${toPosix(p)}`,
+        contentPath: Path.join(path, p)
+      }
+    }
+  })()
+}
+
+async function pathAndType (path) {
+  const stat = await Fs.promises.stat(path)
+  return { path, type: stat.isDirectory() ? 'dir' : 'file' }
+}
+
+const toPosix = path => path.replace(/\\/g, '/')

src/add-from-fs/index.browser.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = () => () => { throw new Error('unavailable in the browser') }

src/add-from-fs/index.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const globSource = require('./glob-source')
+
+module.exports = configure(({ ky }) => {
+  const add = require('../add')({ ky })
+  return (path, options) => add(globSource(path, options), options)
+})

src/add-from-url.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const kyDefault = require('ky-universal').default
+const configure = require('./lib/configure')
+const toIterable = require('./lib/stream-to-iterable')
+
+module.exports = configure(({ ky }) => {
+  const add = require('./add')({ ky })
+
+  return (url, options) => (async function * () {
+    options = options || {}
+
+    const { body } = await kyDefault.get(url)
+
+    const input = {
+      path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''),
+      content: toIterable(body)
+    }
+
+    for await (const file of add(input, options)) {
+      yield file
+    }
+  })()
+})

src/add/form-data.browser.js

@@ -0,0 +1,30 @@
+'use strict'
+/* eslint-env browser */
+
+const normaliseInput = require('./normalise-input')
+
+exports.toFormData = async (input) => {
+  const files = normaliseInput(input)
+  const formData = new FormData()
+  let i = 0
+
+  for await (const file of files) {
+    if (file.content) {
+      // In the browser there's _currently_ no streaming upload, buffer up our
+      // async iterator chunks and append a big Blob :(
+      // One day, this will be browser streams
+      const bufs = []
+      for await (const chunk of file.content) {
+        bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk)
+      }
+
+      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), file.path)
+    } else {
+      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), file.path)
+    }
+
+    i++
+  }
+
+  return formData
+}

src/add/form-data.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const FormData = require('form-data')
+const { Buffer } = require('buffer')
+const toStream = require('it-to-stream')
+const normaliseInput = require('./normalise-input')
+
+exports.toFormData = async (input) => {
+  const files = normaliseInput(input)
+  const formData = new FormData()
+  let i = 0
+
+  for await (const file of files) {
+    if (file.content) {
+      // In Node.js, FormData can be passed a stream so no need to buffer
+      formData.append(
+        `file-${i}`,
+        // FIXME: add a `path` property to the stream so `form-data` doesn't set
+        // a Content-Length header that is only the sum of the size of the
+        // header/footer when knownLength option (below) is null.
+        Object.assign(
+          toStream.readable(file.content),
+          { path: file.path || `file-${i}` }
+        ),
+        {
+          filepath: encodeURIComponent(file.path),
+          contentType: 'application/octet-stream',
+          knownLength: file.content.length // Send Content-Length header if known
+        }
+      )
+    } else {
+      formData.append(`dir-${i}`, Buffer.alloc(0), {
+        filepath: encodeURIComponent(file.path),
+        contentType: 'application/x-directory'
+      })
+    }
+
+    i++
+  }
+
+  return formData
+}

src/add/index.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const ndjson = require('iterable-ndjson')
+const configure = require('../lib/configure')
+const toIterable = require('../lib/stream-to-iterable')
+const { toFormData } = require('./form-data')
+const toCamel = require('../lib/object-to-camel')
+
+module.exports = configure(({ ky }) => {
+  return (input, options) => (async function * () {
+    options = options || {}
+
+    const searchParams = new URLSearchParams(options.searchParams)
+
+    searchParams.set('stream-channels', true)
+    if (options.chunker) searchParams.set('chunker', options.chunker)
+    if (options.cidVersion) searchParams.set('cid-version', options.cidVersion)
+    if (options.cidBase) searchParams.set('cid-base', options.cidBase)
+    if (options.enableShardingExperiment != null) searchParams.set('enable-sharding-experiment', options.enableShardingExperiment)
+    if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+    if (options.onlyHash != null) searchParams.set('only-hash', options.onlyHash)
+    if (options.pin != null) searchParams.set('pin', options.pin)
+    if (options.progress) searchParams.set('progress', true)
+    if (options.quiet != null) searchParams.set('quiet', options.quiet)
+    if (options.quieter != null) searchParams.set('quieter', options.quieter)
+    if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves)
+    if (options.shardSplitThreshold) searchParams.set('shard-split-threshold', options.shardSplitThreshold)
+    if (options.silent) searchParams.set('silent', options.silent)
+    if (options.trickle != null) searchParams.set('trickle', options.trickle)
+    if (options.wrapWithDirectory != null) searchParams.set('wrap-with-directory', options.wrapWithDirectory)
+
+    const res = await ky.post('add', {
+      timeout: options.timeout,
+      signal: options.signal,
+      headers: options.headers,
+      searchParams,
+      body: await toFormData(input)
+    })
+
+    for await (let file of ndjson(toIterable(res.body))) {
+      file = toCamel(file)
+      // console.log(file)
+      if (options.progress && file.bytes) {
+        options.progress(file.bytes)
+      } else {
+        yield toCoreInterface(file)
+      }
+    }
+  })()
+})
+
+function toCoreInterface ({ name, hash, size }) {
+  return { path: name, hash, size: parseInt(size) }
+}