feat: convert to async iterators (#15)

* Remove support for Node.js streams and pull streams
* Adds a `urlSource` utility that will be exported by `js-ipfs-http-client` and `js-ipfs` along side `globSource`

BREAKING CHANGE: Support for Node.js streams and Pull Streams has been removed

Author: Alan Shaw

package.json

@@ -29,23 +29,18 @@
     "buffer": "^5.2.1",
     "err-code": "^2.0.0",
     "fs-extra": "^8.1.0",
-    "is-buffer": "^2.0.3",
     "is-electron": "^2.2.0",
-    "is-pull-stream": "0.0.0",
-    "is-stream": "^2.0.0",
-    "it-glob": "0.0.4",
-    "kind-of": "^6.0.2",
-    "pull-stream-to-async-iterator": "^1.0.2",
-    "readable-stream": "^3.4.0"
+    "it-glob": "0.0.6",
+    "ky": "^0.15.0",
+    "ky-universal": "^0.3.0",
+    "stream-to-it": "^0.2.0"
   },
   "devDependencies": {
     "aegir": "^20.3.0",
     "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
-    "dirty-chai": "^2.0.1",
-    "pull-stream": "^3.6.13",
-    "readable-stream-2": "npm:readable-stream@^2.0.0"
+    "dirty-chai": "^2.0.1"
   },
   "contributors": [
     "Alan Shaw <alan.shaw@protocol.ai>",

src/files/add-input-validation.js

@@ -4,7 +4,6 @@ const fs = require('fs-extra')
 const glob = require('it-glob')
 const Path = require('path')
 const errCode = require('err-code')
-const kindOf = require('kind-of')
 
 /**
 * Create an async iterator that yields paths that match requested file paths.
@@ -20,7 +19,7 @@ const kindOf = require('kind-of')
 module.exports = async function * globSource (paths, options) {
   options = options || {}
 
-  if (kindOf(paths) === 'string') {
+  if (typeof paths === 'string') {
     paths = [paths]
   }
 

src/files/glob-source.js

@@ -2,11 +2,7 @@
 
 const errCode = require('err-code')
 const { Buffer } = require('buffer')
-const pullStreamToIterable = require('pull-stream-to-async-iterator')
-const { isSource } = require('is-pull-stream')
 const globalThis = require('../globalthis')
-const { Readable } = require('stream')
-const Readable3 = require('readable-stream')
 
 /*
  * Transform one of:
@@ -21,8 +17,6 @@ const Readable3 = require('readable-stream')
  * { path, content: Iterable<Number> } [single file]
  * { path, content: Iterable<Bytes> } [single file]
  * { path, content: AsyncIterable<Bytes> } [single file]
- * { path, content: PullStream<Bytes> } [single file]
- * { path, content: Readable<Bytes> } [single file]
  * Iterable<Number> [single file]
  * Iterable<Bytes> [single file]
  * Iterable<Bloby> [multiple files]
@@ -33,8 +27,6 @@ const Readable3 = require('readable-stream')
  * Iterable<{ path, content: Iterable<Number> }> [multiple files]
  * Iterable<{ path, content: Iterable<Bytes> }> [multiple files]
  * Iterable<{ path, content: AsyncIterable<Bytes> }> [multiple files]
- * Iterable<{ path, content: PullStream<Bytes> }> [multiple files]
- * Iterable<{ path, content: Readable<Bytes> }> [multiple files]
  * AsyncIterable<Bytes> [single file]
  * AsyncIterable<Bloby> [multiple files]
  * AsyncIterable<String> [multiple files]
@@ -44,30 +36,6 @@ const Readable3 = require('readable-stream')
  * AsyncIterable<{ path, content: Iterable<Number> }> [multiple files]
  * AsyncIterable<{ path, content: Iterable<Bytes> }> [multiple files]
  * AsyncIterable<{ path, content: AsyncIterable<Bytes> }> [multiple files]
- * AsyncIterable<{ path, content: PullStream<Bytes> }> [multiple files]
- * AsyncIterable<{ path, content: Readable<Bytes> }> [multiple files]
- * PullStream<Bytes> [single file]
- * PullStream<Bloby> [multiple files]
- * PullStream<String> [multiple files]
- * PullStream<{ path, content: Bytes }> [multiple files]
- * PullStream<{ path, content: Bloby }> [multiple files]
- * PullStream<{ path, content: String }> [multiple files]
- * PullStream<{ path, content: Iterable<Number> }> [multiple files]
- * PullStream<{ path, content: Iterable<Bytes> }> [multiple files]
- * PullStream<{ path, content: AsyncIterable<Bytes> }> [multiple files]
- * PullStream<{ path, content: PullStream<Bytes> }> [multiple files]
- * PullStream<{ path, content: Readable<Bytes> }> [multiple files]
- * Readable<Bytes> [single file]
- * Readable<Bloby> [multiple files]
- * Readable<String> [multiple files]
- * Readable<{ path, content: Bytes }> [multiple files]
- * Readable<{ path, content: Bloby }> [multiple files]
- * Readable<{ path, content: String }> [multiple files]
- * Readable<{ path, content: Iterable<Number> }> [multiple files]
- * Readable<{ path, content: Iterable<Bytes> }> [multiple files]
- * Readable<{ path, content: AsyncIterable<Bytes> }> [multiple files]
- * Readable<{ path, content: PullStream<Bytes> }> [multiple files]
- * Readable<{ path, content: Readable<Bytes> }> [multiple files]
  * ```
  * Into:
  *
@@ -99,11 +67,6 @@ module.exports = function normaliseInput (input) {
     })()
   }
 
-  // Readable<?>
-  if (isOldReadable(input)) {
-    input = upgradeOldStream(input)
-  }
-
   // Iterable<?>
   if (input[Symbol.iterator]) {
     return (async function * () { // eslint-disable-line require-await
@@ -176,37 +139,6 @@ module.exports = function normaliseInput (input) {
     })()
   }
 
-  // PullStream<?>
-  if (isSource(input)) {
-    return (async function * () {
-      const iterator = pullStreamToIterable(input)[Symbol.asyncIterator]()
-      const first = await iterator.next()
-      if (first.done) return iterator
-
-      // PullStream<Bytes>
-      if (isBytes(first.value)) {
-        yield toFileObject((async function * () { // eslint-disable-line require-await
-          yield first.value
-          yield * iterator
-        })())
-        return
-      }
-
-      // PullStream<Bloby>
-      // PullStream<String>
-      // PullStream<{ path, content }>
-      if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') {
-        yield toFileObject(first.value)
-        for await (const obj of iterator) {
-          yield toFileObject(obj)
-        }
-        return
-      }
-
-      throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
-    })()
-  }
-
   throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
 }
 
@@ -235,11 +167,6 @@ function toAsyncIterable (input) {
     return blobToAsyncGenerator(input)
   }
 
-  // Readable<?>
-  if (isOldReadable(input)) {
-    input = upgradeOldStream(input)
-  }
-
   // Iterator<?>
   if (input[Symbol.iterator]) {
     return (async function * () { // eslint-disable-line require-await
@@ -278,22 +205,9 @@ function toAsyncIterable (input) {
     })()
   }
 
-  // PullStream<Bytes>
-  if (isSource(input)) {
-    return pullStreamToIterable(input)
-  }
-
   throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
 }
 
-function isOldReadable (obj) {
-  if (obj[Symbol.iterator] || obj[Symbol.asyncIterator]) {
-    return false
-  }
-
-  return Boolean(obj.readable)
-}
-
 function toBuffer (chunk) {
   return isBytes(chunk) ? chunk : Buffer.from(chunk)
 }
@@ -311,17 +225,6 @@ function isFileObject (obj) {
   return typeof obj === 'object' && (obj.path || obj.content)
 }
 
-function upgradeOldStream (stream) {
-  if (stream[Symbol.asyncIterator] || stream[Symbol.iterator]) {
-    return stream
-  }
-
-  // in the browser the stream.Readable is not an async iterator but readble-stream@3 is...
-  stream[Symbol.asyncIterator] = Readable.prototype[Symbol.asyncIterator] || Readable3.prototype[Symbol.asyncIterator]
-
-  return stream
-}
-
 function blobToAsyncGenerator (blob) {
   if (typeof blob.stream === 'function') {
     // firefox < 69 does not support blob.stream()

src/files/normalise-input.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const { default: ky } = require('ky-universal')
+const toIterable = require('stream-to-it/source')
+
+module.exports = async function * urlSource (url, options) {
+  options = options || {}
+
+  const { body } = await ky.get(url)
+
+  yield {
+    path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''),
+    content: toIterable(body)
+  }
+}