chore: split core, http api server and cli out from ipfs module

Decomposes the main ipfs module into separate core, http api server and
cli modules which encapsulate the different parts of ipfs.

This allows us to:

1. Omit modules/code/depencies if they aren't going to be used
2. Reduces the amount of tests being run in series

I'm hoping this will get us to sub-15 minute builds.

Fixes #2678
Fixes #2877

examples/browser-exchange-files/package.json

@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-all": "^1.0.1",
+    "it-all": "^1.0.2",
     "test-ipfs-example": "^2.0.3"
   },
   "browser": {

examples/browser-exchange-files/test.js

@@ -12,7 +12,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client')
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   }
 })
 const {

examples/browser-http-client-upload-file/test.js

@@ -5,7 +5,7 @@ const execa = require('execa')
 const { createFactory } = require('ipfsd-ctl')
 const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
-  ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+  ipfsBin: require.resolve('ipfs/src/cli.js')
 })
 const {
   startServer

examples/browser-ipns-publish/package.json

@@ -25,7 +25,7 @@
     "last 2 versions and not dead and > 2%"
   ],
   "devDependencies": {
-    "delay": "^4.3.0",
+    "delay": "^4.4.0",
     "execa": "^4.0.0",
     "ipfsd-ctl": "^7.0.0",
     "go-ipfs": "^0.6.0",

examples/browser-readablestream/package.json

@@ -21,7 +21,7 @@
   },
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-to-stream": "^0.1.1",
+    "it-to-stream": "^0.1.2",
     "videostream": "^3.2.0"
   }
 }

examples/circuit-relaying/package.json

@@ -14,7 +14,7 @@
   "author": "Dmitriy Ryajov <dryajov@gmail.com>",
   "license": "MIT",
   "dependencies": {
-    "delay": "^4.3.0",
+    "delay": "^4.4.0",
     "ipfs": "^0.50.1",
     "ipfs-pubsub-room": "^2.0.1",
     "uint8arrays": "^1.1.0"

examples/custom-ipfs-repo/package.json

@@ -13,7 +13,7 @@
     "datastore-fs": "^2.0.0",
     "ipfs": "^0.50.1",
     "ipfs-repo": "^6.0.3",
-    "it-all": "^1.0.1"
+    "it-all": "^1.0.2"
   },
   "devDependencies": {
     "execa": "^4.0.0",

examples/explore-ethereum-blockchain/test.js

@@ -8,7 +8,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client')
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   }
 })
 

examples/http-client-browser-pubsub/test.js

@@ -7,7 +7,7 @@ const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
 }, {
   js: {
-    ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+    ipfsBin: require.resolve('ipfs/src/cli.js')
   },
   go: {
     ipfsBin: require('go-ipfs').path(),

examples/http-client-bundle-webpack/test.js

@@ -5,7 +5,7 @@ const execa = require('execa')
 const { createFactory } = require('ipfsd-ctl')
 const df = createFactory({
   ipfsHttpModule: require('ipfs-http-client'),
-  ipfsBin: require.resolve('ipfs/src/cli/bin.js')
+  ipfsBin: require.resolve('ipfs/src/cli.js')
 })
 const {
   startServer

examples/ipfs-101/package.json

@@ -11,7 +11,7 @@
   "license": "MIT",
   "dependencies": {
     "ipfs": "^0.50.1",
-    "it-all": "^1.0.1",
+    "it-all": "^1.0.2",
     "uint8arrays": "^1.1.0"
   },
   "devDependencies": {

examples/running-multiple-nodes/test.js

@@ -25,7 +25,7 @@ async function startCliNode () {
       IPFS_PATH: repoDir
     }
   }
-  const ipfs = require.resolve('ipfs/src/cli/bin.js')
+  const ipfs = require.resolve('ipfs/src/cli.js')
 
   await execa(ipfs, ['init'], opts)
   await execa(ipfs, ['config', 'Addresses.Swarm', '--json', JSON.stringify([`/ip4/0.0.0.0/tcp/0`, `/ip4/127.0.0.1/tcp/0/ws`])], opts)

examples/traverse-ipld-graphs/package.json

@@ -15,7 +15,7 @@
   "dependencies": {
     "cids": "^1.0.0",
     "ipfs": "^0.50.1",
-    "ipld-block": "^0.10.0",
+    "ipld-block": "^0.10.1",
     "ipld-dag-pb": "^0.20.0",
     "multihashing-async": "^2.0.1"
   }

packages/interface-ipfs-core/package.json

@@ -36,17 +36,17 @@
     "chai-as-promised": "^7.1.1",
     "chai-subset": "^1.6.0",
     "cids": "^1.0.0",
-    "delay": "^4.3.0",
+    "delay": "^4.4.0",
     "dirty-chai": "^2.0.1",
-    "ipfs-unixfs": "^2.0.2",
-    "ipfs-unixfs-importer": "^3.0.2",
+    "ipfs-unixfs": "^2.0.3",
+    "ipfs-unixfs-importer": "^3.0.4",
     "ipfs-utils": "^3.0.0",
-    "ipld-block": "^0.10.0",
+    "ipld-block": "^0.10.1",
     "ipld-dag-cbor": "^0.17.0",
     "ipld-dag-pb": "^0.20.0",
     "is-ipfs": "^2.0.0",
     "iso-random-stream": "^1.1.1",
-    "it-all": "^1.0.1",
+    "it-all": "^1.0.2",
     "it-drain": "^1.0.1",
     "it-last": "^1.0.2",
     "it-map": "^1.0.2",
@@ -55,8 +55,8 @@
     "multiaddr": "^8.0.0",
     "multibase": "^3.0.0",
     "multihashing-async": "^2.0.1",
-    "nanoid": "^3.0.2",
-    "peer-id": "^0.14.0",
+    "nanoid": "^3.1.12",
+    "peer-id": "^0.14.1",
     "readable-stream": "^3.4.0",
     "temp-write": "^4.0.0",
     "uint8arrays": "^1.1.0"

packages/ipfs-cli/package.json

@@ -0,0 +1,100 @@
+{
+  "name": "ipfs-cli",
+  "version": "0.0.1",
+  "description": "JavaScript implementation of the IPFS specification",
+  "keywords": [
+    "IPFS"
+  ],
+  "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs#readme",
+  "bugs": "https://github.com/ipfs/js-ipfs/issues",
+  "license": "(Apache-2.0 OR MIT)",
+  "leadMaintainer": "Alex Potsides <alex@achingbrain.net>",
+  "files": [
+    "src",
+    "dist"
+  ],
+  "main": "src/index.js",
+  "browser": {
+    "./src/core/runtime/init-assets-nodejs.js": "./src/core/runtime/init-assets-browser.js",
+    "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
+    "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
+    "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
+    "./src/core/runtime/libp2p-pubsub-routers-nodejs.js": "./src/core/runtime/libp2p-pubsub-routers-browser.js",
+    "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
+    "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
+    "./src/core/runtime/ipld-nodejs.js": "./src/core/runtime/ipld-browser.js",
+    "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
+    "ipfs-utils/src/files/glob-source": false
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ipfs/js-ipfs.git"
+  },
+  "scripts": {
+    "lint": "aegir lint",
+    "build": "aegir build",
+    "test": "aegir test -t node",
+    "coverage": "nyc --reporter=text --reporter=lcov npm run test:node",
+    "clean": "rm -rf ./dist",
+    "dep-check": "aegir dep-check"
+  },
+  "dependencies": {
+    "bignumber.js": "^9.0.0",
+    "byteman": "^1.3.5",
+    "cid-tool": "^1.0.0",
+    "cids": "^1.0.0",
+    "debug": "^4.1.1",
+    "dlv": "^1.1.3",
+    "electron-webrtc": "^0.3.0",
+    "err-code": "^2.0.3",
+    "execa": "^4.0.3",
+    "get-folder-size": "^2.0.1",
+    "ipfs-core": "^0.0.1",
+    "ipfs-core-utils": "^0.4.0",
+    "ipfs-http-client": "^47.0.1",
+    "ipfs-http-server": "^0.0.1",
+    "ipfs-repo": "^6.0.3",
+    "ipfs-utils": "^3.0.0",
+    "ipld-dag-cbor": "^0.17.0",
+    "ipld-dag-pb": "^0.20.0",
+    "it-all": "^1.0.2",
+    "it-concat": "^1.0.1",
+    "it-first": "^1.0.2",
+    "it-glob": "0.0.8",
+    "it-pipe": "^1.1.0",
+    "jsondiffpatch": "^0.4.1",
+    "just-safe-set": "^2.1.0",
+    "libp2p": "^0.29.0",
+    "libp2p-crypto": "^0.18.0",
+    "libp2p-delegated-content-routing": "^0.7.0",
+    "libp2p-delegated-peer-routing": "^0.7.0",
+    "libp2p-webrtc-star": "^0.20.1",
+    "mafmt": "^8.0.0",
+    "multiaddr": "^8.0.0",
+    "multiaddr-to-uri": "^6.0.0",
+    "multibase": "^3.0.0",
+    "multihashing-async": "^2.0.1",
+    "parse-duration": "^0.4.4",
+    "peer-id": "^0.14.1",
+    "pretty-bytes": "^5.4.1",
+    "progress": "^2.0.3",
+    "stream-to-it": "^0.2.2",
+    "streaming-iterables": "^5.0.2",
+    "uint8arrays": "^1.1.0",
+    "yargs": "^16.0.3"
+  },
+  "devDependencies": {
+    "aegir": "^26.0.0",
+    "nanoid": "^3.1.12",
+    "ncp": "^2.0.0",
+    "rimraf": "^3.0.2",
+    "sinon": "^9.0.3",
+    "string-argv": "^0.3.1",
+    "temp-write": "^4.0.0",
+    "wrtc": "^0.4.6"
+  },
+  "optionalDependencies": {
+    "prom-client": "^12.0.0",
+    "prometheus-gc-stats": "^0.6.0"
+  }
+}

packages/ipfs/src/cli/commands/add.js → packages/ipfs-cli/src/commands/add.js

@@ -11,7 +11,7 @@ const {
   coerceMtime,
   coerceMtimeNsecs
 } = require('../utils')
-const { cidToString } = require('../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const globSource = require('ipfs-utils/src/files/glob-source')
 const parseDuration = require('parse-duration').default
 

packages/ipfs/src/cli/commands/bitswap/stat.js → packages/ipfs-cli/src/commands/bitswap/stat.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const prettyBytes = require('pretty-bytes')
 const parseDuration = require('parse-duration').default
 

packages/ipfs/src/cli/commands/bitswap/unwant.js → packages/ipfs-cli/src/commands/bitswap/unwant.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/bitswap/wantlist.js → packages/ipfs-cli/src/commands/bitswap/wantlist.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/block/put.js → packages/ipfs-cli/src/commands/block/put.js

@@ -3,7 +3,7 @@
 const fs = require('fs')
 const multibase = require('multibase')
 const concat = require('it-concat')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/block/stat.js → packages/ipfs-cli/src/commands/block/stat.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/daemon.js → packages/ipfs-cli/src/commands/daemon.js

@@ -48,7 +48,7 @@ module.exports = {
   async handler (argv) {
     const { print, repoPath } = argv.ctx
     print('Initializing IPFS daemon...')
-    print(`js-ipfs version: ${require('../../../package.json').version}`)
+    print(`js-ipfs version: ${require('../../package.json').version}`)
     print(`System version: ${os.arch()}/${os.platform()}`)
     print(`Node.js version: ${process.versions.node}`)
 
@@ -65,7 +65,7 @@ module.exports = {
     }
 
     // Required inline to reduce startup time
-    const Daemon = require('../../cli/daemon')
+    const Daemon = require('../../src/daemon')
     const daemon = new Daemon({
       config,
       silent: argv.silent,

packages/ipfs/src/cli/commands/dag/put.js → packages/ipfs-cli/src/commands/dag/put.js

@@ -6,7 +6,7 @@ const dagCBOR = require('ipld-dag-cbor')
 const dagPB = require('ipld-dag-pb')
 const concat = require('it-concat')
 const CID = require('cids')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 const uint8ArrayToString = require('uint8arrays/to-string')
 

packages/ipfs/src/cli/commands/init.js → packages/ipfs-cli/src/commands/init.js

@@ -66,7 +66,7 @@ module.exports = {
     print(`initializing ipfs node at ${repoPath}`)
 
     // Required inline to reduce startup time
-    const IPFS = require('../../core')
+    const IPFS = require('ipfs-core')
     const Repo = require('ipfs-repo')
 
     const node = await IPFS.create({

packages/ipfs/src/cli/commands/ls.js → packages/ipfs-cli/src/commands/ls.js

@@ -2,7 +2,7 @@
 
 const multibase = require('multibase')
 const { rightpad } = require('../utils')
-const { cidToString } = require('../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const formatMode = require('ipfs-core-utils/src/files/format-mode')
 const formatMtime = require('ipfs-core-utils/src/files/format-mtime')
 const parseDuration = require('parse-duration').default

packages/ipfs/src/cli/commands/object/get.js → packages/ipfs-cli/src/commands/object/get.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 const uint8ArrayToString = require('uint8arrays/to-string')
 

packages/ipfs/src/cli/commands/object/links.js → packages/ipfs-cli/src/commands/object/links.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/object/new.js → packages/ipfs-cli/src/commands/object/new.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const multibase = require('multibase')
-const { cidToString } = require('../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/object/patch/add-link.js → packages/ipfs-cli/src/commands/object/patch/add-link.js

@@ -3,7 +3,7 @@
 const dagPB = require('ipld-dag-pb')
 const DAGLink = dagPB.DAGLink
 const multibase = require('multibase')
-const { cidToString } = require('../../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {

packages/ipfs/src/cli/commands/object/patch/append-data.js → packages/ipfs-cli/src/commands/object/patch/append-data.js

@@ -3,7 +3,7 @@
 const concat = require('it-concat')
 const fs = require('fs')
 const multibase = require('multibase')
-const { cidToString } = require('../../../../utils/cid')
+const { cidToString } = require('ipfs-core-utils/src/cid')
 const parseDuration = require('parse-duration').default
 
 module.exports = {