feat: improved support for remote API

package.json

@@ -14,7 +14,7 @@
     "test": "run-s -cl test:unit test:build test:e2e",
     "test:unit": "react-scripts test --env=jsdom --runInBand --watchAll=false",
     "test:unit:watch": "react-scripts test --env=jsdom",
-    "test:e2e": "cross-env JEST_PUPPETEER_CONFIG=test/e2e/jest-puppeteer.config.js jest -c test/e2e/jest.config.js --runInBand",
+    "test:e2e": "cross-env JEST_PUPPETEER_CONFIG=test/e2e/jest-puppeteer.config.js jest --verbose -c test/e2e/jest.config.js --runInBand",
     "test:build": "shx test -f build/index.html || run-s build",
     "test:coverage": "react-scripts test --coverage",
     "analyze": "webpack-bundle-analyzer build/stats.json",
@@ -103,6 +103,7 @@
     "@svgr/cli": "^5.4.0",
     "@types/node": "^14.0.27",
     "babel-eslint": "^10.1.0",
+    "basic-auth": "^2.0.1",
     "big.js": "^5.2.2",
     "bundlesize": "^0.18.0",
     "cross-env": "^6.0.3",
@@ -116,14 +117,15 @@
     "fake-indexeddb": "^3.1.2",
     "get-port": "^5.1.1",
     "go-ipfs": "0.6.0",
+    "http-proxy": "^1.18.1",
     "http-server": "^0.12.3",
     "ipfs": "^0.48.1",
-    "ipfsd-ctl": "^5.0.0",
+    "ipfsd-ctl": "^7.0.0",
     "is-pull-stream": "0.0.0",
     "jest-puppeteer": "^4.4.0",
     "multihashing-async": "^1.0.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^5.2.1",
     "run-script-os": "^1.1.1",
     "shx": "^0.3.2",
     "typescript": "3.9.7",

src/bundles/config.js

@@ -98,9 +98,11 @@ function getURLFromAddress (name, config) {
     const address = Array.isArray(config.Addresses[name])
       ? config.Addresses[name][0]
       : config.Addresses[name]
-    return toUri(address).replace('tcp://', 'http://')
+    const url = toUri(address, { assumeHttp: true })
+    if (new URL(url).port === 0) throw Error('port set to 0, not deterministic')
+    return url
   } catch (error) {
-    console.log(`Failed to get url from Addresses.${name}`, error)
+    console.log(`Failed to get url from config at Addresses.${name}`, error)
     return null
   }
 }

src/bundles/ipfs-provider.js

@@ -59,7 +59,7 @@ const update = (state, message) => {
         ready: true,
         failed: false,
         provider: message.payload.provider,
-        apiAddress: message.payload.apiAddress || state.apiAddress
+        apiAddress: asAPIAddress(message.payload.apiAddress || state.apiAddress)
       }
     }
     case 'IPFS_STOPPED': {
@@ -104,33 +104,49 @@ const readAPIAddressSetting = () => {
   return setting == null ? null : asAPIAddress(setting)
 }
 
-const asAPIAddress = (value) => asMultiaddress(value) || asURL(value)
+const asAPIAddress = (value) => asHttpClientOptions(value) || asMultiaddress(value) || asURL(value)
 
 /**
- * Attempts to turn cast given value into `URL` instance. Return either `URL`
- * instance or `null`.
+ * Attempts to turn cast given value into URL.
+ * Return either string instance or `null`.
  * @param {any} value
  * @returns {string|null}
  */
 const asURL = (value) => {
+  if (!value || typeof value !== 'string') return null
   try {
     return new URL(value).toString()
-  } catch (_) {
-    return null
-  }
+  } catch (_) {}
+  return null
 }
 
 /**
- * Attempts to turn cast given value into `URL` instance. Return either `URL`
- * instance or `null`.
+ * Attempts to turn cast given value into Multiaddr.
+ * Return either string instance or `null`.
  */
 const asMultiaddress = (value) => {
-  if (value != null) {
-    try {
-      return multiaddr(value).toString()
-    } catch (_) {}
-  }
+  if (!value) return null
+  try {
+    return multiaddr(value).toString()
+  } catch (_) {}
+  return null
+}
 
+/**
+ * Attempts to turn cast given value into options object compatible with ipfs-http-client constructor.
+ * Return either string with JSON or `null`.
+ * @param {any} value
+ * @returns {string|null}
+ */
+const asHttpClientOptions = (value) => {
+  if (!value) return null
+  try {
+    value = JSON.parse(value)
+  } catch (_) {}
+  // https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#importing-the-module-and-usage
+  if (value && (value.host || value.url || value.protocol || value.port || value.headers)) {
+    return JSON.stringify(value)
+  }
   return null
 }
 
@@ -239,7 +255,36 @@ const initIPFS = async (store) => {
   store.dispatch({ type: 'IPFS_INIT_STARTED' })
 
   /** @type {Model} */
-  const { apiAddress } = store.getState().ipfs
+  let { apiAddress } = store.getState().ipfs
+
+  try {
+    // if a custom JSON config is present, use it instead of multiaddr or URL
+    apiAddress = JSON.parse(apiAddress)
+  } catch (_) { }
+
+  // TODO: move this to ipfs-provider package, use apiAddress as-is
+  try {
+    const uri = new URL(apiAddress)
+    const { username, password } = uri
+    if (username && password) {
+      // ipfs-http-client does not support URIs with inlined credentials,
+      // so we manually convert string URL to options object
+      // and move basic auth to Authorization header
+      uri.username = ''
+      uri.password = ''
+      apiAddress = {
+        url: uri.toString(),
+        /* TODO: 'url' is useful, but undocumented option: update docs of http-client to list `url` as alternative to host etc
+        host: uri.hostname,
+        port: uri.port,
+        protocol: uri.protocol.split(':').shift(),
+        */
+        headers: {
+          authorization: `Basic ${btoa(username + ':' + password)}`
+        }
+      }
+    }
+  } catch (_) { }
 
   try {
     const result = await getIpfs({

src/components/api-address-form/ApiAddressForm.js

@@ -27,7 +27,7 @@ const ApiAddressForm = ({ t, doUpdateIpfsApiAddress, ipfsApiAddress = '' }) => {
         className='w-100 lh-copy monospace f5 pl1 pv1 mb2 charcoal input-reset ba b--black-20 br1 focus-outline'
         onChange={onChange}
         onKeyPress={onKeyPress}
-        value={value} />
+        value={value || ''} />
       <div className='tr'>
         <Button className="tc">{t('apiAddressForm.submitButton')}</Button>
       </div>

src/status/NodeInfoAdvanced.js

@@ -46,13 +46,13 @@ const NodeInfoAdvanced = ({ t, identity, ipfsProvider, ipfsApiAddress, gatewayUr
         <Definition advanced term={t('gateway')} desc={gatewayUrl} />
         {ipfsProvider === 'httpClient'
           ? <Definition advanced term={t('api')} desc={
-            isMultiaddr(ipfsApiAddress)
-              ? (
-                <div className="flex items-center">
-                  <Address value={ipfsApiAddress} />
-                  <a className='ml2 link blue sans-serif fw6' href="#/settings">{t('apiEdit')}</a>
-                </div>)
-              : ipfsApiAddress
+            (<div className="flex items-center">
+              {isMultiaddr(ipfsApiAddress)
+                ? (<Address value={ipfsApiAddress} />)
+                : ipfsApiAddress
+              }
+              <a className='ml2 link blue sans-serif fw6' href="#/settings">{t('apiEdit')}</a>
+            </div>)
           } />
           : <Definition advanced term={t('api')} desc={<ProviderLink name={ipfsProvider} />} />
         }

test/e2e/remote-api.test.js

@@ -0,0 +1,173 @@
+/* global webuiUrl, page, waitForTitle, describe, it, expect, beforeAll */
+
+const { createController } = require('ipfsd-ctl')
+const getPort = require('get-port')
+const http = require('http')
+const httpProxy = require('http-proxy')
+const basicAuth = require('basic-auth')
+const toUri = require('multiaddr-to-uri')
+
+// Why do we support and test Basic Auth?
+// -----------------------------------
+// Some users choose to access remote API.
+// It requires setting up reverse proxy with correct CORS  and Basic Auth headers,
+// but when done properly, should work. This test sets up a proxy which
+// acts as properly protected and configured remote API to ensure there are no
+// regressions for this difficult to test use case.
+
+describe('Remote API with CORS and Basic Auth', () => {
+  let ipfsd
+  let proxyd
+  let user
+  let password
+  let proxyPort
+  let startProxyServer
+  beforeAll(async () => {
+    await page.goto(webuiUrl)
+    // spawn an ephemeral local node to ensure we connect to a different, remote node
+    ipfsd = await createController({
+      type: 'go',
+      ipfsBin: require('go-ipfs').path(),
+      ipfsHttpModule: require('ipfs-http-client'),
+      test: true,
+      disposable: true
+    })
+
+    // set up proxy in front of remote API to provide CORS and Basic Auth
+    proxyPort = await getPort()
+    user = 'user'
+    password = 'pass'
+
+    const proxy = httpProxy.createProxyServer()
+    const remoteApiUrl = toUri(ipfsd.apiAddr.toString(), { assumeHttp: true })
+    proxy.on('proxyReq', (proxyReq, req, res, options) => {
+      // swap Origin before passing to the real API
+      // This way internal check of go-ipfs does get triggered (403 Forbidden on Origin mismatch)
+      proxyReq.setHeader('Origin', remoteApiUrl)
+      proxyReq.setHeader('Referer', remoteApiUrl)
+      proxyReq.setHeader('Host', new URL(remoteApiUrl).host)
+    })
+
+    proxy.on('error', function (err, req, res) {
+      res.writeHead(500, { 'Content-Type': 'text/plain' })
+      res.end(`proxyd error: ${JSON.stringify(err)}`)
+    })
+
+    proxyd = http.createServer((req, res) => {
+      // console.log(`${req.method}\t\t${req.url}`)
+
+      res.oldWriteHead = res.writeHead
+      res.writeHead = function (statusCode, headers) {
+        // hardcoded liberal CORS for easier testing
+        res.setHeader('Access-Control-Allow-Origin', '*')
+        // usual suspects + 'authorization' header
+        res.setHeader('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Output, X-Content-Length, authorization')
+        res.oldWriteHead(statusCode)
+      }
+
+      const auth = basicAuth(req)
+      const preflight = req.method === 'OPTIONS' // need preflight working
+      if (!preflight && !(auth && auth.name === user && auth.pass === password)) {
+        res.writeHead(401, {
+          'WWW-Authenticate': 'Basic realm="IPFS API"'
+        })
+        res.end('Access denied')
+      } else {
+        proxy.web(req, res, { target: remoteApiUrl })
+      }
+    })
+    startProxyServer = async (port) => {
+      return new Promise((resolve, reject) => {
+        proxyd.on('listening', () => resolve())
+        proxyd.on('error', (e) => reject(e))
+        proxyd.listen(port)
+      })
+    }
+    await startProxyServer(proxyPort)
+  })
+
+  beforeEach(async () => {
+    // set API endpoint to one without any service
+    //  to avoid false-positives when running test on localhost with go-ipfs on default ports
+    await switchIpfsApiEndpointViaLocalStorage(`/ip4/127.0.0.1/tcp/${await getPort()}`)
+  })
+
+  const switchIpfsApiEndpointViaLocalStorage = async (endpoint) => {
+    await page.goto(webuiUrl)
+    if (endpoint) {
+      await page.evaluate((a) => localStorage.setItem('ipfsApi', a), endpoint)
+    } else {
+      await page.evaluate(() => localStorage.removeItem('ipfsApi'))
+    }
+    await waitForIpfsApiEndpoint(endpoint)
+    await page.reload()
+  }
+  const waitForIpfsApiEndpoint = async (endpoint) => {
+    endpoint = (endpoint ? `'${endpoint}'` : 'null')
+    await page.waitForFunction(`localStorage.getItem('ipfsApi') === ${endpoint}`)
+  }
+  const proxyConnectionConfirmation = async (proxyPort) => {
+    // confirm API section  on Status pageincludes expected address
+    const link = 'a[href="#/"]'
+    await page.waitForSelector(link)
+    await page.click(link)
+    await waitForTitle('Status | IPFS')
+    await page.waitForSelector('summary', { visible: true })
+    await expect(page).toClick('summary', { text: 'Advanced' })
+    await expect(page).toMatch(`http://127.0.0.1:${proxyPort}`)
+    // confirm webui is actually connected to expected node :^)
+    const { id } = await ipfsd.api.id()
+    await expect(page).toMatch(id)
+  }
+  const nodeBtoa = (b) => Buffer.from(b).toString('base64')
+
+  it('should work when localStorage[ipfsApi] is set to URL with inlined Basic Auth credentials', async () => {
+    await switchIpfsApiEndpointViaLocalStorage(`http://${user}:${password}@127.0.0.1:${proxyPort}`)
+    await proxyConnectionConfirmation(proxyPort)
+  })
+
+  it('should work when localStorage[ipfsApi] is set to a string with a custom ipfs-http-client config', async () => {
+    // options object accepted by the constructor of ipfs-http-client
+    const apiOptions = JSON.stringify({
+      url: `http://127.0.0.1:${proxyPort}/api/v0`,
+      headers: {
+        authorization: `Basic ${nodeBtoa(user + ':' + password)}`
+      }
+    })
+    await switchIpfsApiEndpointViaLocalStorage(apiOptions)
+    await proxyConnectionConfirmation(proxyPort)
+  })
+
+  const switchIpfsApiEndpointViaSettings = async (endpoint) => {
+    await page.goto(webuiUrl + '#/settings')
+    const selector = 'input[id="api-address"]'
+    await page.waitForSelector(selector)
+    await page.evaluate(s => { document.querySelector(s).value = '' }, selector)
+    await page.type(selector, endpoint)
+    await page.keyboard.type('\n')
+    // value could be a complex JSON, a partial match of unique port will do
+    await page.waitForFunction(`localStorage.getItem('ipfsApi').includes(${String(proxyPort)})`)
+    await page.reload()
+  }
+
+  it('should work when URL with inlined credentials is entered at the Settings screen', async () => {
+    const basicAuthApiAddr = `http://${user}:${password}@127.0.0.1:${proxyPort}`
+    await switchIpfsApiEndpointViaSettings(basicAuthApiAddr)
+    await proxyConnectionConfirmation(proxyPort)
+  })
+
+  it('should work when JSON with ipfs-http-client config is entered at the Settings screen', async () => {
+    const apiOptions = JSON.stringify({
+      url: `http://127.0.0.1:${proxyPort}/api/v0`,
+      headers: {
+        authorization: `Basic ${nodeBtoa(user + ':' + password)}`
+      }
+    })
+    await switchIpfsApiEndpointViaSettings(apiOptions)
+    await proxyConnectionConfirmation(proxyPort)
+  })
+
+  afterAll(async () => {
+    await Promise.all([ipfsd.stop(), proxyd.close()])
+  })
+})

test/e2e/setup/global-after-env.js

@@ -1,6 +1,6 @@
 const expect = require('expect-puppeteer')
 
 // increase timeouts for CI
-const timeout = 30 * 1000
+const timeout = 60 * 1000
 jest.setTimeout(timeout)
 expect.setDefaultOptions({ timeout })

test/e2e/setup/test-environment.js

@@ -2,7 +2,7 @@
 const PuppeteerEnvironment = require('jest-environment-puppeteer')
 const expect = require('expect-puppeteer')
 
-expect.setDefaultOptions({ timeout: 30 * 1000 })
+expect.setDefaultOptions({ timeout: 60 * 1000 })
 
 class WebuiTestEnvironment extends PuppeteerEnvironment {
   async setup () {
@@ -24,6 +24,7 @@ class WebuiTestEnvironment extends PuppeteerEnvironment {
     // and uses the address if present. below sets it to the address of HTTP API
     await page.evaluate((apiMultiaddr) =>
       localStorage.setItem('ipfsApi', apiMultiaddr), apiMultiaddr)
+    await page.waitForFunction(`localStorage.getItem('ipfsApi') === '${apiMultiaddr}'`)
     await page.reload()
 
     // open Status page, confirm working connection to API