feat: improved support for remote API (#1613)

* feat: improved support for remote API

This change adds support for connecting to remote API that is protected
by HTTP Basic Auth and enables entering custom configuration as a hidden
feature for advanced users.

Terse summary:
- custom API address is stored in and read from localStorage[ipfsApi]
- ipfsApi is a string or null
- if ipfsApi is null, ipfs-provider will try default /ip4/127.0.0.1/tcp/5001 and same-origin
- ipfsApi tring can be multiaddr, URL or a JSON with constructor
  config for ipfs-http-client
- user can enter URL with inlined basic auth  user and password, we
  convert them to advanced config before passing to http client.
- E2E tests for using remote API with basic auth
  (URL or JSON, by setting localStorage[ipfsApi] or via Settings screen)

* fix: hide JSON when custom API config is used

#1613 (comment)

* testi(e2e): multiaddr and JSON API address

* test(e2e): API set to multiaddr and URL

This makes tests DRY and adds regression tests for API set to multiaddr
and regular URL in addition to existing basic auth JSON/URL.

* chore: ipfs-provider 1.1.0

This update ensures that if user passes a custom config for
ipfs-http-client, the object is not mutated by the http client.

* refactor: avoid redundant JSON serialization

This removes double JSON serialization, as noted in
#1613 (comment)
and cleans up a bunch of tests to only use documented config keys
for ipfs-http-client.

package.json

@@ -14,7 +14,7 @@
     "test": "run-s -cl test:unit test:build test:e2e",
     "test:unit": "react-scripts test --env=jsdom --runInBand --watchAll=false",
     "test:unit:watch": "react-scripts test --env=jsdom",
-    "test:e2e": "cross-env JEST_PUPPETEER_CONFIG=test/e2e/jest-puppeteer.config.js jest -c test/e2e/jest.config.js --runInBand",
+    "test:e2e": "cross-env JEST_PUPPETEER_CONFIG=test/e2e/jest-puppeteer.config.js jest --verbose -c test/e2e/jest.config.js --runInBand",
     "test:build": "shx test -f build/index.html || run-s build",
     "test:coverage": "react-scripts test --coverage",
     "analyze": "webpack-bundle-analyzer build/stats.json",
@@ -48,7 +48,7 @@
     "ipfs-css": "^1.2.0",
     "ipfs-geoip": "^5.0.1",
     "ipfs-http-client": "^46.0.0",
-    "ipfs-provider": "^1.0.0",
+    "ipfs-provider": "^1.1.0",
     "ipld-explorer-components": "1.6.0",
     "is-binary": "^0.1.0",
     "is-ipfs": "^1.0.3",
@@ -80,7 +80,7 @@
     "react-loadable": "^5.5.0",
     "react-overlays": "^2.1.1",
     "react-router-dom": "^5.2.0",
-    "react-scripts": "^3.4.0",
+    "react-scripts": "^3.4.3",
     "react-spring": "^8.0.27",
     "react-test-renderer": "^16.13.1",
     "react-virtualized": "^9.21.2",
@@ -104,6 +104,7 @@
     "@svgr/cli": "^5.4.0",
     "@types/node": "^14.0.27",
     "babel-eslint": "^10.1.0",
+    "basic-auth": "^2.0.1",
     "big.js": "^5.2.2",
     "bundlesize": "^0.18.0",
     "cross-env": "^6.0.3",
@@ -117,14 +118,15 @@
     "fake-indexeddb": "^3.1.2",
     "get-port": "^5.1.1",
     "go-ipfs": "0.6.0",
+    "http-proxy": "^1.18.1",
     "http-server": "^0.12.3",
     "ipfs": "^0.48.1",
-    "ipfsd-ctl": "^5.0.0",
+    "ipfsd-ctl": "^7.0.0",
     "is-pull-stream": "0.0.0",
     "jest-puppeteer": "^4.4.0",
     "multihashing-async": "^1.0.0",
     "npm-run-all": "^4.1.5",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^5.2.1",
     "run-script-os": "^1.1.1",
     "shx": "^0.3.2",
     "typescript": "3.9.7",

public/locales/en/status.json

@@ -12,6 +12,7 @@
     "paragraph1": "Hosting <0>{repoSize} of files</0>",
     "paragraph2": "{count, plural, one {Discovered <0>1 peer</0>} other {Discovered <0>{peersCount} peers</0>}}"
   },
+  "customApiConfig": "Custom JSON configuration",
   "AskToEnable": {
     "label": "Help improve this app by sending anonymous usage data."
   },

src/bundles/config.js

@@ -98,9 +98,11 @@ function getURLFromAddress (name, config) {
     const address = Array.isArray(config.Addresses[name])
       ? config.Addresses[name][0]
       : config.Addresses[name]
-    return toUri(address).replace('tcp://', 'http://')
+    const url = toUri(address, { assumeHttp: true })
+    if (new URL(url).port === 0) throw Error('port set to 0, not deterministic')
+    return url
   } catch (error) {
-    console.log(`Failed to get url from Addresses.${name}`, error)
+    console.log(`Failed to get url from config at Addresses.${name}`, error)
     return null
   }
 }

src/bundles/ipfs-provider.js

@@ -60,7 +60,7 @@ const update = (state, message) => {
         ready: true,
         failed: false,
         provider: message.payload.provider,
-        apiAddress: message.payload.apiAddress || state.apiAddress
+        apiAddress: asAPIOptions(message.payload.apiAddress || state.apiAddress)
       }
     }
     case 'IPFS_STOPPED': {
@@ -102,14 +102,17 @@ const init = () => {
  */
 const readAPIAddressSetting = () => {
   const setting = readSetting('ipfsApi')
-  return setting == null ? null : asAPIAddress(setting)
+  return setting == null ? null : asAPIOptions(setting)
 }
 
-const asAPIAddress = (value) => asMultiaddress(value) || asURL(value)
+/**
+ * @returns {object|string|null}
+ */
+const asAPIOptions = (value) => asHttpClientOptions(value) || asMultiaddress(value) || asURL(value)
 
 /**
- * Attempts to turn cast given value into `URL` instance. Return either `URL`
- * instance or `null`.
+ * Attempts to turn cast given value into URL.
+ * Return either string instance or `null`.
  * @param {any} value
  * @returns {string|null}
  */
@@ -122,16 +125,53 @@ const asURL = (value) => {
 }
 
 /**
- * Attempts to turn cast given value into `URL` instance. Return either `URL`
- * instance or `null`.
+ * Attempts to turn cast given value into Multiaddr.
+ * Return either string instance or `null`.
+ * @param {any} value
+ * @returns {string|null}
  */
 const asMultiaddress = (value) => {
-  if (value != null) {
+  // ignore empty string, as it will produce '/'
+  if (value != null && value !== '') {
     try {
       return multiaddr(value).toString()
     } catch (_) {}
   }
+  return null
+}
 
+/**
+ * Attempts to turn cast given value into options object compatible with ipfs-http-client constructor.
+ * Return either string with JSON or `null`.
+ * @param {any} value
+ * @returns {object|null}
+ */
+const asHttpClientOptions = (value) => {
+  try {
+    value = JSON.parse(value)
+  } catch (_) {}
+
+  // turn URL with inlined basic auth into client options object
+  try {
+    const uri = new URL(value)
+    const { username, password } = uri
+    if (username && password) {
+      value = {
+        host: uri.hostname,
+        port: uri.port || (uri.protocol === 'https:' ? '443' : '80'),
+        protocol: uri.protocol.split(':').shift(),
+        apiPath: (uri.pathname !== '/' ? uri.pathname : 'api/v0'),
+        headers: {
+          authorization: `Basic ${btoa(username + ':' + password)}`
+        }
+      }
+    }
+  } catch (_) { }
+
+  // https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-http-client#importing-the-module-and-usage
+  if (value && (value.host || value.apiPath || value.protocol || value.port || value.headers)) {
+    return value
+  }
   return null
 }
 
@@ -223,7 +263,7 @@ const bundle = {
   },
 
   doUpdateIpfsApiAddress: (address) => async (store) => {
-    const apiAddress = asAPIAddress(address)
+    const apiAddress = asAPIOptions(address)
     if (apiAddress == null) {
       store.dispatch({ type: 'IPFS_API_ADDRESS_INVALID' })
     } else {

src/components/api-address-form/ApiAddressForm.js

@@ -3,8 +3,8 @@ import { connect } from 'redux-bundler-react'
 import { withTranslation } from 'react-i18next'
 import Button from '../button/Button'
 
-const ApiAddressForm = ({ t, doUpdateIpfsApiAddress, ipfsApiAddress = '' }) => {
-  const [value, setValue] = useState(ipfsApiAddress)
+const ApiAddressForm = ({ t, doUpdateIpfsApiAddress, ipfsApiAddress }) => {
+  const [value, setValue] = useState(asAPIString(ipfsApiAddress))
 
   const onChange = (event) => setValue(event.target.value)
 
@@ -21,20 +21,31 @@ const ApiAddressForm = ({ t, doUpdateIpfsApiAddress, ipfsApiAddress = '' }) => {
 
   return (
     <form onSubmit={onSubmit}>
-      <input id='api-address'
+      <input
+        id='api-address'
         aria-label={t('apiAddressForm.apiLabel')}
         type='text'
         className='w-100 lh-copy monospace f5 pl1 pv1 mb2 charcoal input-reset ba b--black-20 br1 focus-outline'
         onChange={onChange}
         onKeyPress={onKeyPress}
-        value={value} />
+        value={value}
+      />
       <div className='tr'>
-        <Button className="tc">{t('actions.submit')}</Button>
+        <Button className='tc'>{t('actions.submit')}</Button>
       </div>
     </form>
   )
 }
 
+/**
+ * @returns {string}
+ */
+const asAPIString = (value) => {
+  if (value == null) return ''
+  if (typeof value === 'string') return value
+  return JSON.stringify(value)
+}
+
 export default connect(
   'doUpdateIpfsApiAddress',
   'selectIpfsApiAddress',

src/status/NodeInfoAdvanced.js

@@ -40,19 +40,26 @@ const NodeInfoAdvanced = ({ t, identity, ipfsProvider, ipfsApiAddress, gatewayUr
     ev.preventDefault()
   }
 
+  const asAPIString = (value) => {
+    // hide raw JSON if advanced config is present in the string
+    return typeof value !== 'string'
+      ? t('customApiConfig')
+      : value
+  }
+
   return (
     <Details className='mt3 f6' summaryText={t('app:terms.advanced')} open={isNodeInfoOpen} onClick={handleSummaryClick}>
       <DefinitionList className='mt3'>
         <Definition advanced term={t('app:terms.gateway')} desc={gatewayUrl} />
         {ipfsProvider === 'httpClient'
           ? <Definition advanced term={t('app:terms.api')} desc={
-            isMultiaddr(ipfsApiAddress)
-              ? (
-                <div className="flex items-center">
-                  <Address value={ipfsApiAddress} />
-                  <a className='ml2 link blue sans-serif fw6' href="#/settings">{t('app:actions.edit')}</a>
-                </div>)
-              : ipfsApiAddress
+            (<div id="http-api-address" className="flex items-center">
+              {isMultiaddr(ipfsApiAddress)
+                ? (<Address value={ipfsApiAddress} />)
+                : asAPIString(ipfsApiAddress)
+              }
+              <a className='ml2 link blue sans-serif fw6' href="#/settings">{t('app:actions.edit')}</a>
+            </div>)
           } />
           : <Definition advanced term={t('app:terms.api')} desc={<ProviderLink name={ipfsProvider} />} />
         }