Merge pull request #22 from netlify/large-deploy-handling

Support async deploys

README.md

@@ -146,6 +146,7 @@ Optional `opts` include:
   deployTimeout: 1.2e6, // 20 mins
   parallelHash: 100, // number of parallel hashing calls
   parallelUpload: 4, // number of files to upload in parallel
+  maxRetry: 5, // number of times to try on failed file uploads
   filter: filename => { /* return false to filter a file from the deploy */ },
   tmpDir: tempy.directory(), // a temporary directory to zip loose files into
   statusCb: statusObj => {

package.json

@@ -30,6 +30,7 @@
   "dependencies": {
     "@netlify/open-api": "^0.4.1",
     "archiver": "^3.0.0",
+    "backoff": "^2.5.0",
     "clean-deep": "^3.0.2",
     "flush-write-stream": "^1.0.3",
     "folder-walker": "^3.2.0",

src/deploy/index.js

@@ -3,6 +3,7 @@ const hashFiles = require('./hash-files')
 const hashFns = require('./hash-fns')
 const cleanDeep = require('clean-deep')
 const tempy = require('tempy')
+const { waitForDiff } = require('./util')
 
 const { waitForDeploy, getUploadList, defaultFilter } = require('./util')
 
@@ -18,6 +19,8 @@ module.exports = async (api, siteId, dir, opts) => {
       concurrentHash: 100, // concurrent file hash calls
       concurrentUpload: 15, // Number of concurrent uploads
       filter: defaultFilter,
+      syncFileLimit: 7000, // number of files
+      maxRetry: 5, // number of times to retry an upload
       statusCb: statusObj => {
         /* default to noop */
         /* statusObj: {
@@ -64,11 +67,14 @@ module.exports = async (api, siteId, dir, opts) => {
     body: {
       files,
       functions,
+      async: Object.keys(files).length > opts.syncFileLimit,
       draft: opts.draft
     }
   })
 
   let deploy = await api.createSiteDeploy(deployParams)
+  if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, opts.deployTimeout)
+
   const { id: deployId, required: requiredFiles, required_functions: requiredFns } = deploy
 
   statusCb({
@@ -88,7 +94,7 @@ module.exports = async (api, siteId, dir, opts) => {
     msg: 'Waiting for deploy to go live...',
     phase: 'start'
   })
-  deploy = await waitForDeploy(api, deployId, opts.deployTimeout)
+  deploy = await waitForDeploy(api, deployId, siteId, opts.deployTimeout)
 
   statusCb({
     type: 'wait-for-deploy',

src/deploy/upload-files.js

@@ -1,9 +1,10 @@
 const pMap = require('p-map')
 const fs = require('fs')
+const backoff = require('backoff')
 
 module.exports = uploadFiles
-async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb }) {
-  if (!concurrentUpload || !statusCb) throw new Error('Missing required option concurrentUpload')
+async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) {
+  if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload')
   statusCb({
     type: 'upload',
     msg: `Uploading ${uploadList.length} files`,
@@ -22,20 +23,28 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status
     let response
     switch (assetType) {
       case 'file': {
-        response = await api.uploadDeployFile({
-          body: readStream,
-          deployId,
-          path: encodeURI(normalizedPath)
-        })
+        response = await retryUpload(
+          () =>
+            api.uploadDeployFile({
+              body: readStream,
+              deployId,
+              path: encodeURI(normalizedPath)
+            }),
+          maxRetry
+        )
         break
       }
       case 'function': {
-        response = await api.uploadDeployFunction({
-          body: readStream,
-          deployId,
-          name: encodeURI(normalizedPath),
-          runtime
-        })
+        response = await await retryUpload(
+          () =>
+            api.uploadDeployFunction({
+              body: readStream,
+              deployId,
+              name: encodeURI(normalizedPath),
+              runtime
+            }),
+          maxRetry
+        )
         break
       }
       default: {
@@ -56,3 +65,42 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status
   })
   return results
 }
+
+function retryUpload(uploadFn, maxRetry) {
+  return new Promise((resolve, reject) => {
+    let lastError
+    const fibonacciBackoff = backoff.fibonacci({
+      randomisationFactor: 0.5,
+      initialDelay: 100,
+      maxDelay: 10000
+    })
+    fibonacciBackoff.failAfter(maxRetry)
+
+    fibonacciBackoff.on('backoff', (number, delay) => {
+      // Do something when backoff starts, e.g. show to the
+      // user the delay before next reconnection attempt.
+    })
+
+    fibonacciBackoff.on('ready', tryUpload)
+
+    fibonacciBackoff.on('fail', () => {
+      reject(lastError)
+    })
+
+    function tryUpload(number, delay) {
+      uploadFn()
+        .then(results => resolve(results))
+        .catch(e => {
+          lastError = e
+          if (e.name === 'FetchError') {
+            console.log(e)
+            fibonacciBackoff.backoff()
+          } else {
+            return reject(e)
+          }
+        })
+    }
+
+    tryUpload(0, 0)
+  })
+}

src/deploy/util.js

@@ -33,9 +33,47 @@ exports.normalizePath = relname => {
   )
 }
 
+exports.waitForDiff = waitForDiff
+// poll an async deployId until its done diffing
+async function waitForDiff(api, deployId, siteId, timeout) {
+  let deploy // capture ready deploy during poll
+
+  await pWaitFor(loadDeploy, {
+    interval: 1000,
+    timeout,
+    message: 'Timeout while waiting for deploy'
+  })
+
+  return deploy
+
+  async function loadDeploy() {
+    const d = await api.getSiteDeploy({ siteId, deployId })
+
+    switch (d.state) {
+      // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33
+      case 'error': {
+        const deployError = new Error(`Deploy ${deployId} had an error`)
+        deployError.deploy = d
+        throw deployError
+      }
+      case 'prepared':
+      case 'uploading':
+      case 'uploaded':
+      case 'ready': {
+        deploy = d
+        return true
+      }
+      case 'preparing':
+      default: {
+        return false
+      }
+    }
+  }
+}
+
 // Poll a deployId until its ready
 exports.waitForDeploy = waitForDeploy
-async function waitForDeploy(api, deployId, timeout) {
+async function waitForDeploy(api, deployId, siteId, timeout) {
   let deploy // capture ready deploy during poll
 
   await pWaitFor(loadDeploy, {
@@ -47,12 +85,25 @@ async function waitForDeploy(api, deployId, timeout) {
   return deploy
 
   async function loadDeploy() {
-    const d = await api.getDeploy({ deployId })
-    if (d.state === 'ready') {
-      deploy = d
-      return true
-    } else {
-      return false
+    const d = await api.getSiteDeploy({ siteId, deployId })
+    switch (d.state) {
+      // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33
+      case 'error': {
+        const deployError = new Error(`Deploy ${deployId} had an error`)
+        deployError.deploy = d
+        throw deployError
+      }
+      case 'ready': {
+        deploy = d
+        return true
+      }
+      case 'preparing':
+      case 'prepared':
+      case 'uploaded':
+      case 'uploading':
+      default: {
+        return false
+      }
     }
   }
 }
@@ -77,3 +128,5 @@ exports.isExe = stat => {
 
   return Boolean(mode & 0o0001 || (mode & 0o0010 && isGroup) || (mode & 0o0100 && isUser))
 }
+
+exports.retry = async (fn, errHandler, opts) => {}