diff --git a/README.md b/README.md index 003ae01..4dcd2d0 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Optional `opts` include: deployTimeout: 1.2e6, // 20 mins parallelHash: 100, // number of parallel hashing calls parallelUpload: 4, // number of files to upload in parallel + maxRetry: 5, // number of times to try on failed file uploads filter: filename => { /* return false to filter a file from the deploy */ }, tmpDir: tempy.directory(), // a temporary directory to zip loose files into statusCb: statusObj => { diff --git a/package.json b/package.json index 7c53cc3..d966b86 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "@netlify/open-api": "^0.4.1", "archiver": "^3.0.0", + "backoff": "^2.5.0", "clean-deep": "^3.0.2", "flush-write-stream": "^1.0.3", "folder-walker": "^3.2.0", diff --git a/src/deploy/index.js b/src/deploy/index.js index 8a156ec..9fb8a6e 100644 --- a/src/deploy/index.js +++ b/src/deploy/index.js @@ -3,6 +3,7 @@ const hashFiles = require('./hash-files') const hashFns = require('./hash-fns') const cleanDeep = require('clean-deep') const tempy = require('tempy') +const { waitForDiff } = require('./util') const { waitForDeploy, getUploadList, defaultFilter } = require('./util') @@ -18,6 +19,8 @@ module.exports = async (api, siteId, dir, opts) => { concurrentHash: 100, // concurrent file hash calls concurrentUpload: 15, // Number of concurrent uploads filter: defaultFilter, + syncFileLimit: 7000, // number of files + maxRetry: 5, // number of times to retry an upload statusCb: statusObj => { /* default to noop */ /* statusObj: { @@ -64,11 +67,14 @@ module.exports = async (api, siteId, dir, opts) => { body: { files, functions, + async: Object.keys(files).length > opts.syncFileLimit, draft: opts.draft } }) let deploy = await api.createSiteDeploy(deployParams) + if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, opts.deployTimeout) + const { id: deployId, required: requiredFiles, required_functions: requiredFns } = deploy statusCb({ @@ -88,7 +94,7 @@ module.exports = async (api, siteId, dir, opts) => { msg: 'Waiting for deploy to go live...', phase: 'start' }) - deploy = await waitForDeploy(api, deployId, opts.deployTimeout) + deploy = await waitForDeploy(api, deployId, siteId, opts.deployTimeout) statusCb({ type: 'wait-for-deploy', diff --git a/src/deploy/upload-files.js b/src/deploy/upload-files.js index 3b2ee0c..0c1a756 100644 --- a/src/deploy/upload-files.js +++ b/src/deploy/upload-files.js @@ -1,9 +1,10 @@ const pMap = require('p-map') const fs = require('fs') +const backoff = require('backoff') module.exports = uploadFiles -async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb }) { - if (!concurrentUpload || !statusCb) throw new Error('Missing required option concurrentUpload') +async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) { + if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload') statusCb({ type: 'upload', msg: `Uploading ${uploadList.length} files`, @@ -22,20 +23,28 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status let response switch (assetType) { case 'file': { - response = await api.uploadDeployFile({ - body: readStream, - deployId, - path: encodeURI(normalizedPath) - }) + response = await retryUpload( + () => + api.uploadDeployFile({ + body: readStream, + deployId, + path: encodeURI(normalizedPath) + }), + maxRetry + ) break } case 'function': { - response = await api.uploadDeployFunction({ - body: readStream, - deployId, - name: encodeURI(normalizedPath), - runtime - }) + response = await await retryUpload( + () => + api.uploadDeployFunction({ + body: readStream, + deployId, + name: encodeURI(normalizedPath), + runtime + }), + maxRetry + ) break } default: { @@ -56,3 +65,42 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status }) return results } + +function retryUpload(uploadFn, maxRetry) { + return new Promise((resolve, reject) => { + let lastError + const fibonacciBackoff = backoff.fibonacci({ + randomisationFactor: 0.5, + initialDelay: 100, + maxDelay: 10000 + }) + fibonacciBackoff.failAfter(maxRetry) + + fibonacciBackoff.on('backoff', (number, delay) => { + // Do something when backoff starts, e.g. show to the + // user the delay before next reconnection attempt. + }) + + fibonacciBackoff.on('ready', tryUpload) + + fibonacciBackoff.on('fail', () => { + reject(lastError) + }) + + function tryUpload(number, delay) { + uploadFn() + .then(results => resolve(results)) + .catch(e => { + lastError = e + if (e.name === 'FetchError') { + console.log(e) + fibonacciBackoff.backoff() + } else { + return reject(e) + } + }) + } + + tryUpload(0, 0) + }) +} diff --git a/src/deploy/util.js b/src/deploy/util.js index 0036443..85a39f7 100644 --- a/src/deploy/util.js +++ b/src/deploy/util.js @@ -33,9 +33,47 @@ exports.normalizePath = relname => { ) } +exports.waitForDiff = waitForDiff +// poll an async deployId until its done diffing +async function waitForDiff(api, deployId, siteId, timeout) { + let deploy // capture ready deploy during poll + + await pWaitFor(loadDeploy, { + interval: 1000, + timeout, + message: 'Timeout while waiting for deploy' + }) + + return deploy + + async function loadDeploy() { + const d = await api.getSiteDeploy({ siteId, deployId }) + + switch (d.state) { + // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33 + case 'error': { + const deployError = new Error(`Deploy ${deployId} had an error`) + deployError.deploy = d + throw deployError + } + case 'prepared': + case 'uploading': + case 'uploaded': + case 'ready': { + deploy = d + return true + } + case 'preparing': + default: { + return false + } + } + } +} + // Poll a deployId until its ready exports.waitForDeploy = waitForDeploy -async function waitForDeploy(api, deployId, timeout) { +async function waitForDeploy(api, deployId, siteId, timeout) { let deploy // capture ready deploy during poll await pWaitFor(loadDeploy, { @@ -47,12 +85,25 @@ async function waitForDeploy(api, deployId, timeout) { return deploy async function loadDeploy() { - const d = await api.getDeploy({ deployId }) - if (d.state === 'ready') { - deploy = d - return true - } else { - return false + const d = await api.getSiteDeploy({ siteId, deployId }) + switch (d.state) { + // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33 + case 'error': { + const deployError = new Error(`Deploy ${deployId} had an error`) + deployError.deploy = d + throw deployError + } + case 'ready': { + deploy = d + return true + } + case 'preparing': + case 'prepared': + case 'uploaded': + case 'uploading': + default: { + return false + } } } } @@ -77,3 +128,5 @@ exports.isExe = stat => { return Boolean(mode & 0o0001 || (mode & 0o0010 && isGroup) || (mode & 0o0100 && isUser)) } + +exports.retry = async (fn, errHandler, opts) => {}