From 7bd2f59def31df127f7163a8b55ad7c590dbe8d8 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 20 Sep 2018 17:32:41 -0700 Subject: [PATCH 1/4] Support async deploys --- src/deploy/index.js | 5 ++++ src/deploy/util.js | 59 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/deploy/index.js b/src/deploy/index.js index 8a156ec..a11572b 100644 --- a/src/deploy/index.js +++ b/src/deploy/index.js @@ -3,6 +3,7 @@ const hashFiles = require('./hash-files') const hashFns = require('./hash-fns') const cleanDeep = require('clean-deep') const tempy = require('tempy') +const { waitForDiff } = require('./util') const { waitForDeploy, getUploadList, defaultFilter } = require('./util') @@ -18,6 +19,7 @@ module.exports = async (api, siteId, dir, opts) => { concurrentHash: 100, // concurrent file hash calls concurrentUpload: 15, // Number of concurrent uploads filter: defaultFilter, + syncFileLimit: 7000, statusCb: statusObj => { /* default to noop */ /* statusObj: { @@ -64,11 +66,14 @@ module.exports = async (api, siteId, dir, opts) => { body: { files, functions, + async: Object.keys(files).length > opts.syncFileLimit, draft: opts.draft } }) let deploy = await api.createSiteDeploy(deployParams) + deploy = await waitForDiff(api, deploy.id, opts.deployTimeout) + const { id: deployId, required: requiredFiles, required_functions: requiredFns } = deploy statusCb({ diff --git a/src/deploy/util.js b/src/deploy/util.js index 0036443..0e6d2ac 100644 --- a/src/deploy/util.js +++ b/src/deploy/util.js @@ -33,6 +33,41 @@ exports.normalizePath = relname => { ) } +exports.waitForDiff = waitForDiff +// poll an async deployId until its done diffing +async function waitForDiff(api, deployId, timeout) { + let deploy // capture ready deploy during poll + + await pWaitFor(loadDeploy, { + interval: 1000, + timeout, + message: 'Timeout while waiting for deploy' + }) + + return deploy + + async function loadDeploy() { + const d = await api.getDeploy({ deployId }) + switch (d.state) { + case 'error': { + const deployError = new Error(`Deploy ${deployId} had an error`) + deployError.deploy = d + throw deployError + } + case 'prepared': + case 'uploaded': + case 'ready': { + deploy = d + return true + } + case 'preparing': + default: { + return false + } + } + } +} + // Poll a deployId until its ready exports.waitForDeploy = waitForDeploy async function waitForDeploy(api, deployId, timeout) { @@ -48,11 +83,23 @@ async function waitForDeploy(api, deployId, timeout) { async function loadDeploy() { const d = await api.getDeploy({ deployId }) - if (d.state === 'ready') { - deploy = d - return true - } else { - return false + switch (d.state) { + case 'error': { + const deployError = new Error(`Deploy ${deployId} had an error`) + deployError.deploy = d + throw deployError + } + case 'ready': { + deploy = d + return true + } + case 'preparing': + case 'prepared': + case 'uploaded': + case 'uploading': + default: { + return false + } } } } @@ -77,3 +124,5 @@ exports.isExe = stat => { return Boolean(mode & 0o0001 || (mode & 0o0010 && isGroup) || (mode & 0o0100 && isUser)) } + +exports.retry = async (fn, errHandler, opts) => {} From def61d8904d4ebc1048db2cb579447e0e873dfd1 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Thu, 20 Sep 2018 18:47:05 -0700 Subject: [PATCH 2/4] Fix async/sync bugs --- src/deploy/index.js | 4 ++-- src/deploy/util.js | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/deploy/index.js b/src/deploy/index.js index a11572b..20f13e2 100644 --- a/src/deploy/index.js +++ b/src/deploy/index.js @@ -72,7 +72,7 @@ module.exports = async (api, siteId, dir, opts) => { }) let deploy = await api.createSiteDeploy(deployParams) - deploy = await waitForDiff(api, deploy.id, opts.deployTimeout) + if (deployParams.body.async) deploy = await waitForDiff(api, deploy.id, siteId, opts.deployTimeout) const { id: deployId, required: requiredFiles, required_functions: requiredFns } = deploy @@ -93,7 +93,7 @@ module.exports = async (api, siteId, dir, opts) => { msg: 'Waiting for deploy to go live...', phase: 'start' }) - deploy = await waitForDeploy(api, deployId, opts.deployTimeout) + deploy = await waitForDeploy(api, deployId, siteId, opts.deployTimeout) statusCb({ type: 'wait-for-deploy', diff --git a/src/deploy/util.js b/src/deploy/util.js index 0e6d2ac..85a39f7 100644 --- a/src/deploy/util.js +++ b/src/deploy/util.js @@ -35,7 +35,7 @@ exports.normalizePath = relname => { exports.waitForDiff = waitForDiff // poll an async deployId until its done diffing -async function waitForDiff(api, deployId, timeout) { +async function waitForDiff(api, deployId, siteId, timeout) { let deploy // capture ready deploy during poll await pWaitFor(loadDeploy, { @@ -47,14 +47,17 @@ async function waitForDiff(api, deployId, timeout) { return deploy async function loadDeploy() { - const d = await api.getDeploy({ deployId }) + const d = await api.getSiteDeploy({ siteId, deployId }) + switch (d.state) { + // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33 case 'error': { const deployError = new Error(`Deploy ${deployId} had an error`) deployError.deploy = d throw deployError } case 'prepared': + case 'uploading': case 'uploaded': case 'ready': { deploy = d @@ -70,7 +73,7 @@ async function waitForDiff(api, deployId, timeout) { // Poll a deployId until its ready exports.waitForDeploy = waitForDeploy -async function waitForDeploy(api, deployId, timeout) { +async function waitForDeploy(api, deployId, siteId, timeout) { let deploy // capture ready deploy during poll await pWaitFor(loadDeploy, { @@ -82,8 +85,9 @@ async function waitForDeploy(api, deployId, timeout) { return deploy async function loadDeploy() { - const d = await api.getDeploy({ deployId }) + const d = await api.getSiteDeploy({ siteId, deployId }) switch (d.state) { + // https://github.com/netlify/bitballoon/blob/master/app/models/deploy.rb#L21-L33 case 'error': { const deployError = new Error(`Deploy ${deployId} had an error`) deployError.deploy = d From 415a3c8266b540e4e003166ef9740abdfce0804a Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Fri, 21 Sep 2018 16:52:33 -0700 Subject: [PATCH 3/4] Implement upload retry with Fibonacci backoff --- README.md | 1 + package.json | 1 + src/deploy/index.js | 3 +- src/deploy/upload-files.js | 75 +++++++++++++++++++++++++++++++------- 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 003ae01..4dcd2d0 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Optional `opts` include: deployTimeout: 1.2e6, // 20 mins parallelHash: 100, // number of parallel hashing calls parallelUpload: 4, // number of files to upload in parallel + maxRetry: 5, // number of times to try on failed file uploads filter: filename => { /* return false to filter a file from the deploy */ }, tmpDir: tempy.directory(), // a temporary directory to zip loose files into statusCb: statusObj => { diff --git a/package.json b/package.json index 7c53cc3..d966b86 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "@netlify/open-api": "^0.4.1", "archiver": "^3.0.0", + "backoff": "^2.5.0", "clean-deep": "^3.0.2", "flush-write-stream": "^1.0.3", "folder-walker": "^3.2.0", diff --git a/src/deploy/index.js b/src/deploy/index.js index 20f13e2..9fb8a6e 100644 --- a/src/deploy/index.js +++ b/src/deploy/index.js @@ -19,7 +19,8 @@ module.exports = async (api, siteId, dir, opts) => { concurrentHash: 100, // concurrent file hash calls concurrentUpload: 15, // Number of concurrent uploads filter: defaultFilter, - syncFileLimit: 7000, + syncFileLimit: 7000, // number of files + maxRetry: 5, // number of times to retry an upload statusCb: statusObj => { /* default to noop */ /* statusObj: { diff --git a/src/deploy/upload-files.js b/src/deploy/upload-files.js index 3b2ee0c..b83cafc 100644 --- a/src/deploy/upload-files.js +++ b/src/deploy/upload-files.js @@ -1,9 +1,10 @@ const pMap = require('p-map') const fs = require('fs') +const backoff = require('backoff') module.exports = uploadFiles -async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb }) { - if (!concurrentUpload || !statusCb) throw new Error('Missing required option concurrentUpload') +async function uploadFiles(api, deployId, uploadList, { concurrentUpload, statusCb, maxRetry }) { + if (!concurrentUpload || !statusCb || !maxRetry) throw new Error('Missing required option concurrentUpload') statusCb({ type: 'upload', msg: `Uploading ${uploadList.length} files`, @@ -22,20 +23,28 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status let response switch (assetType) { case 'file': { - response = await api.uploadDeployFile({ - body: readStream, - deployId, - path: encodeURI(normalizedPath) - }) + response = await retryUpload( + () => + api.uploadDeployFile({ + body: readStream, + deployId, + path: encodeURI(normalizedPath) + }), + maxRetry + ) break } case 'function': { - response = await api.uploadDeployFunction({ - body: readStream, - deployId, - name: encodeURI(normalizedPath), - runtime - }) + response = await await retryUpload( + () => + api.uploadDeployFunction({ + body: readStream, + deployId, + name: encodeURI(normalizedPath), + runtime + }), + maxRetry + ) break } default: { @@ -56,3 +65,43 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status }) return results } + +function retryUpload(uploadFn, maxRetry) { + return new Promise((resolve, reject) => { + const fibonacciBackoff = backoff.fibonacci({ + randomisationFactor: 0.5, + initialDelay: 100, + maxDelay: 10000 + }) + fibonacciBackoff.failAfter(maxRetry) + + fibonacciBackoff.on('backoff', (number, delay) => { + // Do something when backoff starts, e.g. show to the + // user the delay before next reconnection attempt. + console.log(number + ' ' + delay + 'ms') + }) + + fibonacciBackoff.on('ready', tryUpload) + + fibonacciBackoff.on('fail', () => { + // Do something when the maximum number of backoffs is + // reached, e.g. ask the user to check its connection. + console.log('fail') + }) + + function tryUpload(number, delay) { + uploadFn() + .then(results => resolve(results)) + .catch(e => { + if (e.name === 'FetchError') { + console.log(e) + fibonacciBackoff.backoff() + } else { + return reject(e) + } + }) + } + + tryUpload(0, 0) + }) +} From 877667c9d8abedae9e29fe6fb904f9e4d56693a3 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Fri, 21 Sep 2018 16:56:38 -0700 Subject: [PATCH 4/4] Refactor error handling --- src/deploy/upload-files.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/deploy/upload-files.js b/src/deploy/upload-files.js index b83cafc..0c1a756 100644 --- a/src/deploy/upload-files.js +++ b/src/deploy/upload-files.js @@ -68,6 +68,7 @@ async function uploadFiles(api, deployId, uploadList, { concurrentUpload, status function retryUpload(uploadFn, maxRetry) { return new Promise((resolve, reject) => { + let lastError const fibonacciBackoff = backoff.fibonacci({ randomisationFactor: 0.5, initialDelay: 100, @@ -78,21 +79,19 @@ function retryUpload(uploadFn, maxRetry) { fibonacciBackoff.on('backoff', (number, delay) => { // Do something when backoff starts, e.g. show to the // user the delay before next reconnection attempt. - console.log(number + ' ' + delay + 'ms') }) fibonacciBackoff.on('ready', tryUpload) fibonacciBackoff.on('fail', () => { - // Do something when the maximum number of backoffs is - // reached, e.g. ask the user to check its connection. - console.log('fail') + reject(lastError) }) function tryUpload(number, delay) { uploadFn() .then(results => resolve(results)) .catch(e => { + lastError = e if (e.name === 'FetchError') { console.log(e) fibonacciBackoff.backoff()