From 9c0f11b4e07081348b8a296e2ba6eb079ee669ca Mon Sep 17 00:00:00 2001 From: "Benjamin E. Coe" Date: Mon, 6 Apr 2020 18:55:20 -0700 Subject: [PATCH] test: standardize retry logic (#371) --- automl/test/export_dataset.test.js | 6 +++- automl/test/import_dataset.test.js | 15 +--------- automl/test/translate_create_model.test.js | 11 ++++++-- automl/test/util.js | 28 +++++++++++++++++++ .../vision_object_detection_predict.test.js | 14 +--------- 5 files changed, 43 insertions(+), 31 deletions(-) create mode 100644 automl/test/util.js diff --git a/automl/test/export_dataset.test.js b/automl/test/export_dataset.test.js index 274efee999..3eb686a95a 100644 --- a/automl/test/export_dataset.test.js +++ b/automl/test/export_dataset.test.js @@ -27,11 +27,15 @@ const DATASET_ID = 'TRL8522556519449886720'; const EXPORT_DATASET_REGION_TAG = 'export_dataset'; const LOCATION = 'us-central1'; +const {delay} = require('./util'); + describe('Automl Translate Dataset Tests', () => { const client = new AutoMlClient(); const prefix = 'TEST_EXPORT_OUTPUT'; - it('should export a datset', async () => { + it('should export a datset', async function() { + this.retries(4); + await delay(this.test); const projectId = await client.getProjectId(); const bucketName = `${projectId}-automl-translate`; const export_output = execSync( diff --git a/automl/test/import_dataset.test.js b/automl/test/import_dataset.test.js index e96dee43c2..fb67829b23 100644 --- a/automl/test/import_dataset.test.js +++ b/automl/test/import_dataset.test.js @@ -26,20 +26,7 @@ const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); const IMPORT_DATASET_REGION_TAG = 'import_dataset'; const LOCATION = 'us-central1'; const TWENTY_MINUTES_IN_SECONDS = 60 * 20; - -// If two suites of tests are running parallel, importing and creating -// datasets can fail, with: -// No other operations should be working on projects/1046198160504/*. -const delay = async test => { - const retries = test.currentRetry(); - if (retries === 0) return; // no retry on the first failure. - // see: https://cloud.google.com/storage/docs/exponential-backoff: - const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000; - return new Promise(done => { - console.info(`retrying "${test.title}" in ${ms}ms`); - setTimeout(done, ms); - }); -}; +const {delay} = require('./util'); describe('Automl Import Dataset Test', () => { const client = new AutoMlClient(); diff --git a/automl/test/translate_create_model.test.js b/automl/test/translate_create_model.test.js index 5b028ed74f..87f46a0c86 100644 --- a/automl/test/translate_create_model.test.js +++ b/automl/test/translate_create_model.test.js @@ -15,7 +15,7 @@ 'use strict'; const {assert} = require('chai'); -const {describe, it, after} = require('mocha'); +const {describe, it, afterEach} = require('mocha'); const {AutoMlClient} = require('@google-cloud/automl').v1; const cp = require('child_process'); @@ -26,11 +26,16 @@ const CREATE_MODEL_REGION_TAG = 'translate_create_model'; const LOCATION = 'us-central1'; const DATASET_ID = 'TRL8522556519449886720'; +const {delay} = require('./util'); + describe('Automl Translate Create Model Tests', () => { const client = new AutoMlClient(); let operationId; - it('should create a model', async () => { + it('should create a model', async function() { + this.retries(5); + await delay(this.test); + const projectId = await client.getProjectId(); const create_output = execSync( `node ${CREATE_MODEL_REGION_TAG}.js ${projectId} ${LOCATION} ${DATASET_ID} translation_test_create_model` @@ -43,7 +48,7 @@ describe('Automl Translate Create Model Tests', () => { .split('\n')[0]; }); - after('cancel model training', async () => { + afterEach('cancel model training', async () => { await client.operationsClient.cancelOperation({name: operationId}); }); }); diff --git a/automl/test/util.js b/automl/test/util.js new file mode 100644 index 0000000000..97b804a43c --- /dev/null +++ b/automl/test/util.js @@ -0,0 +1,28 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ML tests frequently run into concurrency and quota issues, for which +// retrying with a backoff is a good strategy: +module.exports = { + async delay(test) { + const retries = test.currentRetry(); + if (retries === 0) return; // no retry on the first failure. + // see: https://cloud.google.com/storage/docs/exponential-backoff: + const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000; + return new Promise(done => { + console.info(`retrying "${test.title}" in ${ms}ms`); + setTimeout(done, ms); + }); + }, +}; diff --git a/automl/test/vision_object_detection_predict.test.js b/automl/test/vision_object_detection_predict.test.js index d20f963fe7..1d1cceb832 100644 --- a/automl/test/vision_object_detection_predict.test.js +++ b/automl/test/vision_object_detection_predict.test.js @@ -26,19 +26,7 @@ const MODEL_ID = 'IOD1656537412546854912'; const PREDICT_REGION_TAG = 'vision_object_detection_predict'; const LOCATION = 'us-central1'; -// If two suites of tests are running parallel, importing and creating -// datasets can fail, with: -// "Another DEPLOY model operation is running on the model". -const delay = async test => { - const retries = test.currentRetry(); - if (retries === 0) return; // no retry on the first failure. - // see: https://cloud.google.com/storage/docs/exponential-backoff: - const ms = Math.pow(2, retries) * 1000 + Math.random() * 2000; - return new Promise(done => { - console.info(`retrying "${test.title}" in ${ms}ms`); - setTimeout(done, ms); - }); -}; +const {delay} = require('./util'); describe('Automl Vision Object Detection Predict Test', () => { const client = new AutoMlClient();