Skip to content

Commit cbcfe8d

Browse files
irataxyirataxy
committed
fix: add backoff for test retries and get job state queries (#131)
* fix: add backoff for test retries and get job state queries * Propagate changes from last commit
1 parent f18aa9b commit cbcfe8d

File tree

1 file changed

+132
-42
lines changed

1 file changed

+132
-42
lines changed

‎media/transcoder/test/transcoder.test.js‎

Lines changed: 132 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,19 @@ const cwd = path.join(__dirname, '..');
5858
const videoFile = `testdata/${testFileName}`;
5959
const overlayFile = `testdata/${testOverlayFileName}`;
6060

61+
const delay = async (test, addMs) => {
62+
const retries = test.currentRetry();
63+
await new Promise(r => setTimeout(r, addMs));
64+
// No retry on the first failure.
65+
if (retries === 0) return;
66+
// See: https://cloud.google.com/storage/docs/exponential-backoff
67+
const ms = Math.pow(2, retries) * 10000 + Math.random() * 1000;
68+
return new Promise(done => {
69+
console.info(`retrying "${test.title}" in ${ms}ms`);
70+
setTimeout(done, ms);
71+
});
72+
};
73+
6174
function wait(ms) {
6275
return new Promise(resolve => {
6376
setTimeout(() => {
@@ -193,12 +206,23 @@ describe('Job functions preset', () => {
193206
it('should check that the job succeeded', async function () {
194207
this.retries(5);
195208
createJobFromPreset();
196-
await wait(90000);
197-
const output = execSync(
198-
`node getJobState.js ${projectId} ${location} ${presetJobId}`,
199-
{cwd}
200-
);
201-
assert.ok(output.includes('Job state: SUCCEEDED'));
209+
await delay(this.test, 30000);
210+
211+
let getAttempts = 0;
212+
while (getAttempts < 5) {
213+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
214+
await wait(ms);
215+
const output = execSync(
216+
`node getJobState.js ${projectId} ${location} ${presetJobId}`,
217+
{cwd}
218+
);
219+
if (output.includes('Job state: SUCCEEDED')) {
220+
assert.ok(true);
221+
return;
222+
}
223+
getAttempts++;
224+
}
225+
assert.ok(false);
202226
});
203227
});
204228

@@ -251,12 +275,23 @@ describe('Job functions template', () => {
251275

252276
it('should check that the job succeeded', async function () {
253277
this.retries(5);
254-
await wait(90000);
255-
const output = execSync(
256-
`node getJobState.js ${projectId} ${location} ${this.templateJobId}`,
257-
{cwd}
258-
);
259-
assert.ok(output.includes('Job state: SUCCEEDED'));
278+
await delay(this.test, 30000);
279+
280+
let getAttempts = 0;
281+
while (getAttempts < 5) {
282+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
283+
await wait(ms);
284+
const output = execSync(
285+
`node getJobState.js ${projectId} ${location} ${this.templateJobId}`,
286+
{cwd}
287+
);
288+
if (output.includes('Job state: SUCCEEDED')) {
289+
assert.ok(true);
290+
return;
291+
}
292+
getAttempts++;
293+
}
294+
assert.ok(false);
260295
});
261296
});
262297

@@ -299,12 +334,23 @@ describe('Job functions adhoc', () => {
299334

300335
it('should check that the job succeeded', async function () {
301336
this.retries(5);
302-
await wait(90000);
303-
const output = execSync(
304-
`node getJobState.js ${projectId} ${location} ${this.adhocJobId}`,
305-
{cwd}
306-
);
307-
assert.ok(output.includes('Job state: SUCCEEDED'));
337+
await delay(this.test, 30000);
338+
339+
let getAttempts = 0;
340+
while (getAttempts < 5) {
341+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
342+
await wait(ms);
343+
const output = execSync(
344+
`node getJobState.js ${projectId} ${location} ${this.adhocJobId}`,
345+
{cwd}
346+
);
347+
if (output.includes('Job state: SUCCEEDED')) {
348+
assert.ok(true);
349+
return;
350+
}
351+
getAttempts++;
352+
}
353+
assert.ok(false);
308354
});
309355
});
310356

@@ -339,12 +385,23 @@ describe('Job with static overlay functions', () => {
339385

340386
it('should check that the job succeeded', async function () {
341387
this.retries(5);
342-
await wait(90000);
343-
const output = execSync(
344-
`node getJobState.js ${projectId} ${location} ${this.staticOverlayJobId}`,
345-
{cwd}
346-
);
347-
assert.ok(output.includes('Job state: SUCCEEDED'));
388+
await delay(this.test, 30000);
389+
390+
let getAttempts = 0;
391+
while (getAttempts < 5) {
392+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
393+
await wait(ms);
394+
const output = execSync(
395+
`node getJobState.js ${projectId} ${location} ${this.staticOverlayJobId}`,
396+
{cwd}
397+
);
398+
if (output.includes('Job state: SUCCEEDED')) {
399+
assert.ok(true);
400+
return;
401+
}
402+
getAttempts++;
403+
}
404+
assert.ok(false);
348405
});
349406
});
350407

@@ -379,12 +436,23 @@ describe('Job with animated overlay functions', () => {
379436

380437
it('should check that the job succeeded', async function () {
381438
this.retries(5);
382-
await wait(90000);
383-
const output = execSync(
384-
`node getJobState.js ${projectId} ${location} ${this.animatedOverlayJobId}`,
385-
{cwd}
386-
);
387-
assert.ok(output.includes('Job state: SUCCEEDED'));
439+
await delay(this.test, 30000);
440+
441+
let getAttempts = 0;
442+
while (getAttempts < 5) {
443+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
444+
await wait(ms);
445+
const output = execSync(
446+
`node getJobState.js ${projectId} ${location} ${this.animatedOverlayJobId}`,
447+
{cwd}
448+
);
449+
if (output.includes('Job state: SUCCEEDED')) {
450+
assert.ok(true);
451+
return;
452+
}
453+
getAttempts++;
454+
}
455+
assert.ok(false);
388456
});
389457
});
390458

@@ -419,12 +487,23 @@ describe('Job with set number of images spritesheet', () => {
419487

420488
it('should check that the job succeeded', async function () {
421489
this.retries(5);
422-
await wait(90000);
423-
const output = execSync(
424-
`node getJobState.js ${projectId} ${location} ${this.setNumberSpritesheetJobId}`,
425-
{cwd}
426-
);
427-
assert.ok(output.includes('Job state: SUCCEEDED'));
490+
await delay(this.test, 30000);
491+
492+
let getAttempts = 0;
493+
while (getAttempts < 5) {
494+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
495+
await wait(ms);
496+
const output = execSync(
497+
`node getJobState.js ${projectId} ${location} ${this.setNumberSpritesheetJobId}`,
498+
{cwd}
499+
);
500+
if (output.includes('Job state: SUCCEEDED')) {
501+
assert.ok(true);
502+
return;
503+
}
504+
getAttempts++;
505+
}
506+
assert.ok(false);
428507
});
429508

430509
it('should check that the spritesheet files exist in the bucket', async () => {
@@ -476,12 +555,23 @@ describe('Job with periodic images spritesheet', () => {
476555

477556
it('should check that the job succeeded', async function () {
478557
this.retries(5);
479-
await wait(90000);
480-
const output = execSync(
481-
`node getJobState.js ${projectId} ${location} ${this.periodicSpritesheetJobId}`,
482-
{cwd}
483-
);
484-
assert.ok(output.includes('Job state: SUCCEEDED'));
558+
await delay(this.test, 30000);
559+
560+
let getAttempts = 0;
561+
while (getAttempts < 5) {
562+
const ms = Math.pow(2, getAttempts + 1) * 10000 + Math.random() * 1000;
563+
await wait(ms);
564+
const output = execSync(
565+
`node getJobState.js ${projectId} ${location} ${this.periodicSpritesheetJobId}`,
566+
{cwd}
567+
);
568+
if (output.includes('Job state: SUCCEEDED')) {
569+
assert.ok(true);
570+
return;
571+
}
572+
getAttempts++;
573+
}
574+
assert.ok(false);
485575
});
486576

487577
it('should check that the spritesheet files exist in the bucket', async () => {

0 commit comments

Comments
 (0)