Skip to content
This repository was archived by the owner on Jul 29, 2024. It is now read-only.

Commit a13a2d3

Browse files
hankduanhankduan
committed
feat(scripts): add framework to test protractor
1 parent 6a88642 commit a13a2d3

15 files changed

+470
-21
lines changed

‎package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
},
4747
"main": "lib/protractor.js",
4848
"scripts": {
49-
"pretest": "node_modules/.bin/jshint lib spec",
49+
"pretest": "node_modules/.bin/jshint lib spec scripts",
5050
"test": "scripts/test.js",
5151
"start": "testapp/scripts/web-server.js"
5252
},

‎scripts/test.js

Lines changed: 59 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/usr/bin/env node
22

3+
var Executor = require('./test/test_util').Executor;
34
var glob = require('glob').sync;
4-
var spawn = require('child_process').spawn;
55

6-
var scripts = [
6+
var passingTests = [
77
'node lib/cli.js spec/basicConf.js',
88
'node lib/cli.js spec/multiConf.js',
99
'node lib/cli.js spec/altRootConf.js',
@@ -22,29 +22,68 @@ var scripts = [
2222
'node lib/cli.js spec/suitesConf.js --suite okmany,okspec'
2323
];
2424

25-
scripts.push(
25+
passingTests.push(
2626
'node node_modules/.bin/minijasminenode ' +
2727
glob('spec/unit/*.js').join(' ') + ' ' +
2828
glob('docgen/spec/*.js').join(' '));
2929

30-
var failed = false;
30+
var executor = new Executor();
3131

32-
(function runTests(i) {
33-
if (i < scripts.length) {
34-
console.log('node ' + scripts[i]);
35-
var args = scripts[i].split(/\s/);
32+
passingTests.forEach(function(passing_test) {
33+
executor.addCommandlineTest(passing_test)
34+
.assertExitCodeOnly();
35+
});
3636

37-
var test = spawn(args[0], args.slice(1), {stdio: 'inherit'});
38-
test.on('error', function(err) {
39-
throw err;
37+
/*************************
38+
*Below are failure tests*
39+
*************************/
40+
41+
// assert stacktrace shows line of failure
42+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/singleFailureConf.js')
43+
.expectExitCode(1)
44+
.expectErrors({
45+
stackTrace: 'single_failure_spec1.js:5:32'
4046
});
41-
test.on('exit', function(code) {
42-
if (code != 0) {
43-
failed = true;
44-
}
45-
runTests(i + 1);
47+
48+
// assert timeout works
49+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/timeoutConf.js')
50+
.expectExitCode(1)
51+
.expectErrors({
52+
message: 'timeout: timed out after 1 msec waiting for spec to complete'
53+
})
54+
.expectTestDuration(0, 100);
55+
56+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
57+
.expectExitCode(11)
58+
.expectErrors({
59+
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
4660
});
47-
} else {
48-
process.exit(failed ? 1 : 0);
49-
}
50-
}(0));
61+
62+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/multiFailureConf.js')
63+
.expectExitCode(1)
64+
.expectErrors([{
65+
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
66+
stacktrace: 'single_failure_spec1.js:5:32'
67+
}, {
68+
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
69+
stacktrace: 'single_failure_spec2.js:5:32'
70+
}]);
71+
72+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/shardedFailureConf.js')
73+
.expectExitCode(1)
74+
.expectErrors([{
75+
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
76+
stacktrace: 'single_failure_spec1.js:5:32'
77+
}, {
78+
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
79+
stacktrace: 'single_failure_spec2.js:5:32'
80+
}]);
81+
82+
executor.addCommandlineTest('node lib/cli.js spec/errorTest/mochaFailureConf.js')
83+
.expectExitCode(1)
84+
.expectErrors([{
85+
message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
86+
stacktrace: 'mocha_failure_spec.js:11:20'
87+
}]);
88+
89+
executor.execute();

‎scripts/test/test_util.js

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
#!/usr/bin/env node
2+
3+
var child_process = require('child_process'),
4+
q = require('q'),
5+
fs = require('fs');
6+
7+
var CommandlineTest = function(command) {
8+
var self = this;
9+
this.command_ = command;
10+
this.expectedExitCode_ = 0;
11+
this.stdioOnlyOnFailures_ = true;
12+
this.expectedErrors_ = [];
13+
this.assertExitCodeOnly_ = false;
14+
15+
// If stdioOnlyOnFailures_ is true, do not stream stdio unless test failed.
16+
// This is to prevent tests with expected failures from polluting the output.
17+
this.alwaysEnableStdio = function() {
18+
self.stdioOnlyOnFailures_ = false;
19+
return self;
20+
};
21+
22+
// Only assert the exit code and not failures.
23+
// This must be true if the command you're running does not support
24+
// the flag '--resultJsonOutputFile'.
25+
this.assertExitCodeOnly = function() {
26+
self.assertExitCodeOnly_ = true;
27+
return self;
28+
};
29+
30+
// Set the expected exit code for the test command.
31+
this.expectExitCode = function(exitCode) {
32+
self.expectedExitCode_ = exitCode;
33+
return self;
34+
};
35+
36+
// Set the expected total test duration in milliseconds.
37+
this.expectTestDuration = function(min, max) {
38+
self.expectedMinTestDuration_ = min;
39+
self.expectedMaxTestDuration_ = max;
40+
return self;
41+
};
42+
43+
/**
44+
* Add expected error(s) for the test command.
45+
* Input is an object or list of objects of the following form:
46+
* {
47+
* message: string, // optional regex
48+
* stackTrace: string, //optional regex
49+
* }
50+
*/
51+
this.expectErrors = function(expectedErrors) {
52+
if (expectedErrors instanceof Array) {
53+
self.expectedErrors_ = self.expectedErrors_.concat(expectedErrors);
54+
} else {
55+
self.expectedErrors_.push(expectedErrors);
56+
}
57+
return self;
58+
};
59+
60+
this.run = function() {
61+
var start = new Date().getTime();
62+
var testOutputPath = 'test_output_' + start + '.tmp';
63+
var output = '';
64+
65+
var flushAndFail = function(errorMsg) {
66+
process.stdout.write(output);
67+
throw new Error(errorMsg);
68+
};
69+
70+
return q.promise(function(resolve, reject) {
71+
if (!self.assertExitCodeOnly_) {
72+
self.command_ = self.command_ + ' --resultJsonOutputFile ' + testOutputPath;
73+
}
74+
var args = self.command_.split(/\s/);
75+
76+
var test_process;
77+
78+
if (self.stdioOnlyOnFailures_) {
79+
test_process = child_process.spawn(args[0], args.slice(1));
80+
81+
test_process.stdout.on('data', function(data) {
82+
output += data;
83+
});
84+
85+
test_process.stderr.on('data', function(data) {
86+
output += data;
87+
});
88+
} else {
89+
test_process = child_process.spawn(args[0], args.slice(1), {stdio:'inherit'});
90+
}
91+
92+
test_process.on('error', function(err) {
93+
reject(err);
94+
});
95+
96+
test_process.on('exit', function(exitCode) {
97+
resolve(exitCode);
98+
});
99+
}).then(function(exitCode) {
100+
if (self.expectedExitCode_ !== exitCode) {
101+
flushAndFail('expecting exit code: ' + self.expectedExitCode_ +
102+
', actual: ' + exitCode);
103+
}
104+
105+
// Skip the rest if we are only verify exit code.
106+
// Note: we're expecting a file populated by '--resultJsonOutputFile' after
107+
// this point.
108+
if (self.assertExitCodeOnly_) {
109+
return;
110+
}
111+
112+
var raw_data = fs.readFileSync(testOutputPath);
113+
var testOutput = JSON.parse(raw_data);
114+
115+
var actualErrors = [];
116+
var duration = 0;
117+
testOutput.forEach(function(specResult) {
118+
duration += specResult.duration;
119+
specResult.assertions.forEach(function(assertion) {
120+
if (!assertion.passed) {
121+
actualErrors.push(assertion);
122+
}
123+
});
124+
});
125+
126+
self.expectedErrors_.forEach(function(expectedError) {
127+
var found = false;
128+
for (var i = 0; i < actualErrors.length; ++i) {
129+
var actualError = actualErrors[i];
130+
131+
// if expected message is defined and messages don't match
132+
if (expectedError.message) {
133+
if (!actualError.errorMsg ||
134+
!actualError.errorMsg.match(new RegExp(expectedError.message))) {
135+
continue;
136+
}
137+
}
138+
// if expected stackTrace is defined and stackTraces don't match
139+
if (expectedError.stackTrace) {
140+
if (!actualError.stackTrace ||
141+
!actualError.stackTrace.match(new RegExp(expectedError.stackTrace))) {
142+
continue;
143+
}
144+
}
145+
found = true;
146+
break;
147+
}
148+
149+
if (!found) {
150+
if (expectedError.message && expectedError.stackTrace) {
151+
flushAndFail('did not fail with expected error with message: [' +
152+
expectedError.message + '] and stackTrace: [' +
153+
expectedError.stackTrace + ']');
154+
} else if (expectedError.message) {
155+
flushAndFail('did not fail with expected error with message: [' +
156+
expectedError.message + ']');
157+
} else if (expectedError.stackTrace) {
158+
flushAndFail('did not fail with expected error with stackTrace: [' +
159+
expectedError.stackTrace + ']');
160+
}
161+
} else {
162+
actualErrors.splice(i, 1);
163+
}
164+
});
165+
166+
if (actualErrors.length > 0) {
167+
flushAndFail('failed with ' + actualErrors.length + ' unexpected failures');
168+
}
169+
170+
if (self.expectedMinTestDuration_
171+
&& duration < self.expectedMinTestDuration_) {
172+
flushAndFail('expecting test min duration: ' +
173+
self.expectedMinTestDuration_ + ', actual: ' + duration);
174+
}
175+
if (self.expectedMaxTestDuration_
176+
&& duration > self.expectedMaxTestDuration_) {
177+
flushAndFail('expecting test max duration: ' +
178+
self.expectedMaxTestDuration_ + ', actual: ' + duration);
179+
}
180+
}).fin(function() {
181+
try {
182+
fs.unlinkSync(testOutputPath);
183+
} catch (err) {
184+
// don't do anything
185+
}
186+
});
187+
};
188+
};
189+
190+
/**
191+
* Util for running tests and testing functionalities including:
192+
* exitCode, test durations, expected errors, and expected stackTrace
193+
* Note, this will work with any commandline tests, but only if it supports
194+
* the flag '--resultJsonOutputFile', unless only exitCode is being tested.
195+
* For now, this means protractor tests (jasmine/mocha/cucumber).
196+
*/
197+
exports.Executor = function() {
198+
var tests = [];
199+
this.addCommandlineTest = function(command) {
200+
var test = new CommandlineTest(command);
201+
tests.push(test);
202+
return test;
203+
};
204+
205+
this.execute = function() {
206+
var failed = false;
207+
208+
(function runTests(i) {
209+
if (i < tests.length) {
210+
console.log('running: ' + tests[i].command_);
211+
tests[i].run().then(function() {
212+
console.log('>>> \033[1;32mpass\033[0m');
213+
}, function(err) {
214+
failed = true;
215+
console.log('>>> \033[1;31mfail: ' + err.toString() + '\033[0m');
216+
}).fin(function() {
217+
runTests(i + 1);
218+
}).done();
219+
} else {
220+
console.log('Summary: ' + (failed ? 'fail' : 'pass'));
221+
process.exit(failed ? 1 : 0);
222+
}
223+
}(0));
224+
};
225+
};

‎spec/errorTest/afterLaunchChangesExitCodeConf.js

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
var env = require('../environment.js');
2+
3+
exports.config = {
4+
seleniumAddress: env.seleniumAddress,
5+
6+
specs: [
7+
'baseCase/single_failure_spec1.js'
8+
],
9+
10+
multiCapabilities: [{
11+
'browserName': 'chrome'
12+
}],
13+
14+
baseUrl: env.baseUrl,
15+
16+
jasmineNodeOpts: {
17+
isVerbose: true,
18+
showTiming: true,
19+
defaultTimeoutInterval: 90000
20+
},
21+
22+
afterLaunch: function(exitCode) {
23+
return exitCode + 10;
24+
},
25+
26+
};

‎spec/errorTest/baseCase/error_spec.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
describe('finding an element that does not exist', function() {
2+
it('should throw an error', function() {
3+
browser.get('index.html');
4+
var greeting = element(by.binding('INVALID'));
5+
});
6+
});

‎spec/errorTest/baseCase/mocha_failure_spec.js

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Use the external Chai As Promised to deal with resolving promises in
2+
// expectations.
3+
var chai = require('chai');
4+
var chaiAsPromised = require('chai-as-promised');
5+
chai.use(chaiAsPromised);
6+
var expect = chai.expect;
7+
8+
describe('protractor library', function() {
9+
it('should fail', function() {
10+
browser.get('index.html');
11+
expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
12+
});
13+
});

‎spec/errorTest/baseCase/single_failure_spec1.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
describe('single failure spec1', function() {
2+
it('should fail expectation', function() {
3+
browser.get('index.html');
4+
var greeting = element(by.binding('greeting'));
5+
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
6+
});
7+
});

‎spec/errorTest/baseCase/single_failure_spec2.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
describe('single failure spec2', function() {
2+
it('should fail expectation', function() {
3+
browser.get('index.html');
4+
var greeting = element(by.binding('greeting'));
5+
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
6+
});
7+
});

0 commit comments

Comments
 (0)