Skip to content

Commit

Permalink
benchmark runner
Browse files Browse the repository at this point in the history
  • Loading branch information
subtleGradient committed Jan 6, 2014
1 parent 605b42e commit 9182100
Show file tree
Hide file tree
Showing 32 changed files with 1,568 additions and 182 deletions.
2 changes: 2 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ env:
matrix:
- TEST_TYPE=test:full
- TEST_TYPE=lint
- TEST_TYPE=perf:full
- TEST_TYPE=test:coverage
- TEST_TYPE=test:webdriver:saucelabs BROWSER_NAME=ie11
- TEST_TYPE=test:webdriver:saucelabs BROWSER_NAME=ie10
Expand All @@ -43,6 +44,7 @@ matrix:
allow_failures:
- env: TEST_TYPE=lint
- env: TEST_TYPE=test:coverage
- env: TEST_TYPE=perf:full
- env: TEST_TYPE=test:webdriver:saucelabs BROWSER_NAME=ie11
- env: TEST_TYPE=test:webdriver:saucelabs BROWSER_NAME=ie10
- env: TEST_TYPE=test:webdriver:saucelabs BROWSER_NAME=ie9
Expand Down
36 changes: 35 additions & 1 deletion Gruntfile.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ module.exports = function(grunt) {
browserify: require('./grunt/config/browserify'),
populist: require('./grunt/config/populist'),
connect: require('./grunt/config/server')(grunt),
"webdriver-jasmine": require('./grunt/config/webdriver-jasmine.js'),
"webdriver-jasmine": require('./grunt/config/webdriver-jasmine'),
"webdriver-perf": require('./grunt/config/webdriver-perf'),
npm: require('./grunt/config/npm'),
clean: ['./build', './*.gem', './docs/_site', './examples/shared/*.js', '.module-cache'],
jshint: require('./grunt/config/jshint'),
Expand All @@ -39,6 +40,8 @@ module.exports = function(grunt) {
// Alias 'jshint' to 'lint' to better match the workflow we know
grunt.registerTask('lint', ['jshint']);

grunt.registerTask('download-previous-version', require('./grunt/tasks/download-previous-version.js'));

// Register jsx:debug and :release tasks.
grunt.registerMultiTask('jsx', jsxTask);

Expand All @@ -51,6 +54,8 @@ module.exports = function(grunt) {

grunt.registerMultiTask('webdriver-jasmine', webdriverJasmineTasks);

grunt.registerMultiTask('webdriver-perf', require('./grunt/tasks/webdriver-perf'));

grunt.registerMultiTask('npm', npmTask);

grunt.registerTask('npm-react:release', npmReactTasks.buildRelease);
Expand All @@ -67,6 +72,14 @@ module.exports = function(grunt) {
'version-check',
'browserify:withCodeCoverageLogging'
]);
grunt.registerTask('build:perf', [
'jsx:release',
'version-check',
'browserify:transformer',
'browserify:basic',
'browserify:min',
'download-previous-version'
]);
grunt.registerTask('build:test', [
'jsx:test',
'version-check',
Expand All @@ -84,6 +97,12 @@ module.exports = function(grunt) {
'webdriver-jasmine:local'
]);

grunt.registerTask('perf:webdriver:phantomjs', [
'connect',
'webdriver-phantomjs',
'webdriver-perf:local'
]);

grunt.registerTask('test:full', [
'build:test',
'build:basic',
Expand All @@ -98,6 +117,20 @@ module.exports = function(grunt) {
'webdriver-jasmine:saucelabs_chrome'
]);

grunt.registerTask('perf:full', [
'build:perf',

'connect',
'webdriver-phantomjs',
'webdriver-perf:local',

'sauce-tunnel',
'webdriver-perf:saucelabs_firefox',
'webdriver-perf:saucelabs_chrome',
'webdriver-perf:saucelabs_ie11',
'webdriver-perf:saucelabs_ie8',
]);

grunt.registerTask('test:webdriver:saucelabs', [
'build:test',
'build:basic',
Expand Down Expand Up @@ -137,6 +170,7 @@ module.exports = function(grunt) {
'coverage:parse'
]);
grunt.registerTask('test', ['build:test', 'build:basic', 'test:webdriver:phantomjs']);
grunt.registerTask('perf', ['build:perf', 'perf:webdriver:phantomjs']);
grunt.registerTask('npm:test', ['build', 'npm:pack']);

// Optimized build task that does all of our builds. The subtasks will be run
Expand Down
2 changes: 2 additions & 0 deletions grunt/config/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ module.exports = function(grunt){
coverageWriteStream.write(log.message + '\n');
} else if (log.type == 'coverage done') {
grunt.task.run('finalize-coverage-stream');
} else if (log.type == 'perf') {
grunt.event.emit('perf results', log.message);
} else {
grunt.verbose.writeln(log);
}
Expand Down
113 changes: 113 additions & 0 deletions grunt/config/webdriver-all.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
'use strict';

var grunt = require('grunt');

module.exports = function(props){
if (typeof props.url != 'string') {
throw TypeError('expected url string');
}
if ('isDoneTimeout' in props && typeof props.isDoneTimeout != 'number') {
throw TypeError('expected isDoneTimeout to be a number');
}
if ('onStart' in props && typeof props.onStart != 'function') {
throw TypeError('expected onStart to be a function');
}
if ('onComplete' in props && typeof props.onComplete != 'function') {
throw TypeError('expected onComplete to be a function');
}
if ('onError' in props && typeof props.onError != 'function') {
throw TypeError('expected onError to be a function');
}

var exports = {};

exports.local = {
webdriver: {
remote: { protocol: 'http:', hostname: '127.0.0.1', port: 9515, path: '/' }
},
url: props.url,
onStart: props.onStart,
onComplete: props.onComplete,
onError: props.onError,
isDoneTimeout: props.isDoneTimeout
};

if (grunt.option('debug')) {
exports.local.url += (exports.local.url.indexOf('?') == -1 ? '?' : '&') + 'debug=' + grunt.option('debug');
}

exports.saucelabs = {
webdriver: {
remote: {
/* https://github.com/admc/wd/blob/master/README.md#named-parameters */
user: process.env.SAUCE_USERNAME || 'React',
pwd: process.env.SAUCE_ACCESS_KEY || '339d32ca-d594-4570-a3c2-94c50a91919b',
protocol: 'http:',
hostname: 'ondemand.saucelabs.com',
port: '80',
path: '/wd/hub'
}
},
desiredCapabilities: {
"build": process.env.TRAVIS_BUILD_NUMBER || 'dev' + Date.now(),
"tunnel-identifier": process.env.TRAVIS_JOB_NUMBER || 'my awesome tunnel',
"browserName": "chrome"
},
url: exports.local.url,
onStart: function(browser){
grunt.log.writeln("Starting WebDriver Test. Watch results here: http://saucelabs.com/tests/" + browser.sessionID);
if (props.onStart) {
return props.onStart(browser);
}
},
onComplete: exports.local.onComplete,
onError: exports.local.onError,
isDoneTimeout: exports.local.isDoneTimeout
};

/* https://saucelabs.com/platforms */
exports.saucelabs_ios =
exports.saucelabs_ios7 = sauceItUp({ browserName: 'iphone', version: '7', platform:'OS X 10.9' });
exports.saucelabs_ios6_1 = sauceItUp({ browserName: 'iphone', version: '6.1', platform:'OS X 10.8' });
exports.saucelabs_ios6 = sauceItUp({ browserName: 'iphone', version: '6', platform:'OS X 10.8' });
exports.saucelabs_ios5_1 = sauceItUp({ browserName: 'iphone', version: '5.1', platform:'OS X 10.8' });
exports.saucelabs_ios5 = sauceItUp({ browserName: 'iphone', version: '5', platform:'OS X 10.6' });
exports.saucelabs_ios4 = sauceItUp({ browserName: 'iphone', version: '4', platform:'OS X 10.6' });

exports.saucelabs_ipad =
exports.saucelabs_ipad7 = sauceItUp({ browserName: 'ipad', version: '7', platform:'OS X 10.9' });
exports.saucelabs_ipad6_1 = sauceItUp({ browserName: 'ipad', version: '6.1', platform:'OS X 10.8' });
exports.saucelabs_ipad6 = sauceItUp({ browserName: 'ipad', version: '6', platform:'OS X 10.8' });
exports.saucelabs_ipad5_1 = sauceItUp({ browserName: 'ipad', version: '5.1', platform:'OS X 10.8' });
exports.saucelabs_ipad5 = sauceItUp({ browserName: 'ipad', version: '5', platform:'OS X 10.6' });
exports.saucelabs_ipad4 = sauceItUp({ browserName: 'ipad', version: '4', platform:'OS X 10.6' });

exports.saucelabs_android = sauceItUp({ browserName: 'android', version: '4.0', platform:'Linux' });
exports.saucelabs_android_tablet = sauceItUp({ browserName: 'android', version: '4.0', platform:'Linux', 'device-type':'tablet' });

exports.saucelabs_safari = sauceItUp({ browserName: 'safari' });
exports.saucelabs_chrome = sauceItUp({ browserName: 'chrome' });
exports.saucelabs_firefox = sauceItUp({ browserName: 'firefox' });

exports.saucelabs_ie =
exports.saucelabs_ie8 = sauceItUp({ browserName: 'internet explorer', version: 8 });
exports.saucelabs_ie9 = sauceItUp({ browserName: 'internet explorer', version: 9 });
exports.saucelabs_ie10 = sauceItUp({ browserName: 'internet explorer', version: 10 });
exports.saucelabs_ie11 = sauceItUp({ browserName: 'internet explorer', version: 11, platform:'Windows 8.1' });

function sauceItUp(desiredCapabilities) {
desiredCapabilities["build"] = exports.saucelabs.desiredCapabilities["build"];
desiredCapabilities["tunnel-identifier"] = exports.saucelabs.desiredCapabilities["tunnel-identifier"];
return {
webdriver: exports.saucelabs.webdriver,
url: exports.saucelabs.url,
onStart: exports.saucelabs.onStart,
onComplete: exports.saucelabs.onComplete,
onError: exports.saucelabs.onError,
isDoneTimeout: exports.saucelabs.isDoneTimeout,
desiredCapabilities: desiredCapabilities,
};
}

return exports;
}
80 changes: 2 additions & 78 deletions grunt/config/webdriver-jasmine.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,7 @@

var grunt = require('grunt');


exports.local = {
webdriver: {
remote: { protocol: 'http:', hostname: '127.0.0.1', port: 9515, path: '/' }
},
module.exports = require('./webdriver-all')({
url: "http://127.0.0.1:9999/test/index.html",
onComplete: function(passed){
if (!passed){
Expand All @@ -16,76 +12,4 @@ exports.local = {
onError: function(error){
grunt.fatal(error);
}
};

if (grunt.option('debug')) {
exports.local.url += '?debug=' + grunt.option('debug');
}


exports.saucelabs = {
webdriver: {
remote: {
/* https://github.com/admc/wd/blob/master/README.md#named-parameters */
user: process.env.SAUCE_USERNAME || 'React',
pwd: process.env.SAUCE_ACCESS_KEY || '339d32ca-d594-4570-a3c2-94c50a91919b',
protocol: 'http:',
hostname: 'ondemand.saucelabs.com',
port: '80',
path: '/wd/hub'
}
},
desiredCapabilities: {
"build": process.env.TRAVIS_BUILD_NUMBER || 'dev' + Date.now(),
"tunnel-identifier": process.env.TRAVIS_JOB_NUMBER || 'my awesome tunnel',
"browserName": "chrome"
},
url: exports.local.url,
onStart: function(browser){
grunt.log.writeln("Starting WebDriver Test. Watch results here: http://saucelabs.com/tests/" + browser.sessionID);
},
onComplete: exports.local.onComplete,
onError: exports.local.onError
};

/* https://saucelabs.com/docs/platforms */
exports.saucelabs_ios =
exports.saucelabs_ios6_1 = sauceItUp({ browserName: 'iphone', version: '6.1', platform:'OS X 10.8' });
exports.saucelabs_ios6 = sauceItUp({ browserName: 'iphone', version: '6', platform:'OS X 10.8' });
exports.saucelabs_ios5_1 = sauceItUp({ browserName: 'iphone', version: '5.1', platform:'OS X 10.8' });
exports.saucelabs_ios5 = sauceItUp({ browserName: 'iphone', version: '5', platform:'OS X 10.6' });
exports.saucelabs_ios4 = sauceItUp({ browserName: 'iphone', version: '4', platform:'OS X 10.6' });

exports.saucelabs_ipad =
exports.saucelabs_ipad6_1 = sauceItUp({ browserName: 'ipad', version: '6.1', platform:'OS X 10.8' });
exports.saucelabs_ipad6 = sauceItUp({ browserName: 'ipad', version: '6', platform:'OS X 10.8' });
exports.saucelabs_ipad5_1 = sauceItUp({ browserName: 'ipad', version: '5.1', platform:'OS X 10.8' });
exports.saucelabs_ipad5 = sauceItUp({ browserName: 'ipad', version: '5', platform:'OS X 10.6' });
exports.saucelabs_ipad4 = sauceItUp({ browserName: 'ipad', version: '4', platform:'OS X 10.6' });

exports.saucelabs_android = sauceItUp({ browserName: 'android', version: '4.0', platform:'Linux' });
exports.saucelabs_android_tablet = sauceItUp({ browserName: 'android', version: '4.0', platform:'Linux', 'device-type':'tablet' });

exports.saucelabs_safari = sauceItUp({ browserName: 'safari' });
exports.saucelabs_chrome = sauceItUp({ browserName: 'chrome' });
exports.saucelabs_firefox = sauceItUp({ browserName: 'firefox' });

exports.saucelabs_ie =
exports.saucelabs_ie8 = sauceItUp({ browserName: 'internet explorer', version: 8 });
exports.saucelabs_ie9 = sauceItUp({ browserName: 'internet explorer', version: 9 });
exports.saucelabs_ie10 = sauceItUp({ browserName: 'internet explorer', version: 10 });
exports.saucelabs_ie11 = sauceItUp({ browserName: 'internet explorer', version: 11, platform:'Windows 8.1' });


function sauceItUp(desiredCapabilities) {
desiredCapabilities["build"] = exports.saucelabs.desiredCapabilities["build"];
desiredCapabilities["tunnel-identifier"] = exports.saucelabs.desiredCapabilities["tunnel-identifier"];
return {
webdriver: exports.saucelabs.webdriver,
url: exports.saucelabs.url,
onStart: exports.saucelabs.onStart,
onComplete: exports.saucelabs.onComplete,
onError: exports.saucelabs.onError,
desiredCapabilities: desiredCapabilities,
};
}
});
48 changes: 48 additions & 0 deletions grunt/config/webdriver-perf.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
'use strict';

var grunt = require('grunt');

var tests = grunt.file.expand(__dirname + '/../../perf/tests/*');

var maxTime = 5;

var reactVersions = [
'edge',
'previous'
];

var params = []
.concat('headless=false')
.concat('maxTime=' + maxTime)
.concat(tests
.map(function(path){ return path.split(/tests./i).reverse()[0]; })
.map(encodeURIComponent)
.map(function(filename){ return 'test=' + filename; })
)
.concat(reactVersions
.map(encodeURIComponent)
.map(function(version){ return 'react=' + version }
)
);

module.exports = require('./webdriver-all')({

url: "http://127.0.0.1:9999/perf/index.html?" + params.join('&'),

isDoneTimeout: 15 * 60 * 1000,

onStart: function(){
grunt.event.on('perf results', function(results){
console.log(results);
});
},

onComplete: function(completedTestKeys){
grunt.verbose.writeln('onComplete ' + JSON.stringify(completedTestKeys));
},

onError: function(error){
grunt.fatal(error);
}

});
Loading

0 comments on commit 9182100

Please sign in to comment.