feat(plugins): allow plugins.postTest to know what test just ran

Also reorganized plugin tests, as the number of configs was getting out of hand

Author: sjelin

docs/plugins.md

@@ -124,11 +124,12 @@ exports.postResults = function(config) {};
  *
  * @param {Object} config The plugin configuration object.
  * @param {boolean} passed True if the test passed.
+ * @param {Object} testInfo information about the test which just ran.
  *
  * @return Object If an object is returned, it is merged with the Protractor
  *     result object. May return a promise.
  */
-exports.postTest = function(config, passed) {};
+exports.postTest = function(config, passed, testInfo) {};
 
 /**
  * Used when reporting results.

lib/frameworks/README.md

@@ -17,7 +17,13 @@ exports.run = function(runner, specs)
 Requirements
 ------------
 
- - `runner.emit` must be called with `testPass` and `testFail` messages.
+ - `runner.emit` must be called with `testPass` and `testFail` messages.  These
+   messages must be passed a `testInfo` object, with a `name` and `category`
+   property.  The `category` property could be the name of the `describe` block
+   in jasmine/mocha, the `Feature` in cucumber, or the class name in something
+   like jUnit.  The `name` property could be the name of an `it` block in
+   jasmine/mocha, the `Scenario` in cucumber, or the method name in something
+   like jUnit.
 
  - `runner.runTestPreparer` must be called before any tests are run.
 

lib/frameworks/cucumber.js

@@ -70,15 +70,28 @@ exports.run = function(runner, specs) {
   // Add a listener into cucumber so that protractor can find out which
   // steps passed/failed
   var addResultListener = function(formatter) {
+    var feature = { getName: function() { return ''; } };
+    var originalHandleBeforeFeatureEvent = formatter.handleBeforeFeatureEvent;
+    formatter.handleBeforeFeatureEvent = function(event, callback) {
+      feature = event.getPayloadItem('feature');
+      if (typeof originalHandleAfterScenarioEvent == 'function') {
+        originalHandleBeforeFeatureEvent.apply(formatter, arguments);
+      } else {
+        callback();
+      }
+    }
     var originalHandleAfterScenarioEvent = formatter.handleAfterScenarioEvent;
     formatter.handleAfterScenarioEvent = function(event, callback) {
-      var scenario = event.getPayloadItem('scenario');
-      stepResults.description = scenario.getName();
+      var scenarioInfo = {
+        name: event.getPayloadItem('scenario').getName(),
+        category: feature.getName()
+      };
+      stepResults.description = scenarioInfo.name;
       if (scenarioFailed) {
         ++failedCount;
-        runner.emit('testFail');
+        runner.emit('testFail', scenarioInfo);
       } else {
-        runner.emit('testPass');
+        runner.emit('testPass', scenarioInfo);
       }
 
       testResult.push(stepResults);

lib/frameworks/jasmine.js

@@ -30,10 +30,14 @@ exports.run = function(runner, specs) {
     this.startTime = new Date();
   };
   RunnerReporter.prototype.reportSpecResults = function(spec) {
+    var specInfo = {
+      name: spec.suite.getFullName(),
+      category: spec.description
+    };
     if (spec.results().passedCount) {
-      this.emitter.emit('testPass');
+      this.emitter.emit('testPass', specInfo);
     } else if (spec.results().failedCount) {
-      this.emitter.emit('testFail');
+      this.emitter.emit('testFail', specInfo);
     }
 
     var entry = {

lib/frameworks/jasmine2.js

@@ -17,10 +17,14 @@ RunnerReporter.prototype.specStarted = function() {
 };
 
 RunnerReporter.prototype.specDone = function(result) {
+  var specInfo = {
+    name: result.description,
+    category: result.fullName.slice(0, -result.description.length).trim()
+  };
   if (result.status == 'passed') {
-    this.emitter.emit('testPass');
+    this.emitter.emit('testPass', specInfo);
   } else if (result.status == 'failed') {
-    this.emitter.emit('testFail');
+    this.emitter.emit('testFail', specInfo);
     this.failedCount++;
   }
 

lib/frameworks/mocha.js

@@ -65,7 +65,11 @@ exports.run = function(runner, specs) {
     });
 
     mochaRunner.on('pass', function(test) {
-      runner.emit('testPass');
+      var testInfo = {
+        name: test.title,
+        category: test.fullTitle().slice(0, -test.title.length).trim()
+      };
+      runner.emit('testPass', testInfo);
       testResult.push({
         description: test.title,
         assertions: [{
@@ -76,7 +80,11 @@ exports.run = function(runner, specs) {
     });
 
     mochaRunner.on('fail', function(test) {
-      runner.emit('testFail');
+      var testInfo = {
+        name: test.title,
+        category: test.fullTitle().slice(0, -test.title.length).trim()
+      };
+      runner.emit('testFail', testInfo);
       testResult.push({
         description: test.title,
         assertions: [{

lib/runner.js

@@ -316,11 +316,11 @@ Runner.prototype.run = function() {
     // an event emitter).
     var pluginPostTestPromises = [];
 
-    self.on('testPass', function() {
-      pluginPostTestPromises.push(plugins.postTest(true));
+    self.on('testPass', function(testInfo) {
+      pluginPostTestPromises.push(plugins.postTest(true, testInfo));
     });
-    self.on('testFail', function() {
-      pluginPostTestPromises.push(plugins.postTest(false));
+    self.on('testFail', function(testInfo) {
+      pluginPostTestPromises.push(plugins.postTest(false, testInfo));
     });
 
     return require(frameworkPath).run(self, self.config_.specs).

scripts/test.js

@@ -19,8 +19,12 @@ var passingTests = [
   'node lib/cli.js spec/suitesConf.js --suite okmany',
   'node lib/cli.js spec/suitesConf.js --suite okspec',
   'node lib/cli.js spec/suitesConf.js --suite okmany,okspec',
-  'node lib/cli.js spec/pluginsBasicConf.js',
-  'node lib/cli.js spec/pluginsFullConf.js',
+  'node lib/cli.js spec/plugins/smokeConf.js',
+  'node lib/cli.js spec/plugins/multiPluginConf.js',
+  'node lib/cli.js spec/plugins/jasmine1PostTestConf.js',
+  'node lib/cli.js spec/plugins/jasmine2PostTestConf.js',
+  'node lib/cli.js spec/plugins/mochaPostTestConf.js',
+  'node lib/cli.js spec/plugins/cucumberPostTestConf.js',
   'node lib/cli.js spec/interactionConf.js',
   'node lib/cli.js spec/directConnectConf.js',
   'node lib/cli.js spec/restartBrowserBetweenTestsConf.js',

spec/errorTest/pluginsFailingConf.js

@@ -9,7 +9,7 @@ exports.config = {
 
   // Spec patterns are relative to this directory.
   specs: [
-    '../plugins/fail_spec.js'
+    '../plugins/specs/fail_spec.js'
   ],
 
   capabilities: env.capabilities,
@@ -23,8 +23,8 @@ exports.config = {
 
   // Plugin patterns are relative to this directory.
   plugins: [{
-    path: '../plugins/basic_plugin.js'
+    path: '../plugins/plugins/basic_plugin.js'
   }, {
-    path: '../plugins/failing_plugin.js'
+    path: '../plugins/plugins/failing_plugin.js'
   }]
 };

spec/plugins/cucumberPostTestConf.js

@@ -0,0 +1 @@
+exports.config = require('./postTestConfTemplate')('cucumber');

spec/plugins/features/simple.feature

@@ -0,0 +1,3 @@
+Feature: category
+  This is spec does nothing
+  Scenario: name

spec/plugins/jasmine1PostTestConf.js

@@ -0,0 +1 @@
+exports.config = require('./postTestConfTemplate')('jasmine');

spec/plugins/jasmine2PostTestConf.js

@@ -0,0 +1 @@
+exports.config = require('./postTestConfTemplate')('jasmine2');

spec/plugins/mochaPostTestConf.js

@@ -0,0 +1 @@
+exports.config = require('./postTestConfTemplate')('mocha');

spec/pluginsFullConf.js renamed to spec/plugins/multiPluginConf.js

@@ -1,4 +1,4 @@
-var env = require('./environment.js');
+var env = require('../environment.js');
 
 // A small suite to make sure the full functionality of plugins work
 exports.config = {
@@ -8,7 +8,7 @@ exports.config = {
 
   // Spec patterns are relative to this directory.
   specs: [
-    'plugins/bigger_spec.js'
+    'specs/bigger_spec.js'
   ],
 
   capabilities: env.capabilities,
@@ -24,7 +24,7 @@ exports.config = {
   plugins: [{
     path: 'plugins/basic_plugin.js'
   }, {
-    path: 'plugins/test_plugin.js'
+    path: 'plugins/async_plugin.js'
   }, {
     inline: {
       setup: function() {

spec/plugins/test_plugin.js renamed to spec/plugins/plugins/async_plugin.js

@@ -0,0 +1,25 @@
+exports.postTest = function(config, passed, testInfo) {
+  var nameCorrect = testInfo.name == 'name';
+  var categoryCorrect = testInfo.category == 'category';
+  return {
+    failedCount: passed && nameCorrect && categoryCorrect ? 0 : 1,
+    specResults: [{
+      description: 'make sure postTest passed correct information',
+      assertions: [{
+        passed: passed,
+        errorMsg: passed ? undefined : '`passed` should have been `true`, but' +
+            ' got `' + JSON.stringify(passed) + '`'
+      }, {
+        passed: nameCorrect,
+        errorMsg: nameCorrect ? undefined : '`testInfo.name` should have been' +
+            ' `"name"`, but got `' + JSON.stringify(testInfo.name) + '`'
+      }, {
+        passed: categoryCorrect,
+        errorMsg: categoryCorrect ? undefined : '`testInfo.category` should ' +
+            'have been `"category"`, but got `' +
+            JSON.stringify(testInfo.category) + '`'
+      }],
+      duration: 0
+    }]
+  };
+}

spec/plugins/basic_plugin.js renamed to spec/plugins/plugins/basic_plugin.js

@@ -0,0 +1,22 @@
+var env = require('../environment.js');
+
+module.exports = function(framework) {
+  return {
+    mockSelenium: true,
+
+    framework: framework,
+
+    specs: [
+      framework != 'cucumber' ? 'specs/simple_spec.js' :
+          'features/simple.feature'
+    ],
+
+    capabilities: env.capabilities,
+
+    baseUrl: env.baseUrl,
+
+    plugins: [{
+      path: 'plugins/post_test_plugin.js'
+    }]
+  };
+};

spec/plugins/failing_plugin.js renamed to spec/plugins/plugins/failing_plugin.js

@@ -1,4 +1,4 @@
-var env = require('./environment.js');
+var env = require('../environment.js');
 
 // A small suite to make sure the basic functionality of plugins work
 // Tests the (potential) edge case of exactly one plugin being used
@@ -9,7 +9,7 @@ exports.config = {
 
   // Spec patterns are relative to this directory.
   specs: [
-    'plugins/basic_spec.js'
+    'specs/smoke_spec.js'
   ],
 
   capabilities: env.capabilities,

spec/plugins/plugins/post_test_plugin.js

@@ -0,0 +1,4 @@
+describe('category', function() {
+  it('name', function() {
+  });
+});