chore(test): error tests fixed

.travis.yml

@@ -16,17 +16,17 @@ env:
   matrix:
     - JOB=full
     - JOB=smoke
-    # - JOB=bstack
+    - JOB=bstack
 
 matrix:
   allow_failures:
     - env: "JOB=smoke"
-#     - env: "JOB=bstack"
+    - env: "JOB=bstack"
   exclude:
     - env: JOB=smoke
       node_js: "9"
-#     - env: JOB=bstack
-#       node_js: "8"
+    - env: JOB=bstack
+      node_js: "9"
 
 addons:
   apt:

lib/launcher.ts

@@ -221,7 +221,7 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
           }
           taskResults_.add(result);
           task.done();
-          createNextTaskRunner();
+          await createNextTaskRunner();
           // If all tasks are finished
           if (scheduler.numTasksOutstanding() === 0) {
             resolve();
@@ -232,6 +232,8 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
           logger.error('Error:', (err as any).stack || err.message || err);
           await cleanUpAndExit(errorCode ? errorCode : RUNNERS_FAILED_EXIT_CODE);
         }
+      } else {
+        resolve();
       }
     });
   };

scripts/test.js

@@ -38,15 +38,15 @@ const passingTests = [
   'node built/cli.js spec/controlLockConf.js',
   'node built/cli.js spec/customFramework.js',
   'node built/cli.js spec/noGlobalsConf.js',
-  // 'node built/cli.js spec/angular2Conf.js',
+  'node built/cli.js spec/angular2Conf.js',
   'node built/cli.js spec/hybridConf.js',
   'node built/cli.js spec/built/noCFBasicConf.js',
   'node built/cli.js spec/built/noCFBasicConf.js --useBlockingProxy',
   'node built/cli.js spec/built/noCFPluginConf.js',
   'node scripts/driverProviderAttachSession.js',
-  // 'node scripts/errorTest.js',
-  // // Unit tests
-  // 'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
+  'node scripts/errorTest.js',
+  // Unit tests
+  'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
   // Dependency tests
   'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/dependency_test.json',
   // Typings tests
@@ -65,88 +65,88 @@ passingTests.forEach((passing_test) => {
  *************************/
 
 // assert stacktrace shows line of failure
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
-//     .expectExitCode(1)
-//     .expectErrors({
-//       stackTrace: 'single_failure_spec1.js:5:32'
-//     });
-
-// // assert timeout works
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/timeoutConf.js')
-//     .expectExitCode(1)
-//     .expectErrors({
-//       message: 'Timeout - Async callback was not invoked within timeout ' +
-//           'specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.'
-//     })
-//     .expectTestDuration(0, 1000);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
-//     .expectExitCode(11)
-//     .expectErrors({
-//       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
-//     });
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([{
-//       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-//       stacktrace: 'single_failure_spec1.js:5:32'
-//     }, {
-//       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-//       stacktrace: 'single_failure_spec2.js:5:32'
-//     }]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([{
-//       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-//       stacktrace: 'single_failure_spec1.js:5:32'
-//     }, {
-//       message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
-//       stacktrace: 'single_failure_spec2.js:5:32'
-//     }]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([{
-//       message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
-//       stacktrace: 'mocha_failure_spec.js:11:20'
-//     }]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'Expected true to be false'},
-//       {message: 'from setup'},
-//       {message: 'from postTest passing'},
-//       {message: 'from postTest failing'},
-//       {message: 'from teardown'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
-//       {message: 'The following tasks were pending:[\\s\\S]*' +
-//                 '- \\$timeout: function\\(\\) {[\\s\\S]*' +
-//                   '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
-//                 '*}'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
-//                             '--untrackOutstandingTimeouts true')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
-//       {message: 'While waiting for element with locator - ' +
-//                 'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
-//     ]);
-
-// executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
-//     .expectExitCode(1)
-//     .expectErrors([
-//       {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
-//     ]);
+executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors({
+      stackTrace: 'single_failure_spec1.js:5:38'
+    });
+
+// assert timeout works
+executor.addCommandlineTest('node built/cli.js spec/errorTest/timeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors({
+      message: 'Timeout - Async callback was not invoked within timeout ' +
+          'specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.'
+    })
+    .expectTestDuration(0, 1000);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
+    .expectExitCode(11)
+    .expectErrors({
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
+    });
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec1.js:5:38'
+    }, {
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec2.js:5:38'
+    }]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec1.js:5:38'
+    }, {
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec2.js:5:38'
+    }]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
+      stacktrace: 'mocha_failure_spec.js:11:41'
+    }]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'Expected true to be false'},
+      {message: 'from setup'},
+      {message: 'from postTest passing'},
+      {message: 'from postTest failing'},
+      {message: 'from teardown'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
+      {message: 'The following tasks were pending:[\\s\\S]*' +
+                '- \\$timeout: function\\(\\) {[\\s\\S]*' +
+                  '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
+                '*}'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
+                            '--untrackOutstandingTimeouts true')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
+      {message: 'While waiting for element with locator - ' +
+                'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
+    ]);
+
+executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
+    ]);
 
 // If we're running on CircleCI, save stdout and stderr from the test run to a log file.
 if (process.env['CIRCLE_ARTIFACTS']) {

scripts/test_on_travis.sh

@@ -5,11 +5,11 @@ if [ $JOB = "smoke" ]; then
   node bin/protractor spec/ciSmokeConf.js
 elif [ $JOB = "full" ]; then
   node bin/protractor spec/ciFullConf.js
-  if [ $? = "0" ]; then
-    node bin/protractor spec/ciNg2Conf.js
-  else
-  	exit 1
-  fi
+  # if [ $? = "0" ]; then
+  #   node bin/protractor spec/ciNg2Conf.js
+  # else
+  # 	exit 1
+  # fi
 elif [ $JOB = "bstack" ]; then
   node bin/protractor spec/ciBStackConf.js
 else

spec/ciFullConf.js

@@ -9,18 +9,8 @@ exports.config = {
   framework: 'jasmine',
 
   // Spec patterns are relative to this directory.
-  // TODO(selenium4): revert back to basic/*_spec.js
   specs: [
-    'basic/lib_spec.js',
-    'basic/locators_spec.js'
-    // 'basic/elements_spec.js',
-    // 'basic/expected_conditions_spec.js',
-    // 'basic/handling_spec.js'
-    // 'basic/mockmodule_spec.js',
-    // 'basic/navigation_spec.js',
-    // 'basic/polling_spec.js',
-    // 'basic/restart_spec.js',
-    // 'basic/synchronize_spec.js',
+    'basic/*_spec.js',
   ],
 
   // Exclude patterns are relative to this directory.

spec/errorTest/afterLaunchChangesExitCodeConf.js

@@ -2,6 +2,7 @@ var env = require('../environment.js');
 
 exports.config = {
   seleniumAddress: env.seleniumAddress,
+  SELENIUM_PROMISE_MANAGER: false,
 
   framework: 'jasmine',
 

spec/errorTest/baseCase/error_spec.js

@@ -1,6 +1,6 @@
-describe('finding an element that does not exist', function() {
-  it('should throw an error', function() {
-    browser.get('index.html');
+describe('finding an element that does not exist', () => {
+  it('should throw an error', async () => {
+    await browser.get('index.html');
     element(by.binding('INVALID'));  // greeting
   });
 });

spec/errorTest/baseCase/mocha_failure_spec.js

@@ -1,13 +1,13 @@
 // Use the external Chai As Promised to deal with resolving promises in
 // expectations.
-var chai = require('chai');
-var chaiAsPromised = require('chai-as-promised');
+const chai = require('chai');
+const chaiAsPromised = require('chai-as-promised');
 chai.use(chaiAsPromised);
-var expect = chai.expect;
+const expect = chai.expect;
 
-describe('protractor library', function() {
-  it('should fail', function() {
-    browser.get('index.html');
-    expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
+describe('protractor library', () => {
+  it('should fail', async () => {
+    await browser.get('index.html');
+    expect(await browser.getTitle()).to.equal('INTENTIONALLY INCORRECT');
   });
 });

spec/errorTest/baseCase/single_failure_spec1.js

@@ -1,7 +1,7 @@
-describe('single failure spec1', function() {
-  it('should fail expectation', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
+describe('single failure spec1', () => {
+  it('should fail expectation', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
   });
 });

spec/errorTest/baseCase/single_failure_spec2.js

@@ -1,7 +1,7 @@
-describe('single failure spec2', function() {
-  it('should fail expectation', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
+describe('single failure spec2', () => {
+  it('should fail expectation', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
   });
 });

spec/errorTest/baseCase/slow_http_and_timeout_spec.js

@@ -1,27 +1,27 @@
-describe('slow asynchronous events', function() {
-  beforeEach(function() {
-    browser.get('index.html#/async');
+describe('slow asynchronous events', () => {
+  beforeEach(async () => {
+    await browser.get('index.html#/async');
   });
 
-  it('waits for http calls', function() {
-    var status = element(by.binding('slowHttpStatus'));
-    var button = element(by.css('[ng-click="slowHttp()"]'));
+  it('waits for http calls', async () => {
+    const status = element(by.binding('slowHttpStatus'));
+    const button = element(by.css('[ng-click="slowHttp()"]'));
 
-    expect(status.getText()).toEqual('not started');
+    expect(await status.getText()).toEqual('not started');
 
-    button.click();
+    await button.click();
 
-    expect(status.getText()).toEqual('done');
+    expect(await status.getText()).toEqual('done');
   });
 
-  it('waits for $timeout', function() {
-    var status = element(by.binding('slowAngularTimeoutStatus'));
-    var button = element(by.css('[ng-click="slowAngularTimeout()"]'));
+  it('waits for $timeout', async () => {
+    const status = element(by.binding('slowAngularTimeoutStatus'));
+    const button = element(by.css('[ng-click="slowAngularTimeout()"]'));
 
-    expect(status.getText()).toEqual('not started');
+    expect(await status.getText()).toEqual('not started');
 
-    button.click();
+    await button.click();
 
-    expect(status.getText()).toEqual('done');
+    expect(await status.getText()).toEqual('done');
   });
 });

spec/errorTest/baseCase/success_spec.js

@@ -1,7 +1,7 @@
-describe('success spec', function() {
-  it('should pass', function() {
-    browser.get('index.html');
-    var greeting = element(by.binding('greeting'));
-    expect(greeting.getText()).toEqual('Hiya');
+describe('success spec', () => {
+  it('should pass', async () => {
+    await browser.get('index.html');
+    const greeting = element(by.binding('greeting'));
+    expect(await greeting.getText()).toEqual('Hiya');
   });
 });

spec/errorTest/baseCase/timeout_spec.js

@@ -1,5 +1,5 @@
-describe('timeout spec', function() {
-  it('should timeout due to jasmine spec limit', function() {
-    browser.get('index.html#/form');
+describe('timeout spec', () => {
+  it('should timeout due to jasmine spec limit', async () => {
+    await browser.get('index.html#/form');
   }, 1);
 });