Protractor synchronization fails if multiple versions of selenium-webdriver are in node_modules

Bug report

Setup

• Node Version: 6.3.1
• npm Version: 3.10.3
• Protractor Version: 4.0.4 & 4.0.3
• Browser(s): Chrome (52.0.2743.116 (Official Build) (64-bit))
• Operating System and Version Ubuntu 16.04
• Protractor configuration file

var SpecReporter = require('jasmine-spec-reporter');

exports.config = {
    allScriptsTimeout: 11000,
    specs: [
        '../e2e/**/*.e2e-spec.ts'
    ],
    capabilities: {
        'browserName': 'chrome'
    },
    directConnect: true,
    baseUrl: 'http://localhost:4200/',
    framework: 'jasmine',
    jasmineNodeOpts: {
        showColors: true,
        defaultTimeoutInterval: 30000,
        print: function() {}
    },
    useAllAngular2AppRoots: true,
    beforeLaunch: function() {
        require('ts-node').register({
            project: 'e2e'
        });
    },
    onPrepare: function() {
        jasmine.getEnv().addReporter(new SpecReporter());
    }
};

Bug description

Whenever I run my e2e tests, my browser opens and immediately closes. Protractor will then claim, that all tests have passed, which just can't be. I implemented a test like this to test this:

    it('should fail', () => {
        let ec = protractor.ExpectedConditions;
        let e = element(by.css('some-html-tag-which-definetly-doesnt-exist-in-my-app'));

        browser.wait(ec.presenceOf(e));
    });

Although this test should typically timeout, it will pass just fine in a split second.

If I add a spec like expect(true).toBe(false), the test will fail by the way.

I also had a beforeEach set up, with browser.ignoreSynchronization = true. After removing this line, all tests will pass like before, but protractor will fail anyway with this error:

/home/chris/workspace/mem-webclient/node_modules/protractor/built/exitCodes.js:87
if (e.message.indexOf(errMsg) !== -1) {                                                                                                         
              ^                                                                                                                                  

TypeError: Cannot read property 'indexOf' of undefined
    at Function.ErrorHandler.isError (/home/chris/workspace/mem-webclient/node_modules/protractor/built/exitCodes.js:87:30)
    at Function.ErrorHandler.parseError (/home/chris/workspace/mem-webclient/node_modules/protractor/built/exitCodes.js:98:26)
    at process.<anonymous> (/home/chris/workspace/mem-webclient/node_modules/protractor/built/launcher.js:169:54)
    at emitOne (events.js:96:13)
    at process.emit (events.js:188:7)
    at process.emit (/home/chris/workspace/mem-webclient/node_modules/ts-node/node_modules/source-map-support/source-map-support.js:419:21)
    at process.emit (/home/chris/workspace/mem-webclient/node_modules/protractor/node_modules/source-map-support/source-map-support.js:419:21)
    at process._fatalException (bootstrap_node.js:254:26)

/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:87
if (e.message.indexOf(errMsg) !== -1) {                                                                                                         
              ^                                                                                                                                  

TypeError: Cannot read property 'indexOf' of undefined
    at Function.ErrorHandler.isError (/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:87:30)
    at Function.ErrorHandler.parseError (/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:98:26)
    at process.<anonymous> (/home/chris/workspace/my-webclient/node_modules/protractor/built/launcher.js:169:54)
    at emitOne (events.js:96:13)
    at process.emit (events.js:188:7)
    at process.emit (/home/chris/workspace/my-webclient/node_modules/ts-node/node_modules/source-map-support/source-map-support.js:419:21)
    at process.emit (/home/chris/workspace/my-webclient/node_modules/protractor/node_modules/source-map-support/source-map-support.js:419:21)
    at process._fatalException (bootstrap_node.js:254:26)

(The error message above this sentence is copied from below referenced issue, since I currently can't access my own logs. I will insert my actual error message above this one as soon as I have time)

This also seems to be highly relevant, since it describes the very same behaviour: mgechev/angular-seed#1185
Sadly, this issue was closed without further investigation.

Since this issue seems to be quite rare, I assume that it is caused by some faulty dependency or so. Therefore I thought that maybe also my package.json file could help:

"dependencies": {
    "@angular/common": "2.0.0-rc.5",
    "@angular/compiler": "2.0.0-rc.5",
    "@angular/core": "2.0.0-rc.5",
    "@angular/forms": "0.3.0",
    "@angular/http": "2.0.0-rc.5",
    "@angular/platform-browser": "2.0.0-rc.5",
    "@angular/platform-browser-dynamic": "2.0.0-rc.5",
    "@angular/router": "3.0.0-rc.1",
    "@angular2-material/button": "2.0.0-alpha.7-4",
    "@angular2-material/card": "2.0.0-alpha.7-4",
    "@angular2-material/core": "2.0.0-alpha.7-4",
    "@angular2-material/grid-list": "2.0.0-alpha.7-4",
    "@angular2-material/list": "2.0.0-alpha.7-4",
    "@angular2-material/menu": "2.0.0-alpha.7-4",
    "@angular2-material/toolbar": "2.0.0-alpha.7-4",
    "angular2-moment": "1.0.0-beta.1",
    "core-js": "2.4.1",
    "moment": "2.14.1",
    "rxjs": "5.0.0-beta.11",
    "ts-helpers": "1.1.1",
    "zone.js": "0.6.12"
},
"devDependencies": {
    "@types/hammerjs": "2.0.32",
    "@types/jasmine": "2.2.33",
    "@types/protractor": "1.5.18",
    "angular-cli": "1.0.0-beta.11-webpack.8",
    "concurrently": "2.2.0",
    "codelyzer": "0.0.28",
    "jasmine-core": "2.5.0",
    "jasmine-spec-reporter": "2.7.0",
    "karma": "1.2.0",
    "karma-chrome-launcher": "2.0.0",
    "karma-jasmine": "1.0.2",
    "karma-remap-istanbul": "0.2.1",
    "protractor": "4.0.4",
    "ts-node": "1.3.0",
    "tslint": "3.15.1",
    "typescript": "2.0.0"
}

Reproduction

See #3505 (comment)

If you need more information, please let me know. I will try to get you everything you need to reproduce and hopefully eventually fix this

[cnishina]

• When you turn ignoreSynchronization to false true (updated: originally listed as false), ExpectedConditions will not work.
• Having an unhelpful stacktrace because the error.message does not exist appears to not be correct. This should be investigated.

When you turn ignoreSynchronization to false, ExpectedConditions will not work.

It did work once with ignoreSynchronization = true. I think that false is the default. This did not work due to strange timeouts I could not fix in 3 days of work. I opened a StackOverflow question for this, but did not get any response so far.

Having an unhelpful stacktrace because the error.message does not exist appears to not be correct. This should be investigated.

While I agree with that statement, this is not the core problem. Like I wrote in the bug report, protractor will just pass all tests. By the way, this will also happen if I don't even have the application running. I can literally see that a browser window opens with the chrome error page - still the tests pass.

Please let me know what I can do for you to help your investigation

As promised, here is the actual error message I get

/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:87
if (e.message.indexOf(errMsg) !== -1) {                                                                                                         
              ^                                                                                                                                  

TypeError: Cannot read property 'indexOf' of undefined
    at Function.ErrorHandler.isError (/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:87:30)
    at Function.ErrorHandler.parseError (/home/chris/workspace/my-webclient/node_modules/protractor/built/exitCodes.js:98:26)
    at process.<anonymous> (/home/chris/workspace/my-webclient/node_modules/protractor/built/launcher.js:169:54)
    at emitOne (events.js:96:13)
    at process.emit (events.js:188:7)
    at process.emit (/home/chris/workspace/my-webclient/node_modules/ts-node/node_modules/source-map-support/source-map-support.js:419:21)
    at process.emit (/home/chris/workspace/my-webclient/node_modules/protractor/node_modules/source-map-support/source-map-support.js:419:21)
    at process._fatalException (bootstrap_node.js:254:26)

But I think that this is a rather small issue compared to blindly passing tests

[heathkit]

Would it be possible for you to post a an example repository with the issue? I started with mgechev/angular2-seed and set it up to use ts-node in the protractor config the way you have it, but I'm still not able to reproduce the problem.

I will try. It's late now here, I will do so as soon as i wake up. By the way, I am not even using angular-seed. I am running on angular-cli

I found a quick way for you to reproduce this bug.

# terminal 1
# clone angular2-seed repo
$ git clone https://github.com/mgechev/angular2-seed.git
$ cd angular2-seed
# checkout the issue mentioned in mgechev/angular2-seed#1185 to be faulty
$ git checkout 9a8bf12eef0f28d310ecc0e568f6c5235422749e
# sidenote: At this point I opened package.json and changed 
# "dependencies.zone.js": "^0.6.12"
# to
# "dependencies.zone.js": "0.6.12"
# because 0.6.17 is buggy. Don't know if this changes anything
$ npm install
# once after install
$ npm run webdriver-update
# then anytime to reproduce
$ npm run webdriver-start
# terminal 2
$ npm run serve.e2e
# terminal 3
$ npm run e2e

npm run e2e will succeed immediately, and you will see the error with indexOf. You can now continue to further test this behaviour by doing this:

# terminal 2: Quit server using CTRL+C
$ ^C
# terminal 3:
$ npm run e2e

No webserver running, but tests will still pass. Now go to /src/client/app/app.component.e2e-spec.ts and apply this diff:

// Line 7
  it('should have a title', () => {
// Line 8 [---]
    expect(browser.getTitle()).toEqual('Welcome to angular2-seed!');
// Line 8 [+++]
    expect(browser.getTitle()).toEqual('surely not the correct title!');
// Line 9
  });
// ...

You will see that the tests will still pass, no matter whether you started the webserver or not. If you then do the following:

// Line 7
  it('should have a title', () => {
// Line 8 [---]
    expect(browser.getTitle()).toEqual('surely not the correct title!');
// Line 8 [+++]
    expect(true).toBe(false);
// Line 9
  });
// ...

The test will fail (assuming that you've started the webserver, otherwise it didn't recompile).

I hope this helps somehow. Please let me know if I can help any further or if you need anything.

Update: Same bevaiour on Debian 8.5, node 6.4.0 and npm 3.10.3

[heathkit]

Thanks for the repro steps. I am able to reproduce this. I fixed the error handling, and this is the actual error:

[10:45:45] E/launcher - Error while waiting for Protractor to sync with the page: "window.getAllAngularTestabilities is not a function"
[10:45:45] E/launcher - Error
    at ElementArrayFinder.applyAction_ (/Users/heathkit/src/repro-3505/node_modules/protractor/built/element.js:395:27)
    at ElementArrayFinder._this.(anonymous function) [as getText] (/Users/heathkit/src/repro-3505/node_modules/protractor/built/element.js:97:30)
    at ElementFinder._this.(anonymous function) [as getText] (/Users/heathkit/src/repro-3505/node_modules/protractor/built/element.js:729:22)
    at Object.<anonymous> (app/+home/home.component.e2e-spec.ts:12:42)
    at /Users/heathkit/src/repro-3505/node_modules/jasminewd2/index.js:96:23
    at new Promise (/Users/heathkit/src/repro-3505/node_modules/selenium-webdriver/lib/promise.js:1043:7)
    at controlFlowExecute (/Users/heathkit/src/repro-3505/node_modules/jasminewd2/index.js:82:18)
    at TaskQueue.execute_ (/Users/heathkit/src/repro-3505/node_modules/selenium-webdriver/lib/promise.js:2790:14)
    at TaskQueue.executeNext_ (/Users/heathkit/src/repro-3505/node_modules/selenium-webdriver/lib/promise.js:2773:21)
    at /Users/heathkit/src/repro-3505/node_modules/selenium-webdriver/lib/promise.js:2697:25
[10:45:45] E/launcher - Process exited with error code 199

Also, it's worth noting that even though the tests are marked as passing (which is a bug), protractor exits with a non-zero exit code, indicating something went wrong. I'll try to figure out what's going on.

That's strange, because if it would return a non-zero exit code, my CI tests wouldn't pass.

Update: I have tested what you said, and for me, it does return 0 as exit code. I have a script in my package.json called e2e, which is just an alias for protractor. If I run the following in the command line:

$ npm run e2e config/protractor.conf.js && echo "previous exit code was 0"

I get the following output:

Spec started
  app
    ✓ should load                                                                                                                                                     
    # ......                                                                                                                     
    ✓ this test should definetly fail                                                                                                                                                         

Executed 6 of 6 specs SUCCESS in 0.125 sec.                                                                                                                               
[19:53:34] I/launcher - 0 instance(s) of WebDriver still running                                                                                                          
[19:53:34] I/launcher - chrome #01 passed                                                                                                                                 
previous exit code was 0                       

I would also get the default npm error stuff if the script that was ran using a npm run command would return non-zero

[heathkit]

It looks like this is related to the upgrade to selenium-webdriver 2.53.2, but it's a little hard to track down. If you have a minute, could you try installing protractor with 'npm install -g protractor' and launching your tests with that one?