Run all Closure tests on Travis using protractor.

.travis.yml

@@ -3,22 +3,25 @@
 # Use of this source code is governed by the Apache License, Version 2.0.
 # See the LICENSE file for details.
 
-# Will upgrade to new container-based infrastructure
+# Use new container-based infrastructure.
 sudo: false
 
+# Cache node modules across runs.
 cache:
   - node_modules
 
-before_install:
-  # emulate a GUI environment
-  - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
-  # 10 minutes
-  - "export PROTRACTOR_TIMEOUT=600000"
-
 install:
   - ./scripts/ci/install_closure_deps.sh
 
 script:
   - ./scripts/ci/compile_closure.sh
   - ./scripts/ci/lint_pull_request.sh
+  - travis_wait 30 ./scripts/ci/run_all_tests.sh
+
+addons:
+  sauce_connect: true
+
+env:
+  global:
+    - secure: G3YiCL+uZVLv0XdMjPhczNM5k6vBBqUqsvjArQZmJmf/zKwEP49vv7ag2f//9msc4XrzXs06OzHW3tI8nq7JJnGrupTcJBeZ6Vcg06Z1W2pG8/dSNJ0dRC493ICIBpU8VW2moYF1aDdY1oktdII7xQ705nBcRJ9VcLqG1Km2BpY=
+    - secure: kqXS2ih32mv1Xw0l8bs+QeMRc7wdVxVMzsZ7LdtLbTfAKdHlF/FyshLXpbLjPJDdIndyBZfH1U+KJI2SfqRhyo/kvuiOlz0okeh+paRumCjradPYlTXkr/Ji91gSRa/HObupEoYOKoERjZlK/eH9FXYBKx63LSwV6qiAppQdiSo=

package.json

@@ -20,7 +20,7 @@
   },
   "homepage": "https://developers.google.com/closure/library/",
   "devDependencies": {
-    "http-server": "^0.8.0",
+    "promise": "^7.0.4",
     "protractor": "^2.1.0"
   }
 }

protractor.conf.js

@@ -1,27 +1,30 @@
-var httpServer = require('http-server');
-var PROTRACTOR_TIMEOUT = process.env.PROTRACTOR_TIMEOUT ?
-    parseInt(process.env.PROTRACTOR_TIMEOUT, 10) : 10 * 60 * 1000;
+// See https://github.com/angular/protractor/blob/master/docs/referenceConf.js
+// for full protractor config reference.
 exports.config = {
-  directConnect: false,
-  seleniumAddress: 'http://localhost:4444/wd/hub',
+  sauceUser: process.env.SAUCE_USERNAME,
 
-  // Capabilities to be passed to the webdriver instance.
+  sauceKey: process.env.SAUCE_ACCESS_KEY,
+
+  // Options specific to which browser tests are run on.
   capabilities: {
-    'browserName': 'firefox'
+    'browserName': 'chrome',
+    'tunnel-identifier': process.env.TRAVIS_JOB_NUMBER,
+    'build': process.env.TRAVIS_BUILD_NUMBER,
+    'name': process.env.TRAVIS_PULL_REQUEST == 'false' ?
+                'CO-' + process.env.TRAVIS_BRANCH + '-' +
+                    process.env.TRAVIS_COMMIT :
+                'PR-' + process.env.TRAVIS_PULL_REQUEST + '-' +
+                    process.env.TRAVIS_BRANCH + '-' + process.env.TRAVIS_COMMIT
   },
 
+  // Testing framework used for spec file.
   framework: 'jasmine2',
 
+  // Relative path to spec (i.e., tests).
   specs: ['protractor_spec.js'],
 
   jasmineNodeOpts: {
-    //default timeout plus 10 seconds to make sure spec times out before runner
-    defaultTimeoutInterval: PROTRACTOR_TIMEOUT + 10000
-  },
-
-  beforeLaunch: function () {
-    httpServer.createServer({
-      showDir: false
-    }).listen('8080', 'localhost');
+    // Timeout in ms before a test fails. 30 minutes.
+    defaultTimeoutInterval: 30 * 60 * 1000
   }
 };

protractor_spec.js

@@ -1,27 +1,125 @@
-var EC = protractor.ExpectedConditions;
-var PROTRACTOR_TIMEOUT = process.env.PROTRACTOR_TIMEOUT ?
-    parseInt(process.env.PROTRACTOR_TIMEOUT, 10) : 10 * 60 * 1000
-
-describe('alltests', function() {
-  beforeEach(function() { browser.ignoreSynchronization = true; });
-
-  it('should successfully run all tests', function(done) {
-    browser.get('http://localhost:8080/all_tests.html');
-    $('.goog-testrunner-buttons button:first-child').click();
-    var summary = $('.goog-testrunner-progress-summary');
-    //Same timeout as parent global timeout
-    browser.wait(EC.textToBePresentInElement(summary, 'passed'), PROTRACTOR_TIMEOUT)
-        .then(function() {
-          console.log('done waiting');
-          summary.getText().then(function(text) {
-            $('.goog-testrunner-report')
-                .getText()
-                .then(function(failures) {
-                  console.log(failures);
-                  expect(text.split('\n')[1]).toContain('0 failed');
-                  done();
-                });
+// TODO(joeltine): Remove promise module when stable node version supports it
+// natively.
+var Promise = require('promise');
+var allTests = require('./alltests');
+
+// Timeout for individual test package to complete.
+var TEST_TIMEOUT = 45 * 1000;
+var TEST_SERVER = 'http://localhost:8080';
+var IGNORED_TESTS = ['closure/goog/i18n/currency_test.html'];
+
+describe('Run all Closure unit tests', function() {
+  var removeIgnoredTests = function(tests) {
+    for (var i = 0; i < IGNORED_TESTS.length; i++) {
+      var index = tests.indexOf(IGNORED_TESTS[i]);
+      if (index != -1) {
+        tests.splice(index, 1);
+      }
+    }
+    return tests;
+  };
+
+  beforeAll(function() {
+    allTests = removeIgnoredTests(allTests);
+  });
+
+  beforeEach(function() {
+    // Ignores synchronization with angular loading. Since we don't use angular,
+    // enable it.
+    browser.ignoreSynchronization = true;
+  });
+
+  // Polls currently loaded test page for test completion. Returns Promise that
+  // will resolve when test is finished.
+  var waitForTestSuiteCompletion = function(testPath) {
+    var testStartTime = +new Date();
+
+    var waitForTest = function(resolve, reject) {
+      // executeScript runs the passed method in the "window" context of
+      // the current test. JSUnit exposes hooks into the test's status through
+      // the "G_testRunner" global object.
+      browser.executeScript(function() {
+               if (window['G_testRunner'] &&
+                   window['G_testRunner']['isFinished']()) {
+                 var status = {};
+                 status['isFinished'] = true;
+                 status['isSuccess'] = window['G_testRunner']['isSuccess']();
+                 status['report'] = window['G_testRunner']['getReport']();
+                 return status;
+               } else {
+                 return {'isFinished': false};
+               }
+             })
+          .then(
+              function(status) {
+                if (status && status.isFinished) {
+                  resolve(status);
+                } else {
+                  var currTime = +new Date();
+                  if (currTime - testStartTime > TEST_TIMEOUT) {
+                    status.isSuccess = false;
+                    status.report = testPath + ' timed out after ' +
+                                    (TEST_TIMEOUT / 1000) + 's!';
+                    // resolve so tests continue running.
+                    resolve(status);
+                  } else {
+                    // Check every 300ms for completion.
+                    setTimeout(waitForTest.bind(undefined, resolve, reject),
+                               300);
+                  }
+                }
+              },
+              function(err) { reject(err); });
+    };
+
+    return new Promise(function(resolve, reject) {
+      waitForTest(resolve, reject);
+    });
+  };
+
+  it('should run all tests with 0 failures', function(done) {
+    var failureReports = [];
+
+    // Navigates to testPath to invoke tests. Upon completion inspects returned
+    // test status and keeps track of the total number failed tests.
+    var runNextTest = function(testPath) {
+      return browser.navigate()
+          .to(TEST_SERVER + '/' + testPath)
+          .then(function() { return waitForTestSuiteCompletion(testPath); })
+          .then(function(status) {
+            if (!status.isSuccess) {
+              failureReports.push(status.report);
+            }
+
+            return status;
           });
-        });
+    };
+
+    // Chains the next test to the completion of the previous through its
+    // promise.
+    var chainNextTest = function(promise, test) {
+      return promise.then(function() { runNextTest(test); });
+    };
+
+    var testPromise = null;
+    for (var i = 0; i < allTests.length; i++) {
+      if (testPromise != null) {
+        testPromise = chainNextTest(testPromise, allTests[i]);
+      } else {
+        testPromise = runNextTest(allTests[i]);
+      }
+    }
+
+    testPromise.then(function() {
+      var totalFailures = failureReports.length;
+      if (totalFailures > 0) {
+        console.error('There was ' + totalFailures + ' test failure(s)!');
+        for (var i = 0; i < failureReports.length; i++) {
+          console.error(failureReports[i]);
+        }
+      }
+      expect(failureReports.length).toBe(0);
+      done();
+    });
   });
 });

scripts/ci/compile_closure.sh

@@ -19,4 +19,7 @@ java -jar ../closure-compiler/build/compiler.jar \
   --js='!**osapi/osapi.js' \
   --js='!**svgpan/svgpan.js' \
   --js='!**alltests.js' \
+  --js='!**\./node_modules**.js' \
+  --js='!**protractor_spec.js' \
+  --js='!**protractor.conf.js' \
   --js_output_file=$(mktemp);

scripts/ci/install_closure_deps.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
+#
+# Script to install all necessary dependencies for running Closure tests,
+# linting, and compiling.
+
+set -ex
 
 # Install closure compiler and linter.
 cd ..
@@ -9,3 +14,6 @@ ant jar
 cd ../closure-linter
 python ./setup.py install --user
 cd ../closure-library
+
+# Installs node "devDependencies" found in package.json.
+npm install

scripts/ci/lint_pull_request.sh

@@ -11,7 +11,11 @@ CHANGED_FILES=$(git diff --name-only --diff-filter=AM master..$CURRENT_BRANCH |
 
 if [[ -n "$CHANGED_FILES" ]]; then
   set -x
-  $USER_BASE/bin/gjslint --strict --jslint_error=all --exclude_files=deps.js,alltests.js $CHANGED_FILES
+  $USER_BASE/bin/gjslint \
+    --strict \
+    --jslint_error=all \
+    --exclude_files=deps.js,alltests.js,protractor.conf.js,protractor_spec.js \
+    $CHANGED_FILES;
 else
   echo "No .js files found to lint in this Pull Request."
 fi

scripts/ci/run_all_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Script to install dependencies and bootstrap protractor to run all Closure
+# tests.
+
+HTTP_PORT=8080
+
+# Make sure to kill both servers on exit (e.g., CTRL+C).
+trap 'fuser -k $HTTP_PORT/tcp' EXIT
+
+# Start web server, with POST support.
+python scripts/http/simple_http_server.py 2> /dev/null & sleep 5
+
+# Invoke protractor to run tests.
+./node_modules/.bin/protractor protractor.conf.js

scripts/http/simple_http_server.py

@@ -0,0 +1,26 @@
+import SimpleHTTPServer
+import SocketServer
+import cgi
+import sys
+
+PORT = 8080
+
+# Simple server to respond to both POST and GET requests. POST requests will
+# just respond as normal GETs.
+class ServerHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+
+    def do_GET(self):
+        SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
+
+    def do_POST(self):
+        SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
+
+Handler = ServerHandler
+
+# Allows use to restart server immediately after restarting it.
+SocketServer.ThreadingTCPServer.allow_reuse_address = True
+
+httpd = SocketServer.TCPServer(("", PORT), Handler)
+
+print ("Serving at: http://%s:%s" % ("localhost", PORT))
+httpd.serve_forever()