Reconcile v0.12 with new CI

Makefile

@@ -6,6 +6,7 @@ NINJA ?= ninja
 DESTDIR ?=
 SIGN ?=
 PREFIX ?= /usr/local
+FLAKY_TESTS ?= run
 
 NODE ?= ./node
 
@@ -127,6 +128,9 @@ test-all-http1: test-build
 test-all-valgrind: test-build
 	$(PYTHON) tools/test.py --mode=debug,release --valgrind
 
+test-ci:
+	$(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --arch=$(DESTCPU) --flaky-tests=$(FLAKY_TESTS) simple message internet
+
 test-release: test-build
 	$(PYTHON) tools/test.py --mode=release
 
@@ -240,6 +244,11 @@ docopen: out/doc/api/all.html
 docclean:
 	-rm -rf out/doc
 
+run-ci:
+	$(PYTHON) ./configure --without-snapshot $(CONFIG_FLAGS)
+	$(MAKE)
+	$(MAKE) test-ci
+
 RAWVER=$(shell $(PYTHON) tools/getnodeversion.py)
 VERSION=v$(RAWVER)
 NODE_DOC_VERSION=$(VERSION)
@@ -439,4 +448,9 @@ cpplint:
 
 lint: jslint cpplint
 
-.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean check uninstall install install-includes install-bin all staticlib dynamiclib test test-all test-addons build-addons website-upload pkg blog blogclean tar binary release-only bench-http-simple bench-idle bench-all bench bench-misc bench-array bench-buffer bench-net bench-http bench-fs bench-tls
+.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean \
+	check uninstall install install-includes install-bin all staticlib \
+	dynamiclib test test-all test-addons build-addons website-upload pkg \
+	blog blogclean tar binary release-only bench-http-simple bench-idle \
+	bench-all bench bench-misc bench-array bench-buffer bench-net \
+	bench-http bench-fs bench-tls run-ci

tools/test.py

@@ -29,6 +29,7 @@
 
 
 import imp
+import logging
 import optparse
 import os
 import platform
@@ -45,6 +46,8 @@
 from datetime import datetime
 from Queue import Queue, Empty
 
+logger = logging.getLogger('testrunner')
+
 VERBOSE = False
 
 
@@ -65,7 +68,9 @@ def __init__(self, cases, flaky_tests_mode):
     self.remaining = len(cases)
     self.total = len(cases)
     self.failed = [ ]
+    self.flaky_failed = [ ]
     self.crashed = 0
+    self.flaky_crashed = 0
     self.terminate = False
     self.lock = threading.Lock()
 
@@ -126,9 +131,14 @@ def RunSingle(self):
         return
       self.lock.acquire()
       if output.UnexpectedOutput():
-        self.failed.append(output)
-        if output.HasCrashed():
-          self.crashed += 1
+        if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare":
+          self.flaky_failed.append(output)
+          if output.HasCrashed():
+            self.flaky_crashed += 1
+        else:
+          self.failed.append(output)
+          if output.HasCrashed():
+            self.crashed += 1
       else:
         self.succeeded += 1
       self.remaining -= 1
@@ -225,7 +235,7 @@ def HasRun(self, output):
 class TapProgressIndicator(SimpleProgressIndicator):
 
   def Starting(self):
-    print '1..%i' % len(self.cases)
+    logger.info('1..%i' % len(self.cases))
     self._done = 0
 
   def AboutToRun(self, case):
@@ -238,26 +248,26 @@ def HasRun(self, output):
       status_line = 'not ok %i - %s' % (self._done, command)
       if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare":
         status_line = status_line + " # TODO : Fix flaky test"
-      print status_line
+      logger.info(status_line)
       for l in output.output.stderr.splitlines():
-        print '#' + l
+        logger.info('#' + l)
       for l in output.output.stdout.splitlines():
-        print '#' + l
+        logger.info('#' + l)
     else:
       status_line = 'ok %i - %s' % (self._done, command)
       if FLAKY in output.test.outcomes:
         status_line = status_line + " # TODO : Fix flaky test"
-      print status_line
+      logger.info(status_line)
 
     duration = output.test.duration
 
     # total_seconds() was added in 2.7
     total_seconds = (duration.microseconds +
       (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6
 
-    print '  ---'
-    print '  duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000)
-    print '  ...'
+    logger.info('  ---')
+    logger.info('  duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000))
+    logger.info('  ...')
 
   def Done(self):
     pass
@@ -1192,6 +1202,8 @@ def BuildOptions():
       default='release')
   result.add_option("-v", "--verbose", help="Verbose output",
       default=False, action="store_true")
+  result.add_option('--logfile', dest='logfile',
+      help='write test output to file. NOTE: this only applies the tap progress indicator')
   result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons",
       default=[], action="append")
   result.add_option("-p", "--progress",
@@ -1368,6 +1380,13 @@ def Main():
     parser.print_help()
     return 1
 
+  ch = logging.StreamHandler(sys.stdout)
+  logger.addHandler(ch)
+  logger.setLevel(logging.INFO)
+  if options.logfile:
+    fh = logging.FileHandler(options.logfile)
+    logger.addHandler(fh)
+
   workspace = abspath(join(dirname(sys.argv[0]), '..'))
   suites = GetSuites(join(workspace, 'test'))
   repositories = [TestRepository(join(workspace, 'test', name)) for name in suites]

vcbuild.bat

@@ -38,6 +38,7 @@ set noperfctr_msi_arg=
 set i18n_arg=
 set download_arg=
 set build_release=
+set flaky_tests_arg=
 
 :next-arg
 if "%1"=="" goto args-done
@@ -61,7 +62,8 @@ if /i "%1"=="test-simple"   set test=test-simple&goto arg-ok
 if /i "%1"=="test-message"  set test=test-message&goto arg-ok
 if /i "%1"=="test-gc"       set test=test-gc&set buildnodeweak=1&goto arg-ok
 if /i "%1"=="test-all"      set test=test-all&set buildnodeweak=1&goto arg-ok
-if /i "%1"=="test"          set test=test&goto arg-ok
+if /i "%1"=="test-ci"       set test=test-ci&set nosnapshot=1&goto arg-ok
+if /i "%1"=="test"          set test=test&set jslint=1&goto arg-ok
 @rem Include small-icu support with MSI installer
 if /i "%1"=="msi"           set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok
 if /i "%1"=="upload"        set upload=1&goto arg-ok
@@ -71,6 +73,7 @@ if /i "%1"=="full-icu"      set i18n_arg=%1&goto arg-ok
 if /i "%1"=="intl-none"     set i18n_arg=%1&goto arg-ok
 if /i "%1"=="download-all"  set download_arg="--download=all"&goto arg-ok
 if /i "%1"=="build-release" set build_release=1&goto arg-ok
+if /i "%1"=="ignore-flaky"  set flaky_tests_arg=--flaky-tests=dontcare&goto arg-ok
 
 echo Warning: ignoring invalid command line option `%1`.
 
@@ -81,7 +84,6 @@ goto next-arg
 
 :args-done
 if defined upload goto upload
-if defined jslint goto jslint
 
 if defined build_release (
   set nosnapshot=1
@@ -197,12 +199,15 @@ if errorlevel 1 echo Failed to sign msi&goto exit
 
 :run
 @rem Run tests if requested.
-if "%test%"=="" goto exit
+if "%test%"=="" goto jslint
 
 if "%config%"=="Debug" set test_args=--mode=debug
 if "%config%"=="Release" set test_args=--mode=release
 
+set test_args=%test_args% --arch=%target_arch% 
+
 if "%test%"=="test" set test_args=%test_args% simple message
+if "%test%"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap %flaky_tests_arg% simple message internet 
 if "%test%"=="test-internet" set test_args=%test_args% internet
 if "%test%"=="test-pummel" set test_args=%test_args% pummel
 if "%test%"=="test-simple" set test_args=%test_args% simple
@@ -224,8 +229,7 @@ goto exit
 :run-tests
 echo running 'python tools/test.py %test_args%'
 python tools/test.py %test_args%
-if "%test%"=="test" goto jslint
-goto exit
+goto jslint
 
 :create-msvs-files-failed
 echo Failed to create vc project files. 
@@ -243,6 +247,7 @@ scp Release\node.pdb node@nodejs.org:~/web/nodejs.org/dist/v%NODE_VERSION%/node.
 goto exit
 
 :jslint
+if not defined jslint goto exit
 echo running jslint
 set PYTHONPATH=tools/closure_linter/
 python tools/closure_linter/closure_linter/gjslint.py --unix_mode --strict --nojsdoc -r lib/ -r src/ --exclude_files lib/punycode.js