Add a unit test for run_test_cases.py.

Make Progress to output to stderr, it automatically flushes and it doesn't
interfere with the output, making testing easier and stdout more usable in
general, like if piping the results.

NOTRY=true
R=cmp@chromium.org
BUG=
TEST=new integration test


Review URL: https://chromiumcodereview.appspot.com/10533085

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@141686 0039d316-1c4b-4281-b951-d872f2087c98

tools/isolate/data/gtest_fake/gtest_fake.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Simulate a google-test executable.
+
+http://code.google.com/p/googletest/
+"""
+
+import optparse
+import sys
+
+
+TESTS = {
+  'Foo': ['Bar1', 'Bar2', 'Bar3'],
+  'Baz': ['Fail'],
+}
+TOTAL = sum(len(v) for v in TESTS.itervalues())
+
+
+def get_test_output(test_name):
+  fixture, case = test_name.split('.', 1)
+  return (
+    '[==========] Running 1 test from 1 test case.\n'
+    '[----------] Global test environment set-up.\n'
+    '[----------] 1 test from %(fixture)s\n'
+    '[ RUN      ] %(fixture)s.%(case)s\n'
+    '[       OK ] %(fixture)s.%(case)s (0 ms)\n'
+    '[----------] 1 test from %(fixture)s (0 ms total)\n'
+    '\n') % {
+      'fixture': fixture,
+      'case': case,
+    }
+
+
+def get_footer(number):
+  return (
+    '[----------] Global test environment tear-down\n'
+    '[==========] %(number)d test from %(total)d test case ran. (0 ms total)\n'
+    '[  PASSED  ] %(number)d test.\n'
+    '\n'
+    '  YOU HAVE 5 DISABLED TESTS\n'
+    '\n'
+    '  YOU HAVE 2 tests with ignored failures (FAILS prefix)\n') % {
+      'number': number,
+      'total': TOTAL,
+    }
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.add_option('--gtest_list_tests', action='store_true')
+  parser.add_option('--gtest_filter')
+  options, args = parser.parse_args()
+  if args:
+    parser.error('Failed to process args %s' % args)
+
+  if options.gtest_list_tests:
+    for fixture, cases in TESTS.iteritems():
+      print '%s.' % fixture
+      for case in cases:
+        print '  ' + case
+    print '  YOU HAVE 2 tests with ignored failures (FAILS prefix)'
+    print ''
+    return 0
+
+  if options.gtest_filter:
+    # Simulate running one test.
+    print 'Note: Google Test filter = %s\n' % options.gtest_filter
+    print get_test_output(options.gtest_filter)
+    print get_footer(1)
+    # Make Baz.Fail fail.
+    return options.gtest_filter == 'Baz.Fail'
+
+  for fixture, cases in TESTS.iteritems():
+    for case in cases:
+      print get_test_output('%s.%s' % (fixture, case))
+  print get_footer(TOTAL)
+  return 1
+
+
+if __name__ == '__main__':
+  sys.exit(main())

tools/isolate/run_test_cases.py

@@ -117,7 +117,8 @@ def get_test_cases(executable, whitelist, blacklist):
   return tests
 
 
-def run_test_cases(executable, whitelist, blacklist, jobs, timeout, stats_only):
+def run_test_cases(
+    executable, whitelist, blacklist, jobs, timeout, stats_only, no_dump):
   """Traces test cases one by one."""
   test_cases = get_test_cases(executable, whitelist, blacklist)
   if not test_cases:
@@ -131,8 +132,9 @@ def run_test_cases(executable, whitelist, blacklist, jobs, timeout, stats_only):
     results = pool.join(progress, 0.1)
     duration = time.time() - progress.start
   results = dict((item[0]['test_case'], item) for item in results)
-  trace_inputs.write_json('%s.run_test_cases' % executable, results, False)
-  print ''
+  if not no_dump:
+    trace_inputs.write_json('%s.run_test_cases' % executable, results, False)
+  sys.stderr.write('\n')
   total = len(results)
   if not total:
     return 1
@@ -207,6 +209,10 @@ def main():
       action='count',
       default=int(os.environ.get('ISOLATE_DEBUG', 0)),
       help='Use multiple times')
+  parser.add_option(
+      '--no-dump',
+      action='store_true',
+      help='do not generate a .test_cases file')
   options, args = parser.parse_args()
   levels = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG]
   logging.basicConfig(
@@ -224,7 +230,8 @@ def main():
       options.blacklist,
       options.jobs,
       options.timeout,
-      options.stats)
+      options.stats,
+      options.no_dump)
 
 
 if __name__ == '__main__':

tools/isolate/run_test_cases_test.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import re
+import subprocess
+import sys
+import unittest
+
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+sys.path.append(os.path.join(ROOT_DIR, 'data', 'gtest_fake'))
+import gtest_fake
+
+
+class TraceTestCases(unittest.TestCase):
+  def test_simple(self):
+    target = os.path.join(ROOT_DIR, 'data', 'gtest_fake', 'gtest_fake.py')
+    cmd = [
+        sys.executable,
+        os.path.join(ROOT_DIR, 'run_test_cases.py'),
+        '--no-dump',
+        target,
+    ]
+    proc = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # pylint is confused.
+    out, err = proc.communicate() or ('', '')
+    self.assertEquals(0, proc.returncode)
+    expected = (
+      'Note: Google Test filter = Baz.Fail\n'
+      '\n'
+      '%(test_output)s\n'
+      '%(test_footer)s\n'
+      '\n'
+      'Success:    3 75.00%%\n'
+      'Flaky:      0  0.00%%\n'
+      'Fail:       1 25.00%%\n') % {
+        'test_output': gtest_fake.get_test_output('Baz.Fail'),
+        'test_footer': gtest_fake.get_footer(1),
+      }
+
+    self.assertTrue(
+        out.startswith(expected),
+        '\n'.join(['XXX', expected, 'XXX', out[:len(expected)], 'XXX']))
+    remaining_actual = out[len(expected):]
+    regexp = (
+        r'\d+\.\ds Done running 4 tests with 6 executions. \d+\.\d test/s'
+        + '\n')
+    self.assertTrue(re.match(regexp, remaining_actual), remaining_actual)
+    # Progress junk went to stderr.
+    self.assertTrue(err.startswith('\r'), err)
+
+
+if __name__ == '__main__':
+  VERBOSE = '-v' in sys.argv
+  logging.basicConfig(level=logging.DEBUG if VERBOSE else logging.ERROR)
+  unittest.main()

tools/isolate/worker_pool.py

@@ -136,8 +136,7 @@ def print_update(self):
           self.next_line,
           ' ' * max(0, len(self.last_printed_line) - len(self.next_line)))
       self.last_printed_line = self.next_line
-    sys.stdout.write(line)
-    sys.stdout.flush()
+    sys.stderr.write(line)
 
   def increase_count(self):
     with self.lock: