Re-land "Add new test runner harness." (#2)

Re-land #2 changes:

 * export labels are fixed for the CFI build
 * crash test disabled because of flakiness and issues with asan

Re-land changes:

 * Unit test is suppressed in ASAN
 * --deqp-case is fixed
 * Debug layer errors should correctly work with failure expectations

Original message:

The ANGLE test harness is a harness around GoogleTest that provides
functionality similar to the Chromium test harness. It supports:

 * splitting a test set into shards
 * catching and reporting crashes and timeouts
 * outputting to the Chromium JSON test results format
 * multi-process execution

Unit tests are added in test_utils_unittest.cpp.

Bug: angleproject:3162
Bug: chromium:1030192
Change-Id: I71d66a407ea0e53d73cbe75b5b4bfb9e73791534
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1965091
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Jonah Ryan-Davis <jonahr@google.com>

Author: null77

gni/angle.gni

@@ -189,11 +189,14 @@ set_defaults("angle_test") {
   public_deps = []
   sources = []
   data = []
+  defines = []
   main = ""
   suppressed_configs = angle_remove_configs
 
-  # TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
-  if (build_with_chromium) {
+  # By default use the Chromium harness in Chromium. Can be overriden in a target.
+  standalone_harness = !build_with_chromium
+
+  if (!standalone_harness) {
     suppressed_configs -= [ "//build/config/compiler:default_include_dirs" ]
   }
 
@@ -286,18 +289,6 @@ template("angle_static_library") {
 }
 
 template("angle_test") {
-  _googletest_deps = [
-    "//testing/gmock",
-    "//testing/gtest",
-    "//third_party/googletest:gmock",
-    "//third_party/googletest:gtest",
-  ]
-
-  # TODO(jmadill): Migrate to standalone harness. http://anglebug.com/3162
-  if (build_with_chromium) {
-    _googletest_deps += [ "//base/test:test_support" ]
-  }
-
   test(target_name) {
     forward_variables_from(invoker,
                            "*",
@@ -311,32 +302,51 @@ template("angle_test") {
     forward_variables_from(invoker, [ "visibility" ])
 
     configs += invoker.configs
-    configs -= invoker.suppressed_configs
-    configs -= [ angle_root + ":constructor_and_destructor_warnings" ]
-    configs -= [ angle_root + ":extra_warnings" ]
+    configs -= invoker.suppressed_configs + [
+                 "$angle_root:constructor_and_destructor_warnings",
+                 "$angle_root:extra_warnings",
+               ]
 
     if (is_linux && !is_component_build) {
       # Set rpath to find shared libs in a non-component build.
       configs += [ "//build/config/gcc:rpath_for_built_shared_libraries" ]
     }
 
     if (is_android) {
-      configs += [ angle_root + ":build_id_config" ]
-      if (build_with_chromium) {
-        configs -= [ "//build/config/android:hide_all_but_jni" ]
-      }
+      configs += [ "$angle_root:build_id_config" ]
     }
 
-    deps += _googletest_deps + [
-              "$angle_root:angle_common",
-              "$angle_root:includes",
-              "$angle_root/util:angle_test_utils",
-            ]
-
-    if (build_with_chromium) {
-      sources += [ "//gpu/${invoker.main}.cc" ]
+    deps += [
+      "$angle_root:angle_common",
+      "$angle_root:includes",
+      "$angle_root/third_party/rapidjson:rapidjson",
+      "$angle_root/util:angle_test_utils",
+      "//testing/gmock",
+      "//testing/gtest",
+      "//third_party/googletest:gmock",
+      "//third_party/googletest:gtest",
+    ]
+
+    sources += [
+      "$angle_root/src/tests/test_utils/runner/TestSuite.cpp",
+      "$angle_root/src/tests/test_utils/runner/TestSuite.h",
+    ]
+
+    # To use the Chromium test infrastructure we must currently use the //base test launcher.
+    # Eventually we could switch to using standalone testing. See http://crbug.com/837741
+    if (standalone_harness) {
+      if (invoker.main != "") {
+        sources += [ "${invoker.main}.cpp" ]
+      }
     } else {
-      sources += [ "${invoker.main}.cpp" ]
+      if (invoker.main != "") {
+        sources += [ "//gpu/${invoker.main}.cc" ]
+      }
+      deps += [ "//base/test:test_support" ]
+
+      if (is_android) {
+        configs -= [ "//build/config/android:hide_all_but_jni" ]
+      }
     }
   }
 }

src/common/platform.h

@@ -123,4 +123,11 @@
 #    endif
 #endif
 
+// Define ANGLE_WITH_ASAN macro.
+#if defined(__has_feature)
+#    if __has_feature(address_sanitizer)
+#        define ANGLE_WITH_ASAN 1
+#    endif
+#endif
+
 #endif  // COMMON_PLATFORM_H_

src/common/system_utils.h

@@ -17,6 +17,8 @@ namespace angle
 std::string GetExecutablePath();
 std::string GetExecutableDirectory();
 const char *GetSharedLibraryExtension();
+const char *GetExecutableExtension();
+char GetPathSeparator();
 Optional<std::string> GetCWD();
 bool SetCWD(const char *dirName);
 bool SetEnvironmentVar(const char *variableName, const char *value);

src/common/system_utils_posix.cpp

@@ -125,4 +125,14 @@ void BreakDebugger()
     // See https://cs.chromium.org/chromium/src/base/debug/debugger_posix.cc
     abort();
 }
+
+const char *GetExecutableExtension()
+{
+    return "";
+}
+
+char GetPathSeparator()
+{
+    return '/';
+}
 }  // namespace angle

src/common/system_utils_win.cpp

@@ -166,4 +166,13 @@ void BreakDebugger()
     __debugbreak();
 }
 
+const char *GetExecutableExtension()
+{
+    return ".exe";
+}
+
+char GetPathSeparator()
+{
+    return '\\';
+}
 }  // namespace angle

src/tests/BUILD.gn

@@ -13,11 +13,18 @@ declare_args() {
   build_angle_gles1_conform_tests = false
 }
 
-angle_executable("test_utils_unittest_helper") {
-  sources = test_utils_unittest_helper_sources
+angle_test("test_utils_unittest_helper") {
+  standalone_harness = true
+
+  sources = [
+    "../../util/test_utils_unittest_helper.cpp",
+    "../../util/test_utils_unittest_helper.h",
+    "test_utils/angle_test_instantiate.h",
+    "test_utils/runner/TestSuite_unittest.cpp",
+  ]
 
   deps = [
-    "${angle_root}:angle_common",
+    "$angle_root:angle_common",
   ]
 }
 

src/tests/angle_deqp_tests_main.cpp

@@ -8,6 +8,8 @@
 
 #include <gtest/gtest.h>
 
+#include "test_utils/runner/TestSuite.h"
+
 // Defined in angle_deqp_gtest.cpp. Declared here so we don't need to make a header that we import
 // in Chromium.
 namespace angle
@@ -18,7 +20,7 @@ void InitTestHarness(int *argc, char **argv);
 int main(int argc, char **argv)
 {
     angle::InitTestHarness(&argc, argv);
-    testing::InitGoogleTest(&argc, argv);
+    angle::TestSuite testSuite(&argc, argv);
     int rt = RUN_ALL_TESTS();
     return rt;
 }

src/tests/angle_end2end_tests_main.cpp

@@ -5,13 +5,14 @@
 //
 
 #include "gtest/gtest.h"
+#include "test_utils/runner/TestSuite.h"
 
 void ANGLEProcessTestArgs(int *argc, char *argv[]);
 
 int main(int argc, char **argv)
 {
+    angle::TestSuite testSuite(&argc, argv);
     ANGLEProcessTestArgs(&argc, argv);
-    testing::InitGoogleTest(&argc, argv);
     int rt = RUN_ALL_TESTS();
     return rt;
 }

src/tests/angle_perftests_main.cpp

@@ -9,13 +9,14 @@
 
 #include <gtest/gtest.h>
 
+#include "test_utils/runner/TestSuite.h"
+
 void ANGLEProcessPerfTestArgs(int *argc, char **argv);
 
 int main(int argc, char **argv)
 {
+    angle::TestSuite testSuite(&argc, argv);
     ANGLEProcessPerfTestArgs(&argc, argv);
-    testing::InitGoogleTest(&argc, argv);
-    testing::AddGlobalTestEnvironment(new testing::Environment());
     int rt = RUN_ALL_TESTS();
     return rt;
 }

src/tests/angle_unittest_main.cpp

@@ -6,6 +6,7 @@
 
 #include "GLSLANG/ShaderLang.h"
 #include "gtest/gtest.h"
+#include "test_utils/runner/TestSuite.h"
 
 class CompilerTestEnvironment : public testing::Environment
 {
@@ -29,8 +30,7 @@ class CompilerTestEnvironment : public testing::Environment
 
 int main(int argc, char **argv)
 {
-    testing::InitGoogleTest(&argc, argv);
+    angle::TestSuite testSuite(&argc, argv);
     testing::AddGlobalTestEnvironment(new CompilerTestEnvironment());
-    int rt = RUN_ALL_TESTS();
-    return rt;
+    return testSuite.run();
 }