Felt add integration (#17190)

* adding arguments to felt for running different type of tests

* adding test type enums to felt

* more changes to felt for testing branches

* more additions to code

* more changes to felt

* adding code for cloning web_drivers

* validating test directories. running the integration tests. update the readme file partially

* Removing todo lists used for development

* addressing reviewers comments

* remove unused imports

* addressing more reviewer comments

* addressing more reviewer comments

* addressing reviewer comments.

* addressing reviewer comments.

* fixing typos

* using chromedriverinstaller as a library. Fixing the commit number used from web_installers repo

* clean drivers directory after tests finish

* addressing more reviewer comments

* throwing all critical exceptions instead of logging and returning boolean flags

* adding todos for items we can do for making felt easier to use for local development

* do not run integration tests on CIs. Added an issue to the TODO.

* changing todo's with issues.

Author: Nurhan Turgut

lib/web_ui/dev/README.md

@@ -31,12 +31,24 @@ felt build [-w] -j 100
 If you are a Google employee, you can use an internal instance of Goma to parallelize your builds. Because Goma compiles code on remote servers, this option is effective even on low-powered laptops.
 
 ## Running web engine tests
-To run all tests on Chrome:
+To run all tests on Chrome. This will run both integration tests and the unit tests:
 
 ```
 felt test
 ```
 
+To run unit tests only:
+
+```
+felt test --unit-tests-only
+```
+
+To run integration tests only. For now these tests are only available on Chrome Desktop browsers.
+
+```
+felt test --integration-tests-only
+```
+
 To run tests on Firefox (this will work only on a Linux device):
 
 ```
@@ -55,7 +67,7 @@ To run tests on Safari use the following command. It works on MacOS devices and
 felt test --browser=safari
 ```
 
-To run tests on Windows Edge use the following command. It works on Windows devices and it uses the Edge installed on the OS. 
+To run tests on Windows Edge use the following command. It works on Windows devices and it uses the Edge installed on the OS.
 
 ```
 felt_windows.bat test --browser=edge

lib/web_ui/dev/common.dart

@@ -27,6 +27,15 @@ class BrowserInstallerException implements Exception {
   String toString() => message;
 }
 
+class DriverException implements Exception {
+  DriverException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => message;
+}
+
 abstract class PlatformBinding {
   static PlatformBinding get instance {
     if (_instance == null) {
@@ -77,11 +86,10 @@ class _WindowsBinding implements PlatformBinding {
   @override
   String getFirefoxDownloadUrl(String version) =>
       'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/win64/en-US/'
-          '${getFirefoxDownloadFilename(version)}';
+      '${getFirefoxDownloadFilename(version)}';
 
   @override
-  String getFirefoxDownloadFilename(String version) =>
-      'firefox-${version}.exe';
+  String getFirefoxDownloadFilename(String version) => 'firefox-${version}.exe';
 
   @override
   String getFirefoxExecutablePath(io.Directory versionDir) =>
@@ -117,7 +125,7 @@ class _LinuxBinding implements PlatformBinding {
   @override
   String getFirefoxDownloadUrl(String version) =>
       'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/linux-x86_64/en-US/'
-          '${getFirefoxDownloadFilename(version)}';
+      '${getFirefoxDownloadFilename(version)}';
 
   @override
   String getFirefoxDownloadFilename(String version) =>
@@ -161,16 +169,15 @@ class _MacBinding implements PlatformBinding {
 
   @override
   String getFirefoxDownloadUrl(String version) =>
-    'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/mac/en-US/'
-        '${getFirefoxDownloadFilename(version)}';
+      'https://download-installer.cdn.mozilla.net/pub/firefox/releases/${version}/mac/en-US/'
+      '${getFirefoxDownloadFilename(version)}';
 
   @override
-  String getFirefoxDownloadFilename(String version) =>
-      'Firefox ${version}.dmg';
+  String getFirefoxDownloadFilename(String version) => 'Firefox ${version}.dmg';
 
   @override
   String getFirefoxExecutablePath(io.Directory versionDir) =>
-    path.join(versionDir.path, 'Firefox.app','Contents','MacOS', 'firefox');
+      path.join(versionDir.path, 'Firefox.app', 'Contents', 'MacOS', 'firefox');
 
   @override
   String getFirefoxLatestVersionUrl() =>
@@ -243,3 +250,14 @@ class DevNull implements StringSink {
 }
 
 bool get isCirrus => io.Platform.environment['CIRRUS_CI'] == 'true';
+
+/// There might be proccesses started during the tests.
+///
+/// Use this list to store those Processes, for cleaning up before shutdown.
+final List<io.Process> processesToCleanUp = List<io.Process>();
+
+/// There might be temporary directories created during the tests.
+///
+/// Use this list to store those directories and for deleteing them before
+/// shutdown.
+final List<io.Directory> temporaryDirectories = List<io.Directory>();

lib/web_ui/dev/environment.dart

@@ -22,6 +22,7 @@ class Environment {
     final io.Directory hostDebugUnoptDir = io.Directory(pathlib.join(outDir.path, 'host_debug_unopt'));
     final io.Directory dartSdkDir = io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk'));
     final io.Directory webUiRootDir = io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui'));
+    final io.Directory integrationTestsDir = io.Directory(pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web'));
 
     for (io.Directory expectedDirectory in <io.Directory>[engineSrcDir, outDir, hostDebugUnoptDir, dartSdkDir, webUiRootDir]) {
       if (!expectedDirectory.existsSync()) {
@@ -34,6 +35,7 @@ class Environment {
       self: self,
       webUiRootDir: webUiRootDir,
       engineSrcDir: engineSrcDir,
+      integrationTestsDir: integrationTestsDir,
       outDir: outDir,
       hostDebugUnoptDir: hostDebugUnoptDir,
       dartSdkDir: dartSdkDir,
@@ -44,6 +46,7 @@ class Environment {
     this.self,
     this.webUiRootDir,
     this.engineSrcDir,
+    this.integrationTestsDir,
     this.outDir,
     this.hostDebugUnoptDir,
     this.dartSdkDir,
@@ -58,6 +61,9 @@ class Environment {
   /// Path to the engine's "src" directory.
   final io.Directory engineSrcDir;
 
+  /// Path to the web integration tests.
+  final io.Directory integrationTestsDir;
+
   /// Path to the engine's "out" directory.
   ///
   /// This is where you'll find the ninja output, such as the Dart SDK.

lib/web_ui/dev/felt

@@ -56,9 +56,9 @@ install_deps() {
 KERNEL_NAME=`uname`
 if [[ $KERNEL_NAME == *"Darwin"* ]]
 then
-     echo "Running on MacOS. Increase the user limits"
-     ulimit -n 50000
-     ulimit -u 4096
+  echo "Running on MacOS. Increase the user limits"
+  ulimit -n 50000
+  ulimit -u 4096
 fi
 
 if [[ "$FELT_USE_SNAPSHOT" == "false" || "$FELT_USE_SNAPSHOT" == "0" ]]; then

lib/web_ui/dev/felt.dart

@@ -9,6 +9,7 @@ import 'package:args/command_runner.dart';
 
 import 'build.dart';
 import 'clean.dart';
+import 'common.dart';
 import 'licenses.dart';
 import 'test_runner.dart';
 
@@ -41,12 +42,29 @@ void main(List<String> args) async {
     io.exit(64); // Exit code 64 indicates a usage error.
   } catch (e) {
     rethrow;
+  } finally {
+    _cleanup();
   }
 
   // Sometimes the Dart VM refuses to quit.
   io.exit(io.exitCode);
 }
 
+void _cleanup() {
+  // Cleanup remaining processes if any.
+  if (processesToCleanUp.length > 0) {
+    for (io.Process process in processesToCleanUp) {
+      process.kill();
+    }
+  }
+  // Delete temporary directories.
+  if (temporaryDirectories.length > 0) {
+    for (io.Directory directory in temporaryDirectories) {
+      directory.deleteSync(recursive: true);
+    }
+  }
+}
+
 void _listenToShutdownSignals() {
   io.ProcessSignal.sigint.watch().listen((_) {
     print('Received SIGINT. Shutting down.');