Headless test runner

Package.swift

@@ -63,6 +63,7 @@ let package = Package(
         .product(name: "Vapor", package: "vapor"),
         "CartonHelpers",
         "SwiftToolchain",
+        "WebDriverClient",
       ]
     ),
     .target(
@@ -84,6 +85,10 @@ let package = Package(
         "WasmTransformer",
       ]
     ),
+    .target(name: "WebDriverClient", dependencies: [
+      .product(name: "AsyncHTTPClient", package: "async-http-client"),
+      .product(name: "NIOFoundationCompat", package: "swift-nio"),
+    ]),
     // This target is used only for release automation tasks and
     // should not be installed by `carton` users.
     .executableTarget(
@@ -112,5 +117,6 @@ let package = Package(
         .product(name: "TSCTestSupport", package: "swift-tools-support-core"),
       ]
     ),
+    .testTarget(name: "WebDriverClientTests", dependencies: ["WebDriverClient"]),
   ]
 )

README.md

@@ -93,6 +93,9 @@ either: `wasmer`, `node` or `defaultBrowser`. Code that depends on
 [JavaScriptKit](https://github.com/swiftwasm/JavaScriptKit) should pass either `--environment node` or
 `--environment defaultBrowser` options, depending on whether it needs Web APIs to work. Otherwise
 the test run will not succeed, since JavaScript environment is not available with `--environment wasmer`.
+If you want to run your test suite on CI or without GUI but on browser, you can pass `--headless` flag.
+It enables [WebDriver](https://w3c.github.io/webdriver/)-based headless browser testing. Note that you
+need to install a WebDriver executable in `PATH` before running tests.
 You can use the command with a prebuilt test bundle binary instead of building it in carton by passing
 `--prebuilt-test-bundle-path <your binary path>`.
 

Sources/CartonCLI/Commands/Dev.swift

@@ -112,7 +112,7 @@ struct Dev: AsyncParsableCommand {
 
     let sources = try paths.flatMap { try localFileSystem.traverseRecursively($0) }
 
-    try await Server(
+    let server = try await Server(
       .init(
         builder: Builder(
           arguments: build.arguments,
@@ -124,7 +124,6 @@ struct Dev: AsyncParsableCommand {
         ),
         mainWasmPath: build.mainWasmPath,
         verbose: verbose,
-        shouldSkipAutoOpen: skipAutoOpen,
         port: port,
         host: host,
         customIndexPath: customIndexPage.map { AbsolutePath($0, relativeTo: localFileSystem.currentWorkingDirectory!) },
@@ -134,6 +133,11 @@ struct Dev: AsyncParsableCommand {
         entrypoint: Self.entrypoint,
         terminal: terminal
       )
-    ).run()
+    )
+    let localURL = try await server.start()
+    if !skipAutoOpen {
+      openInSystemBrowser(url: localURL)
+    }
+    try await server.waitUntilStop()
   }
 }

Sources/CartonCLI/Commands/Test.swift

@@ -22,6 +22,10 @@ extension Environment: ExpressibleByArgument {}
 
 extension SanitizeVariant: ExpressibleByArgument {}
 
+struct TestError: Error, CustomStringConvertible {
+  let description: String
+}
+
 struct Test: AsyncParsableCommand {
 
   static let configuration = CommandConfiguration(abstract: "Run the tests in a WASI environment.")
@@ -41,6 +45,12 @@ struct Test: AsyncParsableCommand {
   )
   private var environment = Environment.wasmer
 
+  /// It is implemented as a separate flag instead of a `--environment` variant because `--environment`
+  /// is designed to accept specific browser names in the future like `--environment firefox`. 
+  /// Then `--headless` should be able to be used with `defaultBrowser` and other browser values.
+  @Flag(help: "When running browser tests, run the browser in headless mode")
+  var headless: Bool = false
+
   @Option(help: "Turn on runtime checks for various behavior.")
   private var sanitize: SanitizeVariant?
 
@@ -71,6 +81,12 @@ struct Test: AsyncParsableCommand {
     )
   }
 
+  func validate() throws {
+    if headless && environment != .defaultBrowser {
+      throw TestError(description: "The `--headless` flag can be applied only for browser environments")
+    }
+  }
+
   func run() async throws {
     let terminal = InteractiveWriter.stdout
     let toolchain = try await Toolchain(localFileSystem, terminal)
@@ -101,6 +117,7 @@ struct Test: AsyncParsableCommand {
         testFilePath: bundlePath,
         host: host,
         port: port,
+        headless: headless,
         // swiftlint:disable:next force_try
         manifest: try! toolchain.manifest.get(),
         terminal: terminal

Sources/CartonCLI/Commands/TestRunners/BrowserTestRunner.swift

@@ -12,39 +12,165 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import AsyncHTTPClient
 import CartonHelpers
 import CartonKit
 import Foundation
+import NIOCore
+import NIOPosix
 import PackageModel
 import TSCBasic
+import WebDriverClient
 
 private enum Constants {
   static let entrypoint = Entrypoint(fileName: "test.js", sha256: testEntrypointSHA256)
 }
 
+enum BrowserTestRunnerError: Error, CustomStringConvertible {
+  case invalidRemoteURL(String)
+  case failedToFindWebDriver
+
+  var description: String {
+    switch self {
+    case let .invalidRemoteURL(url): return "Invalid remote URL: \(url)"
+    case .failedToFindWebDriver:
+      return """
+      Failed to find WebDriver executable or remote URL to a running driver process.
+      Please make sure that you are satisfied with one of the followings (in order of priority)
+      1. Set `WEBDRIVER_REMOTE_URL` with the address of remote WebDriver like `WEBDRIVER_REMOTE_URL=http://localhost:9515`.
+      2. Set `WEBDRIVER_PATH` with the path to your WebDriver executable.
+      3. `chromedriver`, `geckodriver`, `safaridriver`, or `msedgedriver` has been installed in `PATH`
+      """
+    }
+  }
+}
+
 struct BrowserTestRunner: TestRunner {
   let testFilePath: AbsolutePath
   let host: String
   let port: Int
+  let headless: Bool
   let manifest: Manifest
   let terminal: InteractiveWriter
+  let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1)
+  let httpClient: HTTPClient
+
+  init(
+    testFilePath: AbsolutePath,
+    host: String,
+    port: Int,
+    headless: Bool,
+    manifest: Manifest,
+    terminal: InteractiveWriter
+  ) {
+    self.testFilePath = testFilePath
+    self.host = host
+    self.port = port
+    self.headless = headless
+    self.manifest = manifest
+    self.terminal = terminal
+    httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup))
+  }
+
+  typealias Disposer = () -> ()
+
+  func findAvailablePort() async throws -> SocketAddress {
+    let bootstrap = ServerBootstrap(group: eventLoopGroup)
+    let address = try SocketAddress.makeAddressResolvingHost("127.0.0.1", port: 0)
+    let channel = try await bootstrap.bind(to: address).get()
+    let localAddr = channel.localAddress!
+    try await channel.close()
+    return localAddr
+  }
+
+  func launchDriver(executablePath: String) async throws -> (URL, Disposer) {
+    let address = try await findAvailablePort()
+    let process = Process(arguments: [
+      executablePath, "--port=\(address.port!)",
+    ])
+    terminal.logLookup("Launch WebDriver executable: ", executablePath)
+    try process.launch()
+    let disposer = { process.signal(SIGKILL) }
+    return (URL(string: "http://\(address.ipAddress!):\(address.port!)")!, disposer)
+  }
+
+  func selectWebDriver() async throws -> (URL, Disposer) {
+    let strategies: [() async throws -> (URL, Disposer)?] = [
+      {
+        terminal.logLookup("- checking WebDriver endpoint: ", "WEBDRIVER_REMOTE_URL")
+        guard let value = ProcessInfo.processInfo.environment["WEBDRIVER_REMOTE_URL"] else {
+          return nil
+        }
+        guard let url = URL(string: value) else {
+          throw BrowserTestRunnerError.invalidRemoteURL(value)
+        }
+        return (url, {})
+      },
+      {
+        terminal.logLookup("- checking WebDriver executable: ", "WEBDRIVER_PATH")
+        guard let executable = ProcessEnv.vars["WEBDRIVER_PATH"] else {
+          return nil
+        }
+        let (url, disposer) = try await launchDriver(executablePath: executable)
+        return (url, disposer)
+      },
+      {
+        let driverCandidates = [
+          "chromedriver", "geckodriver", "safaridriver", "msedgedriver",
+        ]
+        terminal.logLookup("- checking WebDriver executable in PATH: ", driverCandidates.joined(separator: ", "))
+        guard let found = driverCandidates.lazy.compactMap({ Process.findExecutable($0) }).first else {
+          return nil
+        }
+        return try await launchDriver(executablePath: found.pathString)
+      },
+    ]
+    for strategy in strategies {
+      if let (url, disposer) = try await strategy() {
+        return (url, disposer)
+      }
+    }
+    throw BrowserTestRunnerError.failedToFindWebDriver
+  }
 
   func run() async throws {
+    defer { try! httpClient.syncShutdown() }
     try Constants.entrypoint.check(on: localFileSystem, terminal)
-    try await Server(
+    let server = try await Server(
       .init(
         builder: nil,
         mainWasmPath: testFilePath,
         verbose: true,
-        shouldSkipAutoOpen: false,
         port: port,
         host: host,
         customIndexPath: nil,
         manifest: manifest,
         product: nil,
         entrypoint: Constants.entrypoint,
         terminal: terminal
-      )
-    ).run()
+      ),
+      .shared(eventLoopGroup)
+    )
+    let localURL = try await server.start()
+    var disposer: () async throws -> () = {}
+    do {
+      if headless {
+        let (endpoint, clientDisposer) = try await selectWebDriver()
+        let client = try await WebDriverClient.newSession(endpoint: endpoint, httpClient: httpClient)
+        disposer = {
+          try await client.closeSession()
+          clientDisposer()
+        }
+        try await client.goto(url: localURL)
+      } else {
+        disposer = {}
+        openInSystemBrowser(url: localURL)
+      }
+      try await server.waitUntilStop()
+      try await disposer()
+    } catch {
+      try await disposer()
+      throw error
+    }
   }
 }