Document available probe endpoints and how to write tests (VirtusLab#12)

* Describe writing tests and available probe endpoints
* Set non-headless mode as the default

README.md

@@ -8,8 +8,63 @@ The framework itself comprises two components:
 - driver - responsible for controlling the workspace and IDE startup
 - probe - used to interact with the IDE  
 
+#### Overview
 
-#### Configuration
+A single test case consists of a configuration and workflow specification. 
+Configuration can either be:
+
+A) loaded from file,
+```scala
+private val config = Config.fromClasspath("path/to/file")
+
+@Test def test = IntelliJFixture.fromConfig(config).run {intelliJ => 
+    // workflow steps
+}
+```
+B) provided as a string, or
+```scala
+private val config = Config.fromString("""probe { workspace.path = /foo/bar } """)
+@Test def test = IntelliJFixture.fromConfig(config).run {intelliJ => 
+    // workflow steps
+}
+```
+C) specified programmatically.
+```scala
+private val fixture = IntelliJFixture(
+  workspaceTemplate = WorkspaceTemplate.fromFile(path),
+  version = IntelliJVersion("202.5792.28-EAP-SNAPSHOT"),
+  plugins = List(Plugin("org.intellij.scala", "2020.2.7"))
+)
+
+@Test def test = fixture.run {intelliJ => 
+    // workflow steps
+} 
+```
+
+Workflow can only be defined programmatically, since it comprises sequence of intertwined:
+1. probe commands,
+2. IDE state queries,
+3. workspace manipulation,
+4. custom verification logic.
+
+```scala
+@Test def test = fixture.run { intelliJ =>
+  val buildSbt = intelliJ.workspace.resolve("build.sbt")
+  Files.write(buildSbt, """name := "foo" """)
+
+  val projectRef = intelliJ.probe.openProject(buildSbt)
+  val structure = intelliJ.probe.projectModel(projectRef)
+
+  assertEquals("foo", structure.name)  
+}
+``` 
+
+To see the full list of probe endpoints see 
+[Commands](docs/endpoints/commands.md) or [Queries](docs/endpoints/queries.md).
+
+Note, that any communication with the probe is synchronous.
+
+#### Configuration 
 
 1. [Driver](docs/driver.md)
 2. [Resolvers](docs/custom-resolvers.md)

api/src/main/scala/org/virtuslab/ideprobe/protocol/Endpoints.scala

@@ -7,28 +7,32 @@ import org.virtuslab.ideprobe.jsonrpc.PayloadJsonFormat._
 import pureconfig.generic.auto._
 
 object Endpoints {
+
+  // commands
+  val AwaitIdle = Request[Unit, Unit]("awaitIdle")
   val AwaitNotification = Request[String, IdeNotification]("notification/await")
-  val PID = Request[Unit, Long]("pid")
-  val Ping = Request[Unit, Unit]("ping")
-  val Shutdown = Notification[Unit]("shutdown")
-  val Plugins = Request[Unit, Seq[InstalledPlugin]]("plugins")
-  val OpenProject = Request[Path, ProjectRef]("project/open")
+  val Build = Request[BuildParams, BuildResult]("build")
   val CloseProject = Request[ProjectRef, Unit]("project/close")
-  val FileReferences = Request[FileRef, Seq[Reference]]("file/references")
-  val Messages = Request[Unit, Seq[IdeMessage]]("messages")
+  val Find = Request[NavigationQuery, List[NavigationTarget]]("find")
   val InvokeActionAsync = Request[String, Unit]("action/invokeAsync")
   val InvokeAction = Request[String, Unit]("action/invoke")
-  val Freezes = Request[Unit, Seq[Freeze]]("freezes")
-  val ModuleSdk = Request[ModuleRef, Option[String]]("module/sdk")
-  val ProjectSdk = Request[ProjectRef, Option[String]]("project/sdk")
-  val ListOpenProjects = Request[Unit, Seq[ProjectRef]]("projects/all")
-  val ProjectModel = Request[ProjectRef, Project]("project/model")
-  val AwaitIdle = Request[Unit, Unit]("awaitIdle")
-  val Build = Request[BuildParams, BuildResult]("build")
+  val OpenProject = Request[Path, ProjectRef]("project/open")
   val Run = Request[ApplicationRunConfiguration, ProcessResult]("run/application")
   val RunJUnit = Request[JUnitRunConfiguration, TestsRunResult]("run/junit")
+  val Shutdown = Notification[Unit]("shutdown")
   val SyncFiles = Request[Unit, Unit]("fs/sync")
-  val Find = Request[NavigationQuery, List[NavigationTarget]]("find")
   val TakeScreenshot = Request[String, Unit]("screenshot")
+
+  // queries
+  val FileReferences = Request[FileRef, Seq[Reference]]("file/references")
+  val Freezes = Request[Unit, Seq[Freeze]]("freezes")
+  val ListOpenProjects = Request[Unit, Seq[ProjectRef]]("projects/all")
+  val Messages = Request[Unit, Seq[IdeMessage]]("messages")
+  val ModuleSdk = Request[ModuleRef, Option[String]]("module/sdk")
+  val PID = Request[Unit, Long]("pid")
+  val Ping = Request[Unit, Unit]("ping")
+  val Plugins = Request[Unit, Seq[InstalledPlugin]]("plugins")
+  val ProjectSdk = Request[ProjectRef, Option[String]]("project/sdk")
+  val ProjectModel = Request[ProjectRef, Project]("project/model")
   val VcsRoots = Request[ProjectRef, Seq[VcsRoot]]("project/vcsRoots")
 }

docs/driver.md

@@ -22,9 +22,9 @@ By default, the driver doesn't fail the test upon detecting any errors or freeze
 
 
 #### Headless mode
-`driver.headless = true`
+`driver.headless = false`
 
-By default, the driver launches the IDE in the headless mode. Note, that the behavior of the IDE can differ between headless and non-headless modes.  
+By default, the driver launches the IDE in the non-headless mode. Note, that the behavior of the IDE can differ between headless and non-headless modes.  
 
 #### Virtual Machine options
 `driver.vmOptions = ["-Xmx4096m"]`

docs/endpoints/commands.md

@@ -0,0 +1,13 @@
+#### Probe commands
+
+- Await Notification - waits until the IDE issues a notification with a specified title
+- Await Idle - waits until there is no more background tasks running
+- Build - builds the specified file, module or project
+- Close Project - closes the specified project
+- Invoke Action Async - invokes the specified actions without waiting for it to finish
+- Invoke Action - invokes the specified actions and waits for it to finish
+- Open Project - opens the specified file as a project
+- Run - runs main class using the specified configuration
+- Run JUnit - runs the specified JUnit configuration
+- Shutdown - starts the process of shutting down the IDE
+- Take screenshot - saves the current view of the IDE alongside the automatically captured screenshots 

docs/endpoints/queries.md

@@ -0,0 +1,12 @@
+#### Probe queries
+
+- File References - returns all the files referenced by the specified file
+- Find - returns all navigable elements matching the specified pattern in the specified project
+- Freezes - returns the list of all freezes detected by the IDE
+- List Open Projects - lists currently open projects
+- Messages - returns the log of all messages produced by the IDE
+- Module SDK - returns the SDK of the specified module
+- PID - returns the Process ID of the IDE
+- Plugins - returns the list of all installed plugins
+- Project SDK - returns the SDK of the specified project
+- Sync Files - refreshes the file cache (useful, when those were modified outside of IDE)

driver/bindings/junit/src/main/scala/org/virtuslab/ideprobe/IntelliJFixture.scala

@@ -30,8 +30,8 @@ final case class IntelliJFixture(
     copy(afterWorkspaceSetup = afterWorkspaceSetup :+ action)
   }
 
-  def withDisplay: IntelliJFixture = {
-    copy(factory = factory.withConfig(factory.config.copy(headless = false)))
+  def headless: IntelliJFixture = {
+    copy(factory = factory.withConfig(factory.config.copy(headless = true)))
   }
 
   def run = new SingleRunIntelliJ(this)

driver/sources/src/main/scala/org/virtuslab/ideprobe/ProbeDriver.scala

@@ -1,7 +1,6 @@
 package org.virtuslab.ideprobe
 
 import java.nio.file.Path
-
 import org.virtuslab.ideprobe.jsonrpc.JsonRpc.Handler
 import org.virtuslab.ideprobe.jsonrpc.JsonRpc.Method
 import org.virtuslab.ideprobe.jsonrpc.JsonRpcConnection
@@ -26,7 +25,6 @@ import org.virtuslab.ideprobe.protocol.ProjectRef
 import org.virtuslab.ideprobe.protocol.Reference
 import org.virtuslab.ideprobe.protocol.TestsRunResult
 import org.virtuslab.ideprobe.protocol.VcsRoot
-
 import scala.annotation.tailrec
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
@@ -43,8 +41,6 @@ final class ProbeDriver(protected val connection: JsonRpcConnection)(implicit pr
 
   def listOpenProjects(): Seq[ProjectRef] = send(Endpoints.ListOpenProjects)
 
-  def openProject(path: Path): ProjectRef = send(Endpoints.OpenProject, path)
-
   def openProjectWithName(path: Path, expectedName: String): ProjectRef = {
     val expectedRef = ProjectRef(expectedName)
     val projectRef = openProject(path)
@@ -100,64 +96,140 @@ final class ProbeDriver(protected val connection: JsonRpcConnection)(implicit pr
     }
   }
 
-  def closeProject(name: ProjectRef = ProjectRef.Default): Unit = send(Endpoints.CloseProject, name)
+  /**
+   * Forces the probe to wait until all background tasks are complete before processing next request
+   */
+  def awaitIdle(): Unit = send(Endpoints.AwaitIdle)
 
-  def ping(): Unit = send(Endpoints.Ping)
+  /**
+   * Forces the probe to wait until the specified notification is issued by the IDE
+   */
+  def awaitNotification(title: String): IdeNotification = send(Endpoints.AwaitNotification, title)
 
-  def plugins: Seq[InstalledPlugin] = send(Endpoints.Plugins).toList
+  /**
+   * Builds the specified files, modules or project
+   */
+  def build(scope: BuildScope = BuildScope.project): BuildResult = build(BuildParams(scope, rebuild = false))
 
-  def shutdown(): Unit = send(Endpoints.Shutdown)
+  /**
+   * Closes specified project
+   */
+  def closeProject(name: ProjectRef = ProjectRef.Default): Unit = send(Endpoints.CloseProject, name)
 
+  /**
+   * Invokes the specified actions without waiting for it to finish
+   */
   def invokeActionAsync(id: String): Unit = send(Endpoints.InvokeActionAsync, id)
 
+  /**
+   * Invokes the specified actions and waits for it to finish
+   */
   def invokeAction(id: String): Unit = send(Endpoints.InvokeAction, id)
 
+  /**
+   * Opens specified project
+   */
+  def openProject(path: Path): ProjectRef = send(Endpoints.OpenProject, path)
+
+  /**
+   * Rebuilds the specified files, modules or project
+   */
+  def rebuild(scope: BuildScope = BuildScope.project): BuildResult = build(BuildParams(scope, rebuild = true))
+
+  /**
+   * starts the process of shutting down the IDE
+   */
+  def shutdown(): Unit = send(Endpoints.Shutdown)
+
+  /**
+   * Refreshes the file cache (useful, when those were modified outside of IDE)
+   */
+  def syncFiles(): Unit = send(Endpoints.SyncFiles)
+
+  /**
+   * Finds all the files referenced by the specified file
+   */
   def fileReferences(project: ProjectRef = ProjectRef.Default, path: Path): Seq[Reference] = {
     send(Endpoints.FileReferences, FileRef(project, path))
   }
 
+  /**
+   * Finds all the files referenced by the specified file
+   */
   def fileReferences(fileRef: FileRef): Seq[Reference] = {
     send(Endpoints.FileReferences, fileRef)
   }
 
+  /**
+   * Finds all navigable elements matching the specified pattern in the specified project
+   */
   def find(query: NavigationQuery): List[NavigationTarget] = {
     send(Endpoints.Find, query)
   }
 
+  /**
+   * Returns the list of all errors produced by the IDE
+   */
   def errors: Seq[IdeMessage] = send(Endpoints.Messages).filter(_.isError).toList
 
+  /**
+   * Returns the list of all warnings produced by the IDE
+   */
   def warnings: Seq[IdeMessage] = send(Endpoints.Messages).filter(_.isWarn).toList
 
+  /**
+   * Returns the list of all messages produced by the IDE
+   */
   def messages: Seq[IdeMessage] = send(Endpoints.Messages).toList
 
+  /**
+   * Returns the model of the specified project
+   */
   def projectModel(name: ProjectRef = ProjectRef.Default): Project = send(Endpoints.ProjectModel, name)
 
-  def awaitIdle(): Unit = send(Endpoints.AwaitIdle)
-
-  def syncFiles(): Unit = send(Endpoints.SyncFiles)
-
+  /**
+   * Returns the list of all freezes detected by the IDE
+   */
   def freezes: Seq[Freeze] = send(Endpoints.Freezes)
 
-  def build(scope: BuildScope = BuildScope.project): BuildResult = build(BuildParams(scope, rebuild = false))
-
-  def rebuild(scope: BuildScope = BuildScope.project): BuildResult = build(BuildParams(scope, rebuild = true))
-
-  private def build(params: BuildParams): BuildResult = send(Endpoints.Build, params)
-
-  def awaitNotification(title: String): IdeNotification = send(Endpoints.AwaitNotification, title)
-
+  /**
+   * Runs the specified application configuration
+   */
   def run(runConfiguration: ApplicationRunConfiguration): ProcessResult = send(Endpoints.Run, runConfiguration)
 
+  /**
+   * Runs the specified JUnit configuration
+   */
   def run(runConfiguration: JUnitRunConfiguration): TestsRunResult = send(Endpoints.RunJUnit, runConfiguration)
 
+  /**
+   * Saves the current view of the IDE alongside the automatically captured screenshots
+   * with the specified name suffix
+   */
+  def screenshot(nameSuffix: String = ""): Unit = send(Endpoints.TakeScreenshot, nameSuffix)
+
+  /**
+   * Returns the sdk of the specified project
+   */
   def projectSdk(project: ProjectRef = ProjectRef.Default): Option[String] = send(Endpoints.ProjectSdk, project)
 
+  /**
+   * Returns the sdk of the specified module
+   */
   def moduleSdk(module: ModuleRef): Option[String] = send(Endpoints.ModuleSdk, module)
 
-  def screenshot(nameSuffix: String = ""): Unit = send(Endpoints.TakeScreenshot, nameSuffix)
-
+  /**
+   * Returns the list of VCS roots of the specified project
+   */
   def vcsRoots(project: ProjectRef = ProjectRef.Default): Seq[VcsRoot] = send(Endpoints.VcsRoots, project)
 
+  /**
+   * Returns the list of all installed plugins
+   */
+  def plugins: Seq[InstalledPlugin] = send(Endpoints.Plugins).toList
+
+  def ping(): Unit = send(Endpoints.Ping)
+
   def as[A](extensionPluginId: String, convert: ProbeDriver => A): A = {
     val isLoaded = plugins.exists(_.id == extensionPluginId)
     if (isLoaded) convert(this)
@@ -171,6 +243,8 @@ final class ProbeDriver(protected val connection: JsonRpcConnection)(implicit pr
   def send[R: ClassTag](method: Method[Unit, R]): R = {
     send(method, ())
   }
+
+  private def build(params: BuildParams): BuildResult = send(Endpoints.Build, params)
 }
 
 object ProbeDriver {

driver/sources/src/main/scala/org/virtuslab/ideprobe/ide/intellij/DriverConfig.scala

@@ -6,7 +6,7 @@ import scala.concurrent.duration._
 case class DriverConfig(
     launch: LaunchParameters = LaunchParameters(),
     check: CheckConfig = CheckConfig(),
-    headless: Boolean = true,
+    headless: Boolean = false,
     vmOptions: Seq[String] = Nil
 )