Merge pull request #113 from codeoverflow-org/feature/105-plugin-dependencies

Implement Coursier for plugin dependency resolution and fetching

Author: sebinside

bootstrap/build.sbt

@@ -4,4 +4,6 @@ assemblyJarName in assembly := "ChatOverflow.jar"
 
 libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.1.1"
 libraryDependencies += "org.jline" % "jline-terminal-jansi" % "3.11.0" // used for terminal width
-fork := true
+fork := true
+
+packageBin / includePom := false

build.sbt

@@ -89,6 +89,9 @@ libraryDependencies += "com.fazecast" % "jSerialComm" % "[2.0.0,3.0.0)"
 // Socket.io
 libraryDependencies += "io.socket" % "socket.io-client" % "1.0.0"
 
+// Coursier
+libraryDependencies += "io.get-coursier" %% "coursier" % "2.0.0-RC3-2"
+
 // ---------------------------------------------------------------------------------------------------------------------
 // PLUGIN FRAMEWORK DEFINITIONS
 // ---------------------------------------------------------------------------------------------------------------------
@@ -135,6 +138,7 @@ Compile / packageBin := {
 }
 
 Compile / unmanagedJars := (crossTarget.value ** "chatoverflow-gui*.jar").classpath
+packageBin / includePom := false
 
 fork in run := true // Start ChatOverflow in it's own java process when starting it with 'sbt run'
 

build/src/main/scala/org/codeoverflow/chatoverflow/build/PomInclusionPlugin.scala

@@ -0,0 +1,47 @@
+package org.codeoverflow.chatoverflow.build
+
+import sbt._
+import sbt.Keys._
+import sbt.plugins.JvmPlugin
+
+/**
+ * A sbt plugin to automatically include the dependencies of a sbt project in the jar as a pom file called "dependencies.pom".
+ */
+object PomInclusionPlugin extends AutoPlugin {
+
+  // Everything in autoImport will be visible to sbt project files
+  // They can set this value to false if they don't want to include their dependencies as a pom file
+  object autoImport {
+    val includePom = settingKey[Boolean]("Whether to include a pom file inside the jar with all dependencies.")
+  }
+  import autoImport._
+
+  // We require to have the Compile configuration and the packageBin task to override
+  override def requires = JvmPlugin
+  override def trigger = allRequirements
+
+  // Adds our custom task before the packageBin task
+  override val projectSettings: Seq[Def.Setting[_]] =
+    inConfig(Compile)(Seq(
+      Compile / packageBin := {
+        addPomToOutput.value
+        (Compile / packageBin).value
+      }
+    ))
+
+  // Sets default values
+  override def buildSettings: Seq[Def.Setting[_]] = inConfig(Compile)(
+    includePom in packageBin := true
+  )
+
+  // Just copies the pom resulted by makePom into the directory for compiled classes
+  // That way the file will be included in the jar
+  private lazy val addPomToOutput = Def.taskDyn {
+    if ((includePom in packageBin).value) Def.task {
+      val pomFile = (Compile / makePom).value
+
+      IO.copyFile(pomFile, new File((Compile / classDirectory).value, "dependencies.pom"))
+    } else
+      Def.task {} // if disabled, do nothing
+  }
+}

src/main/scala/org/codeoverflow/chatoverflow/framework/PluginFramework.scala

@@ -7,6 +7,10 @@ import org.codeoverflow.chatoverflow.framework.helper.PluginLoader
 import org.codeoverflow.chatoverflow.framework.manager.PluginManagerStub
 
 import scala.collection.mutable.ListBuffer
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.concurrent.{Await, Future}
+import scala.util.Success
 
 /**
   * The plugin framework holds all plugin types important from the jar files in the plugin folder.
@@ -62,6 +66,8 @@ class PluginFramework(pluginDirectoryPath: String) extends WithLogger {
       logger warn s"PluginType directory '$pluginDirectory' does not exist!"
     } else {
 
+      val futures = ListBuffer[Future[_]]()
+
       // Get (new) jar file urls
       val jarFiles = getNewJarFiles(pluginDirectory)
       logger info s"Found ${jarFiles.length} new plugins."
@@ -81,22 +87,28 @@ class PluginFramework(pluginDirectoryPath: String) extends WithLogger {
           } else {
 
             // Try to test the initiation of the plugin
-            try {
-              plugin.createPluginInstance(new PluginManagerStub)
-              logger info s"Successfully tested instantiation of plugin '${plugin.getName}'"
-              pluginTypes += plugin
-            } catch {
-              // Note that we catch not only exceptions, but also errors like NoSuchMethodError. Deep stuff
-              case _: Error => logger warn s"Error while test init of plugin '${plugin.getName}'."
-              case _: Exception => logger warn s"Exception while test init of plugin '${plugin.getName}'."
+            futures += plugin.getDependencyFuture andThen {
+              case Success(_) =>
+                try {
+                  plugin.createPluginInstance(new PluginManagerStub)
+                  logger info s"Successfully tested instantiation of plugin '${plugin.getName}'"
+                  pluginTypes += plugin
+                } catch {
+                  // Note that we catch not only exceptions, but also errors like NoSuchMethodError. Deep stuff
+                  case _: Error => logger warn s"Error while test init of plugin '${plugin.getName}'."
+                  case _: Exception => logger warn s"Exception while test init of plugin '${plugin.getName}'."
+                }
             }
           }
         }
       }
-    }
 
-    logger info s"Loaded ${pluginTypes.length} plugin types in total: " +
-      s"${pluginTypes.map(pt => s"${pt.getName} (${pt.getAuthor})").mkString(", ")}"
+      // If plugins aren't done within this timeout they can still fetch everything in the background, they just won't be included in this summary
+      futures.foreach(f => Await.ready(f, 5.seconds))
+
+      logger info s"Loaded ${pluginTypes.length} plugin types in total: " +
+        s"${pluginTypes.map(pt => s"${pt.getName} (${pt.getAuthor})").mkString(", ")}"
+    }
   }
 
   /**

src/main/scala/org/codeoverflow/chatoverflow/framework/PluginType.scala

@@ -1,25 +1,31 @@
 package org.codeoverflow.chatoverflow.framework
 
+import java.io.File
+
 import org.codeoverflow.chatoverflow.WithLogger
 import org.codeoverflow.chatoverflow.api.APIVersion
 import org.codeoverflow.chatoverflow.api.plugin.{Plugin, PluginManager}
 import org.codeoverflow.chatoverflow.framework.PluginCompatibilityState.PluginCompatibilityState
 
+import scala.concurrent.Future
+
 /**
-  * A plugin type is a container for all information about a plugin, everything in the 'plugin.xml' and the actual class.
-  * The plugins functionality and meta information can be accessed through this interface.
-  *
-  * @param name            the name of the plugin, used for identifying
-  * @param author          the author of the plugin, used for identifying
-  * @param version         the version of the plugin
-  * @param majorAPIVersion the major api version, with which the plugin was developed
-  * @param minorAPIVersion the minor api version, with which the plugin was developed
-  * @param pluginClass     the class of the plugin, used to create instances of this plugin.
-  *                        Needs to have a constructor with the signature of one PluginManager,
-  *                        otherwise instances can't be created from it.
-  */
+ * A plugin type is a container for all information about a plugin, everything in the 'plugin.xml' and the actual class.
+ * The plugins functionality and meta information can be accessed through this interface.
+ *
+ * @param name               the name of the plugin, used for identifying
+ * @param author             the author of the plugin, used for identifying
+ * @param version            the version of the plugin
+ * @param majorAPIVersion    the major api version, with which the plugin was developed
+ * @param minorAPIVersion    the minor api version, with which the plugin was developed
+ * @param pluginClass        the class of the plugin, used to create instances of this plugin.
+ *                           Needs to have a constructor with the signature of one PluginManager,
+ *                           otherwise instances can't be created from it.
+ * @param pluginDependencies A future that completes when all dependencies, that the plugin has, are available and
+ *                           that returns a seq of the required dependencies files in the local coursier cache.
+ */
 class PluginType(name: String, author: String, version: String, majorAPIVersion: Int, minorAPIVersion: Int,
-                 metadata: PluginMetadata, pluginClass: Class[_ <: Plugin]) extends WithLogger {
+                 metadata: PluginMetadata, pluginClass: Class[_ <: Plugin], pluginDependencies: Future[Seq[File]]) extends WithLogger {
 
   private var pluginVersionState = PluginCompatibilityState.Untested
 
@@ -126,4 +132,12 @@ class PluginType(name: String, author: String, version: String, majorAPIVersion:
     * @return the PluginMetadata instance of this plugin
     */
   def getMetadata: PluginMetadata = metadata
+
+  /**
+   * Returns the future that will result in a sequence of jar files that represents the dependencies
+   * of this plugin including all sub-dependencies.
+   *
+   * @return the dependency future
+   */
+  def getDependencyFuture: Future[Seq[File]] = pluginDependencies
 }

src/main/scala/org/codeoverflow/chatoverflow/framework/helper/CoursierUtils.scala

@@ -0,0 +1,58 @@
+package org.codeoverflow.chatoverflow.framework.helper
+
+import java.io.{File, InputStream}
+
+import coursier.Fetch
+import coursier.cache.{CacheLogger, FileCache}
+import coursier.core.Dependency
+import coursier.maven.PomParser
+import org.codeoverflow.chatoverflow.WithLogger
+
+import scala.io.Source
+
+/**
+ * A utility object containing some common code for use with Coursier.
+ */
+object CoursierUtils extends WithLogger {
+
+  private object CoursierLogger extends CacheLogger {
+    override def downloadedArtifact(url: String, success: Boolean): Unit = {
+      logger debug (if (success)
+        s"Successfully downloaded $url"
+      else
+        s"Failed to download $url")
+    }
+  }
+
+  private val cache = FileCache().noCredentials.withLogger(CoursierLogger)
+
+  /**
+   * Extracts all dependencies out of the provided pom. Throws an exception if the pom is invalid.
+   *
+   * @param is the InputStream from which the pom is read
+   * @return a seq of all found dependencies
+   */
+  def parsePom(is: InputStream): Seq[Dependency] = {
+    val pomFile = Source.fromInputStream(is)
+    val parser = coursier.core.compatibility.xmlParseSax(pomFile.mkString, new PomParser)
+
+    parser.project match {
+      case Right(deps) => deps.dependencies.map(_._2)
+      case Left(errorMsg) => throw new IllegalArgumentException(s"Pom couldn't be parsed: $errorMsg")
+    }
+  }
+
+  /**
+   * Resolves and fetches all passed dependencies and gives back a seq of all local files of these dependencies.
+   *
+   * @param dependencies all dependencies that you want to be fetched
+   * @return all local files for the passed dependencies
+   */
+  def fetchDependencies(dependencies: Seq[Dependency]): Seq[File] = {
+    // IntelliJ may warn you that a implicit is missing. This is one of the many bugs in IntelliJ, the code compiles fine.
+    Fetch()
+      .withCache(cache)
+      .addDependencies(dependencies: _*)
+      .run()
+  }
+}

src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginClassLoader.scala

@@ -2,9 +2,52 @@ package org.codeoverflow.chatoverflow.framework.helper
 
 import java.net.{URL, URLClassLoader}
 
+import org.codeoverflow.chatoverflow.WithLogger
+
+/**
+ * This plugin class loader is used for plugin security policy checks,
+ * to expose the addURL method to the package inorder to add all required dependencies after dependency resolution
+ * and most importantly to isolate the plugin from the normal classpath and only access the classpath if it needs to load the ChatOverflow api.
+ * Also if this PluginClassLoader had access to the classpath the same classes of the classpath would have
+ * higher priority over the classes in this classloader which could be a problem  if a plugin uses a newer version
+ * of a dependency that the framework.
+ *
+ * @param urls Takes an array of urls an creates a simple URLClassLoader with it
+ */
+class PluginClassLoader(urls: Array[URL]) extends URLClassLoader(urls, PluginClassLoader.platformClassloader) {
+  // Note the platform classloader in the constructor of the URLClassLoader as the parent.
+  // That way the classloader skips the app classloader with the classpath when it is asks it's parents for classes.
+
+  protected[helper] override def addURL(url: URL): Unit = super.addURL(url) // just exposes this method to be package-private instead of class internal protected
+
+  override def loadClass(name: String, resolve: Boolean): Class[_] = {
+    if (name.startsWith("org.codeoverflow.chatoverflow.api")) {
+      PluginClassLoader.appClassloader.loadClass(name) // Api needs to be loaded from the classpath
+    } else {
+      super.loadClass(name, resolve) // non api class. load it as normal
+    }
+  }
+}
+
 /**
-  * This plugin class loader does only exist for plugin security policy checks.
-  *
-  * @param urls Takes an array of urls an creates a simple URLClassLoader with it
-  */
-class PluginClassLoader(urls: Array[URL]) extends URLClassLoader(urls)
+ * This companion object holds references to the app classloader (normal classloader, includes java and classpath)
+ * and to the extension/platform classloader depending on the java version that excludes the classpath,
+ * but still includes everything from java.
+ */
+private object PluginClassLoader extends WithLogger {
+  val appClassloader: ClassLoader = this.getClass.getClassLoader
+  val platformClassloader: ClassLoader = {
+    var current = appClassloader
+    while (current != null && !current.getClass.getName.contains("ExtClassLoader") && // ExtClassLoader is java < 9
+      !current.getClass.getName.contains("PlatformClassLoader")) { // PlatformClassLoader is java >= 9
+      current = current.getParent
+    }
+
+    if (current != null) {
+      current
+    } else {
+      logger error "Platform classloader couldn't be found. Falling back to normal app classloader."
+      appClassloader
+    }
+  }
+}

src/main/scala/org/codeoverflow/chatoverflow/framework/helper/PluginLoader.scala

@@ -10,6 +10,9 @@ import org.reflections.scanners.SubTypesScanner
 import org.reflections.util.ConfigurationBuilder
 
 import scala.collection.JavaConverters._
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
 import scala.xml.{Node, SAXParseException, XML}
 
 /**
@@ -80,7 +83,8 @@ class PluginLoader(private val jar: File) extends WithLogger {
         majorString.toInt,
         minorString.toInt,
         PluginMetadata.fromXML(p),
-        cls
+        cls,
+        resolveDependencies(getString(p, "name"))
       ))
     } catch {
       // thrown by getString
@@ -144,4 +148,29 @@ class PluginLoader(private val jar: File) extends WithLogger {
         None
     }
   }
+
+  /**
+   * Creates a future which gets all dependencies from the included dependencies.pom, if existing, fetches them
+   * and adds their jar files to the classloader.
+   *
+   * @param pluginName the name of the plugin, only used for logging
+   * @return a future of all required jars for this plugin
+   */
+  private def resolveDependencies(pluginName: String): Future[Seq[File]] = {
+    val pomIs = classloader.getResourceAsStream("dependencies.pom")
+    if (pomIs == null) {
+      return Future(Seq())
+    }
+
+    Future(CoursierUtils.parsePom(pomIs))
+      .map(dependencies => dependencies.filter(_.module.name.value != "chatoverflow-api_2.12"))
+      .map(dependencies => CoursierUtils.fetchDependencies(dependencies))
+      .andThen {
+        case Success(jarFiles) =>
+          jarFiles.foreach(jar => classloader.addURL(jar.toURI.toURL))
+          logger info s"Dependencies for the plugin $pluginName successfully resolved and fetched if missing."
+        case Failure(exception) =>
+          logger warn s"Couldn't resolve and fetch dependencies for the plugin in $pluginName: $exception"
+      }
+  }
 }

src/main/scala/org/codeoverflow/chatoverflow/instance/PluginInstance.scala

@@ -99,6 +99,11 @@ class PluginInstance(val instanceName: String, pluginType: PluginType) extends W
 
         } else {
 
+          if (!areDependenciesAvailable) {
+            logger error "Dependencies have either failed to resolve and fetch or aren't done yet."
+            return false
+          }
+
           // This is set to false if any connector (aka input/output) is not ready.
           var allConnectorsReady = true
 
@@ -224,6 +229,16 @@ class PluginInstance(val instanceName: String, pluginType: PluginType) extends W
     }
   }
 
+  /**
+   * Returns whether all dependencies are resolved and fetched which is required for the plugin to start.
+   *
+   * @return true if all dependencies are available, false if it has failed or aren't downloaded yet.
+   */
+  def areDependenciesAvailable: Boolean = {
+    val opt = pluginType.getDependencyFuture.value
+    opt.isDefined && opt.get.isSuccess
+  }
+
   /**
     * Returns if the plugin is currently executed (the thread is running)
     *