Merge pull request #2 from codeoverflow-org/master

merge

Author: derNiklaas

bootstrap/build.sbt

@@ -3,4 +3,5 @@ version := "0.1"
 assemblyJarName in assembly := "ChatOverflow.jar"
 
 libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.1.1"
+libraryDependencies += "org.jline" % "jline-terminal-jansi" % "3.11.0" // used for terminal width
 fork := true

bootstrap/src/main/scala/Bootstrap.scala

@@ -34,11 +34,6 @@ object Bootstrap {
         if (javaPath.isDefined) {
           println("Found java installation. Starting ChatOverflow...")
 
-          // Create config folder, if not existent
-          if (!new File("config/").exists()) {
-            new File("config/").mkdir()
-          }
-
           // Start chat overflow!
           val process = new java.lang.ProcessBuilder(javaPath.get, "-cp", s"bin/*${File.pathSeparator}lib/*", chatOverflowMainClass)
             .inheritIO().start()
@@ -85,69 +80,118 @@ object Bootstrap {
   }
 
   /**
-    * Checks if the library folder exists or the reload-flag is set. Triggers the download-process.
+    * Checks if the library folder exists or the reload-flag is set. Triggers the download-process if libraries are missing.
     *
     * @param args the args, the launcher has been called with
     * @return false, if there is a serious problem
     */
   def checkLibraries(args: Array[String]): Boolean = {
 
-    // TODO: Someday in the future, we need incremental library checking to manage updates without full download
-
     val libFolder = new File(s"$currentFolderPath/lib")
-    // Args contains --reload or lib folder is non existent?
-    if ((args.length > 0 && args.head == "--reload") || !libFolder.exists()) {
-
-      // Create or clean directory
-      if (libFolder.exists()) {
-        for (libFile <- libFolder.listFiles()) {
-          try {
-            libFile.delete()
-          } catch {
-            case e: Exception => println(s"Unable to delete file '${libFile.getName}'. Message: ${e.getMessage}")
-          }
-        }
-      } else {
+
+    // Create folder for libs if missing
+    if (!libFolder.exists()) {
+      try {
+        libFolder.mkdir()
+      } catch {
+        case e: Exception => println(s"Unable to create library directory. Message: ${e.getMessage}")
+          return false
+      }
+    }
+
+    // --reload flags instructs to delete all downloaded libraries and to re-download them
+    if (args.contains("--reload")) {
+      for (libFile <- libFolder.listFiles()) {
         try {
-          libFolder.mkdir()
+          libFile.delete()
         } catch {
-          case e: Exception => println(s"Unable to create library directory. Message: ${e.getMessage}")
+          case e: Exception => println(s"Unable to delete file '${libFile.getName}'. Message: ${e.getMessage}")
+            return false
         }
       }
+    }
 
-      // Download all libraries
-      // TODO: Check validity if everything is downloaded
-      println("Downloading libraries...")
-      downloadLibraries()
+    val dependencies = getDependencies
 
-    } else {
-      println("Found libraries folder. Assuming all dependencies are available properly.")
-      true
-    }
+    // Download all libraries
+    // TODO: Check validity if everything is downloaded
+    // try downloading libs and only if it succeeded (returned true) then try to delete older libs
+    downloadMissingLibraries(dependencies) && deleteUndesiredLibraries(dependencies)
   }
 
   /**
-    * Reads the dependency xml file and tries to download every library.
+    * Reads the dependency xml file and tries to download every library that is missing.
     *
     * @return false, if there is a serious problem
     */
-  def downloadLibraries(): Boolean = {
-
-    // Get dependency xml and read dependencies with their download URL
-    val dependencyStream = getClass.getResourceAsStream("/dependencies.xml")
-    val dependencyXML = xml.XML.load(dependencyStream)
-    val dependencies = for (dependency <- dependencyXML \\ "dependency")
-      yield ((dependency \ "name").text.trim, (dependency \ "url").text.trim)
-
-    for (i <- dependencies.indices) {
-      val dependency = dependencies(i)
-      println(s"[${i + 1}/${dependencies.length}] ${dependency._1} (${dependency._2})")
-      if (!downloadLibrary(dependency._1, dependency._2)) {
-        // Second try, just in case
-        downloadLibrary(dependency._1, dependency._2)
+  private def downloadMissingLibraries(dependencies: List[(String, String)]): Boolean = {
+    val pb = new ProgressBar(dependencies.length)
+
+    // using par here to make multiple http requests in parallel, otherwise its awfully slow on internet connections with high RTTs
+    val missing = dependencies.par.filterNot(dep => {
+      val (name, url) = dep
+      pb.countUp()
+      pb.updateDescription(s"$name@$url")
+
+      isLibraryDownloaded(url)
+    }).toList
+
+    pb.finish()
+
+    if (missing.isEmpty) {
+      println("All required libraries are already downloaded.")
+    } else {
+      println(s"Downloading ${missing.length} missing libraries...")
+
+      val pb = new ProgressBar(missing.length)
+
+      for ((name, url) <- missing) {
+        pb.countUp()
+        pb.updateDescription(s"$name@$url")
+
+        if (!downloadLibrary(name, url)) {
+          // Second try, just in case
+          if (!downloadLibrary(name, url)) {
+            return false // error has been displayed, stop bootstrapper from starting with missing lib
+          }
+        }
       }
+
+      pb.finish()
+    }
+    true // everything went fine
+  }
+
+  /**
+    * Deletes all undesired libraries. Currently these are all libs that aren't on the list of dependencies.
+    * The main responsibility is to delete old libs that got updated or libs that aren't required anymore by Chat Overflow.
+    *
+    * @param dependencies the libs that should be kept
+    * @return false, if a file couldn't be deleted
+    */
+  private def deleteUndesiredLibraries(dependencies: List[(String, String)]): Boolean = {
+    val libDir = new File(s"$currentFolderPath/lib")
+    if (libDir.exists() && libDir.isDirectory) {
+      // Desired filenames
+      val libraryFilenames = dependencies.map(d => libraryFile(d._2).getName)
+
+      val undesiredFiles = libDir.listFiles().filterNot(file => libraryFilenames.contains(file.getName)) // filter out libs on the dependency list
+
+      // Count errors while trying to remove undesired files
+      val errorCount = undesiredFiles.count(file => {
+        println(s"Deleting old or unnecessary library at $file")
+        if (file.delete()) {
+          false // no error
+        } else {
+          println(s"Error: Couldn't delete file $file.")
+          true // error
+        }
+      })
+      errorCount == 0 // return false if at least one error occurred
+    } else {
+      // Shouldn't be possible, because this is called from checkLibraries, which creates this directory.
+      true
     }
-    true
   }
 
   /**
@@ -169,7 +213,7 @@ object Bootstrap {
       else {
         // Save file in the lib folder (keeping the name and type)
         try {
-          url #> new File(s"$currentFolderPath/lib/${libraryURL.substring(libraryURL.lastIndexOf("/"))}") !!
+          url #> libraryFile(libraryURL) !!
 
           true
         } catch {
@@ -187,6 +231,53 @@ object Bootstrap {
     }
   }
 
+  /**
+    * Gets all required dependencies from the dependencies.xml in the jar file
+    *
+    * @return a list of tuples that contain the name (e.g. log4j) without org or version and the url.
+    */
+  private def getDependencies: List[(String, String)] = {
+    val stream = getClass.getResourceAsStream("/dependencies.xml")
+    val depXml = xml.XML.load(stream)
+    val dependencies = depXml \\ "dependency"
+    val dependencyTuples = dependencies.map(dep => {
+      val name = (dep \ "name").text.trim
+      val url = (dep \ "url").text.trim
+      (name, url)
+    })
+
+    dependencyTuples.toList
+  }
+
+  /**
+    * Checks whether this library is fully downloaded
+    *
+    * @param libraryURL the url of the library
+    * @return true if it is completely downloaded, false if only partially downloaded or not downloaded at all
+    */
+  private def isLibraryDownloaded(libraryURL: String): Boolean = {
+    val f = libraryFile(libraryURL)
+
+    if (!f.exists()) {
+      false
+    } else {
+      try {
+        // We assume here that the libs don't change at the repo.
+        // While this is true for Maven Central, which is immutable once a file has been uploaded, its not for JCenter.
+        // Updating a released artifact generally isn't valued among developers
+        // and the odds of the updated artifact having the same size is very unlikely.
+        val url = new URL(libraryURL)
+        url.openConnection().getContentLengthLong == f.length()
+      } catch {
+        case _: Exception => false
+      }
+    }
+  }
+
+  private def libraryFile(libraryURL: String): File = {
+    new File(s"$currentFolderPath/lib/${libraryURL.substring(libraryURL.lastIndexOf("/"))}")
+  }
+
   /**
     * Checks, if the installation is valid
     */

bootstrap/src/main/scala/ProgressBar.scala

@@ -0,0 +1,95 @@
+import org.jline.terminal.TerminalBuilder
+
+/**
+  * Progress bar used reporting the status while checking and downloading libs.
+  *
+  * @param max count of events e.g. length of the list which progress is monitored.
+  */
+class ProgressBar(max: Int) {
+  // Width of the terminal, used for size calculations
+  private val width = {
+    val width = TerminalBuilder.builder().dumb(true).build().getWidth
+
+    // Size couldn't be figured out, use a default
+    if (width <= 10)
+      80
+    else
+      width
+  }
+
+  private var count = 0
+  private var description = ""
+
+  // We need to create a empty line so that latest line before creation won't be overwritten by the draw method.
+  println()
+  draw() // Initial draw
+
+  /**
+    * Increases count by 1 and re-draws the progress bar with the updated count.
+    */
+  def countUp(): Unit = {
+    // Thread-safeness when working with parallel collections
+    count.synchronized {
+      count += 1
+      draw()
+    }
+  }
+
+  /**
+    * Updates the description and re-draws the description line.
+    * @param desc the new description
+    */
+  def updateDescription(desc: String): Unit = {
+    // Thread-safeness when working with parallel collections
+    description.synchronized {
+      description = desc
+      drawDescription()
+    }
+  }
+
+  /**
+    * Deletes the description to result in a blank line. The progress bar will still be visible.
+    * After this you can normally print at the beginning of a new line and don't start at the end of the description.
+    */
+  def finish(): Unit = {
+    description = ""
+    drawDescription()
+  }
+
+  /**
+    * Draws the progress bar in the line above the current one.
+    */
+  private def draw(): Unit = {
+    val barWidth = width - 16 // Width of the bar without percentage and counts
+    val percentage = count * 100 / max
+    val equalsSigns = "=" * (barWidth * percentage / 100)
+    val whiteSpaces = " " * (barWidth - equalsSigns.length)
+
+    val content = "%3d%% (%2d|%2d) [%s]".format(percentage, count, max, equalsSigns + ">" + whiteSpaces)
+
+    print(s"\033[1A\r$content\n")
+    //          |   |    |    |
+    // Go up 1 line |    |    |
+    //   Go to beginning of line
+    //                   |    |
+    //          Actual progress bar
+    //                        |
+    //            Go back down for description
+  }
+
+  /**
+    * Draws the description which is located in the current line.
+    */
+  private def drawDescription(): Unit = {
+    // Cap the description at the width of the terminal, otherwise a new line is created and everything would shift.
+    // If the user needs to see a really long url he can just widen his terminal.
+    val content = description.take(width)
+
+    print(s"\r\033[0K$content")
+    //      |     |
+    // Go to beginning
+    //            |
+    // Clear from cursor to end of the line
+  }
+
+}

src/main/scala/org/codeoverflow/chatoverflow/configuration/ConfigurationService.scala

@@ -153,6 +153,12 @@ class ConfigurationService(val configFilePath: String) extends WithLogger {
     * Saves the xml content to the config xml.
     */
   private def saveXML(xmlContent: Node): Unit = {
+    // Create config folder, if not existent
+    val dir = new File(configFilePath).getParentFile
+    if(!dir.exists()) {
+      dir.mkdir()
+    }
+
     val writer = new PrintWriter(configFilePath)
     writer.print(new PrettyPrinter(120, 2).format(xmlContent))
     writer.close()

src/main/scala/org/codeoverflow/chatoverflow/requirement/service/twitch/chat/TwitchChatConnector.scala

@@ -30,18 +30,27 @@ class TwitchChatConnector(override val sourceIdentifier: String) extends Connect
   }
 
   def joinChannel(channel: String): Unit = {
-    bot.send().joinChannel(channel)
-    channels += channel
+    val formattedChan = formatChannel(channel)
+    bot.send().joinChannel(formattedChan)
+    channels += formattedChan
   }
 
   def sendChatMessage(channel: String, chatMessage: String): Unit = {
-    if (!isJoined(channel)) throw new IllegalArgumentException(s"you must join the '$channel' channel, before you can send messages to it")
-    bot.send().message(channel, chatMessage)
+    val formattedChan = formatChannel(channel)
+    if (!isJoined(formattedChan)) throw new IllegalArgumentException(s"you must join the '$channel' channel, before you can send messages to it")
+    bot.send().message(formattedChan, chatMessage)
   }
 
   override def getUniqueTypeString: String = this.getClass.getName
 
-  def isJoined(channel: String): Boolean = channels.contains(channel)
+  def isJoined(channel: String): Boolean = channels.contains(formatChannel(channel))
+
+  /**
+    * Ensures that the channel is in following format: "#lowercasename"
+    * @param chan the unmodified channel
+    * @return the channel in the correct format, changes nothing if already correct
+    */
+  private def formatChannel(chan: String): String = s"#${chan.stripPrefix("#").toLowerCase}"
 
   private def getConfig: Configuration = {