archivesunleashed · ianmilligan1 · Nov 8, 2022 · Nov 8, 2022
diff --git a/pom.xml b/pom.xml
@@ -182,7 +182,15 @@
             <arg>-feature</arg>
             <arg>-explaintypes</arg>
             <arg>-target:jvm-1.8</arg>
+            <arg>-Ywarn-unused-import</arg>
           </args>
+          <compilerPlugins>
+            <compilerPlugin>
+              <groupId>org.scalameta</groupId>
+              <artifactId>semanticdb-scalac_${scala.version}</artifactId>
+              <version>4.6.0</version>
+            </compilerPlugin>
+          </compilerPlugins>
         </configuration>
       </plugin>
       <!-- For license header enforcement. -->
@@ -328,6 +336,11 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>io.github.evis</groupId>
+        <artifactId>scalafix-maven-plugin_${scala.binary.version}</artifactId>
+        <version>0.1.7_0.10.4</version>
+      </plugin>
     </plugins>
   </build>
 

diff --git a/src/main/scala/io/archivesunleashed/SparklingArchiveRecord.scala b/src/main/scala/io/archivesunleashed/SparklingArchiveRecord.scala
@@ -22,11 +22,7 @@ import io.archivesunleashed.matchbox.ExtractDomain
 import org.apache.tika.io.BoundedInputStream
 import org.archive.webservices.sparkling.http.HttpMessage
 import org.archive.webservices.sparkling.io.IOUtil
-import org.archive.webservices.sparkling.util.{
-  ManagedVal,
-  RegexUtil,
-  ValueSupplier
-}
+import org.archive.webservices.sparkling.util.{ManagedVal, ValueSupplier}
 import org.archive.webservices.sparkling.warc.{WarcHeaders, WarcRecord}
 import scala.util.Try
 

diff --git a/src/main/scala/io/archivesunleashed/app/AudioInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/AudioInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object AudioInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/CommandLineApp.scala b/src/main/scala/io/archivesunleashed/app/CommandLineApp.scala
@@ -18,9 +18,9 @@ package io.archivesunleashed.app
 import java.io.File
 import java.nio.file.{Files, Paths}
 
-import io.archivesunleashed.{ArchiveRecord, RecordLoader}
+import io.archivesunleashed.RecordLoader
 import org.apache.log4j.Logger
-import org.apache.spark.sql.{DataFrame, Dataset, Row}
+import org.apache.spark.sql.{Dataset, Row}
 import org.apache.spark.{SparkConf, SparkContext}
 import org.rogach.scallop.exceptions.ScallopException
 import org.rogach.scallop.ScallopConf

diff --git a/src/main/scala/io/archivesunleashed/app/CssInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/CssInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object CssInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/DomainFrequencyExtractor.scala b/src/main/scala/io/archivesunleashed/app/DomainFrequencyExtractor.scala
@@ -16,10 +16,7 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import io.archivesunleashed.udfs.{extractDomain, removePrefixWWW}
-import org.apache.spark.sql.functions.desc
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object DomainFrequencyExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/DomainGraphExtractor.scala b/src/main/scala/io/archivesunleashed/app/DomainGraphExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import io.archivesunleashed.udfs.{extractDomain, removePrefixWWW}
 import org.apache.spark.sql.functions.{desc, substring}
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

diff --git a/src/main/scala/io/archivesunleashed/app/ExtractPopularImagesDF.scala b/src/main/scala/io/archivesunleashed/app/ExtractPopularImagesDF.scala
@@ -15,10 +15,8 @@
  */
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import org.apache.spark.sql.functions.{col, desc, first}
+import org.apache.spark.sql.functions.{desc, first}
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.{RangePartitioner, SparkContext}
 
 /** Extract the most popular images from a DataFrame. */
 object ExtractPopularImagesDF {

diff --git a/src/main/scala/io/archivesunleashed/app/HtmlInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/HtmlInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object HtmlInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/ImageGraphExtractor.scala b/src/main/scala/io/archivesunleashed/app/ImageGraphExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object ImageGraphExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/ImageInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/ImageInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object ImageInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/JsInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/JsInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object JsInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/JsonInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/JsonInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object JsonInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/PDFInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/PDFInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object PDFInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/PlainTextExtractor.scala b/src/main/scala/io/archivesunleashed/app/PlainTextExtractor.scala
@@ -16,8 +16,7 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.udfs.{extractBoilerpipeText}
+import io.archivesunleashed.udfs.extractBoilerpipeText
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.functions.lower
 import scala.language.postfixOps

diff --git a/src/main/scala/io/archivesunleashed/app/PlainTextInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/PlainTextInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object PlainTextInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/PresentationProgramInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/PresentationProgramInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object PresentationProgramInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/SpreadsheetInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/SpreadsheetInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object SpreadsheetInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/VideoInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/VideoInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object VideoInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/WebGraphExtractor.scala b/src/main/scala/io/archivesunleashed/app/WebGraphExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object WebGraphExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/WebPagesExtractor.scala b/src/main/scala/io/archivesunleashed/app/WebPagesExtractor.scala
@@ -16,13 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.udfs.{
-  extractDomain,
-  removeHTML,
-  removeHTTPHeader,
-  removePrefixWWW
-}
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object WebPagesExtractor {
@@ -35,9 +28,6 @@ object WebPagesExtractor {
     */
   def apply(d: DataFrame): Dataset[Row] = {
     val spark = SparkSession.builder().master("local").getOrCreate()
-    // scalastyle:off
-    import spark.implicits._
-    // scalastyle:on
     d
   }
 }
diff --git a/src/main/scala/io/archivesunleashed/app/WordProcessorInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/WordProcessorInformationExtractor.scala
@@ -16,7 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object WordProcessorInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/app/XmlInformationExtractor.scala b/src/main/scala/io/archivesunleashed/app/XmlInformationExtractor.scala
@@ -16,8 +16,6 @@
 
 package io.archivesunleashed.app
 
-import io.archivesunleashed.ArchiveRecord
-import io.archivesunleashed.df.DataFrameLoader
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 
 object XmlInformationExtractor {

diff --git a/src/main/scala/io/archivesunleashed/matchbox/ExtractBoilerpipeText.scala b/src/main/scala/io/archivesunleashed/matchbox/ExtractBoilerpipeText.scala
@@ -16,7 +16,6 @@
 package io.archivesunleashed.matchbox
 
 import de.l3s.boilerpipe.extractors.DefaultExtractor
-import java.io.IOException
 
 /** Extract raw text content from an HTML page, minus "boilerplate" content (using boilerpipe). */
 object ExtractBoilerpipeText {

diff --git a/src/main/scala/io/archivesunleashed/matchbox/ExtractLinks.scala b/src/main/scala/io/archivesunleashed/matchbox/ExtractLinks.scala
@@ -15,7 +15,6 @@
  */
 package io.archivesunleashed.matchbox
 
-import java.io.IOException
 import org.jsoup.Jsoup
 import org.jsoup.select.Elements
 import scala.collection.mutable

diff --git a/src/main/scala/io/archivesunleashed/matchbox/ExtractTextFromPDFs.scala b/src/main/scala/io/archivesunleashed/matchbox/ExtractTextFromPDFs.scala
@@ -15,11 +15,7 @@
  */
 package io.archivesunleashed.matchbox
 
-import java.io.ByteArrayInputStream
-import org.apache.tika.metadata.Metadata
-import org.apache.tika.parser.ParseContext
 import org.apache.tika.parser.pdf.PDFParser
-import org.apache.tika.sax.BodyContentHandler;
 
 /** Exacts texts from PDFs using Apache Tika. */
 object ExtractTextFromPDFs {

diff --git a/src/main/scala/io/archivesunleashed/matchbox/RemoveHTML.scala b/src/main/scala/io/archivesunleashed/matchbox/RemoveHTML.scala
@@ -15,7 +15,6 @@
  */
 package io.archivesunleashed.matchbox
 
-import java.io.IOException
 import org.jsoup.Jsoup
 
 /** Removes HTML markup with JSoup. */

diff --git a/src/main/scala/io/archivesunleashed/matchbox/package.scala b/src/main/scala/io/archivesunleashed/matchbox/package.scala
@@ -17,7 +17,6 @@
 package io.archivesunleashed
 
 import java.io.IOException
-import java.security.MessageDigest
 import scala.xml.Utility.escape
 
 /** Package object which supplies implicits providing common UDF-related functionalities. */

diff --git a/src/main/scala/io/archivesunleashed/package.scala b/src/main/scala/io/archivesunleashed/package.scala
@@ -35,36 +35,33 @@ import io.archivesunleashed.matchbox.{
 }
 import io.archivesunleashed.matchbox.ExtractDate.DateComponent
 import io.archivesunleashed.matchbox.ExtractDate.DateComponent.DateComponent
-import java.net.URI
 import java.net.URL
 
 import org.apache.commons.codec.binary.Hex
 import org.apache.commons.io.FilenameUtils
 import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.functions.{lit, lower, udf}
 import org.apache.spark.sql.types.{
   BinaryType,
   IntegerType,
   StringType,
   StructField,
   StructType
 }
-import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.{RangePartitioner, SerializableWritable, SparkContext}
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.apache.spark.SparkContext
 import org.archive.webservices.sparkling.io.{HdfsIO, IOUtil}
 import org.archive.webservices.sparkling.util.{
   IteratorUtil,
   ManagedVal,
   RddUtil,
   ValueSupplier
 }
-import org.archive.webservices.sparkling.warc.{WarcLoader, WarcRecord}
+import org.archive.webservices.sparkling.warc.WarcLoader
 
 import scala.language.postfixOps
 import scala.reflect.ClassTag
 import scala.util.matching.Regex
-import scala.util.Try
 
 /**
   * Package object which supplies implicits to augment generic RDDs with AUT-specific transformations.

diff --git a/src/main/scala/io/archivesunleashed/udfs/package.scala b/src/main/scala/io/archivesunleashed/udfs/package.scala
@@ -31,11 +31,8 @@ import io.archivesunleashed.matchbox.{
   RemoveHTML,
   RemoveHTTPHeader
 }
-import org.apache.commons.lang3.StringUtils
 import org.apache.spark.sql.expressions.UserDefinedFunction
 import org.apache.spark.sql.functions.udf
-import org.apache.spark.sql.SparkSession
-import scala.util.matching.Regex
 
 /** Package object providing UDFs for DataFrames in Scala and PySpark. * */
 package object udfs extends Serializable {

diff --git a/src/test/scala/io/archivesunleashed/RecordDFTest.scala b/src/test/scala/io/archivesunleashed/RecordDFTest.scala
@@ -34,7 +34,7 @@ import io.archivesunleashed.udfs.{
 }
 import com.google.common.io.Resources
 import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.{Dataset, Row, SparkSession}
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner

diff --git a/src/test/scala/io/archivesunleashed/df/DataFrameLoaderTest.scala b/src/test/scala/io/archivesunleashed/df/DataFrameLoaderTest.scala
@@ -16,7 +16,6 @@
 package io.archivesunleashed.df
 
 import com.google.common.io.Resources
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner

diff --git a/src/test/scala/io/archivesunleashed/df/ExtractAudioDetailsTest.scala b/src/test/scala/io/archivesunleashed/df/ExtractAudioDetailsTest.scala
@@ -18,7 +18,6 @@ package io.archivesunleashed
 
 import com.google.common.io.Resources
 import org.apache.spark.sql.functions.desc
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner

diff --git a/src/test/scala/io/archivesunleashed/df/ExtractDateDFTest.scala b/src/test/scala/io/archivesunleashed/df/ExtractDateDFTest.scala
@@ -17,12 +17,7 @@
 package io.archivesunleashed
 
 import com.google.common.io.Resources
-import io.archivesunleashed.udfs.{
-  extractDate,
-  extractDomain,
-  extractLinks,
-  removePrefixWWW
-}
+import io.archivesunleashed.udfs.{extractDate, extractLinks}
 import org.apache.spark.sql.functions.{array, explode_outer, lower, udf}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}

diff --git a/src/test/scala/io/archivesunleashed/df/ExtractHyperlinksTest.scala b/src/test/scala/io/archivesunleashed/df/ExtractHyperlinksTest.scala
@@ -17,7 +17,7 @@
 package io.archivesunleashed
 
 import com.google.common.io.Resources
-import io.archivesunleashed.udfs.{extractDomain, extractLinks, removePrefixWWW}
+import io.archivesunleashed.udfs.extractLinks
 import org.apache.spark.sql.functions.{array, explode_outer, lower, udf}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}

diff --git a/src/test/scala/io/archivesunleashed/df/ExtractImageDetailsTest.scala b/src/test/scala/io/archivesunleashed/df/ExtractImageDetailsTest.scala
@@ -18,7 +18,6 @@ package io.archivesunleashed
 
 import com.google.common.io.Resources
 import org.apache.spark.sql.functions.desc
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner