Remove Tweet utils. (#323)

- Resolves #322
- Resolves #206
- Resolves #194

pom.xml

@@ -475,16 +475,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>com.twitter</groupId>
-      <artifactId>chill_2.11</artifactId>
-      <version>0.8.4</version>
-    </dependency>
-    <dependency>
-      <groupId>org.json4s</groupId>
-      <artifactId>json4s-jackson_2.11</artifactId>
-      <version>3.2.11</version>
-    </dependency>
     <dependency>
       <groupId>org.scala-lang.modules</groupId>
       <artifactId>scala-parser-combinators_2.11</artifactId>
@@ -505,17 +495,6 @@
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.scalanlp</groupId>
-      <artifactId>breeze_2.11</artifactId>
-      <version>0.13.1</version>
-      <exclusions>
-        <exclusion>
-          <groupId>net.sourceforge.f2j</groupId>
-          <artifactId>arpack_combined_all</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_2.11</artifactId>

src/main/scala/io/archivesunleashed/package.scala

@@ -27,8 +27,6 @@ import org.apache.hadoop.fs.{FileSystem, Path}
 import io.archivesunleashed.matchbox.ExtractDate.DateComponent._
 import org.apache.spark.sql._
 import org.apache.spark.sql.types._
-import org.json4s._
-import org.json4s.jackson.JsonMethods._
 // scalastyle:on: underscore.import
 import org.apache.hadoop.io.LongWritable
 import org.apache.spark.{SerializableWritable, SparkContext}
@@ -68,18 +66,6 @@ package object archivesunleashed {
           ((r._2.getFormat == ArchiveFormat.WARC) && r._2.getRecord.getHeader.getHeaderValue("WARC-Type").equals("response")))
         .map(r => new ArchiveRecordImpl(new SerializableWritable(r._2)))
     }
-
-    /** Creates an Archive Record RDD from tweets.
-      *
-      * @param path the path to the Tweets file
-      * @param sc the apache spark context
-      * @return an RDD of JValue (json objects) for mapping.
-      */
-    def loadTweets(path: String, sc: SparkContext): RDD[JValue] =
-      // scalastyle:off null
-      sc.textFile(path).filter(line => !line.startsWith("{\"delete\":"))
-        .map(line => try { parse(line) } catch { case e: Exception => null }).filter(x => x != null)
-      // scalastyle:on null
   }
 
   /** A Wrapper class around RDD to simplify counting. */

src/test/scala/io/archivesunleashed/RecordLoaderTest.scala

@@ -18,9 +18,6 @@
 package io.archivesunleashed
 
 import com.google.common.io.Resources
-// scalastyle:off underscore.import
-import io.archivesunleashed.util.TweetUtils._
-// scalastyle:on underscore.import
 import org.apache.spark.{SparkConf, SparkContext}
 import org.junit.runner.RunWith
 import org.scalatest.junit.JUnitRunner
@@ -29,8 +26,6 @@ import org.scalatest.{BeforeAndAfter, FunSuite}
 @RunWith(classOf[JUnitRunner])
 class RecordLoaderTest extends FunSuite with BeforeAndAfter {
   private val warcPath = Resources.getResource("warc/example.warc.gz").getPath
-  private val tweetPath = Resources.getResource("arc/tweetsTest.json").getPath
-  private val delTweetPath = Resources.getResource("arc/delTweetsTest.json").getPath
   private val master = "local[4]"
   private val appName = "example-spark"
   private var sc: SparkContext = _
@@ -51,19 +46,6 @@ class RecordLoaderTest extends FunSuite with BeforeAndAfter {
     assert(base(0) == "http://www.archive.org/")
   }
 
-  test("loads Tweets") {
-    val base = RecordLoader.loadTweets(tweetPath, sc)
-      .map(x => x.text())
-      .collect()
-    assert(base(0) == "some text")
-    assert(base(1) == "some more text")
-  }
-
-  test("does not load deleted") {
-    val base = RecordLoader.loadTweets(delTweetPath, sc).collect()
-    assert(base.deep == Array().deep)
-  }
-
   after {
     if (sc != null) {
       sc.stop()