Spark to 2.3.0 and update other dependencies

build.sbt

@@ -5,7 +5,9 @@ val localMaven = "s3://net-mozaws-data-us-west-2-ops-mavenrepo/"
 
 resolvers += "S3 local maven snapshots" at localMavenHttps + "snapshots"
 
-val sparkVersion = "2.0.2"
+val sparkVersion = "2.3.0"
+// Should keep hadoop version in sync with the dependency defined by Spark.
+val hadoopVersion = "2.6.5"
 
 lazy val root = (project in file(".")).
   settings(
@@ -18,28 +20,29 @@ lazy val root = (project in file(".")).
     scalaModuleInfo := scalaModuleInfo.value.map(_.withOverrideScalaVersion(true)),
     libraryDependencies += "com.mozilla.telemetry" %% "moztelemetry" % "1.1-SNAPSHOT",
     libraryDependencies += "com.mozilla.telemetry" %% "spark-hyperloglog" % "2.0.0-SNAPSHOT",
-    libraryDependencies += "org.apache.avro" % "avro" % "1.7.7",
-    libraryDependencies += "org.apache.parquet" % "parquet-avro" % "1.7.0",
-    libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4",
+    libraryDependencies += "org.apache.avro" % "avro" % "1.8.2",
+    libraryDependencies += "org.apache.parquet" % "parquet-avro" % "1.8.3",
     libraryDependencies += "net.sandrogrzicic" %% "scalabuff-runtime" % "1.4.0",
-    libraryDependencies += "com.holdenkarau" %% "spark-testing-base" % "2.0.0_0.4.7" % "test",
-    libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.2",
-    libraryDependencies += "org.json4s" %% "json4s-jackson" % "3.2.10",
-    libraryDependencies += "joda-time" % "joda-time" % "2.9.2",
-    libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "2.7.1" excludeAll(ExclusionRule(organization = "javax.servlet")),
-    libraryDependencies += "org.apache.hadoop" % "hadoop-aws" % "2.7.1" excludeAll(ExclusionRule(organization = "javax.servlet")),
-    libraryDependencies += "com.github.nscala-time" %% "nscala-time" % "2.10.0",
+    libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.7.2",
+    libraryDependencies += "org.json4s" %% "json4s-jackson" % "3.5.4",
+    libraryDependencies += "joda-time" % "joda-time" % "2.10",
+    libraryDependencies += "org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(ExclusionRule(organization = "javax.servlet")),
+    libraryDependencies += "org.apache.hadoop" % "hadoop-aws" % hadoopVersion excludeAll(ExclusionRule(organization = "javax.servlet")),
     libraryDependencies += "org.rogach" %% "scallop" % "3.1.2",
     libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion,
     libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion,
     libraryDependencies += "org.apache.spark" %% "spark-mllib" % sparkVersion,
     libraryDependencies += "org.apache.spark" %% "spark-hive" % sparkVersion,
-    libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.3.0",
+    libraryDependencies += "org.scalaj" %% "scalaj-http" % "2.4.0",
     libraryDependencies += "org.yaml" % "snakeyaml" % "1.21",
-    libraryDependencies += "com.google.protobuf" % "protobuf-java" % "2.5.0",
-    libraryDependencies += "com.github.tomakehurst" % "wiremock-standalone" % "2.8.0"
+    // Test dependencies
+    libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.5" % Test,
+    libraryDependencies += "com.github.tomakehurst" % "wiremock-standalone" % "2.18.0" % Test,
+    libraryDependencies += "com.holdenkarau" %% "spark-testing-base" % "2.3.0_0.9.0" % Test
   )
 
+dependencyOverrides += "com.google.guava" % "guava" % "25.1-jre"
+dependencyOverrides += "com.google.code.findbugs" % "jsr305" % "3.0.2"
 /*
  The HBase client requires protobuf-java 2.5.0 but scalapb uses protobuf-java 3.x
  so we have to force the dependency here. This should be fine as we are using only

run-sbt.sh

@@ -12,15 +12,18 @@ if [ ! -f /.dockerenv ]; then
     exit $?
 fi
 
+# Set options that sbt will pass to the JVM
+export SBT_OPTS="-Xss2M -Djava.security.policy=java.policy"
+
 # Run tests
 if [ $TRAVIS_BRANCH ]; then
    # under travis, submit code coverage data
-   sbt -J-Xss2M "$@"
+   sbt "$@"
    bash <(curl -s https://codecov.io/bash)
 elif [ $# -gt 0 ]; then
    # if args specified, run the tests with those
-   sbt -J-Xss2M "$@"
+   sbt "$@"
 else
    # default: just run the tests
-   sbt -J-Xss2M test
+   sbt test
 fi