Remove outdated Cassandra connector

README.md

@@ -87,7 +87,7 @@ docker run -ti \
        -p "4040:4040" \
        gisaia/spark:latest \
        spark-shell \
-        --packages datastax:spark-cassandra-connector:2.3.1-s_2.11,org.elasticsearch:elasticsearch-spark-20_2.11:7.4.2,org.geotools:gt-referencing:20.1,org.geotools:gt-geometry:20.1,org.geotools:gt-epsg-hsql:20.1 \
+        --packages org.elasticsearch:elasticsearch-spark-20_2.11:7.4.2,org.geotools:gt-referencing:20.1,org.geotools:gt-geometry:20.1,org.geotools:gt-epsg-hsql:20.1 \
         --exclude-packages javax.media:jai_core \
         --repositories https://repo.osgeo.org/repository/release/ \
         --jars /opt/proc/target/scala-2.11/arlas-proc-assembly-0.4.0-SNAPSHOT.jar \

build.sbt

@@ -12,10 +12,6 @@ val sparkSQL = "org.apache.spark" %% "spark-sql" % sparkVersion % "provided"
 val sparkMLlib = "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided"
 val spark = Seq(sparkSQL,sparkMLlib)
 
-val sparkCassandraConnector = "com.datastax.spark" %% "spark-cassandra-connector" % "2.3.2" % "provided"
-
-val cassandra = Seq(sparkCassandraConnector)
-
 val scalaTest = "org.scalatest" %% "scalatest" % "2.2.5" % Test
 val wiremockStandalone = "com.github.tomakehurst" % "wiremock-standalone" % "2.25.1" % Test
 val tests = Seq(scalaTest, wiremockStandalone)
@@ -34,7 +30,6 @@ lazy val arlasProc = (project in file("."))
   .settings(
     name := "arlas-proc",
     libraryDependencies ++= spark,
-    libraryDependencies ++= cassandra,
     libraryDependencies ++= elastic,
     libraryDependencies ++= geotools,
     libraryDependencies ++= arlas,

src/main/scala/io/arlas/data/sql/WritableDataFrame.scala

@@ -22,13 +22,12 @@ package io.arlas.data.sql
 import io.arlas.data.model.DataModel
 import io.arlas.data.transform.ArlasTransformerColumns._
 import io.arlas.data.transform.DataFrameException
-import io.arlas.data.utils.CassandraTool
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.ArrayType
 import org.apache.spark.sql.{Column, DataFrame, SaveMode, SparkSession}
 import org.elasticsearch.spark.sql._
 
-class WritableDataFrame(df: DataFrame) extends TransformableDataFrame(df) with CassandraTool {
+class WritableDataFrame(df: DataFrame) extends TransformableDataFrame(df) {
 
   val PARQUET_BLOCK_SIZE: Int = 256 * 1024 * 1024
   val arlasElasticsearchIdColumn = "arlas_es_id"
@@ -103,20 +102,6 @@ class WritableDataFrame(df: DataFrame) extends TransformableDataFrame(df) with C
                 Map("es.mapping.id" -> esIdColName, "es.mapping.exclude" -> (mappingExcluded :+ "dynamicIndex").mkString(",")))
   }
 
-  def writeToScyllaDB(spark: SparkSession, dataModel: DataModel, target: String): Unit = {
-    val targetKeyspace = target.split('.')(0)
-    val targetTable = target.split('.')(1)
-
-    createCassandraKeyspaceIfNotExists(spark, targetKeyspace)
-    createCassandraTableIfNotExists(df, dataModel, targetKeyspace, targetTable)
-
-    df.write
-      .format("org.apache.spark.sql.cassandra")
-      .options(Map("keyspace" -> targetKeyspace, "table" -> targetTable))
-      .mode(SaveMode.Append)
-      .save()
-  }
-
   def writeToCsv(target: String, delimiter: String = ";", toSingleFile: Boolean = true) = {
 
     val coalescedDF = if (toSingleFile) df.coalesce(1) else df