Skip to content

Commit 112a8a9

Browse files
jtgrabowskijtgrabowski
committed
review remarks
1 parent ff1d020 commit 112a8a9

File tree

8 files changed

+57
-41
lines changed

8 files changed

+57
-41
lines changed

‎java/gradle/oss/build.gradle‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ repositories {
66
mavenCentral()
77
}
88

9+
def sparkVersion = "1.6.2"
10+
def connectorVersion = "1.6.0"
11+
912
// The provided configuration behaves the same as the sbt "provided" keyword which will cause jars to be
1013
// excluded from assembled fat-jar
1114
configurations {
@@ -14,13 +17,12 @@ configurations {
1417
}
1518

1619
// Please make sure that following dependencies have versions corresponding to the ones in your cluster.
17-
// Note that spark-hive and spark-cassandra-connector should be provided with '--packages' flag to
18-
// spark-submit command.
20+
// Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
1921
dependencies {
20-
provided "org.apache.spark:spark-core_2.10:1.6.2"
21-
provided "org.apache.spark:spark-sql_2.10:1.6.2"
22-
provided "org.apache.spark:spark-hive_2.10:1.6.2"
23-
provided "com.datastax.spark:spark-cassandra-connector_2.10:1.6.0"
22+
provided "org.apache.spark:spark-core_2.10:$sparkVersion"
23+
provided "org.apache.spark:spark-sql_2.10:$sparkVersion"
24+
provided "org.apache.spark:spark-hive_2.10:$sparkVersion"
25+
provided "com.datastax.spark:spark-cassandra-connector_2.10:$connectorVersion"
2426
// compile "your:dependecy:1.0.0"
2527
}
2628

‎java/maven/oss/pom.xml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,36 +9,37 @@
99

1010
<properties>
1111
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
12+
<spark.version>1.6.2</spark.version>
13+
<connector.version>1.6.0</connector.version>
1214
</properties>
1315

1416
<!--
15-
Please make sure that following dependencies have versions corresponding to the ones in your cluster.
16-
Note that spark-hive and spark-cassandra-connector should be provided with '&#45;&#45;packages' flag
17-
to spark-submit command.
18-
-->
17+
Please make sure that following dependencies have versions corresponding to the ones in your cluster.
18+
Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
19+
-->
1920
<dependencies>
2021
<dependency>
2122
<groupId>org.apache.spark</groupId>
2223
<artifactId>spark-core_2.10</artifactId>
23-
<version>1.6.2</version>
24+
<version>${spark.version}</version>
2425
<scope>provided</scope>
2526
</dependency>
2627
<dependency>
2728
<groupId>org.apache.spark</groupId>
2829
<artifactId>spark-sql_2.10</artifactId>
29-
<version>1.6.2</version>
30+
<version>${spark.version}</version>
3031
<scope>provided</scope>
3132
</dependency>
3233
<dependency>
3334
<groupId>org.apache.spark</groupId>
3435
<artifactId>spark-hive_2.10</artifactId>
35-
<version>1.6.2</version>
36+
<version>${spark.version}</version>
3637
<scope>provided</scope>
3738
</dependency>
3839
<dependency>
3940
<groupId>com.datastax.spark</groupId>
4041
<artifactId>spark-cassandra-connector_2.10</artifactId>
41-
<version>1.6.0</version>
42+
<version>${connector.version}</version>
4243
<scope>provided</scope>
4344
</dependency>
4445
<!-- Your dependencies, 'provided' are not included in jar -->

‎java/sbt/dse/src/main/java/com/datastax/spark/example/WriteRead.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717

1818
import static com.datastax.spark.connector.japi.CassandraJavaUtil.*;
1919

20+
21+
// For DSE it is not necessary to set connection parameters for spark.master (since it will be done
22+
// automatically)
2023
public class WriteRead
2124
{
2225
public static void main(String[] args)

‎java/sbt/oss/build.sbt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,16 @@ crossPaths := false
66

77
autoScalaLibrary := false
88

9+
val sparkVersion = "1.6.2"
10+
val connectorVersion = "1.6.0"
11+
912
// Please make sure that following dependencies have versions corresponding to the ones in your cluster.
10-
// Note that spark-hive and spark-cassandra-connector should be provided with '--packages' flag to
11-
// spark-submit command.
13+
// Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
1214
libraryDependencies ++= Seq(
13-
"org.apache.spark" %% "spark-core" % "1.6.2" % "provided",
14-
"org.apache.spark" %% "spark-sql" % "1.6.2" % "provided",
15-
"org.apache.spark" %% "spark-hive" % "1.6.2" % "provided",
16-
"com.datastax.spark" %% "spark-cassandra-connector" % "1.6.0" % "provided"
15+
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
16+
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
17+
"org.apache.spark" %% "spark-hive" % sparkVersion % "provided",
18+
"com.datastax.spark" %% "spark-cassandra-connector" % connectorVersion % "provided"
1719
)
1820

1921
//Your dependencies

‎scala/gradle/oss/build.gradle‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ repositories {
66
mavenCentral()
77
}
88

9+
def sparkVersion = "1.6.2"
10+
def connectorVersion = "1.6.0"
11+
912
// The provided configuration behaves the same as the sbt "provided" keyword which will cause jars to be
1013
// excluded from assembled fat-jar
1114
configurations {
@@ -14,13 +17,12 @@ configurations {
1417
}
1518

1619
// Please make sure that following dependencies have versions corresponding to the ones in your cluster.
17-
// Note that spark-hive and spark-cassandra-connector should be provided with '--packages' flag to
18-
// spark-submit command.
20+
// Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
1921
dependencies {
20-
provided "org.apache.spark:spark-core_2.10:1.6.2"
21-
provided "org.apache.spark:spark-sql_2.10:1.6.2"
22-
provided "org.apache.spark:spark-hive_2.10:1.6.2"
23-
provided "com.datastax.spark:spark-cassandra-connector_2.10:1.6.0"
22+
provided "org.apache.spark:spark-core_2.10:$sparkVersion"
23+
provided "org.apache.spark:spark-sql_2.10:$sparkVersion"
24+
provided "org.apache.spark:spark-hive_2.10:$sparkVersion"
25+
provided "com.datastax.spark:spark-cassandra-connector_2.10:$connectorVersion"
2426
// compile "your:dependecy:1.0.0"
2527
}
2628

‎scala/maven/oss/pom.xml‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,13 @@
1010
<properties>
1111
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1212
<scala.version>2.10.6</scala.version>
13+
<spark.version>1.6.2</spark.version>
14+
<connector.version>1.6.0</connector.version>
1315
</properties>
1416

1517
<!--
1618
Please make sure that following dependencies have versions corresponding to the ones in your cluster.
17-
Note that spark-hive and spark-cassandra-connector should be provided with '&#45;&#45;packages' flag
18-
to spark-submit command.
19+
Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
1920
-->
2021
<dependencies>
2122
<dependency>
@@ -26,25 +27,25 @@
2627
<dependency>
2728
<groupId>org.apache.spark</groupId>
2829
<artifactId>spark-core_2.10</artifactId>
29-
<version>1.6.2</version>
30+
<version>${spark.version}</version>
3031
<scope>provided</scope>
3132
</dependency>
3233
<dependency>
3334
<groupId>org.apache.spark</groupId>
3435
<artifactId>spark-sql_2.10</artifactId>
35-
<version>1.6.2</version>
36+
<version>${spark.version}</version>
3637
<scope>provided</scope>
3738
</dependency>
3839
<dependency>
3940
<groupId>org.apache.spark</groupId>
4041
<artifactId>spark-hive_2.10</artifactId>
41-
<version>1.6.2</version>
42+
<version>${spark.version}</version>
4243
<scope>provided</scope>
4344
</dependency>
4445
<dependency>
4546
<groupId>com.datastax.spark</groupId>
4647
<artifactId>spark-cassandra-connector_2.10</artifactId>
47-
<version>1.6.0</version>
48+
<version>${connector.version}</version>
4849
<scope>provided</scope>
4950
</dependency>
5051
<!-- Your dependencies, 'provided' are not included in jar -->

‎scala/sbt/dse/src/main/scala/com/datastax/spark/example/WriteRead.scala‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@ import com.datastax.spark.connector.cql.CassandraConnector
55
import org.apache.spark.sql.hive.HiveContext
66
import org.apache.spark.{SparkConf, SparkContext}
77

8+
9+
// For DSE it is not necessary to set connection parameters for spark.master (since it will be done
10+
// automatically)
811
object WriteRead extends App {
912

1013
val conf = new SparkConf()
1114
.setAppName("Datastax Scala example")
1215

13-
CassandraConnector(conf).withSessionDo{ session =>
16+
CassandraConnector(conf).withSessionDo { session =>
1417
session.execute(
1518
"""CREATE KEYSPACE IF NOT EXISTS ks WITH
1619
| replication = {'class': 'SimpleStrategy', 'replication_factor': 1 }""".stripMargin)
@@ -22,7 +25,7 @@ object WriteRead extends App {
2225
val hiveContext = new HiveContext(sc)
2326

2427
// Write some data to C*
25-
sc.parallelize(1 to 10).map( x => (x,x)).saveToCassandra("ks", "kv")
28+
sc.parallelize(1 to 10).map(x => (x, x)).saveToCassandra("ks", "kv")
2629

2730
// Read Data Using the Spark Context
2831
val scReadData = sc.cassandraTable("ks", "kv").collect
@@ -31,7 +34,7 @@ object WriteRead extends App {
3134
val sqlReadData = hiveContext
3235
.read
3336
.format("org.apache.spark.sql.cassandra")
34-
.options(Map( "table" -> "kv", "keyspace" -> "ks" ))
37+
.options(Map("table" -> "kv", "keyspace" -> "ks"))
3538
.load()
3639
.collect()
3740
.map(row => (row.getInt(0), row.getInt(1)))

‎scala/sbt/oss/build.sbt‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,16 @@ version := "0.1"
44

55
scalaVersion := "2.10.6"
66

7+
val sparkVersion = "1.6.2"
8+
val connectorVersion = "1.6.0"
9+
710
// Please make sure that following dependencies have versions corresponding to the ones in your cluster.
8-
// Note that spark-hive and spark-cassandra-connector should be provided with '--packages' flag to
9-
// spark-submit command.
11+
// Note that spark-cassandra-connector should be provided with '--packages' flag to spark-submit command.
1012
libraryDependencies ++= Seq(
11-
"org.apache.spark" %% "spark-core" % "1.6.2" % "provided",
12-
"org.apache.spark" %% "spark-sql" % "1.6.2" % "provided",
13-
"org.apache.spark" %% "spark-hive" % "1.6.2" % "provided",
14-
"com.datastax.spark" %% "spark-cassandra-connector" % "1.6.0" % "provided"
13+
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
14+
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
15+
"org.apache.spark" %% "spark-hive" % sparkVersion % "provided",
16+
"com.datastax.spark" %% "spark-cassandra-connector" % connectorVersion % "provided"
1517
)
1618

1719
//Your dependencies

0 commit comments

Comments
 (0)