Skip to content

Commit f5f757e

Browse files
sryzapwendell
sryza
authored and
pwendell
committed
SPARK-4375. no longer require -Pscala-2.10
It seems like the winds might have moved away from this approach, but wanted to post the PR anyway because I got it working and to show what it would look like. Author: Sandy Ryza <sandy@cloudera.com> Closes #3239 from sryza/sandy-spark-4375 and squashes the following commits: 0ffbe95 [Sandy Ryza] Enable -Dscala-2.11 in sbt cd42d94 [Sandy Ryza] Update doc f6644c3 [Sandy Ryza] SPARK-4375 take 2
1 parent bbd8f5b commit f5f757e

File tree

7 files changed

+54
-171
lines changed

7 files changed

+54
-171
lines changed

‎docs/building-spark.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,9 @@ mvn -Pyarn -Phive -Phive-thriftserver-0.12.0 -Phadoop-2.4 -Dhadoop.version=2.4.0
113113
{% endhighlight %}
114114

115115
# Building for Scala 2.11
116-
To produce a Spark package compiled with Scala 2.11, use the `-Pscala-2.11` profile:
116+
To produce a Spark package compiled with Scala 2.11, use the `-Dscala-2.11` property:
117117

118-
mvn -Pyarn -Phadoop-2.4 -Pscala-2.11 -DskipTests clean package
118+
mvn -Pyarn -Phadoop-2.4 -Dscala-2.11 -DskipTests clean package
119119

120120
Scala 2.11 support in Spark is experimental and does not support a few features.
121121
Specifically, Spark's external Kafka library and JDBC component are not yet

‎examples/pom.xml‎

Lines changed: 3 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -389,11 +389,11 @@
389389
</properties>
390390
</profile>
391391
<profile>
392-
<!-- We add source directories specific to Scala 2.10 and 2.11 since some examples
393-
work only in one and not the other -->
392+
<!-- We add a source directory specific to Scala 2.10 since Kafka and Algebird
393+
only work with it -->
394394
<id>scala-2.10</id>
395395
<activation>
396-
<activeByDefault>true</activeByDefault>
396+
<property><name>!scala-2.11</name></property>
397397
</activation>
398398
<dependencies>
399399
<dependency>
@@ -427,65 +427,6 @@
427427
</sources>
428428
</configuration>
429429
</execution>
430-
<execution>
431-
<id>add-scala-test-sources</id>
432-
<phase>generate-test-sources</phase>
433-
<goals>
434-
<goal>add-test-source</goal>
435-
</goals>
436-
<configuration>
437-
<sources>
438-
<source>src/test/scala</source>
439-
<source>scala-2.10/src/test/scala</source>
440-
<source>scala-2.10/src/test/java</source>
441-
</sources>
442-
</configuration>
443-
</execution>
444-
</executions>
445-
</plugin>
446-
</plugins>
447-
</build>
448-
</profile>
449-
<profile>
450-
<id>scala-2.11</id>
451-
<activation>
452-
<activeByDefault>false</activeByDefault>
453-
</activation>
454-
<dependencies>
455-
<!-- Streaming Kafka and zeromq modules are disabled for now. -->
456-
</dependencies>
457-
<build>
458-
<plugins>
459-
<plugin>
460-
<groupId>org.codehaus.mojo</groupId>
461-
<artifactId>build-helper-maven-plugin</artifactId>
462-
<executions>
463-
<execution>
464-
<id>add-scala-sources</id>
465-
<phase>generate-sources</phase>
466-
<goals>
467-
<goal>add-source</goal>
468-
</goals>
469-
<configuration>
470-
<sources>
471-
<source>src/main/scala</source>
472-
<source>scala-2.11/src/main/scala</source>
473-
</sources>
474-
</configuration>
475-
</execution>
476-
<execution>
477-
<id>add-scala-test-sources</id>
478-
<phase>generate-test-sources</phase>
479-
<goals>
480-
<goal>add-test-source</goal>
481-
</goals>
482-
<configuration>
483-
<sources>
484-
<source>src/test/scala</source>
485-
<source>scala-2.11/src/test/scala</source>
486-
</sources>
487-
</configuration>
488-
</execution>
489430
</executions>
490431
</plugin>
491432
</plugins>

‎pom.xml‎

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1296,9 +1296,6 @@
12961296

12971297
<profile>
12981298
<id>mapr3</id>
1299-
<activation>
1300-
<activeByDefault>false</activeByDefault>
1301-
</activation>
13021299
<properties>
13031300
<hadoop.version>1.0.3-mapr-3.0.3</hadoop.version>
13041301
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
@@ -1309,9 +1306,6 @@
13091306

13101307
<profile>
13111308
<id>mapr4</id>
1312-
<activation>
1313-
<activeByDefault>false</activeByDefault>
1314-
</activation>
13151309
<properties>
13161310
<hadoop.version>2.3.0-mapr-4.0.0-FCS</hadoop.version>
13171311
<yarn.version>2.3.0-mapr-4.0.0-FCS</yarn.version>
@@ -1341,9 +1335,6 @@
13411335
<!-- Build without Hadoop dependencies that are included in some runtime environments. -->
13421336
<profile>
13431337
<id>hadoop-provided</id>
1344-
<activation>
1345-
<activeByDefault>false</activeByDefault>
1346-
</activation>
13471338
<dependencies>
13481339
<dependency>
13491340
<groupId>org.apache.hadoop</groupId>
@@ -1390,18 +1381,12 @@
13901381
</profile>
13911382
<profile>
13921383
<id>hive-thriftserver</id>
1393-
<activation>
1394-
<activeByDefault>false</activeByDefault>
1395-
</activation>
13961384
<modules>
13971385
<module>sql/hive-thriftserver</module>
13981386
</modules>
13991387
</profile>
14001388
<profile>
14011389
<id>hive-0.12.0</id>
1402-
<activation>
1403-
<activeByDefault>false</activeByDefault>
1404-
</activation>
14051390
<properties>
14061391
<hive.version>0.12.0-protobuf-2.5</hive.version>
14071392
<hive.version.short>0.12.0</hive.version.short>
@@ -1410,9 +1395,6 @@
14101395
</profile>
14111396
<profile>
14121397
<id>hive-0.13.1</id>
1413-
<activation>
1414-
<activeByDefault>false</activeByDefault>
1415-
</activation>
14161398
<properties>
14171399
<hive.version>0.13.1a</hive.version>
14181400
<hive.version.short>0.13.1</hive.version.short>
@@ -1423,7 +1405,7 @@
14231405
<profile>
14241406
<id>scala-2.10</id>
14251407
<activation>
1426-
<activeByDefault>true</activeByDefault>
1408+
<property><name>!scala-2.11</name></property>
14271409
</activation>
14281410
<properties>
14291411
<scala.version>2.10.4</scala.version>
@@ -1439,7 +1421,7 @@
14391421
<profile>
14401422
<id>scala-2.11</id>
14411423
<activation>
1442-
<activeByDefault>false</activeByDefault>
1424+
<property><name>scala-2.11</name></property>
14431425
</activation>
14441426
<properties>
14451427
<scala.version>2.11.2</scala.version>

‎project/SparkBuild.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,11 @@ object SparkBuild extends PomBuild {
100100
"conjunction with environment variable.")
101101
v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
102102
}
103+
103104
if (profiles.exists(_.contains("scala-"))) {
104105
profiles
106+
} else if (System.getProperty("scala-2.11") != null) {
107+
profiles ++ Seq("scala-2.11")
105108
} else {
106109
println("Enabled default scala profile")
107110
profiles ++ Seq("scala-2.10")

‎repl/pom.xml‎

Lines changed: 42 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
<sbt.project.name>repl</sbt.project.name>
3636
<deb.install.path>/usr/share/spark</deb.install.path>
3737
<deb.user>root</deb.user>
38+
<extra.source.dir>scala-2.10/src/main/scala</extra.source.dir>
39+
<extra.testsource.dir>scala-2.10/src/test/scala</extra.testsource.dir>
3840
</properties>
3941

4042
<dependencies>
@@ -122,86 +124,51 @@
122124
</environmentVariables>
123125
</configuration>
124126
</plugin>
127+
<!-- Include a source dir depending on the Scala version -->
128+
<plugin>
129+
<groupId>org.codehaus.mojo</groupId>
130+
<artifactId>build-helper-maven-plugin</artifactId>
131+
<executions>
132+
<execution>
133+
<id>add-scala-sources</id>
134+
<phase>generate-sources</phase>
135+
<goals>
136+
<goal>add-source</goal>
137+
</goals>
138+
<configuration>
139+
<sources>
140+
<source>src/main/scala</source>
141+
<source>${extra.source.dir}</source>
142+
</sources>
143+
</configuration>
144+
</execution>
145+
<execution>
146+
<id>add-scala-test-sources</id>
147+
<phase>generate-test-sources</phase>
148+
<goals>
149+
<goal>add-test-source</goal>
150+
</goals>
151+
<configuration>
152+
<sources>
153+
<source>src/test/scala</source>
154+
<source>${extra.testsource.dir}</source>
155+
</sources>
156+
</configuration>
157+
</execution>
158+
</executions>
159+
</plugin>
125160
</plugins>
126161
</build>
127162
<profiles>
128-
<profile>
129-
<id>scala-2.10</id>
130-
<build>
131-
<plugins>
132-
<plugin>
133-
<groupId>org.codehaus.mojo</groupId>
134-
<artifactId>build-helper-maven-plugin</artifactId>
135-
<executions>
136-
<execution>
137-
<id>add-scala-sources</id>
138-
<phase>generate-sources</phase>
139-
<goals>
140-
<goal>add-source</goal>
141-
</goals>
142-
<configuration>
143-
<sources>
144-
<source>src/main/scala</source>
145-
<source>scala-2.10/src/main/scala</source>
146-
</sources>
147-
</configuration>
148-
</execution>
149-
<execution>
150-
<id>add-scala-test-sources</id>
151-
<phase>generate-test-sources</phase>
152-
<goals>
153-
<goal>add-test-source</goal>
154-
</goals>
155-
<configuration>
156-
<sources>
157-
<source>src/test/scala</source>
158-
<source>scala-2.10/src/test/scala</source>
159-
</sources>
160-
</configuration>
161-
</execution>
162-
</executions>
163-
</plugin>
164-
</plugins>
165-
</build>
166-
</profile>
167163
<profile>
168164
<id>scala-2.11</id>
169-
<build>
170-
<plugins>
171-
<plugin>
172-
<groupId>org.codehaus.mojo</groupId>
173-
<artifactId>build-helper-maven-plugin</artifactId>
174-
<executions>
175-
<execution>
176-
<id>add-scala-sources</id>
177-
<phase>generate-sources</phase>
178-
<goals>
179-
<goal>add-source</goal>
180-
</goals>
181-
<configuration>
182-
<sources>
183-
<source>src/main/scala</source>
184-
<source>scala-2.11/src/main/scala</source>
185-
</sources>
186-
</configuration>
187-
</execution>
188-
<execution>
189-
<id>add-scala-test-sources</id>
190-
<phase>generate-test-sources</phase>
191-
<goals>
192-
<goal>add-test-source</goal>
193-
</goals>
194-
<configuration>
195-
<sources>
196-
<source>src/test/scala</source>
197-
<source>scala-2.11/src/test/scala</source>
198-
</sources>
199-
</configuration>
200-
</execution>
201-
</executions>
202-
</plugin>
203-
</plugins>
204-
</build>
165+
<activation>
166+
<property><name>scala-2.11</name></property>
167+
</activation>
168+
<properties>
169+
<extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
170+
<extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
171+
</properties>
205172
</profile>
206173
</profiles>
207174
</project>

‎sql/catalyst/pom.xml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -100,10 +100,11 @@
100100
</plugins>
101101
</build>
102102
<profiles>
103+
<!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. -->
103104
<profile>
104105
<id>scala-2.10</id>
105106
<activation>
106-
<activeByDefault>true</activeByDefault>
107+
<property><name>!scala-2.11</name></property>
107108
</activation>
108109
<dependencies>
109110
<dependency>
@@ -113,13 +114,5 @@
113114
</dependency>
114115
</dependencies>
115116
</profile>
116-
<profile>
117-
<id>scala-2.11</id>
118-
<activation>
119-
<activeByDefault>false</activeByDefault>
120-
</activation>
121-
<!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. -->
122-
</profile>
123-
124117
</profiles>
125118
</project>

‎sql/hive/pom.xml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,9 +144,6 @@
144144
</profile>
145145
<profile>
146146
<id>hive-0.12.0</id>
147-
<activation>
148-
<activeByDefault>false</activeByDefault>
149-
</activation>
150147
<dependencies>
151148
<dependency>
152149
<groupId>com.twitter</groupId>

0 commit comments

Comments
 (0)