Skip to content

Commit 535b0b6

Browse files
committed
reduce conflicts
2 parents d6c93c5 + a60aea8 commit 535b0b6

File tree

488 files changed

+30005
-8135
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

488 files changed

+30005
-8135
lines changed

assembly/pom.xml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
<deb.pkg.name>spark</deb.pkg.name>
4040
<deb.install.path>/usr/share/spark</deb.install.path>
4141
<deb.user>root</deb.user>
42-
<deb.bin.filemode>744</deb.bin.filemode>
42+
<deb.bin.filemode>755</deb.bin.filemode>
4343
</properties>
4444

4545
<dependencies>
@@ -280,7 +280,7 @@
280280
<user>${deb.user}</user>
281281
<group>${deb.user}</group>
282282
<prefix>${deb.install.path}/conf</prefix>
283-
<filemode>744</filemode>
283+
<filemode>${deb.bin.filemode}</filemode>
284284
</mapper>
285285
</data>
286286
<data>
@@ -302,7 +302,7 @@
302302
<user>${deb.user}</user>
303303
<group>${deb.user}</group>
304304
<prefix>${deb.install.path}/sbin</prefix>
305-
<filemode>744</filemode>
305+
<filemode>${deb.bin.filemode}</filemode>
306306
</mapper>
307307
</data>
308308
<data>
@@ -313,7 +313,7 @@
313313
<user>${deb.user}</user>
314314
<group>${deb.user}</group>
315315
<prefix>${deb.install.path}/python</prefix>
316-
<filemode>744</filemode>
316+
<filemode>${deb.bin.filemode}</filemode>
317317
</mapper>
318318
</data>
319319
</dataSet>

bin/spark-shell.cmd

100755100644
File mode changed.

bin/spark-submit2.cmd

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ set ORIG_ARGS=%*
2525
rem Reset the values of all variables used
2626
set SPARK_SUBMIT_DEPLOY_MODE=client
2727

28-
if not defined %SPARK_CONF_DIR% (
28+
if [%SPARK_CONF_DIR%] == [] (
2929
set SPARK_CONF_DIR=%SPARK_HOME%\conf
3030
)
3131
set SPARK_SUBMIT_PROPERTIES_FILE=%SPARK_CONF_DIR%\spark-defaults.conf

bin/utils.sh

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,17 @@ function gatherSparkSubmitOpts() {
2626
exit 1
2727
fi
2828

29-
# NOTE: If you add or remove spark-sumbmit options,
29+
# NOTE: If you add or remove spark-submit options,
3030
# modify NOT ONLY this script but also SparkSubmitArgument.scala
3131
SUBMISSION_OPTS=()
3232
APPLICATION_OPTS=()
3333
while (($#)); do
3434
case "$1" in
35-
--master | --deploy-mode | --class | --name | --jars | --py-files | --files | \
36-
--conf | --properties-file | --driver-memory | --driver-java-options | \
35+
--master | --deploy-mode | --class | --name | --jars | --packages | --py-files | --files | \
36+
--conf | --repositories | --properties-file | --driver-memory | --driver-java-options | \
3737
--driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
38-
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives)
38+
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives | \
39+
--proxy-user)
3940
if [[ $# -lt 2 ]]; then
4041
"$SUBMIT_USAGE_FUNCTION"
4142
exit 1;

bin/windows-utils.cmd

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ SET opts="\<--master\> \<--deploy-mode\> \<--class\> \<--name\> \<--jars\> \<--p
3232
SET opts="%opts:~1,-1% \<--conf\> \<--properties-file\> \<--driver-memory\> \<--driver-java-options\>"
3333
SET opts="%opts:~1,-1% \<--driver-library-path\> \<--driver-class-path\> \<--executor-memory\>"
3434
SET opts="%opts:~1,-1% \<--driver-cores\> \<--total-executor-cores\> \<--executor-cores\> \<--queue\>"
35-
SET opts="%opts:~1,-1% \<--num-executors\> \<--archives\>"
35+
SET opts="%opts:~1,-1% \<--num-executors\> \<--archives\> \<--packages\> \<--repositories\>"
36+
SET opts="%opts:~1,-1% \<--proxy-user\>"
3637

3738
echo %1 | findstr %opts% >nul
3839
if %ERRORLEVEL% equ 0 (

build/mvn

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,14 @@ install_app() {
3434
local binary="${_DIR}/$3"
3535

3636
# setup `curl` and `wget` silent options if we're running on Jenkins
37-
local curl_opts=""
37+
local curl_opts="-L"
3838
local wget_opts=""
3939
if [ -n "$AMPLAB_JENKINS" ]; then
40-
curl_opts="-s"
41-
wget_opts="--quiet"
40+
curl_opts="-s ${curl_opts}"
41+
wget_opts="--quiet ${wget_opts}"
4242
else
43-
curl_opts="--progress-bar"
44-
wget_opts="--progress=bar:force"
43+
curl_opts="--progress-bar ${curl_opts}"
44+
wget_opts="--progress=bar:force ${wget_opts}"
4545
fi
4646

4747
if [ -z "$3" -o ! -f "$binary" ]; then

build/sbt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,4 +125,32 @@ loadConfigFile() {
125125
[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
126126
[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
127127

128+
exit_status=127
129+
saved_stty=""
130+
131+
restoreSttySettings() {
132+
stty $saved_stty
133+
saved_stty=""
134+
}
135+
136+
onExit() {
137+
if [[ "$saved_stty" != "" ]]; then
138+
restoreSttySettings
139+
fi
140+
exit $exit_status
141+
}
142+
143+
saveSttySettings() {
144+
saved_stty=$(stty -g 2>/dev/null)
145+
if [[ ! $? ]]; then
146+
saved_stty=""
147+
fi
148+
}
149+
150+
saveSttySettings
151+
trap onExit INT
152+
128153
run "$@"
154+
155+
exit_status=$?
156+
onExit

build/sbt-launch-lib.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ execRunner () {
8181
echo ""
8282
}
8383

84-
exec "$@"
84+
"$@"
8585
}
8686

8787
addJava () {

core/pom.xml

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,23 @@
122122
<artifactId>jetty-http</artifactId>
123123
<scope>compile</scope>
124124
</dependency>
125+
<dependency>
126+
<groupId>org.eclipse.jetty</groupId>
127+
<artifactId>jetty-continuation</artifactId>
128+
<scope>compile</scope>
129+
</dependency>
130+
<dependency>
131+
<groupId>org.eclipse.jetty</groupId>
132+
<artifactId>jetty-servlet</artifactId>
133+
<scope>compile</scope>
134+
</dependency>
135+
<!-- Because we mark jetty as provided and shade it, its dependency
136+
orbit is ignored, so we explicitly list it here (see SPARK-5557).-->
137+
<dependency>
138+
<groupId>org.eclipse.jetty.orbit</groupId>
139+
<artifactId>javax.servlet</artifactId>
140+
<version>${orbit.version}</version>
141+
</dependency>
125142

126143
<dependency>
127144
<groupId>org.apache.commons</groupId>
@@ -226,11 +243,30 @@
226243
<groupId>io.dropwizard.metrics</groupId>
227244
<artifactId>metrics-graphite</artifactId>
228245
</dependency>
246+
<dependency>
247+
<groupId>com.fasterxml.jackson.core</groupId>
248+
<artifactId>jackson-databind</artifactId>
249+
</dependency>
250+
<dependency>
251+
<groupId>com.fasterxml.jackson.module</groupId>
252+
<artifactId>jackson-module-scala_2.10</artifactId>
253+
</dependency>
229254
<dependency>
230255
<groupId>org.apache.derby</groupId>
231256
<artifactId>derby</artifactId>
232257
<scope>test</scope>
233258
</dependency>
259+
<dependency>
260+
<groupId>org.apache.ivy</groupId>
261+
<artifactId>ivy</artifactId>
262+
<version>${ivy.version}</version>
263+
</dependency>
264+
<dependency>
265+
<groupId>oro</groupId>
266+
<!-- oro is needed by ivy, but only listed as an optional dependency, so we include it. -->
267+
<artifactId>oro</artifactId>
268+
<version>${oro.version}</version>
269+
</dependency>
234270
<dependency>
235271
<groupId>org.tachyonproject</groupId>
236272
<artifactId>tachyon-client</artifactId>
@@ -377,7 +413,7 @@
377413
<overWriteIfNewer>true</overWriteIfNewer>
378414
<useSubDirectoryPerType>true</useSubDirectoryPerType>
379415
<includeArtifactIds>
380-
guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server
416+
guava,jetty-io,jetty-servlet,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server
381417
</includeArtifactIds>
382418
<silent>true</silent>
383419
</configuration>
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark;
19+
20+
import org.apache.spark.scheduler.*;
21+
22+
/**
23+
* Class that allows users to receive all SparkListener events.
24+
* Users should override the onEvent method.
25+
*
26+
* This is a concrete Java class in order to ensure that we don't forget to update it when adding
27+
* new methods to SparkListener: forgetting to add a method will result in a compilation error (if
28+
* this was a concrete Scala class, default implementations of new event handlers would be inherited
29+
* from the SparkListener trait).
30+
*/
31+
public class SparkFirehoseListener implements SparkListener {
32+
33+
public void onEvent(SparkListenerEvent event) { }
34+
35+
@Override
36+
public final void onStageCompleted(SparkListenerStageCompleted stageCompleted) {
37+
onEvent(stageCompleted);
38+
}
39+
40+
@Override
41+
public final void onStageSubmitted(SparkListenerStageSubmitted stageSubmitted) {
42+
onEvent(stageSubmitted);
43+
}
44+
45+
@Override
46+
public final void onTaskStart(SparkListenerTaskStart taskStart) {
47+
onEvent(taskStart);
48+
}
49+
50+
@Override
51+
public final void onTaskGettingResult(SparkListenerTaskGettingResult taskGettingResult) {
52+
onEvent(taskGettingResult);
53+
}
54+
55+
@Override
56+
public final void onTaskEnd(SparkListenerTaskEnd taskEnd) {
57+
onEvent(taskEnd);
58+
}
59+
60+
@Override
61+
public final void onJobStart(SparkListenerJobStart jobStart) {
62+
onEvent(jobStart);
63+
}
64+
65+
@Override
66+
public final void onJobEnd(SparkListenerJobEnd jobEnd) {
67+
onEvent(jobEnd);
68+
}
69+
70+
@Override
71+
public final void onEnvironmentUpdate(SparkListenerEnvironmentUpdate environmentUpdate) {
72+
onEvent(environmentUpdate);
73+
}
74+
75+
@Override
76+
public final void onBlockManagerAdded(SparkListenerBlockManagerAdded blockManagerAdded) {
77+
onEvent(blockManagerAdded);
78+
}
79+
80+
@Override
81+
public final void onBlockManagerRemoved(SparkListenerBlockManagerRemoved blockManagerRemoved) {
82+
onEvent(blockManagerRemoved);
83+
}
84+
85+
@Override
86+
public final void onUnpersistRDD(SparkListenerUnpersistRDD unpersistRDD) {
87+
onEvent(unpersistRDD);
88+
}
89+
90+
@Override
91+
public final void onApplicationStart(SparkListenerApplicationStart applicationStart) {
92+
onEvent(applicationStart);
93+
}
94+
95+
@Override
96+
public final void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
97+
onEvent(applicationEnd);
98+
}
99+
100+
@Override
101+
public final void onExecutorMetricsUpdate(
102+
SparkListenerExecutorMetricsUpdate executorMetricsUpdate) {
103+
onEvent(executorMetricsUpdate);
104+
}
105+
106+
@Override
107+
public final void onExecutorAdded(SparkListenerExecutorAdded executorAdded) {
108+
onEvent(executorAdded);
109+
}
110+
111+
@Override
112+
public final void onExecutorRemoved(SparkListenerExecutorRemoved executorRemoved) {
113+
onEvent(executorRemoved);
114+
}
115+
}

0 commit comments

Comments
 (0)