Skip to content

Commit 38eb535

Browse files
msjgriffithsmsjgriffiths
committed
Merge pull request #2 from apache/master
Updating to latest Spark master
2 parents b08e865 + 57566d0 commit 38eb535

File tree

641 files changed

+32903
-11437
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

641 files changed

+32903
-11437
lines changed

‎.rat-excludes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
target
2+
cache
23
.gitignore
34
.gitattributes
45
.project
@@ -18,6 +19,7 @@ fairscheduler.xml.template
1819
spark-defaults.conf.template
1920
log4j.properties
2021
log4j.properties.template
22+
metrics.properties
2123
metrics.properties.template
2224
slaves
2325
slaves.template

‎assembly/pom.xml

Lines changed: 10 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@
3636
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
3737
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
3838
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>
39-
<deb.pkg.name>spark</deb.pkg.name>
40-
<deb.install.path>/usr/share/spark</deb.install.path>
41-
<deb.user>root</deb.user>
42-
<deb.bin.filemode>744</deb.bin.filemode>
4339
</properties>
4440

4541
<dependencies>
@@ -118,6 +114,16 @@
118114
<exclude>META-INF/*.RSA</exclude>
119115
</excludes>
120116
</filter>
117+
<filter>
118+
<!-- Exclude libgfortran, libgcc for license issues -->
119+
<artifact>org.jblas:jblas</artifact>
120+
<excludes>
121+
<!-- Linux amd64 is OK; not statically linked -->
122+
<exclude>lib/static/Linux/i386/**</exclude>
123+
<exclude>lib/static/Mac OS X/**</exclude>
124+
<exclude>lib/static/Windows/**</exclude>
125+
</excludes>
126+
</filter>
121127
</filters>
122128
</configuration>
123129
<executions>
@@ -217,113 +223,6 @@
217223
</plugins>
218224
</build>
219225
</profile>
220-
<profile>
221-
<id>deb</id>
222-
<build>
223-
<plugins>
224-
<plugin>
225-
<groupId>org.codehaus.mojo</groupId>
226-
<artifactId>buildnumber-maven-plugin</artifactId>
227-
<version>1.2</version>
228-
<executions>
229-
<execution>
230-
<phase>validate</phase>
231-
<goals>
232-
<goal>create</goal>
233-
</goals>
234-
<configuration>
235-
<shortRevisionLength>8</shortRevisionLength>
236-
</configuration>
237-
</execution>
238-
</executions>
239-
</plugin>
240-
<plugin>
241-
<groupId>org.vafer</groupId>
242-
<artifactId>jdeb</artifactId>
243-
<version>0.11</version>
244-
<executions>
245-
<execution>
246-
<phase>package</phase>
247-
<goals>
248-
<goal>jdeb</goal>
249-
</goals>
250-
<configuration>
251-
<deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb>
252-
<attach>false</attach>
253-
<compression>gzip</compression>
254-
<dataSet>
255-
<data>
256-
<src>${spark.jar}</src>
257-
<type>file</type>
258-
<mapper>
259-
<type>perm</type>
260-
<user>${deb.user}</user>
261-
<group>${deb.user}</group>
262-
<prefix>${deb.install.path}/jars</prefix>
263-
</mapper>
264-
</data>
265-
<data>
266-
<src>${basedir}/src/deb/RELEASE</src>
267-
<type>file</type>
268-
<mapper>
269-
<type>perm</type>
270-
<user>${deb.user}</user>
271-
<group>${deb.user}</group>
272-
<prefix>${deb.install.path}</prefix>
273-
</mapper>
274-
</data>
275-
<data>
276-
<src>${basedir}/../conf</src>
277-
<type>directory</type>
278-
<mapper>
279-
<type>perm</type>
280-
<user>${deb.user}</user>
281-
<group>${deb.user}</group>
282-
<prefix>${deb.install.path}/conf</prefix>
283-
<filemode>744</filemode>
284-
</mapper>
285-
</data>
286-
<data>
287-
<src>${basedir}/../bin</src>
288-
<type>directory</type>
289-
<mapper>
290-
<type>perm</type>
291-
<user>${deb.user}</user>
292-
<group>${deb.user}</group>
293-
<prefix>${deb.install.path}/bin</prefix>
294-
<filemode>${deb.bin.filemode}</filemode>
295-
</mapper>
296-
</data>
297-
<data>
298-
<src>${basedir}/../sbin</src>
299-
<type>directory</type>
300-
<mapper>
301-
<type>perm</type>
302-
<user>${deb.user}</user>
303-
<group>${deb.user}</group>
304-
<prefix>${deb.install.path}/sbin</prefix>
305-
<filemode>744</filemode>
306-
</mapper>
307-
</data>
308-
<data>
309-
<src>${basedir}/../python</src>
310-
<type>directory</type>
311-
<mapper>
312-
<type>perm</type>
313-
<user>${deb.user}</user>
314-
<group>${deb.user}</group>
315-
<prefix>${deb.install.path}/python</prefix>
316-
<filemode>744</filemode>
317-
</mapper>
318-
</data>
319-
</dataSet>
320-
</configuration>
321-
</execution>
322-
</executions>
323-
</plugin>
324-
</plugins>
325-
</build>
326-
</profile>
327226
<profile>
328227
<id>kinesis-asl</id>
329228
<dependencies>

‎assembly/src/deb/RELEASE

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎assembly/src/deb/control/control

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎bin/compute-classpath.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ fi
7676

7777
num_jars=0
7878

79-
for f in ${assembly_folder}/spark-assembly*hadoop*.jar; do
79+
for f in "${assembly_folder}"/spark-assembly*hadoop*.jar; do
8080
if [[ ! -e "$f" ]]; then
8181
echo "Failed to find Spark assembly in $assembly_folder" 1>&2
8282
echo "You need to build Spark before running this program." 1>&2
@@ -88,7 +88,7 @@ done
8888

8989
if [ "$num_jars" -gt "1" ]; then
9090
echo "Found multiple Spark assembly jars in $assembly_folder:" 1>&2
91-
ls ${assembly_folder}/spark-assembly*hadoop*.jar 1>&2
91+
ls "${assembly_folder}"/spark-assembly*hadoop*.jar 1>&2
9292
echo "Please remove all but one jar." 1>&2
9393
exit 1
9494
fi

‎bin/run-example

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ fi
4242

4343
JAR_COUNT=0
4444

45-
for f in ${JAR_PATH}/spark-examples-*hadoop*.jar; do
45+
for f in "${JAR_PATH}"/spark-examples-*hadoop*.jar; do
4646
if [[ ! -e "$f" ]]; then
4747
echo "Failed to find Spark examples assembly in $FWDIR/lib or $FWDIR/examples/target" 1>&2
4848
echo "You need to build Spark before running this program" 1>&2
@@ -54,7 +54,7 @@ done
5454

5555
if [ "$JAR_COUNT" -gt "1" ]; then
5656
echo "Found multiple Spark examples assembly jars in ${JAR_PATH}" 1>&2
57-
ls ${JAR_PATH}/spark-examples-*hadoop*.jar 1>&2
57+
ls "${JAR_PATH}"/spark-examples-*hadoop*.jar 1>&2
5858
echo "Please remove all but one jar." 1>&2
5959
exit 1
6060
fi

‎bin/spark-shell.cmd

100755100644
File mode changed.

‎bin/spark-submit2.cmd

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ set ORIG_ARGS=%*
2525
rem Reset the values of all variables used
2626
set SPARK_SUBMIT_DEPLOY_MODE=client
2727

28-
if not defined %SPARK_CONF_DIR% (
28+
if [%SPARK_CONF_DIR%] == [] (
2929
set SPARK_CONF_DIR=%SPARK_HOME%\conf
3030
)
3131
set SPARK_SUBMIT_PROPERTIES_FILE=%SPARK_CONF_DIR%\spark-defaults.conf

‎bin/utils.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ function gatherSparkSubmitOpts() {
3535
--master | --deploy-mode | --class | --name | --jars | --packages | --py-files | --files | \
3636
--conf | --repositories | --properties-file | --driver-memory | --driver-java-options | \
3737
--driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
38-
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives)
38+
--total-executor-cores | --executor-cores | --queue | --num-executors | --archives | \
39+
--proxy-user)
3940
if [[ $# -lt 2 ]]; then
4041
"$SUBMIT_USAGE_FUNCTION"
4142
exit 1;

‎bin/windows-utils.cmd

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ SET opts="%opts:~1,-1% \<--conf\> \<--properties-file\> \<--driver-memory\> \<--
3333
SET opts="%opts:~1,-1% \<--driver-library-path\> \<--driver-class-path\> \<--executor-memory\>"
3434
SET opts="%opts:~1,-1% \<--driver-cores\> \<--total-executor-cores\> \<--executor-cores\> \<--queue\>"
3535
SET opts="%opts:~1,-1% \<--num-executors\> \<--archives\> \<--packages\> \<--repositories\>"
36+
SET opts="%opts:~1,-1% \<--proxy-user\>"
3637

3738
echo %1 | findstr %opts% >nul
3839
if %ERRORLEVEL% equ 0 (

‎build/mvn

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
2222
# Preserve the calling directory
2323
_CALLING_DIR="$(pwd)"
24+
# Options used during compilation
25+
_COMPILE_JVM_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
2426

2527
# Installs any application tarball given a URL, the expected tarball name,
2628
# and, optionally, a checkable binary path to determine if the binary has
@@ -34,14 +36,14 @@ install_app() {
3436
local binary="${_DIR}/$3"
3537

3638
# setup `curl` and `wget` silent options if we're running on Jenkins
37-
local curl_opts=""
39+
local curl_opts="-L"
3840
local wget_opts=""
3941
if [ -n "$AMPLAB_JENKINS" ]; then
40-
curl_opts="-s"
41-
wget_opts="--quiet"
42+
curl_opts="-s ${curl_opts}"
43+
wget_opts="--quiet ${wget_opts}"
4244
else
43-
curl_opts="--progress-bar"
44-
wget_opts="--progress=bar:force"
45+
curl_opts="--progress-bar ${curl_opts}"
46+
wget_opts="--progress=bar:force ${wget_opts}"
4547
fi
4648

4749
if [ -z "$3" -o ! -f "$binary" ]; then
@@ -136,14 +138,15 @@ cd "${_CALLING_DIR}"
136138
# Now that zinc is ensured to be installed, check its status and, if its
137139
# not running or just installed, start it
138140
if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`${ZINC_BIN} -status`" ]; then
141+
export ZINC_OPTS=${ZINC_OPTS:-"$_COMPILE_JVM_OPTS"}
139142
${ZINC_BIN} -shutdown
140143
${ZINC_BIN} -start -port ${ZINC_PORT} \
141144
-scala-compiler "${SCALA_COMPILER}" \
142145
-scala-library "${SCALA_LIBRARY}" &>/dev/null
143146
fi
144147

145148
# Set any `mvn` options if not already present
146-
export MAVEN_OPTS=${MAVEN_OPTS:-"-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"}
149+
export MAVEN_OPTS=${MAVEN_OPTS:-"$_COMPILE_JVM_OPTS"}
147150

148151
# Last, call the `mvn` command as usual
149152
${MVN_BIN} "$@"

‎build/sbt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,4 +125,32 @@ loadConfigFile() {
125125
[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
126126
[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
127127

128+
exit_status=127
129+
saved_stty=""
130+
131+
restoreSttySettings() {
132+
stty $saved_stty
133+
saved_stty=""
134+
}
135+
136+
onExit() {
137+
if [[ "$saved_stty" != "" ]]; then
138+
restoreSttySettings
139+
fi
140+
exit $exit_status
141+
}
142+
143+
saveSttySettings() {
144+
saved_stty=$(stty -g 2>/dev/null)
145+
if [[ ! $? ]]; then
146+
saved_stty=""
147+
fi
148+
}
149+
150+
saveSttySettings
151+
trap onExit INT
152+
128153
run "$@"
154+
155+
exit_status=$?
156+
onExit

‎build/sbt-launch-lib.bash

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ execRunner () {
8181
echo ""
8282
}
8383

84-
exec "$@"
84+
"$@"
8585
}
8686

8787
addJava () {

‎conf/metrics.properties.template

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,15 @@
122122

123123
#worker.sink.csv.unit=minutes
124124

125+
# Enable Slf4jSink for all instances by class name
126+
#*.sink.slf4j.class=org.apache.spark.metrics.sink.Slf4jSink
127+
128+
# Polling period for Slf4JSink
129+
#*.sink.sl4j.period=1
130+
131+
#*.sink.sl4j.unit=minutes
132+
133+
125134
# Enable jvm source for instance master, worker, driver and executor
126135
#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource
127136

‎core/pom.xml

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,13 @@
132132
<artifactId>jetty-servlet</artifactId>
133133
<scope>compile</scope>
134134
</dependency>
135+
<!-- Because we mark jetty as provided and shade it, its dependency
136+
orbit is ignored, so we explicitly list it here (see SPARK-5557).-->
137+
<dependency>
138+
<groupId>org.eclipse.jetty.orbit</groupId>
139+
<artifactId>javax.servlet</artifactId>
140+
<version>${orbit.version}</version>
141+
</dependency>
135142

136143
<dependency>
137144
<groupId>org.apache.commons</groupId>
@@ -236,6 +243,14 @@
236243
<groupId>io.dropwizard.metrics</groupId>
237244
<artifactId>metrics-graphite</artifactId>
238245
</dependency>
246+
<dependency>
247+
<groupId>com.fasterxml.jackson.core</groupId>
248+
<artifactId>jackson-databind</artifactId>
249+
</dependency>
250+
<dependency>
251+
<groupId>com.fasterxml.jackson.module</groupId>
252+
<artifactId>jackson-module-scala_2.10</artifactId>
253+
</dependency>
239254
<dependency>
240255
<groupId>org.apache.derby</groupId>
241256
<artifactId>derby</artifactId>
@@ -314,16 +329,6 @@
314329
<artifactId>scalacheck_${scala.binary.version}</artifactId>
315330
<scope>test</scope>
316331
</dependency>
317-
<dependency>
318-
<groupId>org.easymock</groupId>
319-
<artifactId>easymockclassextension</artifactId>
320-
<scope>test</scope>
321-
</dependency>
322-
<dependency>
323-
<groupId>asm</groupId>
324-
<artifactId>asm</artifactId>
325-
<scope>test</scope>
326-
</dependency>
327332
<dependency>
328333
<groupId>junit</groupId>
329334
<artifactId>junit</artifactId>

0 commit comments

Comments
 (0)