Skip to content

Commit 83c844f

Browse files
author
Davies Liu
committed
Merge branch 'master' of github.com:apache/spark into whole
2 parents 22e8b3e + 23f966f commit 83c844f

File tree

359 files changed

+18713
-4472
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

359 files changed

+18713
-4472
lines changed

‎.gitattributes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.bat text eol=crlf
2+
*.cmd text eol=crlf

‎.rat-excludes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
target
22
.gitignore
3+
.gitattributes
34
.project
45
.classpath
56
.mima-excludes

‎assembly/pom.xml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -146,10 +146,6 @@
146146
<exclude>com/google/common/base/Present*</exclude>
147147
</excludes>
148148
</relocation>
149-
<relocation>
150-
<pattern>org.apache.commons.math3</pattern>
151-
<shadedPattern>org.spark-project.commons.math3</shadedPattern>
152-
</relocation>
153149
</relocations>
154150
<transformers>
155151
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
@@ -201,12 +197,6 @@
201197
<artifactId>spark-hive_${scala.binary.version}</artifactId>
202198
<version>${project.version}</version>
203199
</dependency>
204-
</dependencies>
205-
</profile>
206-
<profile>
207-
<!-- TODO: Move this to "hive" profile once 0.13 JDBC is supported -->
208-
<id>hive-0.12.0</id>
209-
<dependencies>
210200
<dependency>
211201
<groupId>org.apache.spark</groupId>
212202
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>

‎bin/compute-classpath.cmd

Lines changed: 117 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -1,117 +1,117 @@
1-
@echo off
2-
3-
rem
4-
rem Licensed to the Apache Software Foundation (ASF) under one or more
5-
rem contributor license agreements. See the NOTICE file distributed with
6-
rem this work for additional information regarding copyright ownership.
7-
rem The ASF licenses this file to You under the Apache License, Version 2.0
8-
rem (the "License"); you may not use this file except in compliance with
9-
rem the License. You may obtain a copy of the License at
10-
rem
11-
rem http://www.apache.org/licenses/LICENSE-2.0
12-
rem
13-
rem Unless required by applicable law or agreed to in writing, software
14-
rem distributed under the License is distributed on an "AS IS" BASIS,
15-
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16-
rem See the License for the specific language governing permissions and
17-
rem limitations under the License.
18-
rem
19-
20-
rem This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
21-
rem script and the ExecutorRunner in standalone cluster mode.
22-
23-
rem If we're called from spark-class2.cmd, it already set enabledelayedexpansion and setting
24-
rem it here would stop us from affecting its copy of the CLASSPATH variable; otherwise we
25-
rem need to set it here because we use !datanucleus_jars! below.
26-
if "%DONT_PRINT_CLASSPATH%"=="1" goto skip_delayed_expansion
27-
setlocal enabledelayedexpansion
28-
:skip_delayed_expansion
29-
30-
set SCALA_VERSION=2.10
31-
32-
rem Figure out where the Spark framework is installed
33-
set FWDIR=%~dp0..\
34-
35-
rem Load environment variables from conf\spark-env.cmd, if it exists
36-
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
37-
38-
rem Build up classpath
39-
set CLASSPATH=%SPARK_CLASSPATH%;%SPARK_SUBMIT_CLASSPATH%
40-
41-
if not "x%SPARK_CONF_DIR%"=="x" (
42-
set CLASSPATH=%CLASSPATH%;%SPARK_CONF_DIR%
43-
) else (
44-
set CLASSPATH=%CLASSPATH%;%FWDIR%conf
45-
)
46-
47-
if exist "%FWDIR%RELEASE" (
48-
for %%d in ("%FWDIR%lib\spark-assembly*.jar") do (
49-
set ASSEMBLY_JAR=%%d
50-
)
51-
) else (
52-
for %%d in ("%FWDIR%assembly\target\scala-%SCALA_VERSION%\spark-assembly*hadoop*.jar") do (
53-
set ASSEMBLY_JAR=%%d
54-
)
55-
)
56-
57-
set CLASSPATH=%CLASSPATH%;%ASSEMBLY_JAR%
58-
59-
rem When Hive support is needed, Datanucleus jars must be included on the classpath.
60-
rem Datanucleus jars do not work if only included in the uber jar as plugin.xml metadata is lost.
61-
rem Both sbt and maven will populate "lib_managed/jars/" with the datanucleus jars when Spark is
62-
rem built with Hive, so look for them there.
63-
if exist "%FWDIR%RELEASE" (
64-
set datanucleus_dir=%FWDIR%lib
65-
) else (
66-
set datanucleus_dir=%FWDIR%lib_managed\jars
67-
)
68-
set "datanucleus_jars="
69-
for %%d in ("%datanucleus_dir%\datanucleus-*.jar") do (
70-
set datanucleus_jars=!datanucleus_jars!;%%d
71-
)
72-
set CLASSPATH=%CLASSPATH%;%datanucleus_jars%
73-
74-
set SPARK_CLASSES=%FWDIR%core\target\scala-%SCALA_VERSION%\classes
75-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%repl\target\scala-%SCALA_VERSION%\classes
76-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\classes
77-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%bagel\target\scala-%SCALA_VERSION%\classes
78-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%graphx\target\scala-%SCALA_VERSION%\classes
79-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%streaming\target\scala-%SCALA_VERSION%\classes
80-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%tools\target\scala-%SCALA_VERSION%\classes
81-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\catalyst\target\scala-%SCALA_VERSION%\classes
82-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\core\target\scala-%SCALA_VERSION%\classes
83-
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\hive\target\scala-%SCALA_VERSION%\classes
84-
85-
set SPARK_TEST_CLASSES=%FWDIR%core\target\scala-%SCALA_VERSION%\test-classes
86-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%repl\target\scala-%SCALA_VERSION%\test-classes
87-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\test-classes
88-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%bagel\target\scala-%SCALA_VERSION%\test-classes
89-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%graphx\target\scala-%SCALA_VERSION%\test-classes
90-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%streaming\target\scala-%SCALA_VERSION%\test-classes
91-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\catalyst\target\scala-%SCALA_VERSION%\test-classes
92-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\core\target\scala-%SCALA_VERSION%\test-classes
93-
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\hive\target\scala-%SCALA_VERSION%\test-classes
94-
95-
if "x%SPARK_TESTING%"=="x1" (
96-
rem Add test clases to path - note, add SPARK_CLASSES and SPARK_TEST_CLASSES before CLASSPATH
97-
rem so that local compilation takes precedence over assembled jar
98-
set CLASSPATH=%SPARK_CLASSES%;%SPARK_TEST_CLASSES%;%CLASSPATH%
99-
)
100-
101-
rem Add hadoop conf dir - else FileSystem.*, etc fail
102-
rem Note, this assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
103-
rem the configurtion files.
104-
if "x%HADOOP_CONF_DIR%"=="x" goto no_hadoop_conf_dir
105-
set CLASSPATH=%CLASSPATH%;%HADOOP_CONF_DIR%
106-
:no_hadoop_conf_dir
107-
108-
if "x%YARN_CONF_DIR%"=="x" goto no_yarn_conf_dir
109-
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%
110-
:no_yarn_conf_dir
111-
112-
rem A bit of a hack to allow calling this script within run2.cmd without seeing output
113-
if "%DONT_PRINT_CLASSPATH%"=="1" goto exit
114-
115-
echo %CLASSPATH%
116-
117-
:exit
1+
@echo off
2+
3+
rem
4+
rem Licensed to the Apache Software Foundation (ASF) under one or more
5+
rem contributor license agreements. See the NOTICE file distributed with
6+
rem this work for additional information regarding copyright ownership.
7+
rem The ASF licenses this file to You under the Apache License, Version 2.0
8+
rem (the "License"); you may not use this file except in compliance with
9+
rem the License. You may obtain a copy of the License at
10+
rem
11+
rem http://www.apache.org/licenses/LICENSE-2.0
12+
rem
13+
rem Unless required by applicable law or agreed to in writing, software
14+
rem distributed under the License is distributed on an "AS IS" BASIS,
15+
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
rem See the License for the specific language governing permissions and
17+
rem limitations under the License.
18+
rem
19+
20+
rem This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
21+
rem script and the ExecutorRunner in standalone cluster mode.
22+
23+
rem If we're called from spark-class2.cmd, it already set enabledelayedexpansion and setting
24+
rem it here would stop us from affecting its copy of the CLASSPATH variable; otherwise we
25+
rem need to set it here because we use !datanucleus_jars! below.
26+
if "%DONT_PRINT_CLASSPATH%"=="1" goto skip_delayed_expansion
27+
setlocal enabledelayedexpansion
28+
:skip_delayed_expansion
29+
30+
set SCALA_VERSION=2.10
31+
32+
rem Figure out where the Spark framework is installed
33+
set FWDIR=%~dp0..\
34+
35+
rem Load environment variables from conf\spark-env.cmd, if it exists
36+
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
37+
38+
rem Build up classpath
39+
set CLASSPATH=%SPARK_CLASSPATH%;%SPARK_SUBMIT_CLASSPATH%
40+
41+
if not "x%SPARK_CONF_DIR%"=="x" (
42+
set CLASSPATH=%CLASSPATH%;%SPARK_CONF_DIR%
43+
) else (
44+
set CLASSPATH=%CLASSPATH%;%FWDIR%conf
45+
)
46+
47+
if exist "%FWDIR%RELEASE" (
48+
for %%d in ("%FWDIR%lib\spark-assembly*.jar") do (
49+
set ASSEMBLY_JAR=%%d
50+
)
51+
) else (
52+
for %%d in ("%FWDIR%assembly\target\scala-%SCALA_VERSION%\spark-assembly*hadoop*.jar") do (
53+
set ASSEMBLY_JAR=%%d
54+
)
55+
)
56+
57+
set CLASSPATH=%CLASSPATH%;%ASSEMBLY_JAR%
58+
59+
rem When Hive support is needed, Datanucleus jars must be included on the classpath.
60+
rem Datanucleus jars do not work if only included in the uber jar as plugin.xml metadata is lost.
61+
rem Both sbt and maven will populate "lib_managed/jars/" with the datanucleus jars when Spark is
62+
rem built with Hive, so look for them there.
63+
if exist "%FWDIR%RELEASE" (
64+
set datanucleus_dir=%FWDIR%lib
65+
) else (
66+
set datanucleus_dir=%FWDIR%lib_managed\jars
67+
)
68+
set "datanucleus_jars="
69+
for %%d in ("%datanucleus_dir%\datanucleus-*.jar") do (
70+
set datanucleus_jars=!datanucleus_jars!;%%d
71+
)
72+
set CLASSPATH=%CLASSPATH%;%datanucleus_jars%
73+
74+
set SPARK_CLASSES=%FWDIR%core\target\scala-%SCALA_VERSION%\classes
75+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%repl\target\scala-%SCALA_VERSION%\classes
76+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\classes
77+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%bagel\target\scala-%SCALA_VERSION%\classes
78+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%graphx\target\scala-%SCALA_VERSION%\classes
79+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%streaming\target\scala-%SCALA_VERSION%\classes
80+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%tools\target\scala-%SCALA_VERSION%\classes
81+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\catalyst\target\scala-%SCALA_VERSION%\classes
82+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\core\target\scala-%SCALA_VERSION%\classes
83+
set SPARK_CLASSES=%SPARK_CLASSES%;%FWDIR%sql\hive\target\scala-%SCALA_VERSION%\classes
84+
85+
set SPARK_TEST_CLASSES=%FWDIR%core\target\scala-%SCALA_VERSION%\test-classes
86+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%repl\target\scala-%SCALA_VERSION%\test-classes
87+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\test-classes
88+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%bagel\target\scala-%SCALA_VERSION%\test-classes
89+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%graphx\target\scala-%SCALA_VERSION%\test-classes
90+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%streaming\target\scala-%SCALA_VERSION%\test-classes
91+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\catalyst\target\scala-%SCALA_VERSION%\test-classes
92+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\core\target\scala-%SCALA_VERSION%\test-classes
93+
set SPARK_TEST_CLASSES=%SPARK_TEST_CLASSES%;%FWDIR%sql\hive\target\scala-%SCALA_VERSION%\test-classes
94+
95+
if "x%SPARK_TESTING%"=="x1" (
96+
rem Add test clases to path - note, add SPARK_CLASSES and SPARK_TEST_CLASSES before CLASSPATH
97+
rem so that local compilation takes precedence over assembled jar
98+
set CLASSPATH=%SPARK_CLASSES%;%SPARK_TEST_CLASSES%;%CLASSPATH%
99+
)
100+
101+
rem Add hadoop conf dir - else FileSystem.*, etc fail
102+
rem Note, this assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
103+
rem the configurtion files.
104+
if "x%HADOOP_CONF_DIR%"=="x" goto no_hadoop_conf_dir
105+
set CLASSPATH=%CLASSPATH%;%HADOOP_CONF_DIR%
106+
:no_hadoop_conf_dir
107+
108+
if "x%YARN_CONF_DIR%"=="x" goto no_yarn_conf_dir
109+
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%
110+
:no_yarn_conf_dir
111+
112+
rem A bit of a hack to allow calling this script within run2.cmd without seeing output
113+
if "%DONT_PRINT_CLASSPATH%"=="1" goto exit
114+
115+
echo %CLASSPATH%
116+
117+
:exit

‎bin/spark-class

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,11 @@ case "$1" in
8181
OUR_JAVA_OPTS="$SPARK_JAVA_OPTS $SPARK_SUBMIT_OPTS"
8282
OUR_JAVA_MEM=${SPARK_DRIVER_MEMORY:-$DEFAULT_MEM}
8383
if [ -n "$SPARK_SUBMIT_LIBRARY_PATH" ]; then
84-
OUR_JAVA_OPTS="$OUR_JAVA_OPTS -Djava.library.path=$SPARK_SUBMIT_LIBRARY_PATH"
84+
if [[ $OSTYPE == darwin* ]]; then
85+
export DYLD_LIBRARY_PATH="$SPARK_SUBMIT_LIBRARY_PATH:$DYLD_LIBRARY_PATH"
86+
else
87+
export LD_LIBRARY_PATH="$SPARK_SUBMIT_LIBRARY_PATH:$LD_LIBRARY_PATH"
88+
fi
8589
fi
8690
if [ -n "$SPARK_SUBMIT_DRIVER_MEMORY" ]; then
8791
OUR_JAVA_MEM="$SPARK_SUBMIT_DRIVER_MEMORY"

‎core/pom.xml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,16 @@
4444
</exclusion>
4545
</exclusions>
4646
</dependency>
47+
<dependency>
48+
<groupId>org.apache.spark</groupId>
49+
<artifactId>spark-network-common_2.10</artifactId>
50+
<version>${project.version}</version>
51+
</dependency>
52+
<dependency>
53+
<groupId>org.apache.spark</groupId>
54+
<artifactId>spark-network-shuffle_2.10</artifactId>
55+
<version>${project.version}</version>
56+
</dependency>
4757
<dependency>
4858
<groupId>net.java.dev.jets3t</groupId>
4959
<artifactId>jets3t</artifactId>

‎core/src/main/resources/org/apache/spark/ui/static/additional-metrics.js

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
/* Register functions to show/hide columns based on checkboxes. These need
19+
* to be registered after the page loads. */
20+
$(function() {
21+
$("span.expand-additional-metrics").click(function(){
22+
// Expand the list of additional metrics.
23+
var additionalMetricsDiv = $(this).parent().find('.additional-metrics');
24+
$(additionalMetricsDiv).toggleClass('collapsed');
25+
26+
// Switch the class of the arrow from open to closed.
27+
$(this).find('.expand-additional-metrics-arrow').toggleClass('arrow-open');
28+
$(this).find('.expand-additional-metrics-arrow').toggleClass('arrow-closed');
29+
30+
// If clicking caused the metrics to expand, automatically check all options for additional
31+
// metrics (don't trigger a click when collapsing metrics, because it leads to weird
32+
// toggling behavior).
33+
if (!$(additionalMetricsDiv).hasClass('collapsed')) {
34+
$(this).parent().find('input:checkbox:not(:checked)').trigger('click');
35+
}
36+
});
37+
38+
$("input:checkbox:not(:checked)").each(function() {
39+
var column = "table ." + $(this).attr("name");
40+
$(column).hide();
41+
});
42+
43+
$("input:checkbox").click(function() {
44+
var column = "table ." + $(this).attr("name");
45+
$(column).toggle();
46+
stripeTables();
47+
});
48+
49+
// Trigger a click on the checkbox if a user clicks the label next to it.
50+
$("span.additional-metric-title").click(function() {
51+
$(this).parent().find('input:checkbox').trigger('click');
52+
});
53+
});

‎core/src/main/resources/org/apache/spark/ui/static/table.js

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
/* Adds background colors to stripe table rows. This is necessary (instead of using css or the
19+
* table striping provided by bootstrap) to appropriately stripe tables with hidden rows. */
20+
function stripeTables() {
21+
$("table.table-striped-custom").each(function() {
22+
$(this).find("tr:not(:hidden)").each(function (index) {
23+
if (index % 2 == 1) {
24+
$(this).css("background-color", "#f9f9f9");
25+
} else {
26+
$(this).css("background-color", "#ffffff");
27+
}
28+
});
29+
});
30+
}
31+
32+
/* Stripe all tables after pages finish loading. */
33+
$(function() {
34+
stripeTables();
35+
});

0 commit comments

Comments
 (0)