Skip to content

Commit 3a1da85

Browse files
imback82imback82
committed
Merge branch 'master' into refresh_table
2 parents fe577f5 + 3bf5355 commit 3a1da85

File tree

13 files changed

+349
-55
lines changed

13 files changed

+349
-55
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/sorttable.js‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -97,9 +97,14 @@ sorttable = {
9797
sorttable.reverse(this.sorttable_tbody);
9898
this.className = this.className.replace('sorttable_sorted',
9999
'sorttable_sorted_reverse');
100-
this.removeChild(document.getElementById('sorttable_sortfwdind'));
100+
rowlists = this.parentNode.getElementsByTagName("span");
101+
for (var j=0; j < rowlists.length; j++) {
102+
if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)) {
103+
rowlists[j].parentNode.removeChild(rowlists[j]);
104+
}
105+
}
101106
sortrevind = document.createElement('span');
102-
sortrevind.id = "sorttable_sortrevind";
107+
sortrevind.class = "sorttable_sortrevind";
103108
sortrevind.innerHTML = stIsIE ? '&nbsp<font face="webdings">5</font>' : '&nbsp;&#x25BE;';
104109
this.appendChild(sortrevind);
105110
return;
@@ -110,9 +115,14 @@ sorttable = {
110115
sorttable.reverse(this.sorttable_tbody);
111116
this.className = this.className.replace('sorttable_sorted_reverse',
112117
'sorttable_sorted');
113-
this.removeChild(document.getElementById('sorttable_sortrevind'));
118+
rowlists = this.parentNode.getElementsByTagName("span");
119+
for (var j=0; j < rowlists.length; j++) {
120+
if (rowlists[j].className.search(/\sorttable_sortrevind\b/)) {
121+
rowlists[j].parentNode.removeChild(rowlists[j]);
122+
}
123+
}
114124
sortfwdind = document.createElement('span');
115-
sortfwdind.id = "sorttable_sortfwdind";
125+
sortfwdind.class = "sorttable_sortfwdind";
116126
sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
117127
this.appendChild(sortfwdind);
118128
return;
@@ -126,14 +136,17 @@ sorttable = {
126136
cell.className = cell.className.replace('sorttable_sorted','');
127137
}
128138
});
129-
sortfwdind = document.getElementById('sorttable_sortfwdind');
130-
if (sortfwdind) { sortfwdind.parentNode.removeChild(sortfwdind); }
131-
sortrevind = document.getElementById('sorttable_sortrevind');
132-
if (sortrevind) { sortrevind.parentNode.removeChild(sortrevind); }
139+
rowlists = this.parentNode.getElementsByTagName("span");
140+
for (var j=0; j < rowlists.length; j++) {
141+
if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)
142+
|| rowlists[j].className.search(/\sorttable_sortrevind\b/) ) {
143+
rowlists[j].parentNode.removeChild(rowlists[j]);
144+
}
145+
}
133146

134147
this.className += ' sorttable_sorted';
135148
sortfwdind = document.createElement('span');
136-
sortfwdind.id = "sorttable_sortfwdind";
149+
sortfwdind.class = "sorttable_sortfwdind";
137150
sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
138151
this.appendChild(sortfwdind);
139152

‎core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -317,8 +317,7 @@ private class ErrorServlet extends RestServlet {
317317
versionMismatch = true
318318
s"Unknown protocol version '$unknownVersion'."
319319
case _ =>
320-
// never reached
321-
s"Malformed path $path."
320+
"Malformed path."
322321
}
323322
msg += s" Please submit requests through http://[host]:[port]/$serverVersion/submissions/..."
324323
val error = handleError(msg)

‎docs/sql-getting-started.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,9 @@ For example:
346346

347347
</div>
348348

349+
## Scalar Functions
350+
(to be filled soon)
351+
349352
## Aggregations
350353

351354
The [built-in DataFrames functions](api/scala/index.html#org.apache.spark.sql.functions$) provide common

‎docs/sql-ref-syntax-ddl-create-function.md‎

Lines changed: 150 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,153 @@ license: |
1919
limitations under the License.
2020
---
2121

22-
**This page is under construction**
22+
### Description
23+
The `CREATE FUNCTION` statement is used to create a temporary or permanent function
24+
in Spark. Temporary functions are scoped at a session level where as permanent
25+
functions are created in the persistent catalog and are made available to
26+
all sessions. The resources specified in the `USING` clause are made available
27+
to all executors when they are executed for the first time. In addition to the
28+
SQL interface, spark allows users to create custom user defined scalar and
29+
aggregate functions using Scala, Python and Java APIs. Please refer to
30+
[scalar_functions](sql-getting-started.html#scalar-functions) and
31+
[aggregate functions](sql-getting-started#aggregations) for more information.
32+
33+
### Syntax
34+
{% highlight sql %}
35+
CREATE [ OR REPLACE ] [ TEMPORARY ] FUNCTION [ IF NOT EXISTS ]
36+
function_name AS class_name [ resource_locations ]
37+
{% endhighlight %}
38+
39+
### Parameters
40+
<dl>
41+
<dt><code><em>OR REPLACE</em></code></dt>
42+
<dd>
43+
If specified, the resources for the function are reloaded. This is mainly useful
44+
to pick up any changes made to the implementation of the function. This
45+
parameter is mutually exclusive to <code>IF NOT EXISTS</code> and can not
46+
be specified together.
47+
</dd>
48+
<dt><code><em>TEMPORARY</em></code></dt>
49+
<dd>
50+
Indicates the scope of function being created. When <code>TEMPORARY</code> is specified, the
51+
created function is valid and visible in the current session. No persistent
52+
entry is made in the catalog for these kind of functions.
53+
</dd>
54+
<dt><code><em>IF NOT EXISTS</em></code></dt>
55+
<dd>
56+
If specified, creates the function only when it does not exist. The creation
57+
of function succeeds (no error is thrown) if the specified function already
58+
exists in the system. This parameter is mutually exclusive to <code> OR REPLACE</code>
59+
and can not be specified together.
60+
</dd>
61+
<dt><code><em>function_name</em></code></dt>
62+
<dd>
63+
Specifies a name of funnction to be created. The function name may be
64+
optionally qualified with a database name. <br><br>
65+
<b>Syntax:</b>
66+
<code>
67+
[database_name.]function_name
68+
</code>
69+
</dd>
70+
<dt><code><em>class_name</em></code></dt>
71+
<dd>
72+
Specifies the name of the class that provides the implementation for function to be created.
73+
The implementing class should extend one of the base classes as follows:
74+
<ul>
75+
<li>Should extend <code>UDF</code> or <code>UDAF</code> in <code>org.apache.hadoop.hive.ql.exec</code> package.</li>
76+
<li>Should extend <code>AbstractGenericUDAFResolver</code>, <code>GenericUDF</code>, or
77+
<code>GenericUDTF</code> in <code>org.apache.hadoop.hive.ql.udf.generic</code> package.</li>
78+
<li>Should extend <code>UserDefinedAggregateFunction</code> in <code>org.apache.spark.sql.expressions</code> package.</li>
79+
</ul>
80+
</dd>
81+
<dt><code><em>resource_locations</em></code></dt>
82+
<dd>
83+
Specifies the list of resources that contain the implementation of the function
84+
along with its dependencies. <br><br>
85+
<b>Syntax:</b>
86+
<code>
87+
USING { { (JAR | FILE ) resource_uri} , ...}
88+
</code>
89+
</dd>
90+
</dl>
91+
92+
### Examples
93+
{% highlight sql %}
94+
-- 1. Create a simple UDF `SimpleUdf` that increments the supplied integral value by 10.
95+
-- import org.apache.hadoop.hive.ql.exec.UDF;
96+
-- public class SimpleUdf extends UDF {
97+
-- public int evaluate(int value) {
98+
-- return value + 10;
99+
-- }
100+
-- }
101+
-- 2. Compile and place it in a JAR file called `SimpleUdf.jar` in /tmp.
102+
103+
-- Create a table called `test` and insert two rows.
104+
CREATE TABLE test(c1 INT);
105+
INSERT INTO test VALUES (1), (2);
106+
107+
-- Create a permanent function called `simple_udf`.
108+
CREATE FUNCTION simple_udf AS 'SimpleUdf'
109+
USING JAR '/tmp/SimpleUdf.jar';
110+
111+
-- Verify that the function is in the registry.
112+
SHOW USER FUNCTIONS;
113+
+------------------+
114+
| function|
115+
+------------------+
116+
|default.simple_udf|
117+
+------------------+
118+
119+
-- Invoke the function. Every selected value should be incremented by 10.
120+
SELECT simple_udf(c1) AS function_return_value FROM t1;
121+
+---------------------+
122+
|function_return_value|
123+
+---------------------+
124+
| 11|
125+
| 12|
126+
+---------------------+
127+
128+
-- Created a temporary function.
129+
CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf'
130+
USING JAR '/tmp/SimpleUdf.jar';
131+
132+
-- Verify that the newly created temporary function is in the registry.
133+
-- Please note that the temporary function does not have a qualified
134+
-- database associated with it.
135+
SHOW USER FUNCTIONS;
136+
+------------------+
137+
| function|
138+
+------------------+
139+
|default.simple_udf|
140+
| simple_temp_udf|
141+
+------------------+
142+
143+
-- 1. Modify `SimpleUdf`'s implementation to add supplied integral value by 20.
144+
-- import org.apache.hadoop.hive.ql.exec.UDF;
145+
146+
-- public class SimpleUdfR extends UDF {
147+
-- public int evaluate(int value) {
148+
-- return value + 20;
149+
-- }
150+
-- }
151+
-- 2. Compile and place it in a jar file called `SimpleUdfR.jar` in /tmp.
152+
153+
-- Replace the implementation of `simple_udf`
154+
CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
155+
USING JAR '/tmp/SimpleUdfR.jar';
156+
157+
-- Invoke the function. Every selected value should be incremented by 20.
158+
SELECT simple_udf(c1) AS function_return_value FROM t1;
159+
+---------------------+
160+
|function_return_value|
161+
+---------------------+
162+
| 21|
163+
| 22|
164+
+---------------------+
165+
166+
{% endhighlight %}
167+
168+
### Related statements
169+
- [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
170+
- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
171+
- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)

‎docs/sql-ref-syntax-dml-load.md‎

Lines changed: 100 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
layout: global
3-
title: LOAD
4-
displayTitle: LOAD
3+
title: LOAD DATA
4+
displayTitle: LOAD DATA
55
license: |
66
Licensed to the Apache Software Foundation (ASF) under one or more
77
contributor license agreements. See the NOTICE file distributed with
@@ -19,4 +19,101 @@ license: |
1919
limitations under the License.
2020
---
2121

22-
**This page is under construction**
22+
### Description
23+
`LOAD DATA` statement loads the data into a table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
24+
25+
### Syntax
26+
{% highlight sql %}
27+
LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_name
28+
[ PARTITION ( partition_col_name = partition_col_val [ , ... ] ) ]
29+
{% endhighlight %}
30+
31+
### Parameters
32+
<dl>
33+
<dt><code><em>path</em></code></dt>
34+
<dd>Path of the file system. It can be either an absolute or a relative path.</dd>
35+
</dl>
36+
37+
<dl>
38+
<dt><code><em>table_name</em></code></dt>
39+
<dd>The name of an existing table.</dd>
40+
</dl>
41+
42+
<dl>
43+
<dt><code><em>PARTITION ( partition_col_name = partition_col_val [ , ... ] )</em></code></dt>
44+
<dd>Specifies one or more partition column and value pairs.</dd>
45+
</dl>
46+
47+
<dl>
48+
<dt><code><em>LOCAL</em></code></dt>
49+
<dd>If specified, it causes the <code>INPATH</code> to be resolved against the local file system, instead of the default file system, which is typically a distributed storage.</dd>
50+
</dl>
51+
52+
<dl>
53+
<dt><code><em>OVERWRITE</em></code></dt>
54+
<dd>By default, new data is appended to the table. If <code>OVERWRITE</code> is used, the table is instead overwritten with new data.</dd>
55+
</dl>
56+
57+
### Examples
58+
{% highlight sql %}
59+
-- Example without partition specification.
60+
-- Assuming the students table has already been created and populated.
61+
SELECT * FROM students;
62+
63+
+ -------------- + ------------------------------ + -------------- +
64+
| name | address | student_id |
65+
+ -------------- + ------------------------------ + -------------- +
66+
| Amy Smith | 123 Park Ave, San Jose | 111111 |
67+
+ -------------- + ------------------------------ + -------------- +
68+
69+
CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT);
70+
71+
-- Assuming the students table is in '/user/hive/warehouse/'
72+
LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
73+
74+
SELECT * FROM test_load;
75+
76+
+ -------------- + ------------------------------ + -------------- +
77+
| name | address | student_id |
78+
+ -------------- + ------------------------------ + -------------- +
79+
| Amy Smith | 123 Park Ave, San Jose | 111111 |
80+
+ -------------- + ------------------------------ + -------------- +
81+
82+
-- Example with partition specification.
83+
CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
84+
85+
INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
86+
87+
INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
88+
89+
INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
90+
91+
SELECT * FROM test_partition;
92+
93+
+ ------- + ------- + ----- +
94+
| c1 | c2 | c3 |
95+
+ ------- + --------------- +
96+
| 1 | 2 | 3 |
97+
+ ------- + ------- + ----- +
98+
| 4 | 5 | 6 |
99+
+ ------- + ------- + ----- +
100+
| 7 | 8 | 9 |
101+
+ ------- + ------- + ----- +
102+
103+
CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
104+
105+
-- Assuming the test_partition table is in '/user/hive/warehouse/'
106+
LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
107+
OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
108+
109+
SELECT * FROM test_load_partition;
110+
111+
+ ------- + ------- + ----- +
112+
| c1 | c2 | c3 |
113+
+ ------- + --------------- +
114+
| 1 | 2 | 3 |
115+
+ ------- + ------- + ----- +
116+
117+
118+
{% endhighlight %}
119+

‎sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ statement
194194
('(' key=tablePropertyKey ')')? #showTblProperties
195195
| SHOW COLUMNS (FROM | IN) tableIdentifier
196196
((FROM | IN) db=errorCapturingIdentifier)? #showColumns
197-
| SHOW PARTITIONS tableIdentifier partitionSpec? #showPartitions
197+
| SHOW PARTITIONS multipartIdentifier partitionSpec? #showPartitions
198198
| SHOW identifier? FUNCTIONS
199199
(LIKE? (qualifiedName | pattern=STRING))? #showFunctions
200200
| SHOW CREATE TABLE tableIdentifier #showCreateTable

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2743,6 +2743,23 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
27432743
Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
27442744
}
27452745

2746+
/**
2747+
* A command for users to list the partition names of a table. If partition spec is specified,
2748+
* partitions that match the spec are returned. Otherwise an empty result set is returned.
2749+
*
2750+
* This function creates a [[ShowPartitionsStatement]] logical plan
2751+
*
2752+
* The syntax of using this command in SQL is:
2753+
* {{{
2754+
* SHOW PARTITIONS multi_part_name [partition_spec];
2755+
* }}}
2756+
*/
2757+
override def visitShowPartitions(ctx: ShowPartitionsContext): LogicalPlan = withOrigin(ctx) {
2758+
val table = visitMultipartIdentifier(ctx.multipartIdentifier)
2759+
val partitionKeys = Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)
2760+
ShowPartitionsStatement(table, partitionKeys)
2761+
}
2762+
27462763
/**
27472764
* Create a [[RefreshTableStatement]].
27482765
*

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,13 @@ case class TruncateTableStatement(
324324
tableName: Seq[String],
325325
partitionSpec: Option[TablePartitionSpec]) extends ParsedStatement
326326

327+
/**
328+
* A SHOW PARTITIONS statement, as parsed from SQL
329+
*/
330+
case class ShowPartitionsStatement(
331+
tableName: Seq[String],
332+
partitionSpec: Option[TablePartitionSpec]) extends ParsedStatement
333+
327334
/**
328335
* A REFRESH TABLE statement, as parsed from SQL
329336
*/

0 commit comments

Comments
 (0)