Skip to content

Commit

Permalink
follow comment
Browse files Browse the repository at this point in the history
  • Loading branch information
@AngersZhuuuu
AngersZhuuuu committed Aug 13, 2020
1 parent c58729b commit fc5bbce
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 92 deletions.
48 changes: 24 additions & 24 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1088,19 +1088,19 @@ class PlanParserSuite extends AnalysisTest {
assertEqual(
"""
|SELECT TRANSFORM(a, b, c)
|ROW FORMAT DELIMITED
|FIELDS TERMINATED BY '\t'
|COLLECTION ITEMS TERMINATED BY '\u0002'
|MAP KEYS TERMINATED BY '\u0003'
|LINES TERMINATED BY '\n'
|NULL DEFINED AS 'null'
|USING 'cat' AS (a, b, c)
|ROW FORMAT DELIMITED
|FIELDS TERMINATED BY '\t'
|COLLECTION ITEMS TERMINATED BY '\u0004'
|MAP KEYS TERMINATED BY '\u0005'
|LINES TERMINATED BY '\n'
|NULL DEFINED AS 'NULL'
| ROW FORMAT DELIMITED
| FIELDS TERMINATED BY '\t'
| COLLECTION ITEMS TERMINATED BY '\u0002'
| MAP KEYS TERMINATED BY '\u0003'
| LINES TERMINATED BY '\n'
| NULL DEFINED AS 'null'
| USING 'cat' AS (a, b, c)
| ROW FORMAT DELIMITED
| FIELDS TERMINATED BY '\t'
| COLLECTION ITEMS TERMINATED BY '\u0004'
| MAP KEYS TERMINATED BY '\u0005'
| LINES TERMINATED BY '\n'
| NULL DEFINED AS 'NULL'
|FROM testData
""".stripMargin,
ScriptTransformation(
Expand All @@ -1127,17 +1127,17 @@ class PlanParserSuite extends AnalysisTest {
intercept(
"""
|SELECT TRANSFORM(a, b, c)
|ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
|WITH SERDEPROPERTIES(
| "separatorChar" = "\t",
| "quoteChar" = "'",
| "escapeChar" = "\\")
|USING 'cat' AS (a, b, c)
|ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
|WITH SERDEPROPERTIES(
| "separatorChar" = "\t",
| "quoteChar" = "'",
| "escapeChar" = "\\")
| ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
| WITH SERDEPROPERTIES(
| "separatorChar" = "\t",
| "quoteChar" = "'",
| "escapeChar" = "\\")
| USING 'cat' AS (a, b, c)
| ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
| WITH SERDEPROPERTIES(
| "separatorChar" = "\t",
| "quoteChar" = "'",
| "escapeChar" = "\\")
|FROM testData
""".stripMargin,
"TRANSFORM with serde is only supported in hive mode")
Expand Down
2 changes: 2 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/SparkScriptTransformationExec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import org.apache.spark.util.CircularBuffer
* @param input the set of expression that should be passed to the script.
* @param script the command that should be executed.
* @param output the attributes that are produced by the script.
* @param child Child Operator
* @param ioschema The wrapper classes of input and output schema properties
*/
case class SparkScriptTransformationExec(
input: Seq[Expression],
Expand Down
68 changes: 34 additions & 34 deletions sql/core/src/test/resources/sql-tests/inputs/transform.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,27 @@ FROM t;

-- common supported data types between no serde and serde transform
SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b boolean,
c binary,
d tinyint,
e int,
f smallint,
g long,
h float,
i double,
j decimal(38, 18),
k timestamp,
l date)
FROM t
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b boolean,
c binary,
d tinyint,
e int,
f smallint,
g long,
h float,
i double,
j decimal(38, 18),
k timestamp,
l date)
FROM t
) tmp;

-- common supported data types between no serde and serde transform
SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b string,
c string,
Expand All @@ -54,7 +54,7 @@ SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
j string,
k string,
l string)
FROM t
FROM t
) tmp;

-- SPARK-32388 handle schema less
Expand Down Expand Up @@ -90,25 +90,25 @@ REDUCE a, b USING 'cat' AS (a, b) FROM t;

-- transform with defined row format delimit
SELECT TRANSFORM(a, b, c, null)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
USING 'cat' AS (a, b, c, d)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
FROM t;

SELECT TRANSFORM(a, b, c, null)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
USING 'cat' AS (d)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '||'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '||'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
FROM t;
68 changes: 34 additions & 34 deletions sql/core/src/test/resources/sql-tests/results/transform.sql.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,21 +50,21 @@ Subprocess exited with status 2. Error: python: can't open file 'some_non_existe

-- !query
SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b boolean,
c binary,
d tinyint,
e int,
f smallint,
g long,
h float,
i double,
j decimal(38, 18),
k timestamp,
l date)
FROM t
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b boolean,
c binary,
d tinyint,
e int,
f smallint,
g long,
h float,
i double,
j decimal(38, 18),
k timestamp,
l date)
FROM t
) tmp
-- !query schema
struct<a:string,b:boolean,decode(c, UTF-8):string,d:tinyint,e:int,f:smallint,g:bigint,h:float,i:double,j:decimal(38,18),k:timestamp,l:date>
Expand All @@ -76,8 +76,8 @@ struct<a:string,b:boolean,decode(c, UTF-8):string,d:tinyint,e:int,f:smallint,g:b

-- !query
SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
SELECT TRANSFORM(a, b, c, d, e, f, g, h, i, j, k, l)
USING 'cat' AS (
a string,
b string,
c string,
Expand All @@ -90,7 +90,7 @@ SELECT a, b, decode(c, 'UTF-8'), d, e, f, g, h, i, j, k, l FROM (
j string,
k string,
l string)
FROM t
FROM t
) tmp
-- !query schema
struct<a:string,b:string,decode(CAST(c AS BINARY), UTF-8):string,d:string,e:string,f:string,g:string,h:string,i:string,j:string,k:string,l:string>
Expand Down Expand Up @@ -188,15 +188,15 @@ struct<a:string,b:string>

-- !query
SELECT TRANSFORM(a, b, c, null)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
USING 'cat' AS (a, b, c, d)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
FROM t
-- !query schema
struct<a:string,b:string,c:string,d:string>
Expand All @@ -207,15 +207,15 @@ struct<a:string,b:string,c:string,d:string>

-- !query
SELECT TRANSFORM(a, b, c, null)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
USING 'cat' AS (d)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '||'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '||'
LINES TERMINATED BY '\n'
NULL DEFINED AS 'NULL'
FROM t
-- !query schema
struct<d:string>
Expand Down
2 changes: 2 additions & 0 deletions ...ive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ import org.apache.spark.util.{CircularBuffer, Utils}
* @param input the set of expression that should be passed to the script.
* @param script the command that should be executed.
* @param output the attributes that are produced by the script.
* @param child Child Operator
* @param ioschema The wrapper classes of input and output schema properties
*/
case class HiveScriptTransformationExec(
input: Seq[Expression],
Expand Down

0 comments on commit fc5bbce

Please sign in to comment.