Skip to content

Commit 6a99d63

Browse files
Cythia828Cythia828
authored
test: hive select/describe/load/alterTable syntax unit test (DTStack#158)
Co-authored-by: zhaoge <>
1 parent 97ab76a commit 6a99d63

File tree

5 files changed

+269
-2
lines changed

5 files changed

+269
-2
lines changed

‎test/parser/hive/syntax/fixtures/alterTable.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,8 @@ ALTER TABLE db1.tbl PARTITION (`pt1`=1) ENABLE OFFLINE;
212212

213213
ALTER TABLE db1.tbl PARTITION (`pt1`=1) DISABLE OFFLINE;
214214

215+
ALTER TABLE foo PARTITION (ds='2008-04-08', hr=11) CHANGE COLUMN dec_column_name dec_column_name DECIMAL(38,18);
216+
215217
-- Alter Table/Partition Compact
216218
ALTER TABLE tbl_com COMPACT 'compaction_type';
217219

‎test/parser/hive/syntax/fixtures/describe.sql‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
11
DESCRIBE FORMATTED default.src_table PARTITION (part_col = 100) columnA;
22

3-
DESCRIBE default.src_thrift lintString.$elem$.myint;
3+
DESCRIBE default.src_thrift lintString.$elem$.myint;
4+
5+
DESCRIBE DATABASE EXTENDED db_name;
6+
7+
DESCRIBE SCHEMA db_name;
8+
9+
DESCRIBE CONNECTOR EXTENDED connector_name;
10+
11+
DESCRIBE CONNECTOR connector_name;

‎test/parser/hive/syntax/fixtures/load.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ LOAD DATA INPATH 'hdfs://namenode:9000/user/hive/project/data1' INTO TABLE tbl1;
22

33
LOAD DATA LOCAL INPATH '/user/hive/project/data1'
44
OVERWRITE INTO TABLE tablename
5-
PARTITION (pt1=1, pt2=2)
5+
PARTITION (ds='2008-04-08', pt2=2)
66
INPUTFORMAT 'com.apache.hadoop.textInputFormat'
77
SERDE 'JsonSerDe';

‎test/parser/hive/syntax/fixtures/select.sql‎

Lines changed: 241 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,241 @@
1+
-- Select Syntax
2+
SELECT col1, col2 FROM table_name_1;
3+
4+
SELECT ALL col3, col4 FROM table_name_2 WHERE amount > 10 AND region = "US" GROUP BY col5 ORDER BY col6 CLUSTER BY col7 LIMIT 2,5;
5+
6+
-- WHERE Clause
7+
SELECT * FROM sales WHERE amount > 10 AND region = "US";
8+
9+
-- ALL and DISTINCT Clauses
10+
SELECT DISTINCT col1 FROM t1;
11+
12+
-- Partition Based Queries
13+
SELECT page_views.*
14+
FROM page_views
15+
WHERE page_views.`date` >= '2018-03-01' AND page_views.`date` <= '2018-03-31';
16+
17+
SELECT page_views.*
18+
FROM page_views JOIN dim_users
19+
ON (page_views.user_id = dim_users.id AND page_views.adate >= '2008-03-01' AND page_views.adate <= '2008-03-31');
20+
21+
-- HAVING Clause
22+
SELECT col1 FROM t1 GROUP BY col1 HAVING SUM(col2) > 10;
23+
24+
SELECT col1 FROM (SELECT col1, SUM(col2) AS col2sum FROM t1 GROUP BY col1) t2 WHERE t2.col2sum > 10;
25+
26+
-- LIMIT Clause
27+
SELECT * FROM customers LIMIT 5;
28+
29+
SELECT * FROM customers ORDER BY create_date LIMIT 5;
30+
31+
SELECT * FROM customers ORDER BY create_date LIMIT 2,5;
32+
33+
-- REGEX Column Specification
34+
SELECT `(ds|hr)?+.+` FROM sales;
35+
36+
-- Syntax of Sort By
37+
SELECT key, value FROM src SORT BY key ASC, value DESC;
38+
39+
-- Distribute By
40+
SELECT col1, col2 FROM t1 DISTRIBUTE BY col1 SORT BY col1 ASC, col2 DESC;
41+
42+
-- Join Syntax
43+
SELECT a.* FROM a JOIN b ON (a.id = b.id AND a.department = b.department);
44+
45+
-- MapJoin Restrictions
46+
SELECT /*+ MAPJOIN(b) */ a.key, a.value
47+
FROM a JOIN b ON a.key = b.key;
48+
49+
-- Union Syntax
50+
-- UNION within a FROM Clause
51+
SELECT u.id, actions.adate
52+
FROM (
53+
SELECT av.uid AS uid
54+
FROM action_video av
55+
WHERE av.adate = '2008-06-03'
56+
UNION ALL
57+
SELECT ac.uid AS uid
58+
FROM action_comment ac
59+
WHERE ac.adate = '2008-06-03'
60+
) actions JOIN users u ON (u.id = actions.uid);
61+
62+
-- Unions in DDL and Insert Statements--Applying Subclauses
63+
SELECT key FROM (SELECT key FROM src ORDER BY key LIMIT 10)subq1
64+
UNION
65+
SELECT key FROM (SELECT key FROM src1 ORDER BY key LIMIT 10)subq2;
66+
67+
SELECT key FROM src
68+
UNION
69+
SELECT key FROM src1
70+
ORDER BY key LIMIT 10;
71+
72+
-- Unions in DDL and Insert Statements--Column Aliases for Schema Matching
73+
INSERT OVERWRITE TABLE target_table
74+
SELECT name, id, category FROM source_table_1
75+
UNION ALL
76+
SELECT name, id, "Category159" as category FROM source_table_2;
77+
78+
-- Unions in DDL and Insert Statements--Column Type Conversion
79+
SELECT name, id, cast('2001-01-01' as date) d FROM source_table_1
80+
UNION ALL
81+
SELECT name, id, hiredate as d FROM source_table_2;
82+
83+
-- Sampling Bucketized Table
84+
SELECT *
85+
FROM source TABLESAMPLE(BUCKET 3 OUT OF 32 ON rand()) s;
86+
87+
-- Block Sampling
88+
SELECT *
89+
FROM source TABLESAMPLE(0.1 PERCENT) s;
90+
91+
SELECT *
92+
FROM source TABLESAMPLE(100M) s;
93+
94+
-- Subqueries in the FROM Clause
95+
SELECT t3.col
96+
FROM (
97+
SELECT a+b AS col
98+
FROM t1
99+
UNION ALL
100+
SELECT c+d AS col
101+
FROM t2
102+
) t3;
103+
104+
-- Subqueries in the WHERE Clause
105+
SELECT A
106+
FROM T1
107+
WHERE EXISTS (SELECT B FROM T2 WHERE T1.X = T2.Y);
108+
109+
-- explode (array)
110+
SELECT tf.* FROM (SELECT 0) t lateral view explode(ARRAY('A','B','C')) tf AS col;
111+
-- explode (map)
112+
SELECT tf.* FROM (SELECT 0) t lateral view explode(map('A',10,'B',20,'C',30)) tf AS KEY,VALUE;
113+
-- posexplode (array)
114+
SELECT tf.* FROM (SELECT 0) t lateral view posexplode(ARRAY('A','B','C')) tf AS pos,val;
115+
-- inline (array of structs)
116+
SELECT tf.* FROM (SELECT 0) t lateral view inline(ARRAY(STRUCT('A',10,DATE '2015-01-01'),STRUCT('B',20,DATE '2016-02-02'))) tf AS col1,col2,col3;
117+
-- stack (values)
118+
SELECT tf.* FROM (SELECT 0) t lateral view stack(2,'A',10,DATE '2015-01-01','B',20,DATE '2016-01-01') tf AS col0,col1,col2;
119+
120+
121+
-- Lateral View Syntax
122+
SELECT adid, count(1)
123+
FROM pageAds LATERAL VIEW explode(adid_list) adTable AS adid
124+
GROUP BY adid;
125+
126+
-- Multiple Lateral Views
127+
SELECT * FROM exampleTable
128+
LATERAL VIEW explode(col1) myTable1 AS myCol1
129+
LATERAL VIEW explode(myCol1) myTable2 AS myCol2;
130+
131+
-- Outer Lateral Views
132+
SELECT * FROM src LATERAL VIEW explode(array()) C AS a limit 10;
133+
SELECT * FROM src LATERAL VIEW OUTER explode(array()) C AS a limit 10;
134+
135+
-- Windowing and Analytics Functions
136+
SELECT a, COUNT(b) OVER (PARTITION BY c, d)
137+
FROM T;
138+
139+
SELECT a, SUM(b) OVER (PARTITION BY c, d ORDER BY e, f)
140+
FROM T;
141+
142+
SELECT a, AVG(b) OVER (PARTITION BY c ORDER BY d ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
143+
FROM T;
144+
145+
SELECT a, SUM(b) OVER w
146+
FROM T
147+
WINDOW w AS (PARTITION BY c ORDER BY d ROWS UNBOUNDED PRECEDING);
148+
149+
SELECT a, LEAD(a) OVER (PARTITION BY b ORDER BY C)
150+
FROM T;
151+
152+
SELECT a, LAG(a, 3, 0) OVER (PARTITION BY b ORDER BY C)
153+
FROM T;
154+
155+
SELECT a, COUNT(distinct a) OVER (PARTITION BY b)
156+
FROM T;
157+
158+
-- Common Table Expression
159+
-- chaining CTEs
160+
WITH q1 AS ( SELECT key FROM q2 WHERE key = '5'),
161+
q2 AS ( SELECT key FROM src WHERE key = '5')
162+
SELECT * FROM (SELECT key FROM q1) a;
163+
164+
-- union example
165+
WITH q1 AS (SELECT * FROM src WHERE key= '5'),
166+
q2 AS (SELECT * FROM src s2 WHERE key = '4')
167+
SELECT * FROM q1 union all SELECT * FROM q2;
168+
169+
-- view example
170+
CREATE VIEW v1 AS
171+
WITH q1 AS ( SELECT key FROM src WHERE key = '5')
172+
SELECT * FROM q1;
173+
SELECT * FROM v1;
174+
175+
-- view example, name collision
176+
CREATE VIEW v1 AS
177+
WITH q1 AS ( SELECT key FROM src WHERE key = '5')
178+
SELECT * FROM q1;
179+
WITH q1 AS ( SELECT key FROM src WHERE key = '4')
180+
SELECT * FROM v1;
181+
182+
-- GROUPING SETS clause
183+
-- GROUPING SET queries and the equivalent GROUP BY queries
184+
SELECT a, b, SUM(c) FROM tab1 GROUP BY a, b GROUPING SETS ( (a,b) );
185+
186+
SELECT a, b, SUM( c ) FROM tab1 GROUP BY a, b GROUPING SETS ( (a, b), a, b, ( ) );
187+
188+
-- Grouping__ID function
189+
SELECT key, value, GROUPING__ID, count(*)
190+
FROM T1
191+
GROUP BY key, value WITH ROLLUP;
192+
193+
-- Grouping function
194+
SELECT key, value, GROUPING__ID,
195+
grouping(key, value), grouping(value, key), grouping(key), grouping(value),
196+
count(*)
197+
FROM T1
198+
GROUP BY key, value WITH ROLLUP;
199+
200+
-- LanguageManual Explain -- EXPLAIN Syntax
201+
EXPLAIN
202+
FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key;
203+
204+
-- LanguageManual Explain -- The CBO Clause
205+
EXPLAIN CBO
206+
WITH customer_total_return AS
207+
(SELECT sr_customer_sk AS ctr_customer_sk,
208+
sr_store_sk AS ctr_store_sk,
209+
SUM(SR_FEE) AS ctr_total_return
210+
FROM store_returns, date_dim
211+
WHERE sr_returned_date_sk = d_date_sk
212+
AND d_year =2000
213+
GROUP BY sr_customer_sk, sr_store_sk)
214+
SELECT c_customer_id
215+
FROM customer_total_return ctr1, store, customer
216+
WHERE ctr1.ctr_total_return > (SELECT AVG(ctr_total_return)*1.2
217+
FROM customer_total_return ctr2
218+
WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk)
219+
AND s_store_sk = ctr1.ctr_store_sk
220+
AND s_state = 'NM'
221+
AND ctr1.ctr_customer_sk = c_customer_sk
222+
ORDER BY c_customer_id
223+
LIMIT 100;
224+
225+
-- LanguageManual Explain -- The AST Clause
226+
EXPLAIN AST
227+
FROM src INSERT OVERWRITE TABLE dest_g1 SELECT src.key, sum(substr(src.value,4)) GROUP BY src.key;
228+
229+
-- LanguageManual Explain -- The DEPENDENCY Clause
230+
EXPLAIN DEPENDENCY
231+
SELECT key, count(1) FROM srcpart WHERE ds IS NOT NULL GROUP BY key;
232+
233+
-- LanguageManual Explain -- The AUTHORIZATION Clause
234+
EXPLAIN AUTHORIZATION
235+
SELECT * FROM src JOIN srcpart;
236+
237+
-- LanguageManual Explain -- The LOCKS Clause
238+
EXPLAIN LOCKS UPDATE target SET b = 1 WHERE p IN (SELECT t.q1 FROM source t WHERE t.a1=5);
239+
240+
-- LanguageManual Explain -- User-level Explain Output
241+
EXPLAIN select sum(hash(key)), sum(hash(value)) from src_orc_merge_test_part where ds='2012-01-03' and ts='2012-01-03+14:46:31';

‎test/parser/hive/syntax/selectStatement.test.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import HiveSQL from '../../../../src/parser/hive';
2+
import { readSQL } from '../../../helper';
3+
4+
const parser = new HiveSQL();
5+
6+
const features = {
7+
selects: readSQL(__dirname, 'select.sql'),
8+
};
9+
10+
describe('HiveSQL Select Syntax Tests', () => {
11+
features.selects.forEach((select) => {
12+
it(select, () => {
13+
expect(parser.validate(select).length).toBe(0);
14+
});
15+
});
16+
});

0 commit comments

Comments
 (0)