Skip to content

Commit

Permalink
HIVE-21304: Make bucketing version usage more robust (Zoltan Haindric…
Browse files Browse the repository at this point in the history 
…h reviewed by Jesus Camacho Rodriguez)

Signed-off-by: Zoltan Haindrich <zhaindrich@cloudera.com>
  • Loading branch information
@kgyrtkirk
kgyrtkirk committed May 4, 2020
1 parent 342f8fb commit c34ee9d
Show file tree
Hide file tree
Showing 368 changed files with 5,956 additions and 701 deletions.
3 changes: 3 additions & 0 deletions itests/hive-blobstore/src/test/results/clientpositive/insert_into_dynamic_partitions.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
bucketingVersion: 2
key expressions: _col1 (type: string), _bucket_number (type: string), _col0 (type: int)
null sort order: aaa
numBuckets: 2
sort order: +++
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Expand Down Expand Up @@ -161,6 +163,7 @@ STAGE PLANS:
expressions: KEY._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string)
outputColumnNames: _col0, _col1, _bucket_number
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down
7 changes: 7 additions & 0 deletions itests/hive-blobstore/src/test/results/clientpositive/insert_into_table.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down Expand Up @@ -123,7 +124,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
bucketingVersion: 2
null sort order:
numBuckets: -1
sort order:
Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
tag: -1
Expand Down Expand Up @@ -178,6 +181,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
Expand All @@ -188,6 +192,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>
escape.delim \
Expand Down Expand Up @@ -249,6 +254,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down Expand Up @@ -372,6 +378,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down
30 changes: 30 additions & 0 deletions itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_directory.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 2 Data size: 180 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -100,6 +101,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand Down Expand Up @@ -181,6 +183,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -189,6 +192,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand All @@ -206,6 +210,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand All @@ -215,6 +220,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand All @@ -236,6 +242,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -244,6 +251,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand All @@ -261,6 +269,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand All @@ -270,6 +279,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0,_col1
columns.types int:string
serialization.format 1
Expand Down Expand Up @@ -327,6 +337,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -337,6 +348,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -350,6 +362,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 172 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 2
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -360,6 +373,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand Down Expand Up @@ -441,6 +455,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -449,6 +464,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -466,6 +482,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -475,6 +492,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -496,6 +514,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -504,6 +523,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -521,6 +541,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand All @@ -530,6 +551,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types int
serialization.format 1
Expand Down Expand Up @@ -561,6 +583,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -569,6 +592,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand All @@ -586,6 +610,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand All @@ -595,6 +620,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand All @@ -616,6 +642,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: -1
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand All @@ -624,6 +651,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand All @@ -641,6 +669,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand All @@ -650,6 +679,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types string
serialization.format 1
Expand Down
3 changes: 3 additions & 0 deletions .../hive-blobstore/src/test/results/clientpositive/insert_overwrite_dynamic_partitions.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
bucketingVersion: 2
key expressions: _col1 (type: string), _bucket_number (type: string), _col0 (type: int)
null sort order: aaa
numBuckets: 2
sort order: +++
Map-reduce partition columns: _col1 (type: string)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Expand Down Expand Up @@ -179,6 +181,7 @@ STAGE PLANS:
expressions: KEY._col0 (type: int), KEY._col1 (type: string), KEY._bucket_number (type: string)
outputColumnNames: _col0, _col1, _bucket_number
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down
7 changes: 7 additions & 0 deletions itests/hive-blobstore/src/test/results/clientpositive/insert_overwrite_table.q.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 1
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down Expand Up @@ -131,7 +132,9 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
bucketingVersion: 2
null sort order:
numBuckets: -1
sort order:
Statistics: Num rows: 1 Data size: 424 Basic stats: COMPLETE Column stats: COMPLETE
tag: -1
Expand Down Expand Up @@ -186,6 +189,7 @@ STAGE PLANS:
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 440 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
Expand All @@ -196,6 +200,7 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
properties:
bucketing_version -1
columns _col0
columns.types struct<columntype:string,min:bigint,max:bigint,countnulls:bigint,numdistinctvalues:bigint,ndvbitvector:binary>
escape.delim \
Expand Down Expand Up @@ -257,6 +262,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down Expand Up @@ -380,6 +386,7 @@ STAGE PLANS:
TableScan
GatherStats: false
File Output Operator
bucketingVersion: 2
compressed: false
GlobalTableId: 0
directory: ### BLOBSTORE_STAGING_PATH ###
Expand Down
Loading

0 comments on commit c34ee9d

Please sign in to comment.