File tree Expand file tree Collapse file tree 2 files changed +28
-2
lines changed
main/scala/org/apache/spark/sql/hive
test/scala/org/apache/spark/sql/parquet Expand file tree Collapse file tree 2 files changed +28
-2
lines changed Original file line number Diff line number Diff line change @@ -45,6 +45,9 @@ private[hive] trait HiveStrategies {
45
45
*
46
46
* TODO: Much of this logic is duplicated in HiveTableScan. Ideally we would do some refactoring
47
47
* but since this is after the code freeze for 1.1 all logic is here to minimize disruption.
48
+ *
49
+ * Other issues:
50
+ * - Much of this logic assumes case insensitive resolution.
48
51
*/
49
52
@ Experimental
50
53
object ParquetConversion extends Strategy {
@@ -60,8 +63,14 @@ private[hive] trait HiveStrategies {
60
63
})
61
64
}
62
65
63
- implicit class PhysicalPlanHacks (s : SparkPlan ) {
64
- def fakeOutput (newOutput : Seq [Attribute ]) = OutputFaker (newOutput, s)
66
+ implicit class PhysicalPlanHacks (originalPlan : SparkPlan ) {
67
+ def fakeOutput (newOutput : Seq [Attribute ]) =
68
+ OutputFaker (
69
+ originalPlan.output.map(a =>
70
+ newOutput.find(a.name.toLowerCase == _.name.toLowerCase)
71
+ .getOrElse(
72
+ sys.error(s " Can't find attribute $a to fake in set ${newOutput.mkString(" ," )}" ))),
73
+ originalPlan)
65
74
}
66
75
67
76
def apply (plan : LogicalPlan ): Seq [SparkPlan ] = plan match {
Original file line number Diff line number Diff line change @@ -103,6 +103,23 @@ class ParquetMetastoreSuite extends QueryTest with BeforeAndAfterAll {
103
103
)
104
104
}
105
105
106
+ test(" project partitioning and non-partitioning columns" ) {
107
+ checkAnswer(
108
+ sql(" SELECT stringField, p, count(intField) " +
109
+ " FROM partitioned_parquet GROUP BY p, stringField" ),
110
+ (" part-1" , 1 , 10 ) ::
111
+ (" part-2" , 2 , 10 ) ::
112
+ (" part-3" , 3 , 10 ) ::
113
+ (" part-4" , 4 , 10 ) ::
114
+ (" part-5" , 5 , 10 ) ::
115
+ (" part-6" , 6 , 10 ) ::
116
+ (" part-7" , 7 , 10 ) ::
117
+ (" part-8" , 8 , 10 ) ::
118
+ (" part-9" , 9 , 10 ) ::
119
+ (" part-10" , 10 , 10 ) :: Nil
120
+ )
121
+ }
122
+
106
123
test(" simple count" ) {
107
124
checkAnswer(
108
125
sql(" SELECT COUNT(*) FROM partitioned_parquet" ),
You can’t perform that action at this time.
0 commit comments