Skip to content

Commit b531273

Browse files
lianchengliancheng
committed
Fixed coding style issues in sql/hive
1 parent 0b56f77 commit b531273

18 files changed

+119
-125
lines changed

‎sql/hive/src/main/scala/org/apache/hadoop/mapred/SparkHadoopWriter.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,14 @@ import java.text.NumberFormat
2222
import java.util.Date
2323

2424
import org.apache.hadoop.fs.Path
25+
import org.apache.hadoop.hive.ql.exec.{FileSinkOperator, Utilities}
26+
import org.apache.hadoop.hive.ql.io.{HiveFileFormatUtils, HiveOutputFormat}
27+
import org.apache.hadoop.hive.ql.plan.FileSinkDesc
2528
import org.apache.hadoop.io.Writable
2629

2730
import org.apache.spark.Logging
2831
import org.apache.spark.SerializableWritable
2932

30-
import org.apache.hadoop.hive.ql.exec.{Utilities, FileSinkOperator}
31-
import org.apache.hadoop.hive.ql.io.{HiveFileFormatUtils, HiveOutputFormat}
32-
import org.apache.hadoop.hive.ql.plan.FileSinkDesc
33-
3433
/**
3534
* Internal helper class that saves an RDD using a Hive OutputFormat.
3635
* It is based on [[SparkHadoopWriter]].

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,26 @@
1818
package org.apache.spark.sql
1919
package hive
2020

21-
import java.io.{PrintStream, InputStreamReader, BufferedReader, File}
22-
import java.util.{ArrayList => JArrayList}
2321
import scala.language.implicitConversions
2422

25-
import org.apache.spark.SparkContext
23+
import java.io.{BufferedReader, File, InputStreamReader, PrintStream}
24+
import java.util.{ArrayList => JArrayList}
25+
2626
import org.apache.hadoop.hive.conf.HiveConf
27-
import org.apache.hadoop.hive.ql.session.SessionState
28-
import org.apache.hadoop.hive.ql.processors.{CommandProcessorResponse, CommandProcessorFactory}
29-
import org.apache.hadoop.hive.ql.processors.CommandProcessor
3027
import org.apache.hadoop.hive.ql.Driver
31-
import org.apache.spark.rdd.RDD
32-
33-
import catalyst.analysis.{Analyzer, OverrideCatalog}
34-
import catalyst.expressions.GenericRow
35-
import catalyst.plans.logical.{BaseRelation, LogicalPlan, NativeCommand, ExplainCommand}
36-
import catalyst.types._
28+
import org.apache.hadoop.hive.ql.processors._
29+
import org.apache.hadoop.hive.ql.session.SessionState
3730

31+
import org.apache.spark.SparkContext
32+
import org.apache.spark.rdd.RDD
33+
import org.apache.spark.sql.catalyst.analysis.{Analyzer, OverrideCatalog}
34+
import org.apache.spark.sql.catalyst.expressions.GenericRow
35+
import org.apache.spark.sql.catalyst.plans.logical.{BaseRelation, LogicalPlan}
36+
import org.apache.spark.sql.catalyst.plans.logical.{NativeCommand, ExplainCommand}
37+
import org.apache.spark.sql.catalyst.types._
3838
import org.apache.spark.sql.execution._
3939

40+
/* Implicit conversions */
4041
import scala.collection.JavaConversions._
4142

4243
/**

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ import org.apache.hadoop.hive.ql.plan.TableDesc
2727
import org.apache.hadoop.hive.ql.session.SessionState
2828
import org.apache.hadoop.hive.serde2.Deserializer
2929

30-
import catalyst.analysis.Catalog
31-
import catalyst.expressions._
32-
import catalyst.plans.logical
33-
import catalyst.plans.logical._
34-
import catalyst.rules._
35-
import catalyst.types._
30+
import org.apache.spark.sql.catalyst.analysis.Catalog
31+
import org.apache.spark.sql.catalyst.expressions._
32+
import org.apache.spark.sql.catalyst.plans.logical
33+
import org.apache.spark.sql.catalyst.plans.logical._
34+
import org.apache.spark.sql.catalyst.rules._
35+
import org.apache.spark.sql.catalyst.types._
3636

3737
import scala.collection.JavaConversions._
3838

@@ -45,7 +45,7 @@ class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with Logging {
4545
db: Option[String],
4646
tableName: String,
4747
alias: Option[String]): LogicalPlan = {
48-
val databaseName = db.getOrElse(hive.sessionState.getCurrentDatabase())
48+
val databaseName = db.getOrElse(hive.sessionState.getCurrentDatabase)
4949
val table = client.getTable(databaseName, tableName)
5050
val partitions: Seq[Partition] =
5151
if (table.isPartitioned) {
@@ -91,7 +91,7 @@ class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with Logging {
9191
object CreateTables extends Rule[LogicalPlan] {
9292
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
9393
case InsertIntoCreatedTable(db, tableName, child) =>
94-
val databaseName = db.getOrElse(SessionState.get.getCurrentDatabase())
94+
val databaseName = db.getOrElse(SessionState.get.getCurrentDatabase)
9595

9696
createTable(databaseName, tableName, child.output)
9797

@@ -123,8 +123,8 @@ class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with Logging {
123123
} else {
124124
// Only do the casting when child output data types differ from table output data types.
125125
val castedChildOutput = child.output.zip(table.output).map {
126-
case (input, table) if input.dataType != table.dataType =>
127-
Alias(Cast(input, table.dataType), input.name)()
126+
case (input, output) if input.dataType != output.dataType =>
127+
Alias(Cast(input, output.dataType), input.name)()
128128
case (input, _) => input
129129
}
130130

@@ -135,7 +135,7 @@ class HiveMetastoreCatalog(hive: HiveContext) extends Catalog with Logging {
135135

136136
/**
137137
* UNIMPLEMENTED: It needs to be decided how we will persist in-memory tables to the metastore.
138-
* For now, if this functionallity is desired mix in the in-memory [[OverrideCatalog]].
138+
* For now, if this functionality is desired mix in the in-memory [[OverrideCatalog]].
139139
*/
140140
override def registerTable(
141141
databaseName: Option[String], tableName: String, plan: LogicalPlan): Unit = ???

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,19 @@
1818
package org.apache.spark.sql
1919
package hive
2020

21-
import scala.collection.JavaConversions._
22-
2321
import org.apache.hadoop.hive.ql.lib.Node
2422
import org.apache.hadoop.hive.ql.parse._
2523
import org.apache.hadoop.hive.ql.plan.PlanUtils
2624

27-
import catalyst.analysis._
28-
import catalyst.expressions._
29-
import catalyst.plans._
30-
import catalyst.plans.logical
31-
import catalyst.plans.logical._
32-
import catalyst.types._
25+
import org.apache.spark.sql.catalyst.analysis._
26+
import org.apache.spark.sql.catalyst.expressions._
27+
import org.apache.spark.sql.catalyst.plans._
28+
import org.apache.spark.sql.catalyst.plans.logical
29+
import org.apache.spark.sql.catalyst.plans.logical._
30+
import org.apache.spark.sql.catalyst.types._
31+
32+
/* Implicit conversions */
33+
import scala.collection.JavaConversions._
3334

3435
/**
3536
* Used when we need to start parsing the AST before deciding that we are going to pass the command
@@ -48,7 +49,7 @@ case class AddJar(jarPath: String) extends Command
4849

4950
case class AddFile(filePath: String) extends Command
5051

51-
/** Provides a mapping from HiveQL statments to catalyst logical plans and expression trees. */
52+
/** Provides a mapping from HiveQL statements to catalyst logical plans and expression trees. */
5253
object HiveQl {
5354
protected val nativeCommands = Seq(
5455
"TOK_DESCFUNCTION",
@@ -150,13 +151,13 @@ object HiveQl {
150151
}
151152

152153
/**
153-
* Returns a scala.Seq equivilent to [s] or Nil if [s] is null.
154+
* Returns a scala.Seq equivalent to [s] or Nil if [s] is null.
154155
*/
155156
private def nilIfEmpty[A](s: java.util.List[A]): Seq[A] =
156157
Option(s).map(_.toSeq).getOrElse(Nil)
157158

158159
/**
159-
* Returns this ASTNode with the text changed to `newText``.
160+
* Returns this ASTNode with the text changed to `newText`.
160161
*/
161162
def withText(newText: String): ASTNode = {
162163
n.token.asInstanceOf[org.antlr.runtime.CommonToken].setText(newText)
@@ -667,7 +668,7 @@ object HiveQl {
667668
case Token(allJoinTokens(joinToken),
668669
relation1 ::
669670
relation2 :: other) =>
670-
assert(other.size <= 1, s"Unhandled join child ${other}")
671+
assert(other.size <= 1, s"Unhandled join child $other")
671672
val joinType = joinToken match {
672673
case "TOK_JOIN" => Inner
673674
case "TOK_RIGHTOUTERJOIN" => RightOuter

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveStrategies.scala

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,12 @@
1818
package org.apache.spark.sql
1919
package hive
2020

21-
import catalyst.expressions._
22-
import catalyst.planning._
23-
import catalyst.plans._
24-
import catalyst.plans.logical.{BaseRelation, LogicalPlan}
25-
21+
import org.apache.spark.sql.catalyst.expressions._
22+
import org.apache.spark.sql.catalyst.planning._
23+
import org.apache.spark.sql.catalyst.plans._
24+
import org.apache.spark.sql.catalyst.plans.logical.{BaseRelation, LogicalPlan}
2625
import org.apache.spark.sql.execution._
27-
import org.apache.spark.sql.parquet.{ParquetRelation, InsertIntoParquetTable, ParquetTableScan}
26+
import org.apache.spark.sql.parquet.{InsertIntoParquetTable, ParquetRelation, ParquetTableScan}
2827

2928
trait HiveStrategies {
3029
// Possibly being too clever with types here... or not clever enough.

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/ScriptTransformation.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@
1818
package org.apache.spark.sql
1919
package hive
2020

21-
import java.io.{InputStreamReader, BufferedReader}
21+
import java.io.{BufferedReader, InputStreamReader}
2222

23-
import catalyst.expressions._
23+
import org.apache.spark.sql.catalyst.expressions._
2424
import org.apache.spark.sql.execution._
2525

26+
/* Implicit conversions */
2627
import scala.collection.JavaConversions._
2728

2829
/**

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,18 @@ package org.apache.spark.sql
1919
package hive
2020

2121
import org.apache.hadoop.conf.Configuration
22+
import org.apache.hadoop.fs.{Path, PathFilter}
2223
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants._
24+
import org.apache.hadoop.hive.ql.exec.Utilities
2325
import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Table => HiveTable}
2426
import org.apache.hadoop.hive.ql.plan.TableDesc
2527
import org.apache.hadoop.hive.serde2.Deserializer
26-
import org.apache.hadoop.hive.ql.exec.Utilities
2728
import org.apache.hadoop.io.Writable
28-
import org.apache.hadoop.fs.{Path, PathFilter}
29-
import org.apache.hadoop.mapred.{FileInputFormat, JobConf, InputFormat}
29+
import org.apache.hadoop.mapred.{FileInputFormat, InputFormat, JobConf}
3030

3131
import org.apache.spark.SerializableWritable
3232
import org.apache.spark.broadcast.Broadcast
33-
import org.apache.spark.rdd.{HadoopRDD, UnionRDD, EmptyRDD, RDD}
34-
33+
import org.apache.spark.rdd.{EmptyRDD, HadoopRDD, RDD, UnionRDD}
3534

3635
/**
3736
* A trait for subclasses that handle table scans.
@@ -40,7 +39,6 @@ private[hive] sealed trait TableReader {
4039
def makeRDDForTable(hiveTable: HiveTable): RDD[_]
4140

4241
def makeRDDForPartitionedTable(partitions: Seq[HivePartition]): RDD[_]
43-
4442
}
4543

4644

@@ -57,7 +55,6 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
5755
private val _minSplitsPerRDD = math.max(
5856
sc.hiveconf.getInt("mapred.map.tasks", 1), sc.sparkContext.defaultMinSplits)
5957

60-
6158
// TODO: set aws s3 credentials.
6259

6360
private val _broadcastedHiveConf =
@@ -85,8 +82,8 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
8582
def makeRDDForTable(
8683
hiveTable: HiveTable,
8784
deserializerClass: Class[_ <: Deserializer],
88-
filterOpt: Option[PathFilter]): RDD[_] =
89-
{
85+
filterOpt: Option[PathFilter]): RDD[_] = {
86+
9087
assert(!hiveTable.isPartitioned, """makeRDDForTable() cannot be called on a partitioned table,
9188
since input formats may differ across partitions. Use makeRDDForTablePartitions() instead.""")
9289

@@ -115,6 +112,7 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
115112
sys.error(s"Unable to deserialize non-Writable: $value of ${value.getClass.getName}")
116113
}
117114
}
115+
118116
deserializedHadoopRDD
119117
}
120118

@@ -136,8 +134,8 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
136134
*/
137135
def makeRDDForPartitionedTable(
138136
partitionToDeserializer: Map[HivePartition, Class[_ <: Deserializer]],
139-
filterOpt: Option[PathFilter]): RDD[_] =
140-
{
137+
filterOpt: Option[PathFilter]): RDD[_] = {
138+
141139
val hivePartitionRDDs = partitionToDeserializer.map { case (partition, partDeserializer) =>
142140
val partDesc = Utilities.getPartitionDesc(partition)
143141
val partPath = partition.getPartitionPath
@@ -178,6 +176,7 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
178176
}
179177
}
180178
}.toSeq
179+
181180
// Even if we don't use any partitions, we still need an empty RDD
182181
if (hivePartitionRDDs.size == 0) {
183182
new EmptyRDD[Object](sc.sparkContext)
@@ -207,8 +206,8 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
207206
private def createHadoopRdd(
208207
tableDesc: TableDesc,
209208
path: String,
210-
inputFormatClass: Class[InputFormat[Writable, Writable]])
211-
: RDD[Writable] = {
209+
inputFormatClass: Class[InputFormat[Writable, Writable]]): RDD[Writable] = {
210+
212211
val initializeJobConfFunc = HadoopTableReader.initializeLocalJobConfFunc(path, tableDesc) _
213212

214213
val rdd = new HadoopRDD(
@@ -227,7 +226,6 @@ class HadoopTableReader(@transient _tableDesc: TableDesc, @transient sc: HiveCon
227226
}
228227

229228
private[hive] object HadoopTableReader {
230-
231229
/**
232230
* Curried. After given an argument for 'path', the resulting JobConf => Unit closure is used to
233231
* instantiate a HadoopRDD.

‎sql/hive/src/main/scala/org/apache/spark/sql/hive/TestHive.scala

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,23 +22,22 @@ import java.io.File
2222
import java.util.{Set => JavaSet}
2323

2424
import scala.collection.mutable
25-
import scala.collection.JavaConversions._
2625
import scala.language.implicitConversions
2726

28-
import org.apache.hadoop.hive.metastore.api.{SerDeInfo, StorageDescriptor}
29-
import org.apache.hadoop.hive.metastore.MetaStoreUtils
3027
import org.apache.hadoop.hive.ql.exec.FunctionRegistry
31-
import org.apache.hadoop.hive.ql.io.avro.{AvroContainerOutputFormat, AvroContainerInputFormat}
28+
import org.apache.hadoop.hive.ql.io.avro.{AvroContainerInputFormat, AvroContainerOutputFormat}
3229
import org.apache.hadoop.hive.ql.metadata.Table
33-
import org.apache.hadoop.hive.serde2.avro.AvroSerDe
34-
import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
3530
import org.apache.hadoop.hive.serde2.RegexSerDe
31+
import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
32+
import org.apache.hadoop.hive.serde2.avro.AvroSerDe
3633

37-
import org.apache.spark.{SparkContext, SparkConf}
34+
import org.apache.spark.{SparkConf, SparkContext}
35+
import org.apache.spark.sql.catalyst.analysis._
36+
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, NativeCommand}
37+
import org.apache.spark.sql.catalyst.util._
3838

39-
import catalyst.analysis._
40-
import catalyst.plans.logical.{LogicalPlan, NativeCommand}
41-
import catalyst.util._
39+
/* Implicit conversions */
40+
import scala.collection.JavaConversions._
4241

4342
object TestHive
4443
extends TestHiveContext(new SparkContext("local", "TestSQLContext", new SparkConf()))
@@ -52,7 +51,7 @@ object TestHive
5251
*
5352
* TestHive is singleton object version of this class because instantiating multiple copies of the
5453
* hive metastore seems to lead to weird non-deterministic failures. Therefore, the execution of
55-
* testcases that rely on TestHive must be serialized.
54+
* test cases that rely on TestHive must be serialized.
5655
*/
5756
class TestHiveContext(sc: SparkContext) extends LocalHiveContext(sc) {
5857
self =>

0 commit comments

Comments
 (0)