address comment

apache · Apr 7, 2023 · 6d596fc · 6d596fc
1 parent 8280783
commit 6d596fc
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 9 deletions.
diff --git a/...ql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/...ql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala
@@ -55,7 +55,7 @@ object KyuubiArrowConverters extends SQLConfHelper with Logging {
       start: Int,
       length: Int): Array[Byte] = {
     val in = new ByteArrayInputStream(bytes)
-    val out = new ByteArrayOutputStream()
+    val out = new ByteArrayOutputStream(bytes.length)
 
     try {
       val recordBatch = MessageSerializer.deserializeRecordBatch(

diff --git a/...uubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala b/...uubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala
@@ -210,11 +210,12 @@ object SparkDatasetHelper extends Logging {
    * earlier versions of Spark, this function uses reflective calls to the "offset".
    */
   private def offset(collectLimitExec: CollectLimitExec): Int = {
-    val offset = DynMethods.builder("offset")
-      .impl(collectLimitExec.getClass)
-      .orNoop()
-      .build()
-    Option(offset.invoke[Int](collectLimitExec))
+    Option(
+      DynMethods.builder("offset")
+        .impl(collectLimitExec.getClass)
+        .orNoop()
+        .build()
+        .invoke[Int](collectLimitExec))
       .getOrElse(0)
   }
 

diff --git a/...c/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala b/...c/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.kyuubi.engine.spark.operation
 
 import java.sql.Statement
+import java.util.{Set => JSet}
 
 import org.apache.spark.KyuubiSparkContextHelper
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
@@ -431,7 +432,7 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp
   private def isStaticConfigKey(key: String): Boolean = {
     val staticConfKeys = DynFields.builder()
       .hiddenImpl(SQLConf.getClass, "staticConfKeys")
-      .build[java.util.Set[String]](SQLConf)
+      .build[JSet[String]](SQLConf)
       .get()
     staticConfKeys.contains(key)
   }

diff --git a/pom.xml b/pom.xml
@@ -538,8 +538,8 @@
                         <artifactId>hadoop-client</artifactId>
                     </exclusion>
                     <!--
-                      The module is only used in Kyuubi Spark Extensions, so we don't care about which
-                      version of Log4j it depends on.
+                      The module is only used in Kyuubi Spark Extensions and Engine Spark SQL, so we
+                      don't care about which version of Log4j it depends on.
                      -->
                 </exclusions>
             </dependency>