use zlib when SPARK version less than 3.2

apache · Dec 20, 2023 · 31d4fc1 · 31d4fc1
1 parent e2d9a8e
commit 31d4fc1
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/...sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/...sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala
@@ -182,13 +182,19 @@ class ExecuteStatement(
         saveFileName = Some(s"$savePath/$engineId/$sessionId/$statementId")
         // Rename all col name to avoid duplicate columns
         val colName = range(0, result.schema.size).map(x => "col" + x)
+
+        val compress = if (SPARK_ENGINE_RUNTIME_VERSION >= "3.2") {
+          "zstd"
+        } else {
+          "zlib"
+        }
         // df.write will introduce an extra shuffle for the outermost limit, and hurt performance
         if (resultMaxRows > 0) {
           result.toDF(colName: _*).limit(resultMaxRows).write
-            .option("compression", "zstd").format("orc").save(saveFileName.get)
+            .option("compression", compress).format("orc").save(saveFileName.get)
         } else {
           result.toDF(colName: _*).write
-            .option("compression", "zstd").format("orc").save(saveFileName.get)
+            .option("compression", compress).format("orc").save(saveFileName.get)
         }
         info(s"Save result to $saveFileName")
         fetchOrcStatement = Some(new FetchOrcStatement(spark))