infer row size by schema.defaultSize

apache · Apr 4, 2023 · 885cf2c · 885cf2c
1 parent 4e7ca54
commit 885cf2c
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/...ql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersHelper.scala b/...ql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersHelper.scala
@@ -115,8 +115,8 @@ object ArrowConvertersHelper extends Logging {
           arrowWriter.write(row)
           estimatedBatchSize += (row match {
             case ur: UnsafeRow => ur.getSizeInBytes
-            // Trying to estimate the size of the current row, assuming 16 bytes per value.
-            case ir: InternalRow => ir.numFields * 16
+            // Trying to estimate the size of the current row
+            case _: InternalRow => schema.defaultSize
           })
           rowCountInLastBatch += 1
           rowCount += 1