Skip to content

Commit 0a7ebd7

Browse files
committed
Clarify reason why SqlSerializer2 supports this serializer
1 parent 123b992 commit 0a7ebd7

File tree

2 files changed

+6
-2
lines changed

2 files changed

+6
-2
lines changed

core/src/main/scala/org/apache/spark/serializer/Serializer.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,8 @@ abstract class Serializer {
8686
* serIn.open([obj2bytes] concatenate [obj1bytes]) should return (obj2, obj1)
8787
* }}}
8888
*
89-
* In general, this property should hold for serializers that are stateless.
89+
* In general, this property should hold for serializers that are stateless and that do not
90+
* write special metadata at the beginning or end of the serialization stream.
9091
*
9192
* This API is private to Spark; this method should not be overridden in third-party subclasses
9293
* or called in user code and is subject to removal in future Spark releases.

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlSerializer2.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,10 @@ private[sql] class SparkSqlSerializer2(keySchema: Array[DataType], valueSchema:
155155

156156
def newInstance(): SerializerInstance = new ShuffleSerializerInstance(keySchema, valueSchema)
157157

158-
override def supportsRelocationOfSerializedObjects: Boolean = true
158+
override def supportsRelocationOfSerializedObjects: Boolean = {
159+
// SparkSqlSerializer2 is stateless and writes no stream headers
160+
true
161+
}
159162
}
160163

161164
private[sql] object SparkSqlSerializer2 {

0 commit comments

Comments
 (0)