Skip to content

Commit 7f5f7a9

Browse files
10110346srowen
authored andcommitted
[SPARK-25786][CORE] If the ByteBuffer.hasArray is false , it will throw UnsupportedOperationException for Kryo
## What changes were proposed in this pull request? `deserialize` for kryo, the type of input parameter is ByteBuffer, if it is not backed by an accessible byte array. it will throw `UnsupportedOperationException` Exception Info: ``` java.lang.UnsupportedOperationException was thrown. java.lang.UnsupportedOperationException at java.nio.ByteBuffer.array(ByteBuffer.java:994) at org.apache.spark.serializer.KryoSerializerInstance.deserialize(KryoSerializer.scala:362) ``` ## How was this patch tested? Added a unit test Closes apache#22779 from 10110346/InputStreamKryo. Authored-by: liuxian <liu.xian3@zte.com.cn> Signed-off-by: Sean Owen <sean.owen@databricks.com>
1 parent de84899 commit 7f5f7a9

File tree

2 files changed

+25
-3
lines changed

2 files changed

+25
-3
lines changed

core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ import org.apache.spark.internal.Logging
4242
import org.apache.spark.network.util.ByteUnit
4343
import org.apache.spark.scheduler.{CompressedMapStatus, HighlyCompressedMapStatus}
4444
import org.apache.spark.storage._
45-
import org.apache.spark.util.{BoundedPriorityQueue, SerializableConfiguration, SerializableJobConf, Utils}
45+
import org.apache.spark.util.{BoundedPriorityQueue, ByteBufferInputStream, SerializableConfiguration, SerializableJobConf, Utils}
4646
import org.apache.spark.util.collection.CompactBuffer
4747

4848
/**
@@ -417,7 +417,12 @@ private[spark] class KryoSerializerInstance(
417417
override def deserialize[T: ClassTag](bytes: ByteBuffer): T = {
418418
val kryo = borrowKryo()
419419
try {
420-
input.setBuffer(bytes.array(), bytes.arrayOffset() + bytes.position(), bytes.remaining())
420+
if (bytes.hasArray) {
421+
input.setBuffer(bytes.array(), bytes.arrayOffset() + bytes.position(), bytes.remaining())
422+
} else {
423+
input.setBuffer(new Array[Byte](4096))
424+
input.setInputStream(new ByteBufferInputStream(bytes))
425+
}
421426
kryo.readClassAndObject(input).asInstanceOf[T]
422427
} finally {
423428
releaseKryo(kryo)
@@ -429,7 +434,12 @@ private[spark] class KryoSerializerInstance(
429434
val oldClassLoader = kryo.getClassLoader
430435
try {
431436
kryo.setClassLoader(loader)
432-
input.setBuffer(bytes.array(), bytes.arrayOffset() + bytes.position(), bytes.remaining())
437+
if (bytes.hasArray) {
438+
input.setBuffer(bytes.array(), bytes.arrayOffset() + bytes.position(), bytes.remaining())
439+
} else {
440+
input.setBuffer(new Array[Byte](4096))
441+
input.setInputStream(new ByteBufferInputStream(bytes))
442+
}
433443
kryo.readClassAndObject(input).asInstanceOf[T]
434444
} finally {
435445
kryo.setClassLoader(oldClassLoader)

core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.spark.serializer
1919

2020
import java.io.{ByteArrayInputStream, ByteArrayOutputStream, FileInputStream, FileOutputStream}
21+
import java.nio.ByteBuffer
2122
import java.util.concurrent.Executors
2223

2324
import scala.collection.JavaConverters._
@@ -551,6 +552,17 @@ class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSpar
551552
deserializationStream.close()
552553
assert(serInstance.deserialize[Any](helloHello) === ((hello, hello)))
553554
}
555+
556+
test("SPARK-25786: ByteBuffer.array -- UnsupportedOperationException") {
557+
val serInstance = new KryoSerializer(conf).newInstance().asInstanceOf[KryoSerializerInstance]
558+
val obj = "UnsupportedOperationException"
559+
val serObj = serInstance.serialize(obj)
560+
val byteBuffer = ByteBuffer.allocateDirect(serObj.array().length)
561+
byteBuffer.put(serObj.array())
562+
byteBuffer.flip()
563+
assert(serInstance.deserialize[Any](serObj) === (obj))
564+
assert(serInstance.deserialize[Any](byteBuffer) === (obj))
565+
}
554566
}
555567

556568
class ClassLoaderTestingObject

0 commit comments

Comments
 (0)