Skip to content

Commit 5d4bfb5

Browse files
committed
Make objectStreamReset counter count the last object written too
This makes it precise -- before we'd only reset after (reset + 1) writes
1 parent 18fe865 commit 5d4bfb5

File tree

3 files changed

+4
-5
lines changed

3 files changed

+4
-5
lines changed

core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,10 @@ private[spark] class JavaSerializationStream(out: OutputStream, counterReset: In
4040
*/
4141
def writeObject[T: ClassTag](t: T): SerializationStream = {
4242
objOut.writeObject(t)
43-
if (counterReset >= 0 && counter >= counterReset) {
43+
counter += 1
44+
if (counterReset > 0 && counter >= counterReset) {
4445
objOut.reset()
4546
counter = 0
46-
} else {
47-
counter += 1
4847
}
4948
this
5049
}

core/src/test/scala/org/apache/spark/util/collection/ExternalAppendOnlyMapSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class ExternalAppendOnlyMapSuite extends FunSuite with LocalSparkContext {
3434
val conf = new SparkConf(loadDefaults)
3535
// Make the Java serializer write a reset instruction (TC_RESET) after each object to test
3636
// for a bug we had with bytes written past the last object in a batch (SPARK-2792)
37-
conf.set("spark.serializer.objectStreamReset", "0")
37+
conf.set("spark.serializer.objectStreamReset", "1")
3838
conf.set("spark.serializer", "org.apache.spark.serializer.JavaSerializer")
3939
// Ensure that we actually have multiple batches per spill file
4040
conf.set("spark.shuffle.spill.batchSize", "10")

core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class ExternalSorterSuite extends FunSuite with LocalSparkContext {
2929
val conf = new SparkConf(loadDefaults)
3030
// Make the Java serializer write a reset instruction (TC_RESET) after each object to test
3131
// for a bug we had with bytes written past the last object in a batch (SPARK-2792)
32-
conf.set("spark.serializer.objectStreamReset", "0")
32+
conf.set("spark.serializer.objectStreamReset", "1")
3333
conf.set("spark.serializer", "org.apache.spark.serializer.JavaSerializer")
3434
// Ensure that we actually have multiple batches per spill file
3535
conf.set("spark.shuffle.spill.batchSize", "10")

0 commit comments

Comments
 (0)