@@ -363,8 +363,6 @@ private[joins] object UnsafeHashedRelation {
363
363
private [execution] final class LongToUnsafeRowMap (val mm : TaskMemoryManager , capacity : Int )
364
364
extends MemoryConsumer (mm) with Externalizable with KryoSerializable {
365
365
366
- private val ARRAY_MAX = ByteArrayMethods .MAX_ROUNDED_ARRAY_LENGTH
367
-
368
366
// Whether the keys are stored in dense mode or not.
369
367
private var isDense = false
370
368
@@ -619,18 +617,14 @@ private[execution] final class LongToUnsafeRowMap(val mm: TaskMemoryManager, cap
619
617
}
620
618
621
619
private def grow (inputRowSize : Int ): Unit = {
620
+ // There is 8 bytes for the pointer to next value
622
621
val neededNumWords = (cursor - Platform .LONG_ARRAY_OFFSET + 8 + inputRowSize + 7 ) / 8
623
622
if (neededNumWords > page.length) {
624
623
if (neededNumWords > (1 << 30 )) {
625
624
throw new UnsupportedOperationException (
626
625
" Can not build a HashedRelation that is larger than 8G" )
627
626
}
628
627
val newNumWords = math.max(neededNumWords, math.min(page.length * 2 , 1 << 30 ))
629
- if (newNumWords > ARRAY_MAX ) {
630
- throw new UnsupportedOperationException (
631
- " Cannot grow internal buffer by size " + newNumWords +
632
- " because the size after growing " + " exceeds size limitation " + ARRAY_MAX )
633
- }
634
628
ensureAcquireMemory(newNumWords * 8L )
635
629
val newPage = new Array [Long ](newNumWords.toInt)
636
630
Platform .copyMemory(page, Platform .LONG_ARRAY_OFFSET , newPage, Platform .LONG_ARRAY_OFFSET ,
0 commit comments