Skip to content

[SQL]RowBasedKeyValueBatch reuse valueRow too #15193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SQL]RowBasedKeyValueBatch reuse valueRow too #15193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions ...main/java/org/apache/spark/sql/catalyst/expressions/FixedLengthRowBasedKeyValueBatch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@ public UnsafeRow getKeyRow(int rowId) {
keyRow.pointTo(base, offset, klen);
// set keyRowId so we can check if desired row is cached
keyRowId = rowId;
isValueCached = false;
} else {
isValueCached = true;
}
return keyRow;
}
Expand All @@ -91,10 +94,12 @@ public UnsafeRow getKeyRow(int rowId) {
*/
@Override
protected UnsafeRow getValueFromKey(int rowId) {
if (keyRowId != rowId) {
assert(rowId >= 0);
if (keyRowId == rowId && isValueCached) {
return valueRow;
} else if (keyRowId != rowId) {
getKeyRow(rowId);
}
assert(rowId >= 0);
valueRow.pointTo(base, keyRow.getBaseOffset() + klen, vlen + 4);
return valueRow;
}
Expand Down
3 changes: 3 additions & 0 deletions ...talyst/src/main/java/org/apache/spark/sql/catalyst/expressions/RowBasedKeyValueBatch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ public abstract class RowBasedKeyValueBatch extends MemoryConsumer {
protected final UnsafeRow keyRow;
protected final UnsafeRow valueRow;

// mark valueRow as cached after calling getKeyRow if rowId == keyRowId
protected boolean isValueCached = false;

protected MemoryBlock page = null;
protected Object base = null;
protected final long recordStartOffset;
Expand Down
9 changes: 7 additions & 2 deletions ...n/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ public UnsafeRow getKeyRow(int rowId) {
keyRow.pointTo(base, offset, klen);
// set keyRowId so we can check if desired row is cached
keyRowId = rowId;
isValueCached = false;
} else {
isValueCached = true;
}
return keyRow;
}
Expand All @@ -95,10 +98,12 @@ public UnsafeRow getKeyRow(int rowId) {
*/
@Override
public UnsafeRow getValueFromKey(int rowId) {
if (keyRowId != rowId) {
assert(rowId >= 0);
if (keyRowId == rowId && isValueCached) {
return valueRow;
} else if (keyRowId != rowId) {
getKeyRow(rowId);
}
assert(rowId >= 0);
long offset = keyRow.getBaseOffset();
int klen = keyRow.getSizeInBytes();
int vlen = Platform.getInt(base, offset - 8) - klen - 4;
Expand Down