Skip to content

Commit

Permalink
Update
Browse files Browse the repository at this point in the history
  • Loading branch information
@viirya
viirya committed Jun 20, 2024
1 parent 17be62a commit c7359e6
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 43 deletions.
21 changes: 18 additions & 3 deletions common/src/main/java/org/apache/arrow/c/CometBufferImportTypeVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,12 @@ public List<ArrowBuf> visit(ArrowType.LargeUtf8 type) {
type,
start,
end);
final long len = end - start;
// HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
// As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
// a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
// So we add this hack to always take full length of data buffer by assuming the first offset
// is always 0 which is true for Arrow Java and arrow-rs.
final int long = end;
offsets.getReferenceManager().retain();
return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
}
Expand All @@ -277,7 +282,12 @@ public List<ArrowBuf> visit(ArrowType.Binary type) {
type,
start,
end);
final int len = end - start;
// HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
// As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
// a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
// So we add this hack to always take full length of data buffer by assuming the first offset
// is always 0 which is true for Arrow Java and arrow-rs.
final int len = end;
offsets.getReferenceManager().retain();
return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
}
Expand All @@ -296,7 +306,12 @@ public List<ArrowBuf> visit(ArrowType.LargeBinary type) {
type,
start,
end);
final long len = end - start;
// HACK: For the issue https://github.com/apache/datafusion-comet/issues/540
// As Arrow Java doesn't support `offset` in C Data interface, we cannot correctly import
// a slice of string from arrow-rs to Java Arrow and then export it to arrow-rs again.
// So we add this hack to always take full length of data buffer by assuming the first offset
// is always 0 which is true for Arrow Java and arrow-rs.
final long len = end;
offsets.getReferenceManager().retain();
return Arrays.asList(maybeImportBitmap(type), offsets, importData(type, len));
}
Expand Down
54 changes: 27 additions & 27 deletions core/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 13 additions & 13 deletions core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ include = [

[dependencies]
parquet-format = "4.0.0" # This must be kept in sync with that from parquet crate
arrow = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd", features = ["prettyprint", "ffi", "chrono-tz"] }
arrow-array = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
arrow-buffer = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
arrow-data = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
arrow-schema = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
arrow-string = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd" }
parquet = { git = "https://github.com/viirya/arrow-rs.git", rev = "564f3bd", default-features = false, features = ["experimental"] }
arrow = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90", features = ["prettyprint", "ffi", "chrono-tz"] }
arrow-array = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90" }
arrow-buffer = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90" }
arrow-data = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90" }
arrow-schema = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90" }
arrow-string = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90" }
parquet = { git = "https://github.com/viirya/arrow-rs.git", rev = "5978a90", default-features = false, features = ["experimental"] }
half = { version = "2.4.1", default-features = false }
futures = "0.3.28"
mimalloc = { version = "*", default-features = false, optional = true }
Expand Down Expand Up @@ -71,12 +71,12 @@ itertools = "0.11.0"
chrono = { version = "0.4", default-features = false, features = ["clock"] }
chrono-tz = { version = "0.8" }
paste = "1.0.14"
datafusion-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1" }
datafusion = { default-features = false, git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", features = ["unicode_expressions", "crypto_expressions"] }
datafusion-functions = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", features = ["crypto_expressions"] }
datafusion-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
datafusion-physical-expr-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
datafusion-physical-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "dd5f0e1", default-features = false }
datafusion-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642" }
datafusion = { default-features = false, git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642", features = ["unicode_expressions", "crypto_expressions"] }
datafusion-functions = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642", features = ["crypto_expressions"] }
datafusion-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642", default-features = false }
datafusion-physical-expr-common = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642", default-features = false }
datafusion-physical-expr = { git = "https://github.com/viirya/arrow-datafusion.git", rev = "690a642", default-features = false }
unicode-segmentation = "^1.10.1"
once_cell = "1.18.0"
regex = "1.9.6"
Expand Down

0 comments on commit c7359e6

Please sign in to comment.