forked from milvus-io/milvus-storage
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
28ec02a
commit 14c7b22
Showing
6 changed files
with
458 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| function(build_opendal) | ||
| include(ExternalProject) | ||
| set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}") | ||
| set(OPENDAL_PREFIX ${CMAKE_BINARY_DIR}/thirdparty/opendal_ep) | ||
|
|
||
| file(MAKE_DIRECTORY | ||
| "${OPENDAL_PREFIX}" | ||
| "${OPENDAL_PREFIX}/lib" | ||
| "${OPENDAL_PREFIX}/include" | ||
| ) | ||
|
|
||
| if (CMAKE_BUILD_TYPE STREQUAL "Debug") | ||
| set(OPENDAL_BUILD_TYPE "debug") | ||
| else() | ||
| set(OPENDAL_BUILD_TYPE "release") | ||
| endif() | ||
|
|
||
| ExternalProject_Add( | ||
| opendal_ep | ||
| GIT_REPOSITORY https://github.com/apache/incubator-opendal.git | ||
| GIT_TAG main | ||
| PREFIX ${OPENDAL_PREFIX} | ||
| SOURCE_SUBDIR bindings/c | ||
| CONFIGURE_COMMAND echo "configure for opendal_ep" | ||
| BUILD_COMMAND cargo build --${OPENDAL_BUILD_TYPE} | ||
| BUILD_IN_SOURCE 1 | ||
| INSTALL_COMMAND bash -c "cp ${OPENDAL_PREFIX}/src/opendal_ep/target/${OPENDAL_BUILD_TYPE}/${OPENDAL_NAME} ${OPENDAL_PREFIX}/lib/ && cp ${OPENDAL_PREFIX}/src/opendal_ep/bindings/c/include/opendal.h ${OPENDAL_PREFIX}/include/") | ||
|
|
||
|
|
||
| add_library(opendal STATIC IMPORTED) | ||
| set_target_properties(opendal | ||
| PROPERTIES | ||
| IMPORTED_GLOBAL TRUE | ||
| IMPORTED_LOCATION "${OPENDAL_PREFIX}/lib/${OPENDAL_NAME}" | ||
| INTERFACE_INCLUDE_DIRECTORIES "${OPENDAL_PREFIX}/include") | ||
| add_dependencies(opendal opendal_ep) | ||
| if(APPLE) | ||
| target_link_libraries(opendal INTERFACE "-framework CoreFoundation") | ||
| target_link_libraries(opendal INTERFACE "-framework Security") | ||
| target_link_libraries(opendal INTERFACE "-framework SystemConfiguration") | ||
| endif() | ||
|
|
||
| get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION) | ||
| get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES) | ||
| message("OPENDAL_IMPORTED_LOCATION: ${OPENDAL_IMPORTED_LOCATION}") | ||
| message("OPENDAL_INTERFACE_INCLUDE_DIRECTORIES: ${OPENDAL_INTERFACE_INCLUDE_DIRECTORIES}") | ||
| endfunction() | ||
|
|
||
| if (opendal_FOUND) | ||
| message("Found opendal: ${opendal_INCLUDE_DIRS}") | ||
| else() | ||
| build_opendal() | ||
| endif() | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| #include <arrow/filesystem/filesystem.h> | ||
| #include <arrow/util/macros.h> | ||
| #include <arrow/util/uri.h> | ||
| #include "opendal.h" | ||
|
|
||
| namespace milvus_storage { | ||
|
|
||
| class OpendalOptions { | ||
| public: | ||
| static arrow::Result<OpendalOptions> FromUri(const arrow::internal::Uri& uri, std::string* out_path); | ||
|
|
||
| const std::unordered_map<std::string, std::string>& options() const { return options_; } | ||
|
|
||
| const std::string& at(const std::string& key) const { return options_.at(key); } | ||
|
|
||
| protected: | ||
| std::unordered_map<std::string, std::string> options_; | ||
| }; | ||
|
|
||
| class OpendalFileSystem : public arrow::fs::FileSystem { | ||
| public: | ||
| ~OpendalFileSystem() override; | ||
|
|
||
| std::string type_name() const override { return "opendal"; } | ||
|
|
||
| bool Equals(const FileSystem& other) const override; | ||
|
|
||
| arrow::Result<arrow::fs::FileInfo> GetFileInfo(const std::string& path) override; | ||
| arrow::Result<std::vector<arrow::fs::FileInfo>> GetFileInfo(const arrow::fs::FileSelector& select) override; | ||
| arrow::fs::FileInfoGenerator GetFileInfoGenerator(const arrow::fs::FileSelector& select) override { | ||
| throw std::runtime_error("Not implemented"); | ||
| }; | ||
|
|
||
| arrow::Status CreateDir(const std::string& path, bool recursive = true) override; | ||
|
|
||
| arrow::Status DeleteDir(const std::string& path) override; | ||
| arrow::Status DeleteDirContents(const std::string& path, bool missing_dir_ok = false) override; | ||
| arrow::Status DeleteRootDirContents() override { throw std::runtime_error("Not implemented"); } | ||
|
|
||
| arrow::Status DeleteFile(const std::string& path) override; | ||
|
|
||
| arrow::Status Move(const std::string& src, const std::string& dest) override; | ||
|
|
||
| arrow::Status CopyFile(const std::string& src, const std::string& dest) override; | ||
|
|
||
| arrow::Result<std::shared_ptr<arrow::io::InputStream>> OpenInputStream(const std::string& path) override; | ||
| arrow::Result<std::shared_ptr<arrow::io::RandomAccessFile>> OpenInputFile(const std::string& path) override; | ||
|
|
||
| arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenOutputStream( | ||
| const std::string& path, const std::shared_ptr<const arrow::KeyValueMetadata>& metadata = {}) override; | ||
|
|
||
| arrow::Result<std::shared_ptr<arrow::io::OutputStream>> OpenAppendStream( | ||
| const std::string& path, const std::shared_ptr<const arrow::KeyValueMetadata>& metadata = {}) override; | ||
|
|
||
| /// Create a S3FileSystem instance from the given options. | ||
| static arrow::Result<std::shared_ptr<OpendalFileSystem>> Make( | ||
| const OpendalOptions& options, const arrow::io::IOContext& = arrow::io::default_io_context()); | ||
|
|
||
| protected: | ||
| OpendalFileSystem(const OpendalOptions& options, const arrow::io::IOContext& io_context); | ||
| opendal_operator* operator_; | ||
| OpendalOptions options_; | ||
| }; | ||
|
|
||
| } // namespace milvus_storage |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.