refactor(services/hdfs): split hdfs into separate crate

core/Cargo.toml

@@ -134,7 +134,7 @@ services-gdrive = ["dep:opendal-service-gdrive"]
 services-ghac = ["dep:opendal-service-ghac"]
 services-github = ["dep:opendal-service-github"]
 services-gridfs = ["opendal-core/services-gridfs"]
-services-hdfs = ["opendal-core/services-hdfs"]
+services-hdfs = ["dep:opendal-service-hdfs"]
 services-hdfs-native = ["dep:opendal-service-hdfs-native"]
 services-http = ["dep:opendal-service-http"]
 services-huggingface = ["dep:opendal-service-huggingface"]
@@ -238,6 +238,7 @@ opendal-service-gcs = { path = "services/gcs", version = "0.55.0", optional = tr
 opendal-service-gdrive = { path = "services/gdrive", version = "0.55.0", optional = true, default-features = false }
 opendal-service-ghac = { path = "services/ghac", version = "0.55.0", optional = true, default-features = false }
 opendal-service-github = { path = "services/github", version = "0.55.0", optional = true, default-features = false }
+opendal-service-hdfs = { path = "services/hdfs", version = "0.55.0", optional = true, default-features = false }
 opendal-service-hdfs-native = { path = "services/hdfs-native", version = "0.55.0", optional = true, default-features = false }
 opendal-service-http = { path = "services/http", version = "0.55.0", optional = true, default-features = false }
 opendal-service-huggingface = { path = "services/huggingface", version = "0.55.0", optional = true, default-features = false }

core/core/Cargo.toml

@@ -58,7 +58,8 @@ executors-tokio = ["tokio/rt"]
 layers-dtrace = ["dep:probe"]
 
 services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"]
-services-hdfs = ["dep:hdrs"]
+services-ipmfs = []
+services-lakefs = []
 services-memory = []
 services-mongodb = ["dep:mongodb", "dep:mongodb-internal-macros"]
 services-monoiofs = ["dep:monoio", "dep:flume"]
@@ -107,8 +108,7 @@ sha2 = { workspace = true, optional = true }
 
 # For http based services.
 reqsign = { workspace = true, features = ["reqwest_request"], optional = true }
-# for services-hdfs
-hdrs = { version = "0.3.2", optional = true, features = ["async_file"] }
+
 # for services-moka
 moka = { version = "0.12", optional = true, features = ["future", "sync"] }
 # for services-mongodb

core/core/src/services/mod.rs

@@ -24,11 +24,6 @@ mod gridfs;
 #[cfg(feature = "services-gridfs")]
 pub use gridfs::*;
 
-#[cfg(feature = "services-hdfs")]
-mod hdfs;
-#[cfg(feature = "services-hdfs")]
-pub use self::hdfs::*;
-
 #[cfg(feature = "services-memory")]
 mod memory;
 #[cfg(feature = "services-memory")]

core/services/hdfs/Cargo.toml

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+description = "Apache OpenDAL hdfs service implementation"
+name = "opendal-service-hdfs"
+
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+version = { workspace = true }
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+opendal-core = { path = "../../core", version = "0.55.0", default-features = false }
+
+bytes = { workspace = true }
+ctor = { workspace = true }
+futures = { workspace = true }
+hdrs = { version = "0.3.2", features = ["async_file"] }
+log = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true }

core/core/src/services/hdfs/backend.rs → core/services/hdfs/src/backend.rs

@@ -27,8 +27,8 @@ use super::deleter::HdfsDeleter;
 use super::lister::HdfsLister;
 use super::reader::HdfsReader;
 use super::writer::HdfsWriter;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
 
 #[doc = include_str!("docs.md")]
 #[derive(Debug, Default)]

core/core/src/services/hdfs/config.rs → core/services/hdfs/src/config.rs

@@ -59,10 +59,10 @@ impl Debug for HdfsConfig {
     }
 }
 
-impl crate::Configurator for HdfsConfig {
+impl opendal_core::Configurator for HdfsConfig {
     type Builder = HdfsBuilder;
 
-    fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result<Self> {
+    fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result<Self> {
         let mut map = uri.options().clone();
         if let Some(authority) = uri.authority() {
             map.insert("name_node".to_string(), format!("hdfs://{authority}"));
@@ -85,8 +85,8 @@ impl crate::Configurator for HdfsConfig {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::Configurator;
-    use crate::types::OperatorUri;
+    use opendal_core::Configurator;
+    use opendal_core::OperatorUri;
 
     #[test]
     fn from_uri_sets_name_node_and_root() {

core/core/src/services/hdfs/core.rs → core/services/hdfs/src/core.rs

@@ -20,8 +20,8 @@ use std::io;
 use std::io::SeekFrom;
 use std::sync::Arc;
 
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
 
 /// HdfsCore contains code that directly interacts with HDFS.
 #[derive(Clone)]

core/core/src/services/hdfs/deleter.rs → core/services/hdfs/src/deleter.rs

@@ -19,8 +19,8 @@ use std::io;
 use std::sync::Arc;
 
 use super::core::HdfsCore;
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
 
 pub struct HdfsDeleter {
     core: Arc<HdfsCore>,

core/core/src/services/hdfs/docs.md → core/services/hdfs/src/docs.md

@@ -107,14 +107,11 @@ Enabling the vendored feature ensures that hdrs includes the necessary libhdfs.s
 ### Via Builder
 
 ```rust,no_run
-use std::sync::Arc;
-
-use anyhow::Result;
-use opendal_core::services::Hdfs;
 use opendal_core::Operator;
+use opendal_service_hdfs::Hdfs;
 
 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create fs backend builder.
     let mut builder = Hdfs::default()
         // Set the name node for hdfs.

core/core/src/services/hdfs/mod.rs → core/services/hdfs/src/lib.rs

@@ -18,7 +18,7 @@
 /// Default scheme for hdfs service.
 pub const HDFS_SCHEME: &str = "hdfs";
 
-use crate::types::DEFAULT_OPERATOR_REGISTRY;
+use opendal_core::DEFAULT_OPERATOR_REGISTRY;
 
 mod backend;
 mod config;

core/core/src/services/hdfs/lister.rs → core/services/hdfs/src/lister.rs

@@ -15,10 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::EntryMode;
-use crate::Metadata;
-use crate::Result;
-use crate::raw::*;
+use opendal_core::EntryMode;
+use opendal_core::Metadata;
+use opendal_core::Result;
+use opendal_core::raw::*;
 
 pub struct HdfsLister {
     root: String,

core/core/src/services/hdfs/reader.rs → core/services/hdfs/src/reader.rs

@@ -20,8 +20,8 @@ use futures::AsyncReadExt;
 use hdrs::AsyncFile;
 use tokio::io::ReadBuf;
 
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
 
 pub struct HdfsReader<F> {
     f: F,

core/core/src/services/hdfs/writer.rs → core/services/hdfs/src/writer.rs

@@ -20,8 +20,8 @@ use std::sync::Arc;
 use bytes::Buf;
 use futures::AsyncWriteExt;
 
-use crate::raw::*;
-use crate::*;
+use opendal_core::raw::*;
+use opendal_core::*;
 
 pub struct HdfsWriter<F> {
     target_path: String,

core/src/lib.rs

@@ -74,6 +74,8 @@ pub mod services {
     pub use opendal_service_ghac::*;
     #[cfg(feature = "services-github")]
     pub use opendal_service_github::*;
+    #[cfg(feature = "services-hdfs")]
+    pub use opendal_service_hdfs::*;
     #[cfg(feature = "services-hdfs-native")]
     pub use opendal_service_hdfs_native::*;
     #[cfg(feature = "services-http")]