feat: change download() and download_dir() to conditionally initialize backend-specific configs

Cargo.toml

@@ -13,7 +13,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.2.13"
+version = "1.2.14"
 authors = ["The Dragonfly Developers"]
 homepage = "https://d7y.io/"
 repository = "https://github.com/dragonflyoss/client.git"
@@ -23,14 +23,14 @@ readme = "README.md"
 edition = "2021"
 
 [workspace.dependencies]
-dragonfly-client = { path = "dragonfly-client", version = "1.2.13" }
-dragonfly-client-core = { path = "dragonfly-client-core", version = "1.2.13" }
-dragonfly-client-config = { path = "dragonfly-client-config", version = "1.2.13" }
-dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.2.13" }
-dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.2.13" }
-dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.2.13" }
-dragonfly-client-util = { path = "dragonfly-client-util", version = "1.2.13" }
-dragonfly-client-init = { path = "dragonfly-client-init", version = "1.2.13" }
+dragonfly-client = { path = "dragonfly-client", version = "1.2.14" }
+dragonfly-client-core = { path = "dragonfly-client-core", version = "1.2.14" }
+dragonfly-client-config = { path = "dragonfly-client-config", version = "1.2.14" }
+dragonfly-client-storage = { path = "dragonfly-client-storage", version = "1.2.14" }
+dragonfly-client-backend = { path = "dragonfly-client-backend", version = "1.2.14" }
+dragonfly-client-metric = { path = "dragonfly-client-metric", version = "1.2.14" }
+dragonfly-client-util = { path = "dragonfly-client-util", version = "1.2.14" }
+dragonfly-client-init = { path = "dragonfly-client-init", version = "1.2.14" }
 dragonfly-api = "=2.2.24"
 thiserror = "2.0"
 futures = "0.3.32"

dragonfly-client-backend/src/hdfs.rs

@@ -48,8 +48,8 @@ use tokio_util::io::StreamReader;
 use tracing::{debug, error, instrument};
 use url::Url;
 
-/// HDFS_SCHEME is the scheme of the HDFS.
-pub const HDFS_SCHEME: &str = "hdfs";
+/// SCHEME is the scheme of the HDFS.
+pub const SCHEME: &str = "hdfs";
 
 /// DEFAULT_NAMENODE_PORT is the default port of the HDFS namenode.
 const DEFAULT_NAMENODE_PORT: u16 = 9870;
@@ -66,7 +66,7 @@ impl Hdfs {
     /// Create a new Hdfs instance.
     pub fn new() -> Self {
         Self {
-            scheme: HDFS_SCHEME.to_string(),
+            scheme: SCHEME.to_string(),
         }
     }
 

dragonfly-client-backend/src/hugging_face.rs

@@ -53,8 +53,8 @@ use tokio_util::io::StreamReader;
 use tracing::{debug, error};
 use url::Url;
 
-/// HUGGING_FACE_SCHEME is the URL scheme for Hugging Face backend.
-pub const HUGGING_FACE_SCHEME: &str = "hf";
+/// SCHEME is the URL scheme for Hugging Face backend.
+pub const SCHEME: &str = "hf";
 
 /// HUGGING_FACE_BASE_URL is the base URL for Hugging Face Hub.
 const HUGGING_FACE_BASE_URL: &str = "https://huggingface.co";
@@ -227,7 +227,7 @@ impl HuggingFace {
             .build()?;
 
         Ok(Self {
-            scheme: HUGGING_FACE_SCHEME.to_string(),
+            scheme: SCHEME.to_string(),
             client,
         })
     }
@@ -281,17 +281,16 @@ impl HuggingFace {
     /// use the HF backend (preserving auth and URL semantics).
     fn build_hf_url(parsed_url: &ParsedURL, filename: &str) -> String {
         match parsed_url.repository_type {
-            RepositoryType::Model => format!(
-                "{}://{}/{}",
-                HUGGING_FACE_SCHEME, parsed_url.repository_id, filename
-            ),
+            RepositoryType::Model => {
+                format!("{}://{}/{}", SCHEME, parsed_url.repository_id, filename)
+            }
             RepositoryType::Dataset => format!(
                 "{}://datasets/{}/{}",
-                HUGGING_FACE_SCHEME, parsed_url.repository_id, filename
+                SCHEME, parsed_url.repository_id, filename
             ),
             RepositoryType::Space => format!(
                 "{}://spaces/{}/{}",
-                HUGGING_FACE_SCHEME, parsed_url.repository_id, filename
+                SCHEME, parsed_url.repository_id, filename
             ),
         }
     }

dragonfly-client-backend/src/lib.rs

@@ -475,7 +475,7 @@ impl BackendFactory {
         info!("load [hdfs] builtin backend");
 
         self.backends.insert(
-            model_scope::MODEL_SCOPE_SCHEME.to_string(),
+            model_scope::SCHEME.to_string(),
             Box::new(model_scope::ModelScope::new(self.config.clone())?),
         );
         info!("load [modelscope] builtin backend");

dragonfly-client-backend/src/model_scope.rs

@@ -53,8 +53,8 @@ use tokio_util::io::StreamReader;
 use tracing::{debug, error};
 use url::Url;
 
-/// MODELSCOPE_SCHEME is the URL scheme for ModelScope backend.
-pub const MODEL_SCOPE_SCHEME: &str = "modelscope";
+/// SCHEME is the URL scheme for ModelScope backend.
+pub const SCHEME: &str = "modelscope";
 
 /// MODEL_SCOPE_BASE_URL is the base URL for ModelScope Hub.
 const MODEL_SCOPE_BASE_URL: &str = "https://modelscope.cn";
@@ -231,7 +231,7 @@ impl ModelScope {
             .build()?;
 
         Ok(Self {
-            scheme: MODEL_SCOPE_SCHEME.to_string(),
+            scheme: SCHEME.to_string(),
             client,
         })
     }
@@ -267,13 +267,12 @@ impl ModelScope {
     /// use the ModelScope backend (preserving auth and URL semantics).
     fn build_model_scope_url(parsed_url: &ParsedURL, filename: &str) -> String {
         match parsed_url.repository_type {
-            RepositoryType::Model => format!(
-                "{}://{}/{}",
-                MODEL_SCOPE_SCHEME, parsed_url.repository_id, filename
-            ),
+            RepositoryType::Model => {
+                format!("{}://{}/{}", SCHEME, parsed_url.repository_id, filename)
+            }
             RepositoryType::Dataset => format!(
                 "{}://datasets/{}/{}",
-                MODEL_SCOPE_SCHEME, parsed_url.repository_id, filename
+                SCHEME, parsed_url.repository_id, filename
             ),
         }
     }

dragonfly-client-backend/src/object_storage.rs

@@ -100,6 +100,14 @@ pub enum Scheme {
     COS,
 }
 
+/// Scheme implements the Scheme trait.
+impl Scheme {
+    /// Returns true if the given string is a supported scheme.
+    pub fn is_supported(scheme: &str) -> bool {
+        scheme.parse::<Scheme>().is_ok()
+    }
+}
+
 /// Scheme implements the Display.
 impl fmt::Display for Scheme {
     /// Fmt formats the value using the given formatter.
@@ -922,6 +930,21 @@ mod tests {
     use super::*;
     use dragonfly_api::common::v2::ObjectStorage as ObjectStorageInfo;
 
+    #[test]
+    fn should_return_true_for_supported_schemes() {
+        let supported = vec!["s3", "gs", "abs", "oss", "obs", "cos"];
+        for scheme in supported {
+            assert!(Scheme::is_supported(scheme));
+        }
+    }
+    #[test]
+    fn should_return_false_for_unsupported_schemes() {
+        let unsupported = vec!["http", "https", "ftp", "hdfs", "file", "", "S3", "GCS"];
+        for scheme in unsupported {
+            assert!(!Scheme::is_supported(scheme));
+        }
+    }
+
     #[test]
     fn should_get_parsed_url() {
         let file_key = "test-bucket/file";