feat: Introduce Google Cloud Storage Implementation

quickwit/Cargo.toml

@@ -118,6 +118,7 @@ num_cpus = "1"
 numfmt = "1.1.1"
 once_cell = "1"
 oneshot = "0.1.5"
+opendal = { version ="0.44", default-features = false }
 openssl = { version = "0.10.60", default-features = false }
 openssl-probe = "0.1.5"
 opentelemetry = { version = "0.20", features = ["rt-tokio"] }

quickwit/quickwit-config/src/lib.rs

@@ -70,7 +70,7 @@ pub use crate::node_config::{
 use crate::source_config::serialize::{SourceConfigV0_7, VersionedSourceConfig};
 pub use crate::storage_config::{
     AzureStorageConfig, FileStorageConfig, RamStorageConfig, S3StorageConfig, StorageBackend,
-    StorageBackendFlavor, StorageConfig, StorageConfigs,
+    StorageBackendFlavor, StorageConfig, StorageConfigs, GoogleCloudStorageConfig,
 };
 
 #[derive(utoipa::OpenApi)]

quickwit/quickwit-config/src/storage_config.rs

@@ -37,6 +37,8 @@ pub enum StorageBackend {
     Ram,
     /// Amazon S3 or S3-compatible storage
     S3,
+    /// Google Cloud Storage
+    Google,
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)]
@@ -157,13 +159,14 @@ pub enum StorageConfig {
     File(FileStorageConfig),
     Ram(RamStorageConfig),
     S3(S3StorageConfig),
+    Google(GoogleCloudStorageConfig),
 }
 
 impl StorageConfig {
     pub fn redact(&mut self) {
         match self {
             Self::Azure(azure_storage_config) => azure_storage_config.redact(),
-            Self::File(_) | Self::Ram(_) => {}
+            Self::File(_) | Self::Ram(_) | Self::Google(_) => {}
             Self::S3(s3_storage_config) => s3_storage_config.redact(),
         }
     }
@@ -195,6 +198,13 @@ impl StorageConfig {
             _ => None,
         }
     }
+
+    pub fn as_google(&self) -> Option<&GoogleCloudStorageConfig> {
+        match self {
+            Self::Google(google_cloud_storage_config) => Some(google_cloud_storage_config),
+            _ => None,
+        }
+    }
 }
 
 impl From<AzureStorageConfig> for StorageConfig {
@@ -221,13 +231,20 @@ impl From<S3StorageConfig> for StorageConfig {
     }
 }
 
+impl From<GoogleCloudStorageConfig> for StorageConfig {
+    fn from(google_cloud_storage_config: GoogleCloudStorageConfig) -> Self {
+        Self::Google(google_cloud_storage_config)
+    }
+}
+
 impl StorageConfig {
     pub fn backend(&self) -> StorageBackend {
         match self {
             Self::Azure(_) => StorageBackend::Azure,
             Self::File(_) => StorageBackend::File,
             Self::Ram(_) => StorageBackend::Ram,
             Self::S3(_) => StorageBackend::S3,
+            Self::Google(_) => StorageBackend::Google,
         }
     }
 }
@@ -374,6 +391,26 @@ pub struct FileStorageConfig;
 #[serde(deny_unknown_fields)]
 pub struct RamStorageConfig;
 
+#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GoogleCloudStorageConfig {
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub credential_path: Option<String>,
+}
+
+impl GoogleCloudStorageConfig {
+    pub const GOOGLE_CLOUD_STORAGE_CREDENTIAL_PATH_ENV_VAR: &'static str = "QW_GOOGLE_CLOUD_STORAGE_CREDENTIAL_PATH";
+
+    /// Attempts to find the credential path in the environment variable
+    /// `QW_GOOGLE_CLOUD_STORAGE_CREDENTIAL_PATH` or the config.
+    pub fn resolve_credential_path(&self) -> Option<String> {
+        env::var(Self::GOOGLE_CLOUD_STORAGE_CREDENTIAL_PATH_ENV_VAR)
+            .ok()
+            .or_else(|| self.credential_path.clone())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -533,6 +570,22 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_storage_google_config_serde() {
+        {
+            let google_cloud_storage_config_yaml = r#"
+                credential_path: /path/to/credential.json
+            "#;
+            let google_cloud_storage_config: GoogleCloudStorageConfig =
+                serde_yaml::from_str(google_cloud_storage_config_yaml).unwrap();
+
+            let expected_google_cloud_storage_config = GoogleCloudStorageConfig {
+                credential_path: Some("/path/to/credential.json".to_string()),
+            };
+            assert_eq!(google_cloud_storage_config, expected_google_cloud_storage_config);
+        }
+    }
+
     #[test]
     fn test_storage_s3_config_serde() {
         {

quickwit/quickwit-storage/Cargo.toml

@@ -50,6 +50,8 @@ quickwit-common = { workspace = true }
 quickwit-config = { workspace = true }
 quickwit-proto = { workspace = true }
 
+opendal = { workspace = true, optional = true }
+
 [dev-dependencies]
 mockall = { workspace = true }
 proptest = { workspace = true }
@@ -70,6 +72,9 @@ azure = [
   "azure_storage/enable_reqwest_rustls",
   "azure_storage_blobs/enable_reqwest_rustls",
 ]
+google = [
+  "opendal/services-gcs"
+]
 ci-test = []
 integration-testsuite = [
   "azure",

quickwit/quickwit-storage/src/lib.rs

@@ -43,8 +43,11 @@ pub use self::storage::Storage;
 
 mod bundle_storage;
 mod error;
+
 mod local_file_storage;
 mod object_storage;
+#[cfg(feature = "google")]
+mod opendal_storage;
 mod payload;
 mod prefix_storage;
 mod ram_storage;