Add support and test for HDFS

Author: sbernauer

deploy/crd/trinocatalog.crd.yaml

@@ -33,6 +33,15 @@ spec:
                   properties:
                     hive:
                       properties:
+                        hdfs:
+                          nullable: true
+                          properties:
+                            configMap:
+                              description: Name of the discovery-configmap providing information about the HDFS
+                              type: string
+                          required:
+                            - configMap
+                          type: object
                         metastore:
                           description: "Mandatory connection to a Hive Metastore, which will be used as a storage for metadata"
                           properties:

deploy/helm/trino-operator/crds/crds.yaml

@@ -463,6 +463,15 @@ spec:
                   properties:
                     hive:
                       properties:
+                        hdfs:
+                          nullable: true
+                          properties:
+                            configMap:
+                              description: Name of the discovery-configmap providing information about the HDFS
+                              type: string
+                          required:
+                            - configMap
+                          type: object
                         metastore:
                           description: "Mandatory connection to a Hive Metastore, which will be used as a storage for metadata"
                           properties:

deploy/manifests/crds.yaml

@@ -464,6 +464,15 @@ spec:
                   properties:
                     hive:
                       properties:
+                        hdfs:
+                          nullable: true
+                          properties:
+                            configMap:
+                              description: Name of the discovery-configmap providing information about the HDFS
+                              type: string
+                          required:
+                            - configMap
+                          type: object
                         metastore:
                           description: "Mandatory connection to a Hive Metastore, which will be used as a storage for metadata"
                           properties:

rust/crd/src/catalog.rs

@@ -49,7 +49,7 @@ pub struct HiveConnector {
     /// Connection to an S3 store
     pub s3: Option<S3ConnectionDef>,
     // /// Connection to an HDFS
-    // pub hdfs: Option<HdfsConnection>,
+    pub hdfs: Option<HdfsConnection>,
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
@@ -59,12 +59,9 @@ pub struct MetastoreConnection {
     pub config_map: String,
 }
 
-// #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
-// #[serde(rename_all = "camelCase")]
-// pub struct HdfsConnection {
-//     /// Name of the discovery-configmap providing information about the HDFS
-//     pub config_map: String,
-//     /// Wether Trino should impersonate the users when accessing HDFS
-//     #[serde(default)]
-//     impersonation: bool,
-// }
+#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HdfsConnection {
+    /// Name of the discovery-configmap providing information about the HDFS
+    pub config_map: String,
+}

rust/operator-binary/src/catalog.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap};
+use std::collections::BTreeMap;
 
 use snafu::{ResultExt, Snafu};
 use stackable_operator::{
@@ -8,7 +8,10 @@ use stackable_operator::{
     k8s_openapi::api::core::v1::{ConfigMapKeySelector, EnvVar, EnvVarSource, Volume, VolumeMount},
     kube::ResourceExt,
 };
-use stackable_trino_crd::{catalog::TrinoCatalog, S3_SECRET_DIR_NAME};
+use stackable_trino_crd::{
+    CONFIG_DIR_NAME, RW_CONFIG_DIR_NAME,
+    {catalog::TrinoCatalog, S3_SECRET_DIR_NAME},
+};
 
 use self::from_trino_catalog_error::ResolveS3ConnectionDefSnafu;
 
@@ -87,8 +90,9 @@ impl CatalogConfig {
         catalog: TrinoCatalog,
         client: &Client,
     ) -> Result<CatalogConfig, FromTrinoCatalogError> {
+        let catalog_name = catalog.name();
         let mut config = CatalogConfig {
-            name: catalog.name(),
+            name: catalog_name.to_string(),
             properties: BTreeMap::new(),
             env_bindings: Vec::new(),
             load_env_from_files: BTreeMap::new(),
@@ -144,6 +148,23 @@ impl CatalogConfig {
 
                     // TODO: Handle TLS settings (related to https://github.com/stackabletech/trino-operator/pull/244)
                 }
+
+                if let Some(hdfs) = &hive.hdfs {
+                    config.add_property(
+                        "hive.config.resources",
+                        format!("{RW_CONFIG_DIR_NAME}/catalog/{catalog_name}/hdfs-config/hdfs-site.xml"),
+                    );
+
+                    let volume_name = format!("{catalog_name}-hdfs");
+                    config.volumes.push(
+                        VolumeBuilder::new(&volume_name)
+                            .with_config_map(&hdfs.config_map)
+                            .build(),
+                    );
+                    config
+                        .volume_mounts
+                        .push(VolumeMountBuilder::new(&volume_name, format!("{CONFIG_DIR_NAME}/catalog/{catalog_name}/hdfs-config")).build());
+                }
             }
         }
 

tests/templates/kuttl/smoke/04-assert.yaml

@@ -2,13 +2,13 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
-  name: install-postgres
+  name: install-zookeeper
 timeout: 600
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: hive-postgresql
+  name: zookeeper-server-default
 status:
   readyReplicas: 1
   replicas: 1

tests/templates/kuttl/smoke/04-install-zookeeper.yaml.j2

@@ -0,0 +1,11 @@
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperCluster
+metadata:
+  name: zookeeper
+spec:
+  servers:
+    roleGroups:
+      default:
+        replicas: 1
+  version: {{ test_scenario['values']['zookeeper'] }}

tests/templates/kuttl/smoke/05-assert.yaml

@@ -2,13 +2,29 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
-  name: install-hive
-timeout: 720
+  name: install-hdfs
+timeout: 600
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: hive-metastore-default
+  name: hdfs-namenode-default
+status:
+  readyReplicas: 2
+  replicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hdfs-journalnode-default
+status:
+  readyReplicas: 1
+  replicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hdfs-datanode-default
 status:
   readyReplicas: 1
   replicas: 1

tests/templates/kuttl/smoke/05-install-hdfs.yaml.j2

@@ -0,0 +1,29 @@
+---
+apiVersion: hdfs.stackable.tech/v1alpha1
+kind: HdfsCluster
+metadata:
+  name: hdfs
+spec:
+  version: {{ test_scenario['values']['hdfs'] }}
+  zookeeperConfigMapName: zookeeper-hdfs-znode
+  dfsReplication: 1
+  nameNodes:
+    roleGroups:
+      default:
+        replicas: 2
+  dataNodes:
+    roleGroups:
+      default:
+        replicas: 1
+  journalNodes:
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: zookeeper.stackable.tech/v1alpha1
+kind: ZookeeperZnode
+metadata:
+  name:  zookeeper-hdfs-znode
+spec:
+  clusterRef:
+    name: zookeeper

tests/templates/kuttl/smoke/06-assert.yaml

@@ -2,7 +2,13 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 metadata:
-  name: install-opa
-timeout: 300
-commands:
-  - script: kubectl -n $NAMESPACE rollout status daemonset opa-server-default --timeout 300s
+  name: install-postgres
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: hive-postgresql
+status:
+  readyReplicas: 1
+  replicas: 1