Listener support/external access (#450)

* Remove pod svc controller

* Add listener volume

* Dont deploy rolegroup Service

* Deploy rolegroup services again, namenode/journalnode listeners

* Advertise as pod address

* Run manual tests against 3.3.4, fix børked registration keys

* Use listener addresses to generate discovery

* Fix datanode port for HTTPS

* Use configured listenerclasses

* Move listenerclass into role config

* Remove useless journalnode listenerclass option

* Docs

* Changelog

* Disabled start of test

* Update CRD

* Remove reference to custom image

* Document namenode_listener_refs

* Break out the listener volume name to a constant

* Update tests/templates/kuttl/external-access/operate-on-cluster.sh

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Avoid cloning FQDN override path

* Use the port name constants we already have

* Re-remove redundant role argument from ContainerConfig::volumes

* Bounds-check port number

* Separate data volumes from listener volume

* Add a comment on discovery ordering

* Remove stale TODO

* bash != fish

* Break other volumes out of `container_log_config`

* Update docs/modules/hdfs/pages/usage-guide/listenerclass.adoc

Co-authored-by: Siegfried Weber <mail@siegfriedweber.net>

* Drop external access test limit range

---------

Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Siegfried Weber <mail@siegfriedweber.net>

Author: 4 people

CHANGELOG.md

@@ -7,10 +7,12 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - More CRD documentation ([#433]).
+- Support for exposing HDFS clusters to clients outside of Kubernetes ([#450]).
 
 ### Changed
 
 - `operator-rs` `0.56.1` -> `0.57.0` ([#433]).
+- [BREAKING] `.spec.clusterConfig.listenerClass` has been renamed to `.spec.nameNodes.config.listenerClass`, migration will be required when using `external-unstable` ([#450]).
 - Change default value of `dfs.ha.nn.not-become-active-in-safemode` from `true` to `false` ([#458]).
 
 ### Fixed
@@ -19,6 +21,7 @@ All notable changes to this project will be documented in this file.
   and `dfs.datanode.kerberos.principal` in the discovery ConfigMap in case Kerberos is enabled ([#451]).
 
 [#433]: https://github.com/stackabletech/hdfs-operator/pull/433
+[#450]: https://github.com/stackabletech/hdfs-operator/pull/450
 [#451]: https://github.com/stackabletech/hdfs-operator/pull/451
 [#458]: https://github.com/stackabletech/hdfs-operator/pull/458
 

Cargo.lock

@@ -57,20 +57,6 @@ spec:
                       format: uint8
                       minimum: 0.0
                       type: integer
-                    listenerClass:
-                      default: cluster-internal
-                      description: |-
-                        This field controls which type of Service the Operator creates for this HdfsCluster:
-
-                        * cluster-internal: Use a ClusterIP service
-
-                        * external-unstable: Use a NodePort service
-
-                        This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change.
-                      enum:
-                        - cluster-internal
-                        - external-unstable
-                      type: string
                     vectorAggregatorConfigMapName:
                       description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector.
                       nullable: true
@@ -585,6 +571,10 @@ spec:
                           description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                           nullable: true
                           type: string
+                        listenerClass:
+                          description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. DataNodes should have a direct ListenerClass, such as `cluster-internal` or `external-unstable`.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -4093,6 +4083,10 @@ spec:
                                 description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                                 nullable: true
                                 type: string
+                              listenerClass:
+                                description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. DataNodes should have a direct ListenerClass, such as `cluster-internal` or `external-unstable`.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null
@@ -14669,6 +14663,10 @@ spec:
                           description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                           nullable: true
                           type: string
+                        listenerClass:
+                          description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. NameNodes should have a stable ListenerClass, such as `cluster-internal` or `external-stable`.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -18168,6 +18166,10 @@ spec:
                                 description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                                 nullable: true
                                 type: string
+                              listenerClass:
+                                description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. NameNodes should have a stable ListenerClass, such as `cluster-internal` or `external-stable`.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null

deploy/helm/hdfs-operator/crds/crds.yaml

@@ -88,6 +88,12 @@ rules:
       - events
     verbs:
       - create
+  - apiGroups:
+      - listeners.stackable.tech
+    resources:
+      - listeners
+    verbs:
+      - get
   - apiGroups:
       - {{ include "operator.name" . }}.stackable.tech
     resources:

deploy/helm/hdfs-operator/templates/roles.yaml

@@ -5,16 +5,19 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.3.4
   clusterConfig:
     zookeeperConfigMapName: simple-hdfs-znode
-    listenerClass: external-unstable
     dfsReplication: 1
   nameNodes:
+    config:
+      listenerClass: external-stable
     roleGroups:
       default:
         replicas: 2
   dataNodes:
+    config:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

docs/modules/hdfs/examples/getting_started/hdfs.yaml

@@ -1,15 +1,18 @@
 = Service exposition with ListenerClasses
 
-The Operator deploys a service called `<name>-<role>-<rolegroup>` (where `<name>` is the name of the HdfsCluster, `<role>` is the role and `<rolegroup>` the name of the role group) through which the different HDFS processes can be accessed. Unlike many other Stackable Operators, the HDFS Operator does not deploy role-level Services.
+The operator deploys a xref:listener-operator:listener.adoc[Listener] for each DataNode and NameNode pod. They both default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{data,name}Nodes.config.listenerClass`.
 
-These services can have either the `cluster-internal` or `external-unstable` type. `external-stable` is not supported for HDFS at the moment. Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
-
-This is how the listener class is configured:
+The cluster can be configured to be accessible from outside of Kubernetes like this:
 
 [source,yaml]
 ----
 spec:
-  clusterConfig:
-    listenerClass: cluster-internal  # <1>
+  dataNodes:
+    config:
+      listenerClass: external-unstable # <1>
+  nameNodes:
+    config:
+      listenerClass: external-stable # <2>
 ----
-<1> The default `cluster-internal` setting. 
+<1> DataNode listeners should prioritize having a direct connection, to minimize network transfer overhead.
+<2> NameNode listeners should prioritize having a stable address, since they will be baked into the client configuration.

docs/modules/hdfs/pages/usage-guide/listenerclass.adoc

@@ -17,6 +17,7 @@ stackable-operator.workspace = true
 product-config.workspace = true
 strum.workspace = true
 tracing.workspace = true
+futures.workspace = true
 
 [dev-dependencies]
 serde_yaml.workspace = true

rust/crd/Cargo.toml

@@ -9,9 +9,6 @@ pub const FIELD_MANAGER_SCOPE_POD: &str = "pod-service";
 
 pub const APP_NAME: &str = "hdfs";
 
-pub const LABEL_ENABLE: &str = "hdfs.stackable.tech/pod-service";
-pub const LABEL_STS_POD_NAME: &str = "statefulset.kubernetes.io/pod-name";
-
 pub const HDFS_SITE_XML: &str = "hdfs-site.xml";
 pub const CORE_SITE_XML: &str = "core-site.xml";
 pub const HADOOP_POLICY_XML: &str = "hadoop-policy.xml";
@@ -27,6 +24,8 @@ pub const SERVICE_PORT_NAME_HTTPS: &str = "https";
 pub const SERVICE_PORT_NAME_DATA: &str = "data";
 pub const SERVICE_PORT_NAME_METRICS: &str = "metrics";
 
+pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal";
+
 pub const DEFAULT_NAME_NODE_METRICS_PORT: u16 = 8183;
 pub const DEFAULT_NAME_NODE_HTTP_PORT: u16 = 9870;
 pub const DEFAULT_NAME_NODE_HTTPS_PORT: u16 = 9871;
@@ -77,4 +76,7 @@ pub const JOURNALNODE_ROOT_DATA_DIR: &str = "/stackable/data/journalnode";
 pub const DATANODE_ROOT_DATA_DIR_PREFIX: &str = "/stackable/data/";
 pub const DATANODE_ROOT_DATA_DIR_SUFFIX: &str = "/datanode";
 
+pub const LISTENER_VOLUME_NAME: &str = "listener";
+pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener";
+
 pub const HDFS_UID: i64 = 1000;