From 84909fc24452ad0676d3a755ac25b2653e0aa2b7 Mon Sep 17 00:00:00 2001 From: Tommy Hughes Date: Tue, 19 Nov 2024 16:35:27 -0600 Subject: [PATCH] tls Signed-off-by: Tommy Hughes --- .../api/v1alpha1/featurestore_types.go | 52 +- .../api/v1alpha1/zz_generated.deepcopy.go | 98 ++++ infra/feast-operator/cmd/main.go | 3 + .../crd/bases/feast.dev_featurestores.yaml | 287 +++++++++++ infra/feast-operator/dist/install.yaml | 287 +++++++++++ .../controller/featurestore_controller.go | 2 - .../featurestore_controller_ephemeral_test.go | 4 +- .../featurestore_controller_test.go | 5 +- .../featurestore_controller_tls_test.go | 478 ++++++++++++++++++ .../internal/controller/services/client.go | 31 ++ .../controller/services/repo_config.go | 23 +- .../controller/services/repo_config_test.go | 10 + .../internal/controller/services/services.go | 370 +++++++++----- .../controller/services/services_types.go | 33 +- .../controller/services/suite_test.go | 4 + .../internal/controller/services/tls.go | 245 +++++++++ .../internal/controller/services/tls_test.go | 273 ++++++++++ .../internal/controller/services/util.go | 38 ++ 18 files changed, 2100 insertions(+), 143 deletions(-) create mode 100644 infra/feast-operator/internal/controller/featurestore_controller_tls_test.go create mode 100644 infra/feast-operator/internal/controller/services/tls.go create mode 100644 infra/feast-operator/internal/controller/services/tls_test.go diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 50bf682213f..ad562ed7711 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -72,6 +72,14 @@ type FeatureStoreServices struct { type OfflineStore struct { ServiceConfigs `json:",inline"` Persistence *OfflineStorePersistence `json:"persistence,omitempty"` + TLS *OfflineTlsConfigs `json:"tls,omitempty"` +} + +// OfflineTlsConfigs configures server TLS for the offline feast service. in an openshift cluster, this is configured by default using service serving certificates. +type OfflineTlsConfigs struct { + TlsConfigs `json:",inline"` + // verify the client TLS certificate. + VerifyClient *bool `json:"verifyClient,omitempty"` } // OfflineStorePersistence configures the persistence settings for the offline store service @@ -115,6 +123,7 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ type OnlineStore struct { ServiceConfigs `json:",inline"` Persistence *OnlineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` } // OnlineStorePersistence configures the persistence settings for the online store service @@ -159,6 +168,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ type LocalRegistryConfig struct { ServiceConfigs `json:",inline"` Persistence *RegistryPersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` } // RegistryPersistence configures the persistence settings for the registry service @@ -234,7 +244,8 @@ type RemoteRegistryConfig struct { // Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` Hostname *string `json:"hostname,omitempty"` // Reference to an existing `FeatureStore` CR in the same k8s cluster. - FeastRef *FeatureStoreRef `json:"feastRef,omitempty"` + FeastRef *FeatureStoreRef `json:"feastRef,omitempty"` + TLS *TlsRemoteRegistryConfigs `json:"tls,omitempty"` } // FeatureStoreRef defines which existing FeatureStore's registry should be used @@ -263,6 +274,45 @@ type OptionalConfigs struct { Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } +// TlsConfigs configures server TLS for a feast service. in an openshift cluster, this is configured by default using service serving certificates. +// +kubebuilder:validation:XValidation:rule="(!has(self.disable) || !self.disable) ? has(self.secretRef) : true",message="`secretRef` required if `disable` is false." +type TlsConfigs struct { + // references the local k8s secret where the TLS key and cert reside + SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` + SecretKeyNames SecretKeyNames `json:"secretKeyNames,omitempty"` + // will disable TLS for the feast service. useful in an openshift cluster, for example, where TLS is configured by default + Disable *bool `json:"disable,omitempty"` +} + +// `secretRef` required if `disable` is false. +func (tls *TlsConfigs) IsTLS() bool { + if tls != nil { + if tls.Disable != nil && *tls.Disable { + return false + } else if tls.SecretRef == nil { + return false + } + return true + } + return false +} + +// TlsRemoteRegistryConfigs configures client TLS for a remote feast registry. in an openshift cluster, this is configured by default when the remote feast registry is using service serving certificates. +type TlsRemoteRegistryConfigs struct { + // references the local k8s configmap where the TLS cert resides + ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` + // defines the configmap key name for the client TLS cert. + CertName string `json:"certName"` +} + +// SecretKeyNames defines the secret key names for the TLS key and cert. +type SecretKeyNames struct { + // defaults to "tls.crt" + TlsCrt string `json:"tlsCrt,omitempty"` + // defaults to "tls.key" + TlsKey string `json:"tlsKey,omitempty"` +} + // FeatureStoreStatus defines the observed state of FeatureStore type FeatureStoreStatus struct { // Shows the currently applied feast configuration, including any pertinent defaults diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 196b2147005..fde4a00ae67 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -203,6 +203,11 @@ func (in *LocalRegistryConfig) DeepCopyInto(out *LocalRegistryConfig) { *out = new(RegistryPersistence) (*in).DeepCopyInto(*out) } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TlsConfigs) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalRegistryConfig. @@ -224,6 +229,11 @@ func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { *out = new(OfflineStorePersistence) (*in).DeepCopyInto(*out) } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(OfflineTlsConfigs) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OfflineStore. @@ -301,6 +311,27 @@ func (in *OfflineStorePersistence) DeepCopy() *OfflineStorePersistence { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OfflineTlsConfigs) DeepCopyInto(out *OfflineTlsConfigs) { + *out = *in + in.TlsConfigs.DeepCopyInto(&out.TlsConfigs) + if in.VerifyClient != nil { + in, out := &in.VerifyClient, &out.VerifyClient + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OfflineTlsConfigs. +func (in *OfflineTlsConfigs) DeepCopy() *OfflineTlsConfigs { + if in == nil { + return nil + } + out := new(OfflineTlsConfigs) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = *in @@ -310,6 +341,11 @@ func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = new(OnlineStorePersistence) (*in).DeepCopyInto(*out) } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TlsConfigs) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OnlineStore. @@ -583,6 +619,11 @@ func (in *RemoteRegistryConfig) DeepCopyInto(out *RemoteRegistryConfig) { *out = new(FeatureStoreRef) **out = **in } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(TlsRemoteRegistryConfigs) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteRegistryConfig. @@ -595,6 +636,21 @@ func (in *RemoteRegistryConfig) DeepCopy() *RemoteRegistryConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyNames) DeepCopyInto(out *SecretKeyNames) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyNames. +func (in *SecretKeyNames) DeepCopy() *SecretKeyNames { + if in == nil { + return nil + } + out := new(SecretKeyNames) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceConfigs) DeepCopyInto(out *ServiceConfigs) { *out = *in @@ -626,3 +682,45 @@ func (in *ServiceHostnames) DeepCopy() *ServiceHostnames { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { + *out = *in + if in.SecretRef != nil { + in, out := &in.SecretRef, &out.SecretRef + *out = new(v1.LocalObjectReference) + **out = **in + } + out.SecretKeyNames = in.SecretKeyNames + if in.Disable != nil { + in, out := &in.Disable, &out.Disable + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TlsConfigs. +func (in *TlsConfigs) DeepCopy() *TlsConfigs { + if in == nil { + return nil + } + out := new(TlsConfigs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TlsRemoteRegistryConfigs) DeepCopyInto(out *TlsRemoteRegistryConfigs) { + *out = *in + out.ConfigMapRef = in.ConfigMapRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TlsRemoteRegistryConfigs. +func (in *TlsRemoteRegistryConfigs) DeepCopy() *TlsRemoteRegistryConfigs { + if in == nil { + return nil + } + out := new(TlsRemoteRegistryConfigs) + in.DeepCopyInto(out) + return out +} diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index e132a6a3c9c..23a0309041b 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -38,6 +38,7 @@ import ( feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "github.com/feast-dev/feast/infra/feast-operator/internal/controller" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" //+kubebuilder:scaffold:imports ) @@ -132,6 +133,8 @@ func main() { os.Exit(1) } + services.SetIsOpenShift(mgr.GetConfig()) + if err = (&controller.FeatureStoreReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index f1c7fca8f51..96801f36b00 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -360,6 +360,47 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: OfflineTlsConfigs configures server TLS for the + offline feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + verifyClient: + description: verify the client TLS certificate. + type: boolean + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -675,6 +716,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One selection @@ -996,6 +1075,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -1019,6 +1136,32 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift cluster, + this is configured by default when the remote feast + registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for the + client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap where + the TLS cert resides + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -1355,6 +1498,48 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: OfflineTlsConfigs configures server TLS for + the offline feast service. in an openshift cluster, + this is configured by default using service serving + certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + verifyClient: + description: verify the client TLS certificate. + type: boolean + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -1675,6 +1860,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One @@ -2002,6 +2225,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for + a feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, + where TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -2025,6 +2286,32 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift + cluster, this is configured by default when the + remote feast registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for + the client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap + where the TLS cert resides + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 83181f53b09..4b6625b608f 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -368,6 +368,47 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: OfflineTlsConfigs configures server TLS for the + offline feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + verifyClient: + description: verify the client TLS certificate. + type: boolean + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -683,6 +724,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One selection @@ -1004,6 +1083,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -1027,6 +1144,32 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift cluster, + this is configured by default when the remote feast + registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for the + client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap where + the TLS cert resides + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -1363,6 +1506,48 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: OfflineTlsConfigs configures server TLS for + the offline feast service. in an openshift cluster, + this is configured by default using service serving + certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + verifyClient: + description: verify the client TLS certificate. + type: boolean + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -1683,6 +1868,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One @@ -2010,6 +2233,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for + a feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, + where TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -2033,6 +2294,32 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift + cluster, this is configured by default when the + remote feast registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for + the client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap + where the TLS cert resides + properties: + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 278ea4a78fd..0a0cfd4f1ed 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -77,8 +77,6 @@ func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request } currentStatus := cr.Status.DeepCopy() - // initial status defaults must occur before feast deployment - services.ApplyDefaultsToStatus(cr) result, recErr = r.deployFeast(ctx, cr) if cr.DeletionTimestamp == nil && !reflect.DeepEqual(currentStatus, cr.Status) { if err = r.Client.Status().Update(ctx, cr); err != nil { diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index 913f022022d..d27656ee720 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -45,6 +45,7 @@ import ( var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-ephemeral" + const offlineType = "duckdb" var pullPolicy = corev1.PullAlways var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -58,7 +59,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { featurestore := &feastdevv1alpha1.FeatureStore{} onlineStorePath := "/data/online.db" registryPath := "/data/registry.db" - offlineType := "duckdb" BeforeEach(func() { By("creating the custom resource for the Kind FeatureStore") @@ -210,7 +210,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { svc) Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) - Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetPort)))) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort)))) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 00a6e71c71c..80a550d2f74 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -44,6 +44,7 @@ import ( const feastProject = "test_project" const domain = ".svc.cluster.local:80" +const domainTls = ".svc.cluster.local:443" var image = "test:latest" @@ -184,7 +185,7 @@ var _ = Describe("FeatureStore Controller", func() { svc) Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) - Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetPort)))) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort)))) }) It("should properly encode a feature_store.yaml config", func() { @@ -530,7 +531,7 @@ var _ = Describe("FeatureStore Controller", func() { svc) Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) - Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetPort)))) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort)))) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go new file mode 100644 index 00000000000..80f250a3c3b --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -0,0 +1,478 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/base64" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +var _ = Describe("FeatureStore Controller - Feast service TLS", func() { + Context("When reconciling a FeatureStore resource", func() { + const resourceName = "openshift-tls" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + featurestore := &feastdevv1alpha1.FeatureStore{} + tlsConfigs := feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{Name: "test"}, + } + BeforeEach(func() { + By("creating the custom resource for the Kind FeatureStore") + err := k8sClient.Get(ctx, typeNamespacedName, featurestore) + if err != nil && errors.IsNotFound(err) { + resource := &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + OnlineStore: &feastdevv1alpha1.OnlineStore{ + TLS: &tlsConfigs, + }, + OfflineStore: &feastdevv1alpha1.OfflineStore{ + TLS: &feastdevv1alpha1.OfflineTlsConfigs{ + TlsConfigs: tlsConfigs, + }, + }, + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{ + TLS: &tlsConfigs, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + } + }) + AfterEach(func() { + resource := &feastdevv1alpha1.FeatureStore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + By("Cleanup the specific resource instance FeatureStore") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + + It("should successfully reconcile the resource", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + feast := services.FeastServices{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + } + + Expect(resource.Status).NotTo(BeNil()) + Expect(resource.Status.FeastVersion).To(Equal(feastversion.FeastVersion)) + Expect(resource.Status.ClientConfigMap).To(Equal(feast.GetFeastServiceName(services.ClientFeastType))) + Expect(resource.Status.Applied.FeastProject).To(Equal(resource.Spec.FeastProject)) + Expect(resource.Status.Applied.Services).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry).NotTo(BeNil()) + + Expect(resource.Status.ServiceHostnames.OfflineStore).To(Equal(feast.GetFeastServiceName(services.OfflineFeastType) + "." + resource.Namespace + domainTls)) + Expect(resource.Status.ServiceHostnames.OnlineStore).To(Equal(feast.GetFeastServiceName(services.OnlineFeastType) + "." + resource.Namespace + domainTls)) + Expect(resource.Status.ServiceHostnames.Registry).To(Equal(feast.GetFeastServiceName(services.RegistryFeastType) + "." + resource.Namespace + domainTls)) + + Expect(resource.Status.Conditions).NotTo(BeEmpty()) + cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.RegistryReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ClientReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ClientReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OfflineStoreReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OnlineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) + + Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + + svc := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + svc) + Expect(err).NotTo(HaveOccurred()) + Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpsPort)))) + }) + + It("should properly encode a feature_store.yaml config", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + feast := services.FeastServices{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + } + + req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name}) + Expect(err).NotTo(HaveOccurred()) + labelSelector := labels.NewSelector().Add(*req) + listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector} + deployList := appsv1.DeploymentList{} + err = k8sClient.List(ctx, &deployList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(deployList.Items).To(HaveLen(3)) + + svcList := corev1.ServiceList{} + err = k8sClient.List(ctx, &svcList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(svcList.Items).To(HaveLen(3)) + + cmList := corev1.ConfigMapList{} + err = k8sClient.List(ctx, &cmList, listOpts) + Expect(err).NotTo(HaveOccurred()) + Expect(cmList.Items).To(HaveLen(1)) + + // check registry config + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err := base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfig := &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfig) + Expect(err).NotTo(HaveOccurred()) + testConfig := &services.RepoConfig{ + Project: feastProject, + Provider: services.LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: services.DefaultRegistryEphemeralPath, + }, + } + Expect(repoConfig).To(Equal(testConfig)) + + // check offline config + deploy = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfigOffline := &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfigOffline) + Expect(err).NotTo(HaveOccurred()) + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), + Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", + } + offlineConfig := &services.RepoConfig{ + Project: feastProject, + Provider: services.LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + }, + Registry: regRemote, + } + Expect(repoConfigOffline).To(Equal(offlineConfig)) + + // check online config + deploy = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfigOnline := &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfigOnline) + Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpsPort, + Scheme: services.HttpsScheme, + Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", + } + onlineConfig := &services.RepoConfig{ + Project: feastProject, + Provider: services.LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: offlineRemote, + OnlineStore: services.OnlineStoreConfig{ + Path: services.DefaultOnlineStoreEphemeralPath, + Type: services.OnlineSqliteConfigType, + }, + Registry: regRemote, + } + Expect(repoConfigOnline).To(Equal(onlineConfig)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + + // check client config + cm := &corev1.ConfigMap{} + name := feast.GetFeastServiceName(services.ClientFeastType) + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: resource.Namespace, + }, + cm) + Expect(err).NotTo(HaveOccurred()) + repoConfigClient := &services.RepoConfig{} + err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) + Expect(err).NotTo(HaveOccurred()) + clientConfig := &services.RepoConfig{ + Project: feastProject, + Provider: services.LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: offlineRemote, + OnlineStore: services.OnlineStoreConfig{ + Path: fmt.Sprintf("https://feast-%s-online.default.svc.cluster.local:443", resourceName), + Type: services.OnlineRemoteConfigType, + Cert: services.GetTlsPath(services.OnlineFeastType) + "tls.crt", + }, + Registry: regRemote, + } + Expect(repoConfigClient).To(Equal(clientConfig)) + + // change tls and reconcile + resourceNew := resource.DeepCopy() + disable := true + resourceNew.Spec = feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + OnlineStore: &feastdevv1alpha1.OnlineStore{ + TLS: &feastdevv1alpha1.TlsConfigs{ + Disable: &disable, + }, + }, + OfflineStore: &feastdevv1alpha1.OfflineStore{ + TLS: &feastdevv1alpha1.OfflineTlsConfigs{ + TlsConfigs: tlsConfigs, + }, + }, + }, + } + err = k8sClient.Update(ctx, resourceNew) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + feast.FeatureStore = resource + + // check registry config + deploy = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfig = &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig).To(Equal(testConfig)) + + // check offline config + deploy = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OfflineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfigOffline = &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfigOffline) + Expect(err).NotTo(HaveOccurred()) + regRemote = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } + offlineConfig.Registry = regRemote + Expect(repoConfigOffline).To(Equal(offlineConfig)) + + // check online config + deploy = &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.OnlineFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + + repoConfigOnline = &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfigOnline) + Expect(err).NotTo(HaveOccurred()) + onlineConfig.Registry = regRemote + Expect(repoConfigOnline).To(Equal(onlineConfig)) + }) + }) +}) diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go index 1befd2df194..d4329abbcb4 100644 --- a/infra/feast-operator/internal/controller/services/client.go +++ b/infra/feast-operator/internal/controller/services/client.go @@ -55,3 +55,34 @@ func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { feast.FeatureStore.Status.ClientConfigMap = cm.Name return controllerutil.SetControllerReference(feast.FeatureStore, cm, feast.Scheme) } + +func (feast *FeastServices) createCaConfigMap() error { + logger := log.FromContext(feast.Context) + cm := feast.initCaConfigMap() + if op, err := controllerutil.CreateOrUpdate(feast.Context, feast.Client, cm, controllerutil.MutateFn(func() error { + return feast.setCaConfigMap(cm) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ConfigMap", cm.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) setCaConfigMap(cm *corev1.ConfigMap) error { + cm.Labels = map[string]string{ + NameLabelKey: feast.FeatureStore.Name, + } + cm.Annotations = map[string]string{ + "service.beta.openshift.io/inject-cabundle": "true", + } + return controllerutil.SetControllerReference(feast.FeatureStore, cm, feast.Scheme) +} + +func (feast *FeastServices) initCaConfigMap() *corev1.ConfigMap { + cm := &corev1.ConfigMap{ + ObjectMeta: feast.GetObjectMeta(ClientCaFeastType), + } + cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) + return cm +} diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 899a9157d9b..9a2cf5eaa64 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -24,7 +24,6 @@ import ( feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "gopkg.in/yaml.v3" - corev1 "k8s.io/api/core/v1" ) // GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service @@ -52,7 +51,6 @@ func getServiceRepoConfig(feastType FeastServiceType, featureStore *feastdevv1al appliedSpec := featureStore.Status.Applied repoConfig := getClientRepoConfig(featureStore) - isLocalReg := isLocalRegistry(featureStore) if appliedSpec.Services != nil { services := appliedSpec.Services @@ -75,7 +73,7 @@ func getServiceRepoConfig(feastType FeastServiceType, featureStore *feastdevv1al } case RegistryFeastType: // Registry server only has a `registry` section - if isLocalReg { + if isLocalRegistry(featureStore) { err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) if err != nil { return repoConfig, err @@ -208,6 +206,7 @@ func (feast *FeastServices) getClientFeatureStoreYaml() ([]byte, error) { func getClientRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig { status := featureStore.Status + appliedServices := status.Applied.Services clientRepoConfig := RepoConfig{ Project: status.Applied.FeastProject, Provider: LocalProviderType, @@ -219,11 +218,22 @@ func getClientRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig Host: strings.Split(status.ServiceHostnames.OfflineStore, ":")[0], Port: HttpPort, } + if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS != nil && + (&appliedServices.OfflineStore.TLS.TlsConfigs).IsTLS() { + clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.TlsConfigs.SecretKeyNames.TlsCrt + clientRepoConfig.OfflineStore.Port = HttpsPort + clientRepoConfig.OfflineStore.Scheme = HttpsScheme + } } if len(status.ServiceHostnames.OnlineStore) > 0 { + onlinePath := "://" + status.ServiceHostnames.OnlineStore clientRepoConfig.OnlineStore = OnlineStoreConfig{ Type: OnlineRemoteConfigType, - Path: strings.ToLower(string(corev1.URISchemeHTTP)) + "://" + status.ServiceHostnames.OnlineStore, + Path: HttpScheme + onlinePath, + } + if appliedServices.OnlineStore != nil && appliedServices.OnlineStore.TLS.IsTLS() { + clientRepoConfig.OnlineStore.Cert = GetTlsPath(OnlineFeastType) + appliedServices.OnlineStore.TLS.SecretKeyNames.TlsCrt + clientRepoConfig.OnlineStore.Path = HttpsScheme + onlinePath } } if len(status.ServiceHostnames.Registry) > 0 { @@ -231,6 +241,11 @@ func getClientRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig RegistryType: RegistryRemoteConfigType, Path: status.ServiceHostnames.Registry, } + if localRegistryTls(featureStore) { + clientRepoConfig.Registry.Cert = GetTlsPath(RegistryFeastType) + appliedServices.Registry.Local.TLS.SecretKeyNames.TlsCrt + } else if remoteRegistryTls(featureStore) { + clientRepoConfig.Registry.Cert = GetTlsPath(RegistryFeastType) + appliedServices.Registry.Remote.TLS.CertName + } } return clientRepoConfig } diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index eaeeb4ea09f..14564c76eea 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -282,6 +282,16 @@ func minimalFeatureStore() *feastdevv1alpha1.FeatureStore { } } +func minimalFeatureStoreWithAllServices() *feastdevv1alpha1.FeatureStore { + feast := minimalFeatureStore() + feast.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, + Registry: &feastdevv1alpha1.Registry{}, + } + return feast +} + func emptyMockExtractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { return map[string]interface{}{}, nil } diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 4667c8158ad..c9156cff1f3 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -36,61 +36,63 @@ import ( // Deploy the feast services func (feast *FeastServices) Deploy() error { - if err := feast.setServiceHostnames(); err != nil { + // initial status defaults must be applied before feast deployment + if err := feast.ApplyDefaults(); err != nil { + return err + } + if openshiftTls, err := feast.checkOpenshiftTls(); openshiftTls { + if err := feast.createCaConfigMap(); err != nil { + return err + } + } else if err != nil { return err + } else { + _ = feast.deleteOwnedFeastObj(feast.initCaConfigMap()) } services := feast.FeatureStore.Status.Applied.Services - if services != nil { - if services.OfflineStore != nil { - offlinePersistence := services.OfflineStore.Persistence - - err := feast.validateOfflineStorePersistence(offlinePersistence) - if err != nil { - return err - } - - if err = feast.deployFeastServiceByType(OfflineFeastType); err != nil { - return err - } - } else { - if err := feast.removeFeastServiceByType(OfflineFeastType); err != nil { - return err - } + if feast.isOfflinStore() { + err := feast.validateOfflineStorePersistence(services.OfflineStore.Persistence) + if err != nil { + return err } - if services.OnlineStore != nil { - onlinePersistence := services.OnlineStore.Persistence - - err := feast.validateOnlineStorePersistence(onlinePersistence) - if err != nil { - return err - } + if err = feast.deployFeastServiceByType(OfflineFeastType); err != nil { + return err + } + } else { + if err := feast.removeFeastServiceByType(OfflineFeastType); err != nil { + return err + } + } - if err = feast.deployFeastServiceByType(OnlineFeastType); err != nil { - return err - } - } else { - if err := feast.removeFeastServiceByType(OnlineFeastType); err != nil { - return err - } + if feast.isOnlinStore() { + err := feast.validateOnlineStorePersistence(services.OnlineStore.Persistence) + if err != nil { + return err } - if feast.isLocalRegistry() { - registryPersistence := services.Registry.Local.Persistence + if err = feast.deployFeastServiceByType(OnlineFeastType); err != nil { + return err + } + } else { + if err := feast.removeFeastServiceByType(OnlineFeastType); err != nil { + return err + } + } - err := feast.validateRegistryPersistence(registryPersistence) - if err != nil { - return err - } + if feast.isLocalRegistry() { + err := feast.validateRegistryPersistence(services.Registry.Local.Persistence) + if err != nil { + return err + } - if err = feast.deployFeastServiceByType(RegistryFeastType); err != nil { - return err - } - } else { - if err := feast.removeFeastServiceByType(RegistryFeastType); err != nil { - return err - } + if err = feast.deployFeastServiceByType(RegistryFeastType); err != nil { + return err + } + } else { + if err := feast.removeFeastServiceByType(RegistryFeastType); err != nil { + return err } } @@ -178,28 +180,28 @@ func (feast *FeastServices) deployFeastServiceByType(feastType FeastServiceType) } else { _ = feast.deleteOwnedFeastObj(feast.initPVC(feastType)) } - if err := feast.createServiceAccount(feastType); err != nil { + if err := feast.createService(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createDeployment(feastType); err != nil { + if err := feast.createServiceAccount(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createService(feastType); err != nil { + if err := feast.createDeployment(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } return feast.setFeastServiceCondition(nil, feastType) } func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) error { - if err := feast.deleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { - return err - } if err := feast.deleteOwnedFeastObj(feast.initFeastDeploy(feastType)); err != nil { return err } if err := feast.deleteOwnedFeastObj(feast.initFeastSA(feastType)); err != nil { return err } + if err := feast.deleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { + return err + } if err := feast.deleteOwnedFeastObj(feast.initPVC(feastType)); err != nil { return err } @@ -207,6 +209,17 @@ func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) return nil } +func (feast *FeastServices) ApplyDefaults() error { + ApplyDefaultsToStatus(feast.FeatureStore) + if err := feast.setTlsDefaults(); err != nil { + return err + } + if err := feast.setServiceHostnames(); err != nil { + return err + } + return nil +} + func (feast *FeastServices) createService(feastType FeastServiceType) error { logger := log.FromContext(feast.Context) svc := feast.initFeastSvc(feastType) @@ -254,6 +267,7 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1alpha1.PvcCreate, fea return err } + // PVCs are immutable, so we only create... we don't update an existing one. err = feast.Client.Get(feast.Context, client.ObjectKeyFromObject(pvc), pvc) if err != nil && apierrors.IsNotFound(err) { err = feast.Client.Create(feast.Context, pvc) @@ -272,17 +286,12 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F return err } deploy.Labels = feast.getLabels(feastType) - deploySettings := FeastServiceConstants[feastType] + sa := feast.initFeastSA(feastType) + tls := feast.getTlsConfigs(feastType) serviceConfigs := feast.getServiceConfigs(feastType) defaultServiceConfigs := serviceConfigs.DefaultConfigs - sa := feast.initFeastSA(feastType) + probeHandler := getProbeHandler(feastType, tls) - // standard configs are applied here - probeHandler := corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(int(deploySettings.TargetPort)), - }, - } deploy.Spec = appsv1.DeploymentSpec{ Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), @@ -296,11 +305,11 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F { Name: string(feastType), Image: *defaultServiceConfigs.Image, - Command: deploySettings.Command, + Command: feast.getContainerCommand(feastType), Ports: []corev1.ContainerPort{ { Name: string(feastType), - ContainerPort: deploySettings.TargetPort, + ContainerPort: getTargetPort(feastType, tls), Protocol: corev1.ProtocolTCP, }, }, @@ -318,7 +327,7 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F ReadinessProbe: &corev1.Probe{ ProbeHandler: probeHandler, InitialDelaySeconds: 20, - PeriodSeconds: 10, + PeriodSeconds: 30, }, }, }, @@ -327,28 +336,100 @@ func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType F } // configs are applied here - container := &deploy.Spec.Template.Spec.Containers[0] - applyOptionalContainerConfigs(container, serviceConfigs.OptionalConfigs) + podSpec := &deploy.Spec.Template.Spec + applyOptionalContainerConfigs(&podSpec.Containers[0], serviceConfigs.OptionalConfigs) + feast.mountTlsConfig(feastType, podSpec) if pvcConfig, hasPvcConfig := hasPvcConfig(feast.FeatureStore, feastType); hasPvcConfig { - mountPvcConfig(&deploy.Spec.Template.Spec, pvcConfig, deploy.Name) + mountPvcConfig(podSpec, pvcConfig, deploy.Name) + } + + switch feastType { + case OfflineFeastType: + feast.registryClientPodConfigs(podSpec) + case OnlineFeastType: + feast.registryClientPodConfigs(podSpec) + feast.offlineClientPodConfigs(podSpec) } return controllerutil.SetControllerReference(feast.FeatureStore, deploy, feast.Scheme) } +func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { + deploySettings := FeastServiceConstants[feastType] + targetPort := deploySettings.TargetHttpPort + tls := feast.getTlsConfigs(feastType) + if tls.IsTLS() { + targetPort = deploySettings.TargetHttpsPort + feastTlsPath := GetTlsPath(feastType) + deploySettings.Command = append(deploySettings.Command, []string{"--key", feastTlsPath + tls.SecretKeyNames.TlsKey, + "--cert", feastTlsPath + tls.SecretKeyNames.TlsCrt}...) + } + deploySettings.Command = append(deploySettings.Command, []string{"-p", strconv.Itoa(int(targetPort))}...) + + if feastType == OfflineFeastType { + if tls.IsTLS() && feast.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient != nil { + deploySettings.Command = append(deploySettings.Command, + []string{"--verify_client", strconv.FormatBool(*feast.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient)}...) + } + } + + return deploySettings.Command +} + +func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { + feast.mountTlsConfig(OfflineFeastType, podSpec) +} + +func (feast *FeastServices) registryClientPodConfigs(podSpec *corev1.PodSpec) { + feast.setRegistryClientInitContainer(podSpec) + feast.mountRegistryClientTls(podSpec) +} + +func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { + hostname := feast.FeatureStore.Status.ServiceHostnames.Registry + if len(hostname) > 0 { + grpcurlFlag := "-plaintext" + hostSplit := strings.Split(hostname, ":") + if len(hostSplit) > 1 && hostSplit[1] == "443" { + grpcurlFlag = "-insecure" + } + podSpec.InitContainers = []corev1.Container{ + { + Name: "init-registry", + Image: "fullstorydev/grpcurl:v1.9.1-alpine", + Command: []string{ + "sh", "-c", + "until grpcurl " + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", + }, + }, + } + } +} + func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { svc.Labels = feast.getLabels(feastType) - deploySettings := FeastServiceConstants[feastType] + if feast.isOpenShiftTls(feastType) { + svc.Annotations = map[string]string{ + "service.beta.openshift.io/serving-cert-secret-name": svc.Name + tlsNameSuffix, + } + } + var port int32 = HttpPort + scheme := HttpScheme + tls := feast.getTlsConfigs(feastType) + if tls.IsTLS() { + port = HttpsPort + scheme = HttpsScheme + } svc.Spec = corev1.ServiceSpec{ Selector: svc.GetLabels(), Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { - Name: strings.ToLower(string(corev1.URISchemeHTTP)), - Port: HttpPort, + Name: scheme, + Port: port, Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt(int(deploySettings.TargetPort)), + TargetPort: intstr.FromInt(int(getTargetPort(feastType, tls))), }, }, } @@ -375,21 +456,19 @@ func (feast *FeastServices) createNewPVC(pvcCreate *feastdevv1alpha1.PvcCreate, } func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastdevv1alpha1.ServiceConfigs { - appliedSpec := feast.FeatureStore.Status.Applied + appliedServices := feast.FeatureStore.Status.Applied.Services switch feastType { case OfflineFeastType: - if appliedSpec.Services.OfflineStore != nil { - return appliedSpec.Services.OfflineStore.ServiceConfigs + if feast.isOfflinStore() { + return appliedServices.OfflineStore.ServiceConfigs } case OnlineFeastType: - if appliedSpec.Services.OnlineStore != nil { - return appliedSpec.Services.OnlineStore.ServiceConfigs + if feast.isOnlinStore() { + return appliedServices.OnlineStore.ServiceConfigs } case RegistryFeastType: - if appliedSpec.Services.Registry != nil { - if appliedSpec.Services.Registry.Local != nil { - return appliedSpec.Services.Registry.Local.ServiceConfigs - } + if feast.isLocalRegistry() { + return appliedServices.Registry.Local.ServiceConfigs } } return feastdevv1alpha1.ServiceConfigs{} @@ -418,23 +497,24 @@ func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]str func (feast *FeastServices) setServiceHostnames() error { feast.FeatureStore.Status.ServiceHostnames = feastdevv1alpha1.ServiceHostnames{} - services := feast.FeatureStore.Status.Applied.Services - if services != nil { - domain := svcDomain + ":" + strconv.Itoa(HttpPort) - if services.OfflineStore != nil { - objMeta := feast.GetObjectMeta(OfflineFeastType) - feast.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain - } - if services.OnlineStore != nil { - objMeta := feast.GetObjectMeta(OnlineFeastType) - feast.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain - } - if feast.isLocalRegistry() { - objMeta := feast.GetObjectMeta(RegistryFeastType) - feast.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain - } else if feast.isRemoteRegistry() { - return feast.setRemoteRegistryURL() + domain := svcDomain + ":" + if feast.isOfflinStore() { + objMeta := feast.GetObjectMeta(OfflineFeastType) + port := strconv.Itoa(HttpPort) + if feast.offlineTls() { + port = strconv.Itoa(HttpsPort) } + feast.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + port + } + if feast.isOnlinStore() { + objMeta := feast.GetObjectMeta(OnlineFeastType) + feast.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.FeatureStore.Status.Applied.Services.OnlineStore.TLS) + } + if feast.isLocalRegistry() { + objMeta := feast.GetObjectMeta(RegistryFeastType) + feast.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.FeatureStore.Status.Applied.Services.Registry.Local.TLS) + } else if feast.isRemoteRegistry() { + return feast.setRemoteRegistryURL() } return nil } @@ -458,40 +538,49 @@ func (feast *FeastServices) setRemoteRegistryURL() error { if feast.isRemoteHostnameRegistry() { feast.FeatureStore.Status.ServiceHostnames.Registry = *feast.FeatureStore.Status.Applied.Services.Registry.Remote.Hostname } else if feast.IsRemoteRefRegistry() { + remoteFeast, err := feast.getRemoteRegistryFeastHandler() + if err != nil { + return err + } + // referenced/remote registry must use the local install option and be in a 'Ready' state. + if remoteFeast != nil && + remoteFeast.isLocalRegistry() && + apimeta.IsStatusConditionTrue(remoteFeast.FeatureStore.Status.Conditions, feastdevv1alpha1.RegistryReadyType) { + feast.FeatureStore.Status.ServiceHostnames.Registry = remoteFeast.FeatureStore.Status.ServiceHostnames.Registry + } else { + return errors.New("Remote feast registry of referenced FeatureStore '" + remoteFeast.FeatureStore.Name + "' is not ready") + } + } + return nil +} + +func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, error) { + if feast.IsRemoteRefRegistry() { feastRemoteRef := feast.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef // default to FeatureStore namespace if not set if len(feastRemoteRef.Namespace) == 0 { feastRemoteRef.Namespace = feast.FeatureStore.Namespace } - nsName := types.NamespacedName{Name: feastRemoteRef.Name, Namespace: feastRemoteRef.Namespace} crNsName := client.ObjectKeyFromObject(feast.FeatureStore) if nsName == crNsName { - return errors.New("FeatureStore '" + crNsName.Name + "' can't reference itself in `spec.services.registry.remote.feastRef`") + return nil, errors.New("FeatureStore '" + crNsName.Name + "' can't reference itself in `spec.services.registry.remote.feastRef`") } - remoteFeastObj := &feastdevv1alpha1.FeatureStore{} if err := feast.Client.Get(feast.Context, nsName, remoteFeastObj); err != nil { if apierrors.IsNotFound(err) { - return errors.New("Referenced FeatureStore '" + feastRemoteRef.Name + "' was not found") + return nil, errors.New("Referenced FeatureStore '" + feastRemoteRef.Name + "' was not found") } - return err + return nil, err } - - remoteFeast := FeastServices{ + return &FeastServices{ Client: feast.Client, Context: feast.Context, FeatureStore: remoteFeastObj, Scheme: feast.Scheme, - } - // referenced/remote registry must use the local install option and be in a 'Ready' state. - if remoteFeast.isLocalRegistry() && apimeta.IsStatusConditionTrue(remoteFeastObj.Status.Conditions, feastdevv1alpha1.RegistryReadyType) { - feast.FeatureStore.Status.ServiceHostnames.Registry = remoteFeastObj.Status.ServiceHostnames.Registry - } else { - return errors.New("Remote feast registry of referenced FeatureStore '" + feastRemoteRef.Name + "' is not ready") - } + }, nil } - return nil + return nil, nil } func (feast *FeastServices) isLocalRegistry() bool { @@ -499,8 +588,7 @@ func (feast *FeastServices) isLocalRegistry() bool { } func (feast *FeastServices) isRemoteRegistry() bool { - appliedServices := feast.FeatureStore.Status.Applied.Services - return appliedServices != nil && appliedServices.Registry != nil && appliedServices.Registry.Remote != nil + return isRemoteRegistry(feast.FeatureStore) } func (feast *FeastServices) IsRemoteRefRegistry() bool { @@ -519,6 +607,16 @@ func (feast *FeastServices) isRemoteHostnameRegistry() bool { return false } +func (feast *FeastServices) isOfflinStore() bool { + appliedServices := feast.FeatureStore.Status.Applied.Services + return appliedServices != nil && appliedServices.OfflineStore != nil +} + +func (feast *FeastServices) isOnlinStore() bool { + appliedServices := feast.FeatureStore.Status.Applied.Services + return appliedServices != nil && appliedServices.OnlineStore != nil +} + func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1.Deployment { deploy := &appsv1.Deployment{ ObjectMeta: feast.GetObjectMeta(feastType), @@ -607,28 +705,54 @@ func mergeEnvVarsArrays(envVars1 []corev1.EnvVar, envVars2 *[]corev1.EnvVar) []c } func mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, deployName string) { - container := &podSpec.Containers[0] - var pvcName string - if pvcConfig.Create != nil { - pvcName = deployName - } else { - pvcName = pvcConfig.Ref.Name - } + if podSpec != nil && pvcConfig != nil { + container := &podSpec.Containers[0] + var pvcName string + if pvcConfig.Create != nil { + pvcName = deployName + } else { + pvcName = pvcConfig.Ref.Name + } - podSpec.Volumes = []corev1.Volume{ - { + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: pvcName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcName, }, }, - }, - } - container.VolumeMounts = []corev1.VolumeMount{ - { + }) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ Name: pvcName, MountPath: pvcConfig.MountPath, + }) + } +} + +func getTargetPort(feastType FeastServiceType, tls *feastdevv1alpha1.TlsConfigs) int32 { + if tls.IsTLS() { + return FeastServiceConstants[feastType].TargetHttpsPort + } + return FeastServiceConstants[feastType].TargetHttpPort +} + +func getProbeHandler(feastType FeastServiceType, tls *feastdevv1alpha1.TlsConfigs) corev1.ProbeHandler { + targetPort := getTargetPort(feastType, tls) + if feastType == OnlineFeastType { + probeHandler := corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/health", + Port: intstr.FromInt(int(targetPort)), + }, + } + if tls.IsTLS() { + probeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + return probeHandler + } + return corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(int(targetPort)), }, } } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 1251a2ff9e3..df66d493857 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -35,7 +35,13 @@ const ( DefaultOnlineStoreEphemeralPath = "/tmp/online_store.db" DefaultOnlineStorePvcPath = "online_store.db" svcDomain = ".svc.cluster.local" - HttpPort = 80 + + HttpPort = 80 + HttpsPort = 443 + HttpScheme = "http" + HttpsScheme = "https" + tlsPath = "/tls/" + tlsNameSuffix = "-tls" DefaultOfflineStorageRequest = "20Gi" DefaultOnlineStorageRequest = "5Gi" @@ -45,6 +51,7 @@ const ( OnlineFeastType FeastServiceType = "online" RegistryFeastType FeastServiceType = "registry" ClientFeastType FeastServiceType = "client" + ClientCaFeastType FeastServiceType = "client-ca" OfflineRemoteConfigType OfflineConfigType = "remote" OfflineFilePersistenceDaskConfigType OfflineConfigType = "dask" @@ -72,16 +79,19 @@ var ( FeastServiceConstants = map[FeastServiceType]deploymentSettings{ OfflineFeastType: { - Command: []string{"feast", "serve_offline", "-h", "0.0.0.0"}, - TargetPort: 8815, + Command: []string{"feast", "serve_offline", "-h", "0.0.0.0"}, + TargetHttpPort: 8815, + TargetHttpsPort: 8816, }, OnlineFeastType: { - Command: []string{"feast", "serve", "-h", "0.0.0.0"}, - TargetPort: 6566, + Command: []string{"feast", "serve", "-h", "0.0.0.0"}, + TargetHttpPort: 6566, + TargetHttpsPort: 6567, }, RegistryFeastType: { - Command: []string{"feast", "serve_registry"}, - TargetPort: 6570, + Command: []string{"feast", "serve_registry"}, + TargetHttpPort: 6570, + TargetHttpsPort: 6571, }, } @@ -180,6 +190,8 @@ type OfflineStoreConfig struct { Host string `yaml:"host,omitempty"` Type OfflineConfigType `yaml:"type,omitempty"` Port int `yaml:"port,omitempty"` + Scheme string `yaml:"scheme,omitempty"` + Cert string `yaml:"cert,omitempty"` DBParameters map[string]interface{} `yaml:",inline,omitempty"` } @@ -187,6 +199,7 @@ type OfflineStoreConfig struct { type OnlineStoreConfig struct { Path string `yaml:"path,omitempty"` Type OnlineConfigType `yaml:"type,omitempty"` + Cert string `yaml:"cert,omitempty"` DBParameters map[string]interface{} `yaml:",inline,omitempty"` } @@ -194,11 +207,13 @@ type OnlineStoreConfig struct { type RegistryConfig struct { Path string `yaml:"path,omitempty"` RegistryType RegistryConfigType `yaml:"registry_type,omitempty"` + Cert string `yaml:"cert,omitempty"` S3AdditionalKwargs *map[string]string `yaml:"s3_additional_kwargs,omitempty"` DBParameters map[string]interface{} `yaml:",inline,omitempty"` } type deploymentSettings struct { - Command []string - TargetPort int32 + Command []string + TargetHttpPort int32 + TargetHttpsPort int32 } diff --git a/infra/feast-operator/internal/controller/services/suite_test.go b/infra/feast-operator/internal/controller/services/suite_test.go index 5f76f5e6ff7..e1e485f1bf6 100644 --- a/infra/feast-operator/internal/controller/services/suite_test.go +++ b/infra/feast-operator/internal/controller/services/suite_test.go @@ -84,3 +84,7 @@ var _ = AfterSuite(func() { err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) + +func testSetIsOpenShift() { + isOpenShift = true +} diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go new file mode 100644 index 00000000000..e598d597840 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -0,0 +1,245 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "strconv" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +func (feast *FeastServices) setTlsDefaults() error { + if err := feast.setOpenshiftTls(); err != nil { + return err + } + appliedServices := feast.FeatureStore.Status.Applied.Services + if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { + tlsDefaults(&appliedServices.OfflineStore.TLS.TlsConfigs) + } + if feast.isOnlinStore() { + tlsDefaults(appliedServices.OnlineStore.TLS) + } + if feast.isLocalRegistry() { + tlsDefaults(appliedServices.Registry.Local.TLS) + } + return nil +} + +func (feast *FeastServices) setOpenshiftTls() error { + appliedServices := feast.FeatureStore.Status.Applied.Services + tlsConfigs := &feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{}, + } + if feast.offlineOpenshiftTls() { + appliedServices.OfflineStore.TLS = &feastdevv1alpha1.OfflineTlsConfigs{ + TlsConfigs: *tlsConfigs, + } + appliedServices.OfflineStore.TLS.TlsConfigs.SecretRef.Name = feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix + } + if feast.onlineOpenshiftTls() { + appliedServices.OnlineStore.TLS = tlsConfigs + appliedServices.OnlineStore.TLS.SecretRef.Name = feast.initFeastSvc(OnlineFeastType).Name + tlsNameSuffix + } + if feast.localRegistryOpenshiftTls() { + appliedServices.Registry.Local.TLS = tlsConfigs + appliedServices.Registry.Local.TLS.SecretRef.Name = feast.initFeastSvc(RegistryFeastType).Name + tlsNameSuffix + } else if remote, err := feast.remoteRegistryOpenshiftTls(); remote { + // if the remote registry reference is using openshift's service serving certificates, we can use the injected service CA bundle configMap + if appliedServices.Registry.Remote.TLS == nil { + appliedServices.Registry.Remote.TLS = &feastdevv1alpha1.TlsRemoteRegistryConfigs{ + ConfigMapRef: corev1.LocalObjectReference{ + Name: feast.initCaConfigMap().Name, + }, + CertName: "service-ca.crt", + } + } + } else if err != nil { + return err + } + return nil +} + +func (feast *FeastServices) checkOpenshiftTls() (bool, error) { + if feast.offlineOpenshiftTls() || feast.onlineOpenshiftTls() || feast.localRegistryOpenshiftTls() { + return true, nil + } + return feast.remoteRegistryOpenshiftTls() +} + +func (feast *FeastServices) isOpenShiftTls(feastType FeastServiceType) (isOpenShift bool) { + switch feastType { + case OfflineFeastType: + isOpenShift = feast.offlineOpenshiftTls() + case OnlineFeastType: + isOpenShift = feast.onlineOpenshiftTls() + case RegistryFeastType: + isOpenShift = feast.localRegistryOpenshiftTls() + } + return +} + +func (feast *FeastServices) getTlsConfigs(feastType FeastServiceType) (tls *feastdevv1alpha1.TlsConfigs) { + appliedServices := feast.FeatureStore.Status.Applied.Services + switch feastType { + case OfflineFeastType: + if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { + tls = &appliedServices.OfflineStore.TLS.TlsConfigs + } + case OnlineFeastType: + if feast.isOnlinStore() { + tls = appliedServices.OnlineStore.TLS + } + case RegistryFeastType: + if feast.isLocalRegistry() { + tls = appliedServices.Registry.Local.TLS + } + } + return +} + +// True if running in an openshift cluster and Tls not configured in the service Spec +func (feast *FeastServices) offlineOpenshiftTls() bool { + return isOpenShift && + feast.isOfflinStore() && feast.FeatureStore.Spec.Services.OfflineStore.TLS == nil +} + +// True if running in an openshift cluster and Tls not configured in the service Spec +func (feast *FeastServices) onlineOpenshiftTls() bool { + return isOpenShift && + feast.isOnlinStore() && feast.FeatureStore.Spec.Services.OnlineStore.TLS == nil +} + +// True if running in an openshift cluster and Tls not configured in the service Spec +func (feast *FeastServices) localRegistryOpenshiftTls() bool { + return isOpenShift && + feast.isLocalRegistry() && + (feast.FeatureStore.Spec.Services == nil || + feast.FeatureStore.Spec.Services.Registry == nil || + feast.FeatureStore.Spec.Services.Registry.Local == nil || + feast.FeatureStore.Spec.Services.Registry.Local.TLS == nil) +} + +// True if running in an openshift cluster, and using a remote registry in the same cluster, with no remote Tls set in the service Spec +func (feast *FeastServices) remoteRegistryOpenshiftTls() (bool, error) { + if isOpenShift && feast.isRemoteRegistry() { + remoteFeast, err := feast.getRemoteRegistryFeastHandler() + if err != nil { + return false, err + } + return (remoteFeast != nil && remoteFeast.localRegistryOpenshiftTls() && + feast.FeatureStore.Spec.Services.Registry.Remote.TLS == nil), + nil + } + return false, nil +} + +func (feast *FeastServices) offlineTls() bool { + return feast.isOfflinStore() && + feast.FeatureStore.Status.Applied.Services.OfflineStore.TLS != nil && + (&feast.FeatureStore.Status.Applied.Services.OfflineStore.TLS.TlsConfigs).IsTLS() +} + +func (feast *FeastServices) localRegistryTls() bool { + return localRegistryTls(feast.FeatureStore) +} + +func (feast *FeastServices) remoteRegistryTls() bool { + return remoteRegistryTls(feast.FeatureStore) +} + +func (feast *FeastServices) mountRegistryClientTls(podSpec *corev1.PodSpec) { + if podSpec != nil { + if feast.localRegistryTls() { + feast.mountTlsConfig(RegistryFeastType, podSpec) + } else if feast.remoteRegistryTls() { + mountTlsRemoteRegistryConfig(RegistryFeastType, podSpec, + feast.FeatureStore.Status.Applied.Services.Registry.Remote.TLS) + } + } +} + +func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec *corev1.PodSpec) { + tls := feast.getTlsConfigs(feastType) + if tls.IsTLS() && podSpec != nil { + volName := string(feastType) + tlsNameSuffix + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tls.SecretRef.Name, + }, + }, + }) + container := &podSpec.Containers[0] + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } +} + +func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { + if tls != nil { + volName := string(feastType) + tlsNameSuffix + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: tls.ConfigMapRef, + }, + }, + }) + container := &podSpec.Containers[0] + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } +} + +func getPortStr(tls *feastdevv1alpha1.TlsConfigs) string { + if tls.IsTLS() { + return strconv.Itoa(HttpsPort) + } + return strconv.Itoa(HttpPort) +} + +func tlsDefaults(tls *feastdevv1alpha1.TlsConfigs) { + if tls.IsTLS() { + if len(tls.SecretKeyNames.TlsCrt) == 0 { + tls.SecretKeyNames.TlsCrt = "tls.crt" + } + if len(tls.SecretKeyNames.TlsKey) == 0 { + tls.SecretKeyNames.TlsKey = "tls.key" + } + } +} + +func localRegistryTls(featureStore *feastdevv1alpha1.FeatureStore) bool { + return isLocalRegistry(featureStore) && featureStore.Status.Applied.Services.Registry.Local.TLS.IsTLS() +} + +func remoteRegistryTls(featureStore *feastdevv1alpha1.FeatureStore) bool { + return isRemoteRegistry(featureStore) && featureStore.Status.Applied.Services.Registry.Remote.TLS != nil +} + +func GetTlsPath(feastType FeastServiceType) string { + return tlsPath + string(feastType) + "/" +} diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go new file mode 100644 index 00000000000..93b6c137a85 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -0,0 +1,273 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" +) + +// test tls functions directly +var _ = Describe("TLS Config", func() { + Context("When reconciling a FeatureStore", func() { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(feastdevv1alpha1.AddToScheme(scheme)) + + secretKeyNames := feastdevv1alpha1.SecretKeyNames{ + TlsCrt: "tls.crt", + TlsKey: "tls.key", + } + + It("should set default TLS configs", func() { + By("Having the created resource") + + // registry service w/o tls + feast := FeastServices{ + FeatureStore: minimalFeatureStore(), + Scheme: scheme, + } + err := feast.ApplyDefaults() + Expect(err).To(BeNil()) + + tls := feast.getTlsConfigs(RegistryFeastType) + Expect(tls).To(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + Expect(getPortStr(tls)).To(Equal("80")) + + Expect(feast.offlineTls()).To(BeFalse()) + Expect(feast.remoteRegistryTls()).To(BeFalse()) + Expect(feast.localRegistryTls()).To(BeFalse()) + Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) + openshiftTls, err := feast.checkOpenshiftTls() + Expect(err).To(BeNil()) + Expect(openshiftTls).To(BeFalse()) + + // registry service w/ openshift tls + testSetIsOpenShift() + feast.FeatureStore = minimalFeatureStore() + err = feast.ApplyDefaults() + Expect(err).To(BeNil()) + + tls = feast.getTlsConfigs(OfflineFeastType) + Expect(tls).To(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + tls = feast.getTlsConfigs(OnlineFeastType) + Expect(tls).To(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + tls = feast.getTlsConfigs(RegistryFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeTrue()) + Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) + Expect(getPortStr(tls)).To(Equal("443")) + Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) + + Expect(feast.offlineTls()).To(BeFalse()) + Expect(feast.remoteRegistryTls()).To(BeFalse()) + Expect(feast.localRegistryTls()).To(BeTrue()) + Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) + openshiftTls, err = feast.checkOpenshiftTls() + Expect(err).To(BeNil()) + Expect(openshiftTls).To(BeTrue()) + + // all services w/ openshift tls + feast.FeatureStore = minimalFeatureStoreWithAllServices() + err = feast.ApplyDefaults() + Expect(err).To(BeNil()) + + repoConfig := getClientRepoConfig(feast.FeatureStore) + Expect(repoConfig.OfflineStore.Port).To(Equal(HttpsPort)) + Expect(repoConfig.OfflineStore.Scheme).To(Equal(HttpsScheme)) + Expect(repoConfig.OfflineStore.Cert).To(ContainSubstring(string(OfflineFeastType))) + Expect(repoConfig.OnlineStore.Cert).To(ContainSubstring(string(OnlineFeastType))) + Expect(repoConfig.Registry.Cert).To(ContainSubstring(string(RegistryFeastType))) + + tls = feast.getTlsConfigs(OfflineFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeTrue()) + tls = feast.getTlsConfigs(OnlineFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeTrue()) + tls = feast.getTlsConfigs(RegistryFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) + Expect(tls.IsTLS()).To(BeTrue()) + + Expect(feast.offlineTls()).To(BeTrue()) + Expect(feast.remoteRegistryTls()).To(BeFalse()) + Expect(feast.localRegistryTls()).To(BeTrue()) + Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) + Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeTrue()) + Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) + openshiftTls, err = feast.checkOpenshiftTls() + Expect(err).To(BeNil()) + Expect(openshiftTls).To(BeTrue()) + + // check k8s deployment objects + offlineDeploy := feast.initFeastDeploy(OfflineFeastType) + err = feast.setDeployment(offlineDeploy, OfflineFeastType) + Expect(err).To(BeNil()) + Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) + Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + onlineDeploy := feast.initFeastDeploy(OnlineFeastType) + err = feast.setDeployment(onlineDeploy, OnlineFeastType) + Expect(err).To(BeNil()) + Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) + Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + + // registry service w/ tls and in an openshift cluster + feast.FeatureStore = minimalFeatureStore() + feast.FeatureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + OnlineStore: &feastdevv1alpha1.OnlineStore{ + TLS: &feastdevv1alpha1.TlsConfigs{}, + }, + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{ + TLS: &feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{}, + SecretKeyNames: feastdevv1alpha1.SecretKeyNames{ + TlsCrt: "test.crt", + }, + }, + }, + }, + } + err = feast.ApplyDefaults() + Expect(err).To(BeNil()) + + tls = feast.getTlsConfigs(OfflineFeastType) + Expect(tls).To(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + tls = feast.getTlsConfigs(OnlineFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + tls = feast.getTlsConfigs(RegistryFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeTrue()) + Expect(tls.SecretKeyNames).NotTo(Equal(secretKeyNames)) + Expect(getPortStr(tls)).To(Equal("443")) + Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) + + Expect(feast.offlineTls()).To(BeFalse()) + Expect(feast.remoteRegistryTls()).To(BeFalse()) + Expect(feast.localRegistryTls()).To(BeTrue()) + Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) + openshiftTls, err = feast.checkOpenshiftTls() + Expect(err).To(BeNil()) + Expect(openshiftTls).To(BeFalse()) + + // all services w/ tls and in an openshift cluster + feast.FeatureStore = minimalFeatureStoreWithAllServices() + disable := true + feast.FeatureStore.Spec.Services.OnlineStore.TLS = &feastdevv1alpha1.TlsConfigs{ + Disable: &disable, + } + feast.FeatureStore.Spec.Services.Registry = &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{ + TLS: &feastdevv1alpha1.TlsConfigs{ + Disable: &disable, + }, + }, + } + err = feast.ApplyDefaults() + Expect(err).To(BeNil()) + + repoConfig = getClientRepoConfig(feast.FeatureStore) + Expect(repoConfig.OfflineStore.Port).To(Equal(HttpsPort)) + Expect(repoConfig.OfflineStore.Scheme).To(Equal(HttpsScheme)) + Expect(repoConfig.OfflineStore.Cert).To(ContainSubstring(string(OfflineFeastType))) + Expect(repoConfig.OnlineStore.Cert).NotTo(ContainSubstring(string(OnlineFeastType))) + Expect(repoConfig.Registry.Cert).NotTo(ContainSubstring(string(RegistryFeastType))) + + tls = feast.getTlsConfigs(OfflineFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeTrue()) + Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) + tls = feast.getTlsConfigs(OnlineFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + Expect(tls.SecretKeyNames).NotTo(Equal(secretKeyNames)) + tls = feast.getTlsConfigs(RegistryFeastType) + Expect(tls).NotTo(BeNil()) + Expect(tls.IsTLS()).To(BeFalse()) + Expect(tls.SecretKeyNames).NotTo(Equal(secretKeyNames)) + Expect(getPortStr(tls)).To(Equal("80")) + Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) + + Expect(feast.offlineTls()).To(BeTrue()) + Expect(feast.remoteRegistryTls()).To(BeFalse()) + Expect(feast.localRegistryTls()).To(BeFalse()) + Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) + Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) + Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) + openshiftTls, err = feast.checkOpenshiftTls() + Expect(err).To(BeNil()) + Expect(openshiftTls).To(BeTrue()) + + // check k8s service objects + offlineSvc := feast.initFeastSvc(OfflineFeastType) + Expect(offlineSvc.Annotations).To(BeEmpty()) + err = feast.setService(offlineSvc, OfflineFeastType) + Expect(err).To(BeNil()) + Expect(offlineSvc.Annotations).NotTo(BeEmpty()) + Expect(offlineSvc.Spec.Ports[0].Name).To(Equal(HttpsScheme)) + + onlineSvc := feast.initFeastSvc(OnlineFeastType) + err = feast.setService(onlineSvc, OnlineFeastType) + Expect(err).To(BeNil()) + Expect(onlineSvc.Annotations).To(BeEmpty()) + Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) + + // check k8s deployment objects + offlineDeploy = feast.initFeastDeploy(OfflineFeastType) + err = feast.setDeployment(offlineDeploy, OfflineFeastType) + Expect(err).To(BeNil()) + Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) + Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + onlineDeploy = feast.initFeastDeploy(OnlineFeastType) + err = feast.setDeployment(onlineDeploy, OnlineFeastType) + Expect(err).To(BeNil()) + Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) + Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + }) + }) +}) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 5e2daee6738..9143071d065 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -13,15 +13,24 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" ) +var isOpenShift = false + func isLocalRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { appliedServices := featureStore.Status.Applied.Services return appliedServices != nil && appliedServices.Registry != nil && appliedServices.Registry.Local != nil } +func isRemoteRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { + appliedServices := featureStore.Status.Applied.Services + return appliedServices != nil && appliedServices.Registry != nil && appliedServices.Registry.Remote != nil +} + func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastServiceType) (*feastdevv1alpha1.PvcConfig, bool) { services := featureStore.Status.Applied.Services var pvcConfig *feastdevv1alpha1.PvcConfig = nil @@ -272,3 +281,32 @@ func CopyMap(original map[string]interface{}) map[string]interface{} { return newCopy } + +// IsOpenShift is a global flag that can be safely called across reconciliation cycles, defined at the controller manager start. +func IsOpenShift() bool { + return isOpenShift +} + +// SetIsOpenShift sets the global flag isOpenShift by the controller manager. +// We don't need to keep fetching the API every reconciliation cycle that we need to know about the platform. +func SetIsOpenShift(cfg *rest.Config) { + if cfg == nil { + panic("Rest Config struct is nil, impossible to get cluster information") + } + // adapted from https://github.com/RHsyseng/operator-utils/blob/a226fabb2226a313dd3a16591c5579ebcd8a74b0/internal/platform/platform_versioner.go#L95 + client, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + panic(fmt.Sprintf("Impossible to get new client for config when fetching cluster information: %s", err)) + } + apiList, err := client.ServerGroups() + if err != nil { + panic(fmt.Sprintf("issue occurred while fetching ServerGroups: %s", err)) + } + + for _, v := range apiList.Groups { + if v.Name == "route.openshift.io" { + isOpenShift = true + break + } + } +}