From 572ba961912ada2c07eb6143925d16ab6a6a85a3 Mon Sep 17 00:00:00 2001 From: James Milligan <75740990+james-milligan@users.noreply.github.com> Date: Fri, 24 Feb 2023 14:56:25 +0000 Subject: [PATCH] feat: improve deployment pattern (#344) Signed-off-by: James Milligan Signed-off-by: James Milligan <75740990+james-milligan@users.noreply.github.com> Co-authored-by: Skye Gill --- PROJECT | 9 + .../featureflagconfiguration_types.go | 5 +- .../flagsourceconfiguration_conversion.go | 4 +- .../v1alpha1/flagsourceconfiguration_types.go | 166 ++++-- apis/core/v1alpha1/zz_generated.deepcopy.go | 32 ++ .../featureflagconfiguration_conversion.go | 2 +- .../featureflagconfiguration_types.go | 3 + .../flagsourceconfiguration_conversion.go | 5 +- .../flagsourceconfiguration_conversion.go | 89 ++++ .../v1alpha3/flagsourceconfiguration_types.go | 127 +++++ apis/core/v1alpha3/groupversion_info.go | 36 ++ apis/core/v1alpha3/zz_generated.deepcopy.go | 153 ++++++ chart/open-feature-operator/README.md | 2 + chart/open-feature-operator/values.yaml | 1 + ...feature.dev_featureflagconfigurations.yaml | 8 +- ...nfeature.dev_flagsourceconfigurations.yaml | 348 +++++++++++++ ...tion_in_core_flagsourceconfigurations.yaml | 7 + ...hook_in_core_flagsourceconfigurations.yaml | 16 + config/overlays/helm/manager.yaml | 2 + config/rbac/role.yaml | 12 + ...core_v1alpha1_flagsourceconfiguration.yaml | 12 - ...core_v1alpha2_flagsourceconfiguration.yaml | 12 - config/samples/end-to-end.yaml | 12 +- controllers/controllers_test.go | 104 ++++ .../flagsourceconfiguration_controller.go | 117 ++++- controllers/suite_test.go | 70 ++- docs/README.md | 1 + docs/annotations.md | 28 +- docs/feature_flag_configuration.md | 43 +- docs/flag_source_configuration.md | 108 ++++ docs/flagd_configuration.md | 62 --- docs/getting_started.md | 24 +- go.mod | 1 + go.sum | 3 +- main.go | 32 +- test/e2e/e2e.yml | 12 +- webhooks/pod_webhook.go | 486 +++++++++--------- webhooks/pod_webhook_deprecated.go | 56 ++ webhooks/pod_webhook_test.go | 330 +++++++++--- webhooks/suite_test.go | 21 +- 40 files changed, 2002 insertions(+), 559 deletions(-) create mode 100644 apis/core/v1alpha3/flagsourceconfiguration_conversion.go create mode 100644 apis/core/v1alpha3/flagsourceconfiguration_types.go create mode 100644 apis/core/v1alpha3/groupversion_info.go create mode 100644 apis/core/v1alpha3/zz_generated.deepcopy.go create mode 100644 config/crd/patches/cainjection_in_core_flagsourceconfigurations.yaml create mode 100644 config/crd/patches/webhook_in_core_flagsourceconfigurations.yaml delete mode 100644 config/samples/core_v1alpha1_flagsourceconfiguration.yaml delete mode 100644 config/samples/core_v1alpha2_flagsourceconfiguration.yaml create mode 100644 controllers/controllers_test.go create mode 100644 docs/flag_source_configuration.md delete mode 100644 docs/flagd_configuration.md create mode 100644 webhooks/pod_webhook_deprecated.go diff --git a/PROJECT b/PROJECT index 00591b5e1..aaaa4457a 100644 --- a/PROJECT +++ b/PROJECT @@ -34,4 +34,13 @@ resources: kind: FlagSourceConfiguration path: github.com/open-feature/open-feature-operator/apis/core/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openfeature.dev + group: core + kind: FlagSourceConfiguration + path: github.com/open-feature/open-feature-operator/apis/core/v1alpha3 + version: v1alpha3 version: "3" diff --git a/apis/core/v1alpha1/featureflagconfiguration_types.go b/apis/core/v1alpha1/featureflagconfiguration_types.go index 100432e91..856f99f1f 100644 --- a/apis/core/v1alpha1/featureflagconfiguration_types.go +++ b/apis/core/v1alpha1/featureflagconfiguration_types.go @@ -31,9 +31,12 @@ import ( type FeatureFlagConfigurationSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file + + // ServiceProvider [DEPRECATED]: superseded by FlagSourceConfiguration // +optional // +nullable ServiceProvider *FeatureFlagServiceProvider `json:"serviceProvider"` + // SyncProvider [DEPRECATED]: superseded by FlagSourceConfiguration // +optional // +nullable SyncProvider *FeatureFlagSyncProvider `json:"syncProvider"` @@ -53,7 +56,7 @@ type FlagDSpec struct { } type FeatureFlagSyncProvider struct { - Name SyncProviderType `json:"name"` + Name string `json:"name"` // +optional // +nullable HttpSyncConfiguration *HttpSyncConfiguration `json:"httpSyncConfiguration"` diff --git a/apis/core/v1alpha1/flagsourceconfiguration_conversion.go b/apis/core/v1alpha1/flagsourceconfiguration_conversion.go index 7cf81eca4..5c97e1e54 100644 --- a/apis/core/v1alpha1/flagsourceconfiguration_conversion.go +++ b/apis/core/v1alpha1/flagsourceconfiguration_conversion.go @@ -16,7 +16,9 @@ limitations under the License. package v1alpha1 -import ctrl "sigs.k8s.io/controller-runtime" +import ( + ctrl "sigs.k8s.io/controller-runtime" +) // Hub marks this type as a conversion hub. func (ffc *FlagSourceConfiguration) Hub() {} diff --git a/apis/core/v1alpha1/flagsourceconfiguration_types.go b/apis/core/v1alpha1/flagsourceconfiguration_types.go index a21b32952..3aad44752 100644 --- a/apis/core/v1alpha1/flagsourceconfiguration_types.go +++ b/apis/core/v1alpha1/flagsourceconfiguration_types.go @@ -30,6 +30,7 @@ type SyncProviderType string const ( SidecarEnvVarPrefix string = "SIDECAR_ENV_VAR_PREFIX" + InputConfigurationEnvVarPrefix string = "SIDECAR" SidecarMetricPortEnvVar string = "METRICS_PORT" SidecarPortEnvVar string = "PORT" SidecarSocketPathEnvVar string = "SOCKET_PATH" @@ -40,13 +41,12 @@ const ( SidecarDefaultSyncProviderEnvVar string = "SYNC_PROVIDER" SidecarLogFormatEnvVar string = "LOG_FORMAT" defaultSidecarEnvVarPrefix string = "FLAGD" - InputConfigurationEnvVarPrefix string = "SIDECAR" - defaultMetricPort int32 = 8014 + DefaultMetricPort int32 = 8014 defaultPort int32 = 8013 defaultSocketPath string = "" defaultEvaluator string = "json" defaultImage string = "ghcr.io/open-feature/flagd" - // `INPUT_FLAGD_VERSION` is replaced in the `update-flagd` Makefile target + // INPUT_FLAGD_VERSION` is replaced in the `update-flagd` Makefile target defaultTag string = "INPUT_FLAGD_VERSION" defaultLogFormat string = "json" SyncProviderKubernetes SyncProviderType = "kubernetes" @@ -95,25 +95,87 @@ type FlagSourceConfigurationSpec struct { // +optional DefaultSyncProvider SyncProviderType `json:"defaultSyncProvider"` + // Sources defines the syncProviders and associated configuration to be applied to the sidecar + // +kubebuilder:validation:MinItems=1 + Sources []Source `json:"sources"` + + // EnvVars define the env vars to be applied to the sidecar, any env vars in FeatureFlagConfiguration CRs + // are added at the lowest index, all values will have the EnvVarPrefix applied + // +optional + EnvVars []corev1.EnvVar `json:"envVars"` + + // EnvVarPrefix defines the prefix to be applied to all environment variables applied to the sidecar, default FLAGD + // +optional + EnvVarPrefix string `json:"envVarPrefix"` + + // LogFormat allows for the sidecar log format to be overridden, defaults to 'json' + // +optional + LogFormat string `json:"logFormat"` + + // RolloutOnChange dictates whether annotated deployments will be restarted when configuration changes are + // detected in this CR, defaults to false + // +optional + RolloutOnChange *bool `json:"rolloutOnChange"` +} + +type Source struct { + Source string `json:"source"` + // +optional + Provider SyncProviderType `json:"provider"` + // +optional + HttpSyncBearerToken string `json:"httpSyncBearerToken"` // LogFormat allows for the sidecar log format to be overridden, defaults to 'json' // +optional LogFormat string `json:"logFormat"` } +// FlagSourceConfigurationStatus defines the observed state of FlagSourceConfiguration +type FlagSourceConfigurationStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:resource:shortName="fsc" +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:storageversion + +// FlagSourceConfiguration is the Schema for the FlagSourceConfigurations API +type FlagSourceConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FlagSourceConfigurationSpec `json:"spec,omitempty"` + Status FlagSourceConfigurationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// FlagSourceConfigurationList contains a list of FlagSourceConfiguration +type FlagSourceConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FlagSourceConfiguration `json:"items"` +} + func NewFlagSourceConfigurationSpec() (*FlagSourceConfigurationSpec, error) { fsc := &FlagSourceConfigurationSpec{ - MetricsPort: defaultMetricPort, + MetricsPort: DefaultMetricPort, Port: defaultPort, SocketPath: defaultSocketPath, - SyncProviderArgs: []string{}, Evaluator: defaultEvaluator, Image: defaultImage, Tag: defaultTag, + Sources: []Source{}, + EnvVars: []corev1.EnvVar{}, + SyncProviderArgs: []string{}, DefaultSyncProvider: SyncProviderKubernetes, + EnvVarPrefix: defaultSidecarEnvVarPrefix, LogFormat: defaultLogFormat, + RolloutOnChange: nil, } - if metricsPort := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarMetricPortEnvVar)); metricsPort != "" { + if metricsPort := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarMetricPortEnvVar)); metricsPort != "" { metricsPortI, err := strconv.Atoi(metricsPort) if err != nil { return fsc, fmt.Errorf("unable to parse metrics port value %s to int32: %w", metricsPort, err) @@ -121,7 +183,7 @@ func NewFlagSourceConfigurationSpec() (*FlagSourceConfigurationSpec, error) { fsc.MetricsPort = int32(metricsPortI) } - if port := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarPortEnvVar)); port != "" { + if port := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarPortEnvVar)); port != "" { portI, err := strconv.Atoi(port) if err != nil { return fsc, fmt.Errorf("unable to parse sidecar port value %s to int32: %w", port, err) @@ -129,27 +191,27 @@ func NewFlagSourceConfigurationSpec() (*FlagSourceConfigurationSpec, error) { fsc.Port = int32(portI) } - if socketPath := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarSocketPathEnvVar)); socketPath != "" { + if socketPath := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarSocketPathEnvVar)); socketPath != "" { fsc.SocketPath = socketPath } - if evaluator := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarEvaluatorEnvVar)); evaluator != "" { + if evaluator := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarEvaluatorEnvVar)); evaluator != "" { fsc.Evaluator = evaluator } - if image := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarImageEnvVar)); image != "" { + if image := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarImageEnvVar)); image != "" { fsc.Image = image } - if tag := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarVersionEnvVar)); tag != "" { + if tag := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarVersionEnvVar)); tag != "" { fsc.Tag = tag } - if syncProviderArgs := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarProviderArgsEnvVar)); syncProviderArgs != "" { + if syncProviderArgs := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarProviderArgsEnvVar)); syncProviderArgs != "" { fsc.SyncProviderArgs = strings.Split(syncProviderArgs, ",") // todo: add documentation for this } - if syncProvider := os.Getenv(fmt.Sprintf("%s_%s", InputConfigurationEnvVarPrefix, SidecarDefaultSyncProviderEnvVar)); syncProvider != "" { + if syncProvider := os.Getenv(envVarKey(InputConfigurationEnvVarPrefix, SidecarDefaultSyncProviderEnvVar)); syncProvider != "" { fsc.DefaultSyncProvider = SyncProviderType(syncProvider) } @@ -157,6 +219,10 @@ func NewFlagSourceConfigurationSpec() (*FlagSourceConfigurationSpec, error) { fsc.LogFormat = logFormat } + if envVarPrefix := os.Getenv(SidecarEnvVarPrefix); envVarPrefix != "" { + fsc.EnvVarPrefix = envVarPrefix + } + return fsc, nil } @@ -182,93 +248,75 @@ func (fc *FlagSourceConfigurationSpec) Merge(new *FlagSourceConfigurationSpec) { if new.Tag != "" { fc.Tag = new.Tag } + if len(new.Sources) != 0 { + fc.Sources = append(fc.Sources, new.Sources...) + } + if len(new.EnvVars) != 0 { + fc.EnvVars = append(fc.EnvVars, new.EnvVars...) + } if new.SyncProviderArgs != nil && len(new.SyncProviderArgs) > 0 { fc.SyncProviderArgs = append(fc.SyncProviderArgs, new.SyncProviderArgs...) } + if new.EnvVarPrefix != "" { + fc.EnvVarPrefix = new.EnvVarPrefix + } if new.DefaultSyncProvider != "" { fc.DefaultSyncProvider = new.DefaultSyncProvider } if new.LogFormat != "" { fc.LogFormat = new.LogFormat } + if new.RolloutOnChange != nil { + fc.RolloutOnChange = new.RolloutOnChange + } } func (fc *FlagSourceConfigurationSpec) ToEnvVars() []corev1.EnvVar { envs := []corev1.EnvVar{} - prefix := defaultSidecarEnvVarPrefix - if p := os.Getenv(SidecarEnvVarPrefix); p != "" { - prefix = p + for _, envVar := range fc.EnvVars { + envs = append(envs, corev1.EnvVar{ + Name: envVarKey(fc.EnvVarPrefix, envVar.Name), + Value: envVar.Value, + }) } - if fc.MetricsPort != defaultMetricPort { + if fc.MetricsPort != DefaultMetricPort { envs = append(envs, corev1.EnvVar{ - Name: fmt.Sprintf("%s_%s", prefix, SidecarMetricPortEnvVar), + Name: envVarKey(fc.EnvVarPrefix, SidecarMetricPortEnvVar), Value: fmt.Sprintf("%d", fc.MetricsPort), }) } if fc.Port != defaultPort { envs = append(envs, corev1.EnvVar{ - Name: fmt.Sprintf("%s_%s", prefix, SidecarPortEnvVar), + Name: envVarKey(fc.EnvVarPrefix, SidecarPortEnvVar), Value: fmt.Sprintf("%d", fc.Port), }) } if fc.Evaluator != defaultEvaluator { envs = append(envs, corev1.EnvVar{ - Name: fmt.Sprintf("%s_%s", prefix, SidecarEvaluatorEnvVar), + Name: envVarKey(fc.EnvVarPrefix, SidecarEvaluatorEnvVar), Value: fc.Evaluator, }) } if fc.SocketPath != defaultSocketPath { envs = append(envs, corev1.EnvVar{ - Name: fmt.Sprintf("%s_%s", prefix, SidecarSocketPathEnvVar), + Name: envVarKey(fc.EnvVarPrefix, SidecarSocketPathEnvVar), Value: fc.SocketPath, }) } if fc.LogFormat != defaultLogFormat { envs = append(envs, corev1.EnvVar{ - Name: fmt.Sprintf("%s_%s", prefix, SidecarLogFormatEnvVar), + Name: envVarKey(fc.EnvVarPrefix, SidecarLogFormatEnvVar), Value: fc.LogFormat, }) } - return envs -} - -// FlagSourceConfigurationStatus defines the observed state of FlagSourceConfiguration -type FlagSourceConfigurationStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file -} - -//+kubebuilder:resource:shortName="fsc" -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status -//+kubebuilder:storageversion -// FlagSourceConfiguration is the Schema for the FlagSourceConfigurations API -type FlagSourceConfiguration struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec FlagSourceConfigurationSpec `json:"spec,omitempty"` - Status FlagSourceConfigurationStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// FlagSourceConfigurationList contains a list of FlagSourceConfiguration -type FlagSourceConfigurationList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []FlagSourceConfiguration `json:"items"` -} - -func init() { - SchemeBuilder.Register(&FlagSourceConfiguration{}, &FlagSourceConfigurationList{}) + return envs } func (s SyncProviderType) IsKubernetes() bool { @@ -282,3 +330,11 @@ func (s SyncProviderType) IsHttp() bool { func (s SyncProviderType) IsFilepath() bool { return s == SyncProviderFilepath } + +func envVarKey(prefix string, suffix string) string { + return fmt.Sprintf("%s_%s", prefix, suffix) +} + +func init() { + SchemeBuilder.Register(&FlagSourceConfiguration{}, &FlagSourceConfigurationList{}) +} diff --git a/apis/core/v1alpha1/zz_generated.deepcopy.go b/apis/core/v1alpha1/zz_generated.deepcopy.go index 0cc761578..5214a55ca 100644 --- a/apis/core/v1alpha1/zz_generated.deepcopy.go +++ b/apis/core/v1alpha1/zz_generated.deepcopy.go @@ -259,6 +259,23 @@ func (in *FlagSourceConfigurationSpec) DeepCopyInto(out *FlagSourceConfiguration *out = make([]string, len(*in)) copy(*out, *in) } + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]Source, len(*in)) + copy(*out, *in) + } + if in.EnvVars != nil { + in, out := &in.EnvVars, &out.EnvVars + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RolloutOnChange != nil { + in, out := &in.RolloutOnChange, &out.RolloutOnChange + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlagSourceConfigurationSpec. @@ -300,3 +317,18 @@ func (in *HttpSyncConfiguration) DeepCopy() *HttpSyncConfiguration { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Source) DeepCopyInto(out *Source) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Source. +func (in *Source) DeepCopy() *Source { + if in == nil { + return nil + } + out := new(Source) + in.DeepCopyInto(out) + return out +} diff --git a/apis/core/v1alpha2/featureflagconfiguration_conversion.go b/apis/core/v1alpha2/featureflagconfiguration_conversion.go index 3e5701f2c..aebd98408 100644 --- a/apis/core/v1alpha2/featureflagconfiguration_conversion.go +++ b/apis/core/v1alpha2/featureflagconfiguration_conversion.go @@ -43,7 +43,7 @@ func (src *FeatureFlagConfiguration) ConvertTo(dstRaw conversion.Hub) error { } if src.Spec.SyncProvider != nil { - dst.Spec.SyncProvider = &v1alpha1.FeatureFlagSyncProvider{Name: v1alpha1.SyncProviderType(src.Spec.SyncProvider.Name)} + dst.Spec.SyncProvider = &v1alpha1.FeatureFlagSyncProvider{Name: src.Spec.SyncProvider.Name} if src.Spec.SyncProvider.HttpSyncConfiguration != nil { dst.Spec.SyncProvider.HttpSyncConfiguration = &v1alpha1.HttpSyncConfiguration{ Target: src.Spec.SyncProvider.HttpSyncConfiguration.Target, diff --git a/apis/core/v1alpha2/featureflagconfiguration_types.go b/apis/core/v1alpha2/featureflagconfiguration_types.go index eceb7785c..6039714dc 100644 --- a/apis/core/v1alpha2/featureflagconfiguration_types.go +++ b/apis/core/v1alpha2/featureflagconfiguration_types.go @@ -31,9 +31,12 @@ import ( type FeatureFlagConfigurationSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file + + // ServiceProvider [DEPRECATED]: superseded by FlagSourceConfiguration // +optional // +nullable ServiceProvider *FeatureFlagServiceProvider `json:"serviceProvider"` + // SyncProvider [DEPRECATED]: superseded by FlagSourceConfiguration // +optional // +nullable SyncProvider *FeatureFlagSyncProvider `json:"syncProvider"` diff --git a/apis/core/v1alpha2/flagsourceconfiguration_conversion.go b/apis/core/v1alpha2/flagsourceconfiguration_conversion.go index 8091a33a9..f9d6d8380 100644 --- a/apis/core/v1alpha2/flagsourceconfiguration_conversion.go +++ b/apis/core/v1alpha2/flagsourceconfiguration_conversion.go @@ -36,10 +36,11 @@ func (src *FlagSourceConfiguration) ConvertTo(dstRaw conversion.Hub) error { MetricsPort: src.Spec.MetricsPort, Port: src.Spec.Port, SocketPath: src.Spec.SocketPath, - SyncProviderArgs: src.Spec.SyncProviderArgs, Evaluator: src.Spec.Evaluator, Image: src.Spec.Image, Tag: src.Spec.Tag, + Sources: []v1alpha1.Source{}, + SyncProviderArgs: src.Spec.SyncProviderArgs, LogFormat: src.Spec.LogFormat, DefaultSyncProvider: v1alpha1.SyncProviderType(src.Spec.DefaultSyncProvider), } @@ -54,10 +55,10 @@ func (dst *FlagSourceConfiguration) ConvertFrom(srcRaw conversion.Hub) error { MetricsPort: src.Spec.MetricsPort, Port: src.Spec.Port, SocketPath: src.Spec.SocketPath, - SyncProviderArgs: src.Spec.SyncProviderArgs, Evaluator: src.Spec.Evaluator, Image: src.Spec.Image, Tag: src.Spec.Tag, + SyncProviderArgs: src.Spec.SyncProviderArgs, LogFormat: src.Spec.LogFormat, DefaultSyncProvider: string(src.Spec.DefaultSyncProvider), } diff --git a/apis/core/v1alpha3/flagsourceconfiguration_conversion.go b/apis/core/v1alpha3/flagsourceconfiguration_conversion.go new file mode 100644 index 000000000..9532e8bcd --- /dev/null +++ b/apis/core/v1alpha3/flagsourceconfiguration_conversion.go @@ -0,0 +1,89 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/conversion" +) + +func (ffc *FlagSourceConfiguration) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(ffc). + Complete() +} + +func (src *FlagSourceConfiguration) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1alpha1.FlagSourceConfiguration) + + sources := []v1alpha1.Source{} + for _, sp := range src.Spec.Sources { + sources = append(sources, v1alpha1.Source{ + Source: sp.Source, + HttpSyncBearerToken: sp.HttpSyncBearerToken, + Provider: v1alpha1.SyncProviderType(sp.Provider), + }) + } + + dst.ObjectMeta = src.ObjectMeta + dst.Spec = v1alpha1.FlagSourceConfigurationSpec{ + MetricsPort: src.Spec.MetricsPort, + Port: src.Spec.Port, + SocketPath: src.Spec.SocketPath, + Evaluator: src.Spec.Evaluator, + Image: src.Spec.Image, + Tag: src.Spec.Tag, + Sources: sources, + EnvVars: src.Spec.EnvVars, + DefaultSyncProvider: v1alpha1.SyncProviderType(src.Spec.DefaultSyncProvider), + LogFormat: src.Spec.LogFormat, + EnvVarPrefix: src.Spec.EnvVarPrefix, + RolloutOnChange: src.Spec.RolloutOnChange, + } + return nil +} + +func (dst *FlagSourceConfiguration) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1alpha1.FlagSourceConfiguration) + + sources := []Source{} + for _, sp := range src.Spec.Sources { + sources = append(sources, Source{ + Source: sp.Source, + Provider: string(sp.Provider), + HttpSyncBearerToken: sp.HttpSyncBearerToken, + }) + } + + dst.ObjectMeta = src.ObjectMeta + dst.Spec = FlagSourceConfigurationSpec{ + MetricsPort: src.Spec.MetricsPort, + Port: src.Spec.Port, + SocketPath: src.Spec.SocketPath, + Evaluator: src.Spec.Evaluator, + Image: src.Spec.Image, + Tag: src.Spec.Tag, + Sources: sources, + EnvVars: src.Spec.EnvVars, + DefaultSyncProvider: string(src.Spec.DefaultSyncProvider), + LogFormat: src.Spec.LogFormat, + EnvVarPrefix: src.Spec.EnvVarPrefix, + RolloutOnChange: src.Spec.RolloutOnChange, + } + return nil +} diff --git a/apis/core/v1alpha3/flagsourceconfiguration_types.go b/apis/core/v1alpha3/flagsourceconfiguration_types.go new file mode 100644 index 000000000..8e777e0f9 --- /dev/null +++ b/apis/core/v1alpha3/flagsourceconfiguration_types.go @@ -0,0 +1,127 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type SyncProviderType string + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// FlagSourceConfigurationSpec defines the desired state of FlagSourceConfiguration +type FlagSourceConfigurationSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // MetricsPort defines the port to serve metrics on, defaults to 8014 + // +optional + MetricsPort int32 `json:"metricsPort"` + + // Port defines the port to listen on, defaults to 8013 + // +optional + Port int32 `json:"port"` + + // SocketPath defines the unix socket path to listen on + // +optional + SocketPath string `json:"socketPath"` + + // Evaluator sets an evaluator, defaults to 'json' + // +optional + Evaluator string `json:"evaluator"` + + // Image allows for the sidecar image to be overridden, defaults to 'ghcr.io/open-feature/flagd' + // +optional + Image string `json:"image"` + + // Tag to be appended to the sidecar image, defaults to 'main' + // +optional + Tag string `json:"tag"` + + // SyncProviders define the syncProviders and associated configuration to be applied to the sidecar + // +kubebuilder:validation:MinItems=1 + Sources []Source `json:"sources"` + + // EnvVars define the env vars to be applied to the sidecar, any env vars in FeatureFlagConfiguration CRs + // are added at the lowest index, all values will have the EnvVarPrefix applied, default FLAGD + // +optional + EnvVars []corev1.EnvVar `json:"envVars"` + + // SyncProviderArgs are string arguments passed to all sync providers, defined as key values separated by = + // +optional + SyncProviderArgs []string `json:"syncProviderArgs"` + + // DefaultSyncProvider defines the default sync provider + // +optional + DefaultSyncProvider string `json:"defaultSyncProvider"` + + // LogFormat allows for the sidecar log format to be overridden, defaults to 'json' + // +optional + LogFormat string `json:"logFormat"` + + // EnvVarPrefix defines the prefix to be applied to all environment variables applied to the sidecar, default FLAGD + // +optional + EnvVarPrefix string `json:"envVarPrefix"` + + // RolloutOnChange dictates whether annotated deployments will be restarted when configuration changes are + // detected in this CR, defaults to false + // +optional + RolloutOnChange *bool `json:"rolloutOnChange"` +} + +type Source struct { + Source string `json:"source"` + // +optional + Provider string `json:"provider"` + // +optional + HttpSyncBearerToken string `json:"httpSyncBearerToken"` +} + +// FlagSourceConfigurationStatus defines the observed state of FlagSourceConfiguration +type FlagSourceConfigurationStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:resource:shortName="fsc" +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// FlagSourceConfiguration is the Schema for the FlagSourceConfigurations API +type FlagSourceConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FlagSourceConfigurationSpec `json:"spec,omitempty"` + Status FlagSourceConfigurationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// FlagSourceConfigurationList contains a list of FlagSourceConfiguration +type FlagSourceConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FlagSourceConfiguration `json:"items"` +} + +func init() { + SchemeBuilder.Register(&FlagSourceConfiguration{}, &FlagSourceConfigurationList{}) +} diff --git a/apis/core/v1alpha3/groupversion_info.go b/apis/core/v1alpha3/groupversion_info.go new file mode 100644 index 000000000..c151ed719 --- /dev/null +++ b/apis/core/v1alpha3/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha3 contains API Schema definitions for the core v1alpha3 API group +// +kubebuilder:object:generate=true +// +groupName=core.openfeature.dev +package v1alpha3 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "core.openfeature.dev", Version: "v1alpha3"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/core/v1alpha3/zz_generated.deepcopy.go b/apis/core/v1alpha3/zz_generated.deepcopy.go new file mode 100644 index 000000000..11e2fc2d5 --- /dev/null +++ b/apis/core/v1alpha3/zz_generated.deepcopy.go @@ -0,0 +1,153 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha3 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlagSourceConfiguration) DeepCopyInto(out *FlagSourceConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlagSourceConfiguration. +func (in *FlagSourceConfiguration) DeepCopy() *FlagSourceConfiguration { + if in == nil { + return nil + } + out := new(FlagSourceConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FlagSourceConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlagSourceConfigurationList) DeepCopyInto(out *FlagSourceConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FlagSourceConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlagSourceConfigurationList. +func (in *FlagSourceConfigurationList) DeepCopy() *FlagSourceConfigurationList { + if in == nil { + return nil + } + out := new(FlagSourceConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FlagSourceConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlagSourceConfigurationSpec) DeepCopyInto(out *FlagSourceConfigurationSpec) { + *out = *in + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]Source, len(*in)) + copy(*out, *in) + } + if in.EnvVars != nil { + in, out := &in.EnvVars, &out.EnvVars + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SyncProviderArgs != nil { + in, out := &in.SyncProviderArgs, &out.SyncProviderArgs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.RolloutOnChange != nil { + in, out := &in.RolloutOnChange, &out.RolloutOnChange + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlagSourceConfigurationSpec. +func (in *FlagSourceConfigurationSpec) DeepCopy() *FlagSourceConfigurationSpec { + if in == nil { + return nil + } + out := new(FlagSourceConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FlagSourceConfigurationStatus) DeepCopyInto(out *FlagSourceConfigurationStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlagSourceConfigurationStatus. +func (in *FlagSourceConfigurationStatus) DeepCopy() *FlagSourceConfigurationStatus { + if in == nil { + return nil + } + out := new(FlagSourceConfigurationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Source) DeepCopyInto(out *Source) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Source. +func (in *Source) DeepCopy() *Source { + if in == nil { + return nil + } + out := new(Source) + in.DeepCopyInto(out) + return out +} diff --git a/chart/open-feature-operator/README.md b/chart/open-feature-operator/README.md index ea284a4eb..396145d25 100644 --- a/chart/open-feature-operator/README.md +++ b/chart/open-feature-operator/README.md @@ -43,6 +43,7 @@ The command removes all the Kubernetes components associated with the chart and ### Sidecar configuration + | Value | Default | Explanation | | ----------- | ----------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `sidecarConfiguration.envVarPrefix` | `FLAGD` | Sets the prefix for all environment variables set in the injected sidecar. | @@ -54,6 +55,7 @@ The command removes all the Kubernetes components associated with the chart and | `sidecarConfiguration.providerArgs` | `""` | Used to append arguments to the sidecar startup command. This value is a comma separated string of key values separated by '=', e.g. `key=value,key2=value2` results in the appending of `--sync-provider-args key=value --sync-provider-args key2=value2` | | `sidecarConfiguration.defaultSyncProvider` | `kubernetes` | Sets the value of the `XXX_SYNC_PROVIDER` environment variable for the injected sidecar container. There are 3 valid sync providers: `kubernetes`, `filepath` and `http` | | `sidecarConfiguration.logFormat` | `json` | Sets the value of the `XXX_LOG_FORMAT` environment variable for the injected sidecar container. | +| `sidecarConfiguration.evaluator` | `json` | Sets the value of the `XXX_EVALUATOR` environment variable for the injected sidecar container.| ### Operator resource configuration diff --git a/chart/open-feature-operator/values.yaml b/chart/open-feature-operator/values.yaml index 3a558e160..cb6b1cc52 100644 --- a/chart/open-feature-operator/values.yaml +++ b/chart/open-feature-operator/values.yaml @@ -12,6 +12,7 @@ sidecarConfiguration: providerArgs: "" envVarPrefix: "FLAGD" defaultSyncProvider: kubernetes + evaluator: json logFormat: "json" controllerManager: diff --git a/config/crd/bases/core.openfeature.dev_featureflagconfigurations.yaml b/config/crd/bases/core.openfeature.dev_featureflagconfigurations.yaml index 5066bad4a..dfb0ed59d 100644 --- a/config/crd/bases/core.openfeature.dev_featureflagconfigurations.yaml +++ b/config/crd/bases/core.openfeature.dev_featureflagconfigurations.yaml @@ -165,8 +165,7 @@ spec: type: integer type: object serviceProvider: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - Important: Run "make" to regenerate code after modifying this file' + description: 'ServiceProvider [DEPRECATED]: superseded by FlagSourceConfiguration' nullable: true properties: credentials: @@ -239,6 +238,7 @@ spec: - name type: object syncProvider: + description: 'SyncProvider [DEPRECATED]: superseded by FlagSourceConfiguration' nullable: true properties: httpSyncConfiguration: @@ -443,8 +443,7 @@ spec: type: array type: object serviceProvider: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - Important: Run "make" to regenerate code after modifying this file' + description: 'ServiceProvider [DEPRECATED]: superseded by FlagSourceConfiguration' nullable: true properties: credentials: @@ -517,6 +516,7 @@ spec: - name type: object syncProvider: + description: 'SyncProvider [DEPRECATED]: superseded by FlagSourceConfiguration' nullable: true properties: httpSyncConfiguration: diff --git a/config/crd/bases/core.openfeature.dev_flagsourceconfigurations.yaml b/config/crd/bases/core.openfeature.dev_flagsourceconfigurations.yaml index 0a718c2e3..faeecb751 100644 --- a/config/crd/bases/core.openfeature.dev_flagsourceconfigurations.yaml +++ b/config/crd/bases/core.openfeature.dev_flagsourceconfigurations.yaml @@ -42,6 +42,121 @@ spec: defaultSyncProvider: description: DefaultSyncProvider defines the default sync provider type: string + envVarPrefix: + description: EnvVarPrefix defines the prefix to be applied to all + environment variables applied to the sidecar, default FLAGD + type: string + envVars: + description: EnvVars define the env vars to be applied to the sidecar, + any env vars in FeatureFlagConfiguration CRs are added at the lowest + index, all values will have the EnvVarPrefix applied + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded using + the previously defined environment variables in the container + and any service environment variables. If a variable cannot + be resolved, the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the + string literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists or + not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array evaluator: description: Evaluator sets an evaluator, defaults to 'json' type: string @@ -62,9 +177,34 @@ spec: description: Port defines the port to listen on, defaults to 8013 format: int32 type: integer + rolloutOnChange: + description: RolloutOnChange dictates whether annotated deployments + will be restarted when configuration changes are detected in this + CR, defaults to false + type: boolean socketPath: description: SocketPath defines the unix socket path to listen on type: string + sources: + description: Sources defines the syncProviders and associated configuration + to be applied to the sidecar + items: + properties: + httpSyncBearerToken: + type: string + logFormat: + description: LogFormat allows for the sidecar log format to + be overridden, defaults to 'json' + type: string + provider: + type: string + source: + type: string + required: + - source + type: object + minItems: 1 + type: array syncProviderArgs: description: SyncProviderArgs are string arguments passed to all sync providers, defined as key values separated by = @@ -75,6 +215,8 @@ spec: description: Tag to be appended to the sidecar image, defaults to 'main' type: string + required: + - sources type: object status: description: FlagSourceConfigurationStatus defines the observed state @@ -153,3 +295,209 @@ spec: storage: false subresources: status: {} + - name: v1alpha3 + schema: + openAPIV3Schema: + description: FlagSourceConfiguration is the Schema for the FlagSourceConfigurations + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FlagSourceConfigurationSpec defines the desired state of + FlagSourceConfiguration + properties: + defaultSyncProvider: + description: DefaultSyncProvider defines the default sync provider + type: string + envVarPrefix: + description: EnvVarPrefix defines the prefix to be applied to all + environment variables applied to the sidecar, default FLAGD + type: string + envVars: + description: EnvVars define the env vars to be applied to the sidecar, + any env vars in FeatureFlagConfiguration CRs are added at the lowest + index, all values will have the EnvVarPrefix applied, default FLAGD + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded using + the previously defined environment variables in the container + and any service environment variables. If a variable cannot + be resolved, the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the + string literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists or + not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + evaluator: + description: Evaluator sets an evaluator, defaults to 'json' + type: string + image: + description: Image allows for the sidecar image to be overridden, + defaults to 'ghcr.io/open-feature/flagd' + type: string + logFormat: + description: LogFormat allows for the sidecar log format to be overridden, + defaults to 'json' + type: string + metricsPort: + description: MetricsPort defines the port to serve metrics on, defaults + to 8014 + format: int32 + type: integer + port: + description: Port defines the port to listen on, defaults to 8013 + format: int32 + type: integer + rolloutOnChange: + description: RolloutOnChange dictates whether annotated deployments + will be restarted when configuration changes are detected in this + CR, defaults to false + type: boolean + socketPath: + description: SocketPath defines the unix socket path to listen on + type: string + sources: + description: SyncProviders define the syncProviders and associated + configuration to be applied to the sidecar + items: + properties: + httpSyncBearerToken: + type: string + provider: + type: string + source: + type: string + required: + - source + type: object + minItems: 1 + type: array + syncProviderArgs: + description: SyncProviderArgs are string arguments passed to all sync + providers, defined as key values separated by = + items: + type: string + type: array + tag: + description: Tag to be appended to the sidecar image, defaults to + 'main' + type: string + required: + - sources + type: object + status: + description: FlagSourceConfigurationStatus defines the observed state + of FlagSourceConfiguration + type: object + type: object + served: true + storage: false + subresources: + status: {} diff --git a/config/crd/patches/cainjection_in_core_flagsourceconfigurations.yaml b/config/crd/patches/cainjection_in_core_flagsourceconfigurations.yaml new file mode 100644 index 000000000..11e3fce17 --- /dev/null +++ b/config/crd/patches/cainjection_in_core_flagsourceconfigurations.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: flagsourceconfigurations.core.openfeature.dev diff --git a/config/crd/patches/webhook_in_core_flagsourceconfigurations.yaml b/config/crd/patches/webhook_in_core_flagsourceconfigurations.yaml new file mode 100644 index 000000000..2ae67ba6a --- /dev/null +++ b/config/crd/patches/webhook_in_core_flagsourceconfigurations.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: flagsourceconfigurations.core.openfeature.dev +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/overlays/helm/manager.yaml b/config/overlays/helm/manager.yaml index 2c94565f4..1a37f9460 100644 --- a/config/overlays/helm/manager.yaml +++ b/config/overlays/helm/manager.yaml @@ -34,6 +34,8 @@ spec: value: "{{ .Values.sidecarConfiguration.envVarPrefix }}" - name: SIDECAR_SYNC_PROVIDER value: "{{ .Values.sidecarConfiguration.defaultSyncProvider }}" + - name: SIDECAR_EVALUATOR + value: "{{ .Values.sidecarConfiguration.evaluator }}" - name: SIDECAR_LOG_FORMAT value: "{{ .Values.sidecarConfiguration.logFormat }}" - name: kube-rbac-proxy diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8ee737bf3..f03776569 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -37,6 +37,18 @@ rules: - get - list - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - core.openfeature.dev resources: diff --git a/config/samples/core_v1alpha1_flagsourceconfiguration.yaml b/config/samples/core_v1alpha1_flagsourceconfiguration.yaml deleted file mode 100644 index 20fd71c53..000000000 --- a/config/samples/core_v1alpha1_flagsourceconfiguration.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: core.openfeature.dev/v1alpha1 -kind: FlagSourceConfiguration -metadata: - labels: - app.kubernetes.io/name: flagsourceconfiguration - app.kubernetes.io/instance: flagsourceconfiguration-sample - app.kubernetes.io/part-of: open-feature-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: open-feature-operator - name: flagsourceconfiguration-sample -spec: - # TODO(user): Add fields here diff --git a/config/samples/core_v1alpha2_flagsourceconfiguration.yaml b/config/samples/core_v1alpha2_flagsourceconfiguration.yaml deleted file mode 100644 index a44081893..000000000 --- a/config/samples/core_v1alpha2_flagsourceconfiguration.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: core.openfeature.dev/v1alpha2 -kind: FlagSourceConfiguration -metadata: - labels: - app.kubernetes.io/name: flagsourceconfiguration - app.kubernetes.io/instance: flagsourceconfiguration-sample - app.kubernetes.io/part-of: open-feature-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: open-feature-operator - name: flagsourceconfiguration-sample -spec: - # TODO(user): Add fields here diff --git a/config/samples/end-to-end.yaml b/config/samples/end-to-end.yaml index 093bd8bfc..4403a55a9 100644 --- a/config/samples/end-to-end.yaml +++ b/config/samples/end-to-end.yaml @@ -44,6 +44,16 @@ spec: ] } --- +apiVersion: core.openfeature.dev/v1alpha2 +kind: FlagSourceConfiguration +metadata: + name: end-to-end + namespace: open-feature-demo +spec: + sources: + - source: open-feature-demo/end-to-end + provider: kubernetes +--- # Deployment of a demo-app using our custom resource apiVersion: apps/v1 kind: Deployment @@ -63,7 +73,7 @@ spec: app: open-feature-demo annotations: openfeature.dev/enabled: "true" - openfeature.dev/featureflagconfiguration: "end-to-end" + openfeature.dev/flagsourceconfiguration: "end-to-end" spec: serviceAccountName: open-feature-demo-sa containers: diff --git a/controllers/controllers_test.go b/controllers/controllers_test.go new file mode 100644 index 000000000..862b66cb9 --- /dev/null +++ b/controllers/controllers_test.go @@ -0,0 +1,104 @@ +package controllers + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + testNamespace = "test-namespace" + fsConfigName = "test-config" + deploymentName = "test-deploy" +) + +func createTestDeployment() { + deploy := &appsv1.Deployment{} + deploy.Name = "test-deploy" + deploy.Namespace = testNamespace + deploy.Spec.Template.ObjectMeta.Annotations = map[string]string{} + deploy.Spec.Template.ObjectMeta.Annotations[fmt.Sprintf("%s/%s", OpenFeatureAnnotationRoot, "enabled")] = "true" + deploy.Spec.Template.ObjectMeta.Annotations[fmt.Sprintf("%s/%s", OpenFeatureAnnotationRoot, FlagSourceConfigurationAnnotation)] = fmt.Sprintf("%s/%s", testNamespace, fsConfigName) + deploy.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + } + deploy.Spec.Template.ObjectMeta.Labels = map[string]string{ + "app": "test", + } + deploy.Spec.Template.Spec.Containers = []corev1.Container{ + { + Name: "test", + Image: "busybox", + Args: []string{ + "sleep", + "1000", + }, + }, + } + err := k8sClient.Create(testCtx, deploy) + Expect(err).ShouldNot(HaveOccurred()) +} + +func createTestFSConfig() *v1alpha1.FlagSourceConfiguration { + fsConfig := &v1alpha1.FlagSourceConfiguration{} + rolloutOnChange := true + fsConfig.Namespace = testNamespace + fsConfig.Name = fsConfigName + fsConfig.Spec.Image = deploymentName + fsConfig.Spec.Sources = []v1alpha1.Source{ + { + Source: "not-real.com", + Provider: "http", + }, + } + fsConfig.Spec.RolloutOnChange = &rolloutOnChange + err := k8sClient.Create(testCtx, fsConfig) + Expect(err).ShouldNot(HaveOccurred()) + return fsConfig +} + +func setup() { + ns := &corev1.Namespace{} + ns.Name = testNamespace + err := k8sClient.Create(testCtx, ns) + Expect(err).ShouldNot(HaveOccurred()) +} + +var _ = Describe("flagsourceconfiguration controller tests", func() { + + It("should restart annotated deployments", func() { + config := createTestFSConfig() + createTestDeployment() + + // get deployment + set var equal to restartedAt annotation value + deployment := &appsv1.Deployment{} + err := k8sClient.Get(testCtx, client.ObjectKey{Name: deploymentName, Namespace: testNamespace}, deployment) + Expect(err).ShouldNot(HaveOccurred()) + restartAt := deployment.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] + + // update the fsconfig + config.Spec.Image = "image-2" + err = k8sClient.Update(testCtx, config) + Expect(err).ShouldNot(HaveOccurred()) + + // fetch deployment and test if it has been updated + maxRetries := 5 + notRestartedError := fmt.Errorf("deployment has not been restarted after %d seconds", maxRetries) + for i := 0; i < maxRetries; i++ { + err = k8sClient.Get(testCtx, client.ObjectKey{Name: deploymentName, Namespace: testNamespace}, deployment) + Expect(err).ShouldNot(HaveOccurred()) + if deployment.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] != restartAt { + notRestartedError = nil + } + } + Expect(notRestartedError).ShouldNot(HaveOccurred()) + }) +}) diff --git a/controllers/flagsourceconfiguration_controller.go b/controllers/flagsourceconfiguration_controller.go index 62c854546..d0b691afe 100644 --- a/controllers/flagsourceconfiguration_controller.go +++ b/controllers/flagsourceconfiguration_controller.go @@ -18,45 +18,146 @@ package controllers import ( "context" + "fmt" + "strings" + "time" + appsV1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "github.com/go-logr/logr" corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" ) +const ( + OpenFeatureAnnotationPath = "spec.template.metadata.annotations.openfeature.dev/openfeature.dev" + FlagSourceConfigurationAnnotation = "flagsourceconfiguration" + OpenFeatureAnnotationRoot = "openfeature.dev" +) + // FlagSourceConfigurationReconciler reconciles a FlagSourceConfiguration object type FlagSourceConfigurationReconciler struct { client.Client Scheme *runtime.Scheme + // ReqLogger contains the Logger of this controller + Log logr.Logger } //+kubebuilder:rbac:groups=core.openfeature.dev,resources=flagsourceconfigurations,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core.openfeature.dev,resources=flagsourceconfigurations/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core.openfeature.dev,resources=flagsourceconfigurations/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the FlagSourceConfiguration object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// + // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile func (r *FlagSourceConfigurationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + r.Log = log.FromContext(ctx) + + // Fetch the FlagSourceConfiguration from the cache + fsConfig := &corev1alpha1.FlagSourceConfiguration{} + if err := r.Client.Get(ctx, req.NamespacedName, fsConfig); err != nil { + if errors.IsNotFound(err) { + // taking down all associated K8s resources is handled by K8s + r.Log.Info(fmt.Sprintf("%s resource not found. Ignoring since object must be deleted", req.NamespacedName)) + return r.finishReconcile(nil, false) + } + r.Log.Error(err, fmt.Sprintf("Failed to get the %s", req.NamespacedName)) + return r.finishReconcile(err, false) + } + + if fsConfig.Spec.RolloutOnChange == nil || !*fsConfig.Spec.RolloutOnChange { + return r.finishReconcile(nil, false) + } + + // Object has been updated, so, we can restart any deployments that are using this annotation + // => we know there has been an update because we are using the GenerationChangedPredicate filter + // and our resource exists within the cluster + deployList := &appsV1.DeploymentList{} + if err := r.Client.List(ctx, deployList, client.MatchingFields{ + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPath, FlagSourceConfigurationAnnotation): "true", + }); err != nil { + r.Log.Error(err, fmt.Sprintf("Failed to get the pods with annotation %s/%s", OpenFeatureAnnotationPath, FlagSourceConfigurationAnnotation)) + return r.finishReconcile(err, false) + } - // FlagSourceConfigurations do not currently update cluster state, and only provides configuration for the pod_webhook + // Loop through all deployments containing the openfeature.dev/flagsourceconfiguration annotation + // and trigger a restart for any which have our resource listed as a configuration + for _, deployment := range deployList.Items { + annotations := deployment.Spec.Template.Annotations + annotation, ok := annotations[fmt.Sprintf("%s/%s", OpenFeatureAnnotationRoot, FlagSourceConfigurationAnnotation)] + if !ok { + continue + } + if isUsingConfiguration(fsConfig.Namespace, fsConfig.Name, deployment.Namespace, annotation) { + r.Log.Info(fmt.Sprintf("restarting deployment %s/%s", deployment.Namespace, deployment.Name)) + deployment.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + if err := r.Client.Update(ctx, &deployment); err != nil { + r.Log.V(1).Error(err, fmt.Sprintf("Failed to update Deployment: %s/%s", deployment.Namespace, deployment.Name)) + continue + } + } + } + + return r.finishReconcile(nil, false) +} + +func isUsingConfiguration(namespace string, name string, deploymentNamespace string, annotation string) bool { + s := strings.Split(annotation, ",") // parse annotation list + for _, target := range s { + ss := strings.Split(strings.TrimSpace(target), "/") + if len(ss) != 2 { + target = fmt.Sprintf("%s/%s", deploymentNamespace, target) + } + if target == fmt.Sprintf("%s/%s", namespace, name) { + return true + } + } + return false +} + +func (r *FlagSourceConfigurationReconciler) finishReconcile(err error, requeueImmediate bool) (ctrl.Result, error) { + if err != nil { + interval := reconcileErrorInterval + if requeueImmediate { + interval = 0 + } + r.Log.Error(err, "Finished Reconciling FlagSourceConfiguration with error: %w") + return ctrl.Result{Requeue: true, RequeueAfter: interval}, err + } + r.Log.Info("Finished Reconciling FlagSourceConfiguration") + return ctrl.Result{Requeue: false}, nil +} - return ctrl.Result{}, nil +func FlagSourceConfigurationIndex(o client.Object) []string { + deployment := o.(*appsV1.Deployment) + if deployment.Spec.Template.ObjectMeta.Annotations == nil { + return []string{ + "false", + } + } + if _, ok := deployment.Spec.Template.ObjectMeta.Annotations[fmt.Sprintf("openfeature.dev/%s", FlagSourceConfigurationAnnotation)]; ok { + return []string{ + "true", + } + } + return []string{ + "false", + } } // SetupWithManager sets up the controller with the Manager. func (r *FlagSourceConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1alpha1.FlagSourceConfiguration{}). + // we are only interested in update events for this reconciliation loop + WithEventFilter(predicate.GenerationChangedPredicate{}). Complete(r) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index d5bbec7f1..6bbba1755 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -17,19 +17,26 @@ limitations under the License. package controllers import ( + "context" + "fmt" "path/filepath" "testing" + corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + corev1alpha2 "github.com/open-feature/open-feature-operator/apis/core/v1alpha2" + corev1alpha3 "github.com/open-feature/open-feature-operator/apis/core/v1alpha3" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" + appsV1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - configv1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" //+kubebuilder:scaffold:imports ) @@ -37,8 +44,9 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - k8sClient client.Client - testEnv *envtest.Environment + k8sClient client.Client + testEnv *envtest.Environment + testCtx, testCancel = context.WithCancel(context.Background()) ) func TestAPIs(t *testing.T) { @@ -53,27 +61,71 @@ var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) By("bootstrapping test environment") + + scheme := runtime.NewScheme() + err := clientgoscheme.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + err = corev1alpha1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + err = corev1alpha2.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + err = corev1alpha3.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, + Scheme: scheme, + CRDInstallOptions: envtest.CRDInstallOptions{Scheme: scheme}, } cfg, err := testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = configv1alpha1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + LeaderElection: false, + Host: testEnv.WebhookInstallOptions.LocalServingHost, + Port: testEnv.WebhookInstallOptions.LocalServingPort, + CertDir: testEnv.WebhookInstallOptions.LocalServingCertDir, + }) + Expect(err).ToNot(HaveOccurred()) + + err = mgr.GetFieldIndexer().IndexField( + context.Background(), + &appsV1.Deployment{}, + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPath, FlagSourceConfigurationAnnotation), + FlagSourceConfigurationIndex, + ) + Expect(err).ToNot(HaveOccurred()) + + err = (&FlagSourceConfigurationReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr) + Expect(err).ToNot(HaveOccurred()) + + go func() { + err := mgr.Start(testCtx) + Expect(err).ToNot(HaveOccurred()) + }() + + setup() }, 60) var _ = AfterSuite(func() { By("tearing down the test environment") + testCancel() err := testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) diff --git a/docs/README.md b/docs/README.md index 4a7f75047..33e1208b6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,7 @@ Configuration of the deployed sidecars is handled through the `FeatureFlagConfig - [Annotations](./annotations.md) - [FeatureFlagConfigurations](./feature_flag_configuration.md) +- [FlagSourceConfiguration](./flag_source_configuration.md) ## Other Resources - [Architecture](./architecture.md) diff --git a/docs/annotations.md b/docs/annotations.md index 7cdf67065..75966b810 100644 --- a/docs/annotations.md +++ b/docs/annotations.md @@ -11,32 +11,36 @@ Example: openfeature.dev/enabled: "true" ``` -### `openfeature.dev/featureflagconfiguration` -This annotation specifies the names of the FeatureFlagConfigurations used to configure the injected flagd sidecar. +### `openfeature.dev/flagsourceconfiguration` +This annotation specifies the names of the FlagSourceConfigurations used to configure the injected flagd sidecar. The annotation value is a comma separated list of values following one of 2 patterns: {NAME} or {NAMESPACE}/{NAME}. If no namespace is provided it is assumed that the CR is within the same namespace as the deployed pod. +If multiple CRs are provided, they are merged with the latest taking precedence, for example, in the scenario below, `config-B` will take priority in the merge, replacing duplicated values that are set in `config-A`. + Example: -```yaml - metadata: +``` + metadata: annotations: openfeature.dev/enabled: "true" - openfeature.dev/featureflagconfiguration: "demo, test/demo-2" + openfeature.dev/flagsourceconfiguration:"config-A, config-B"` ``` +### `openfeature.dev/allowkubernetessync` +*This annotation is used internally by the operator.* +This annotation is used to mark pods which should have their permissions backfilled in the event of an upgrade. When the OFO manager pod is started, all `Service Accounts` of any `Pods` with this annotation set to `"true"` will be added to the `flagd-kubernetes-sync` `Cluster Role Binding`. -### `openfeature.dev/flagsourceconfiguration` -This annotation specifies the names of the FlagSourceConfigurations used to configure the injected flagd sidecar. + +### `openfeature.dev/featureflagconfiguration` +*This annotation is deprecated in favour of the `openfeature.dev/flagsourceconfiguration` annotation and should no longer be used.* +This annotation specifies the names of the FeatureFlagConfigurations used to configure the injected flagd sidecar. The annotation value is a comma separated list of values following one of 2 patterns: {NAME} or {NAMESPACE}/{NAME}. If no namespace is provided it is assumed that the CR is within the same namespace as the deployed pod. -If multiple CRs are provided, they are merged with the latest taking precedence, for example, in the scenario below, `config-B` will take priority in the merge, replacing duplicated values that are set in `config-A`. - Example: -``` - metadata: +```yaml + metadata: annotations: openfeature.dev/enabled: "true" openfeature.dev/featureflagconfiguration: "demo, test/demo-2" - openfeature.dev/flagsourceconfiguration:"config-A, config-B"` ``` ### `openfeature.dev` diff --git a/docs/feature_flag_configuration.md b/docs/feature_flag_configuration.md index 48aa22984..19ddb0822 100644 --- a/docs/feature_flag_configuration.md +++ b/docs/feature_flag_configuration.md @@ -8,43 +8,16 @@ kind: FeatureFlagConfiguration metadata: name: featureflagconfiguration-sample spec: - syncProvider: - name: filepath # kubernetes (default), filepath or http - featureFlagSpec: - flags: - foo: - state: "ENABLED" - variants: - bar: "BAR" - baz: "BAZ" - defaultVariant: "bar" + featureFlagSpec: + flags: + foo: + state: "ENABLED" + variants: + bar: "BAR" + baz: "BAZ" + defaultVariant: "bar" ``` -Within the CRD there are 2 main objects, namely the `featureFlagSpec` and the `syncProvider` each offering a different set of configurations for the injected `flagd` sidecars. - ## featureFlagSpec The `featureFlagSpec` is an object representing the flag configurations themselves, the documentation for this object can be found [here](https://github.com/open-feature/flagd/blob/main/docs/configuration/flag_configuration.md). - -## syncProvider - -The `syncProvider` specifies how the flag configuration will be supplied to flagd. It contains 2 optional members, `name` and `httpSyncConfiguration`. - -The default `syncProvider` is `kubernetes`, but can be configured globally at installation time by modifying the `defaultSyncProvider` parameter or by setting the `sidecarConfiguration.defaultSyncProvider` helm value (`helm upgrade -i ofo openfeature/open-feature-operator --set sidecarConfiguration.defaultSyncProvider=filepath`). - -### name - -| Value | Explanation | -| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| "kubernetes" (default) | Instruct flagd to query the Kubernetes API directly for the flag configuration custom resource(s) specified in the workload annotation (`openfeature.dev/featureflagconfiguration`). This configuration requires the flag sidecar (and therefore the workload pod) to be able to access the Kubernetes API. | -| "filepath" | Mounts the flag configuration custom resource(s) specified in the workload annotation (`openfeature.dev/featureflagconfiguration`) as volume mounted ConfigMaps, and configures flagd to watch them. | -| "http" | Retrieves the flag configuration from the specified HTTP endpoint. Open Feature Operator does not automatically provide the specified configuration at this URL. Requires [`httpSyncConfiguration`](#httpSyncConfiguration) to be configured. | - -### httpSyncConfiguration - -httpSyncConfiguration must - -| Value | Explanation | -| ------------- | -------------------------------------------------------- | -| "target" | Target URL for flagd to poll for the flag configuration. | -| "bearerToken" | Authorization token to be included in HTTP request. | diff --git a/docs/flag_source_configuration.md b/docs/flag_source_configuration.md new file mode 100644 index 000000000..bd55c2005 --- /dev/null +++ b/docs/flag_source_configuration.md @@ -0,0 +1,108 @@ +# Flag Source configuration + +The injected sidecar is configured using the `FlagSourceConfiguration` CRD, the `openfeature.dev/flagsourceconfiguration` annotation is used to assign `Pods` with their respective `FlagSourceConfiguration` CRs. The annotation value is a comma separated list of values following one of 2 patterns: {NAME} or {NAMESPACE}/{NAME}. If no namespace is provided, it is assumed that the CR is within the same namespace as the deployed pod, for example: +``` + metadata: + namespace: test-ns + annotations: + openfeature.dev/enabled: "true" + openfeature.dev/flagsourceconfiguration:"config-A, test-ns-2/config-B" +``` +In this example, 2 CRs are being used to configure the injected container (by default the the operator uses the `flagd:main` image), `config-A` (which is assumed to be in the namespace `test-ns`) and `config-B` from the `test-ns-2` namespace, with `config-B` taking precedence in the configuration merge. + +The `FlagSourceConfiguration` version `v1alpha3` CRD defines a CR with the following example structure: + +```yaml +apiVersion: core.openfeature.dev/v1alpha3 +kind: FlagSourceConfiguration +metadata: + name: flagsourceconfiguration-sample +spec: + metricsPort: 8080 + Port: 80 + evaluator: json + image: my-custom-sidecar-image + defaultSyncProvider: filepath + tag: main + sources: + - source: namespace/name + provider: kubernetes + - source: namespace/name2 + - source: not-a-real-host.com + provider: http + envVars: + - name: MY_ENV_VAR + value: my-env-value +``` + +The relevant `FlagSourceConfigurations` are passed to the operator by setting the `openfeature.dev/flagsourceconfiguration` annotation, and is responsible for providing the full configuration of the injected sidecar. + +## FlagSourceConfiguration Fields + +| Field | Behavior | Type | Default | +| ----------- | ----------- | ----------- | ----------- | +| MetricsPort | Defines the port for flagd to serve metrics on | optional `int32`| `8013` | +| Port | Defines the port for flagd to listen on | optional `int32` | `8014` | +| SocketPath | Defines the unix socket path to listen on | optional `string` | `""` | +| SyncProviderArgs | String arguments passed to the sidecar on startup, flagd documentation can be found [here](https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md) | optional `array of strings`, key values separated by `=`, e.g `key=value` | `""` | +| Image | Allows for the sidecar image to be overridden | optional `string` | `ghcr.io/open-feature/flagd` | +| Tag | Tag to be appended to the sidecar image | optional `string` | `main` | +| Sources | An array of objects defining configuration and sources for each sync provider to use within flagd, documentation of the object is directly below this table | optional `array of objects` |`[]` | +| EnvVars | An array of environment variables to be applied to the sidecar, all names become prepended with the EnvVarPrefix | optional `array of environment variables` | `[]` | +| EnvVarPrefix | String value defining the prefix to be applied to all environment variables applied to the sidecar| optional `string` | `FLAGD` | +| DefaultSyncProvider | Defines the default provider to be used, can be set to `kubernetes`, `filepath` or `http`. | optional `string` | `kubernetes` | +| RolloutOnChange | When set to true the operator will trigger a restart of any `Deployments` within the `FlagSourceConfiguration` reconcile loop, updating the injected sidecar with the latest configuration. | optional `boolean` | `false` | + +## Source Fields + +| Field | Behavior | Type | +| ----------- | ----------- | ----------- | +| Source | Defines the URI of the flag source, this can be either a `host:port` or the `namespace/name` of a `FeatureFlagConfiguration` | `string` | +| Provider | Defines the provider to be used, can be set to `kubernetes`, `filepath` or `http`. If not provided the default sync provider is used. | optional `string` | +| HttpSyncBearerToken | Defines the bearer token to be used with a `http` sync. Has no effect if `Provider` is not `http` | optional `string` | + +## Configuration Merging + +When multiple `FlagSourceConfigurations` are provided, the configurations are merged. The last `CR` takes precedence over the first, with any configuration from the deprecated `FlagDSpec` field of the `FeatureFlagConfiguration` CRD taking the lowest priority. + + +```mermaid +flowchart LR + FlagSourceConfiguration-values -->|highest priority| environment-variables -->|lowest priority| defaults +``` + + +An example of this behavior: +``` + metadata: + annotations: + openfeature.dev/enabled: "true" + openfeature.dev/flagsourceconfiguration:"config-A, config-B" +``` +Config-A: +``` +apiVersion: core.openfeature.dev/v1alpha2 +kind: FlagSourceConfiguration +metadata: + name: test-configuration-A +spec: + metricsPort: 8080 + tag: latest +``` +Config-B: +``` +apiVersion: core.openfeature.dev/v1alpha2 +kind: FlagSourceConfiguration +metadata: + name: test-configuration-B +spec: + port: 8000 + tag: main +``` +Results in the following configuration: +``` +spec: + metricsPort: 8080 + port: 8000 + tag: main +``` \ No newline at end of file diff --git a/docs/flagd_configuration.md b/docs/flagd_configuration.md deleted file mode 100644 index 968a306c1..000000000 --- a/docs/flagd_configuration.md +++ /dev/null @@ -1,62 +0,0 @@ -# Flagd Configuration - -The injected flagd sidecar is configured using the `FlagSourceConfiguration` CRD, the `openfeature.dev/flagsourceconfiguration` annotation is used to assign `Pods` with their respective `FlagSourceConfiguration` CRs. The annotation value is a comma separated list of values following one of 2 patterns: {NAME} or {NAMESPACE}/{NAME}. If no namespace is provided, it is assumed that the CR is within the same namespace as the deployed pod, for example: -``` - metadata: - namespace: test-ns - annotations: - openfeature.dev/enabled: "true" - openfeature.dev/flagsourceconfiguration:"config-A, test-ns-2/config-B" -``` -In this example, 2 CRs are being used to configure the injected flagd container, `config-A` (which is assumed to be in the namespace `test-ns`) and `config-B` from the `test-ns-2` namespace, with `config-B` taking precedence in the configuration merge. - -## FlagSourceConfiguration - -The flagd configuration CRD contains the following fields: - -| Field | Behavior | Type | -| ----------- | ----------- | ----------- | -| MetricsPort | Defines the port for flagd to serve metrics on, defaults to 8013 | `int32` | -| Port | Defines the port for flagd to listen on, defaults to 8014 | `int32` | -| SocketPath | Defines the unix socket path to listen on | `string` | -| SyncProviderArgs | String arguments passed to flagd on startup, documentation can be found [here](https://github.com/open-feature/flagd/blob/main/docs/configuration/configuration.md) | `array of strings`, key values separated by `=`, e.g `key=value` | -| Image | Allows for the flagd image to be overridden, defaults to `ghcr.io/open-feature/flagd` | `string` | -| Tag | Tag to be appended to the flagd image, defaults to `main` | `string` | - -## Configuration Merging - -When multiple `FlagSourceConfigurations` are passed the configurations are merged, the last `CR` will take precedence over the first, with any configuration from the deprecated `FlagDSpec` field of the `FeatureFlagConfiguration` CRD taking the lowest priority. -An example of this behavior can be found below: -``` - metadata: - annotations: - openfeature.dev/enabled: "true" - openfeature.dev/flagsourceconfiguration:"config-A, config-B" -``` -Config-A: -``` -apiVersion: core.openfeature.dev/v1alpha2 -kind: FlagSourceConfiguration -metadata: - name: test-configuration -spec: - metricsPort: 8080 - tag: latest -``` -Config-B: -``` -apiVersion: core.openfeature.dev/v1alpha2 -kind: FlagSourceConfiguration -metadata: - name: test-configuration -spec: - port: 8000 - tag: main -``` -Results in the following configuration: -``` -spec: - metricsPort: 8080 - port: 8000 - tag: main -``` \ No newline at end of file diff --git a/docs/getting_started.md b/docs/getting_started.md index 0fd6c027d..84d4411b9 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -50,9 +50,25 @@ spec: targeting: {} ``` -### Reference the deployed FeatureFlagConfiguration within a Deployment spec annotation. +### Reference the deployed FeatureFlagConfiguration within FlagSourceConfiguration. -In this example, a`Deployment` containing a `busybox-curl` container is created. In the example below, the `metadata.annotations` object contains the required annotations for the operator to correctly configure and inject the `flagd` sidecar into each deployed `Pod`. The documentation for these annotations can be found [here](./annotations.md). +The `FlagSourceConfiguration` defined below can be used to assign the `FeatureFlagConfiguration`, as well as any other configuration settings, to the injected sidecars. In this example, the port exposed by the injected container is also set. + +```yaml +apiVersion: core.openfeature.dev/v1alpha2 +kind: FlagSourceConfiguration +metadata: + name: flagsourceconfiguration-sample +spec: + syncProviders: + - source: featureflagconfiguration-sample + provider: kubernetes + port: 8080 +``` + +### Reference the deployed FlagSourceConfiguration within a Deployment spec annotation. + +In this example, a `Deployment` containing a `busybox-curl` container is created. In the configuration below, the `metadata.annotations` object contains the required annotations for the operator to correctly configure and inject the `flagd` sidecar into each deployed `Pod`. The documentation for these annotations can be found [here](./annotations.md). ```yaml apiVersion: apps/v1 @@ -70,7 +86,7 @@ spec: app: my-busybox-curl-app annotations: openfeature.dev/enabled: "true" - openfeature.dev/featureflagconfiguration: "default/featureflagconfiguration-sample" + openfeature.dev/flagsourceconfiguration: "default/flagsourceconfiguration-sample" spec: containers: - name: busybox @@ -116,7 +132,7 @@ Now that we have confirmed that the `flagd` sidecar has been injected and the co ```sh kubectl exec -it busybox-curl-7bd5767999-spf7v sh -curl -X POST "localhost:8013/schema.v1.Service/ResolveString" -d '{"flagKey":"foo","context":{}}' -H "Content-Type: application/json" +curl -X POST "localhost:8080/schema.v1.Service/ResolveString" -d '{"flagKey":"foo","context":{}}' -H "Content-Type: application/json" ``` output: ```sh diff --git a/go.mod b/go.mod index 3e62674d8..c7028f47e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/go-logr/logr v1.2.3 github.com/onsi/ginkgo v1.16.5 + github.com/onsi/ginkgo/v2 v2.1.6 github.com/onsi/gomega v1.20.1 github.com/open-feature/schemas v0.2.8 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index 6fe53131f..bfa985a4c 100644 --- a/go.sum +++ b/go.sum @@ -302,12 +302,11 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/open-feature/schemas v0.0.0-20221123004631-302d0fa1f813 h1:jcf+dYt/EL5kMYrftpzdk4t6iscQKzVo8IIvAkAV+MI= -github.com/open-feature/schemas v0.0.0-20221123004631-302d0fa1f813/go.mod h1:vj+rfTsOLlh5PtGGkAbitnJmFPYuTHXTjOy13kzNgKQ= github.com/open-feature/schemas v0.2.8 h1:oA75hJXpOd9SFgmNI2IAxWZkwzQPUDm7Jyyh3q489wM= github.com/open-feature/schemas v0.2.8/go.mod h1:vj+rfTsOLlh5PtGGkAbitnJmFPYuTHXTjOy13kzNgKQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/main.go b/main.go index 0a18cf79f..993bde887 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "fmt" "os" corev1 "k8s.io/api/core/v1" @@ -38,8 +39,10 @@ import ( corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" corev1alpha2 "github.com/open-feature/open-feature-operator/apis/core/v1alpha2" + corev1alpha3 "github.com/open-feature/open-feature-operator/apis/core/v1alpha3" "github.com/open-feature/open-feature-operator/controllers" webhooks "github.com/open-feature/open-feature-operator/webhooks" + appsV1 "k8s.io/api/apps/v1" //+kubebuilder:scaffold:imports ) @@ -78,6 +81,7 @@ func init() { utilruntime.Must(corev1alpha1.AddToScheme(scheme)) utilruntime.Must(corev1alpha2.AddToScheme(scheme)) + utilruntime.Must(corev1alpha3.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -177,10 +181,30 @@ func main() { if err := mgr.GetFieldIndexer().IndexField( context.Background(), &corev1.Pod{}, - webhooks.OpenFeatureEnabledAnnotationPath, + fmt.Sprintf("%s/%s", webhooks.OpenFeatureAnnotationPath, webhooks.AllowKubernetesSyncAnnotation), webhooks.OpenFeatureEnabledAnnotationIndex, ); err != nil { - setupLog.Error(err, "unable to create indexer", "webhook", webhooks.OpenFeatureEnabledAnnotationPath) + setupLog.Error( + err, + "unable to create indexer", + "webhook", + fmt.Sprintf("%s/%s", webhooks.OpenFeatureAnnotationPath, webhooks.AllowKubernetesSyncAnnotation), + ) + os.Exit(1) + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &appsV1.Deployment{}, + fmt.Sprintf("%s/%s", controllers.OpenFeatureAnnotationPath, controllers.FlagSourceConfigurationAnnotation), + controllers.FlagSourceConfigurationIndex, + ); err != nil { + setupLog.Error( + err, + "unable to create indexer", + "webhook", + fmt.Sprintf("%s/%s", webhooks.OpenFeatureAnnotationPath, webhooks.FlagSourceConfigurationAnnotation), + ) os.Exit(1) } @@ -217,6 +241,10 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "FlagSourceConfiguration") os.Exit(1) } + if err := (&corev1alpha3.FlagSourceConfiguration{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "FlagSourceConfiguration") + os.Exit(1) + } //+kubebuilder:scaffold:builder hookServer := mgr.GetWebhookServer() diff --git a/test/e2e/e2e.yml b/test/e2e/e2e.yml index 45f7b4fde..cc844d61b 100644 --- a/test/e2e/e2e.yml +++ b/test/e2e/e2e.yml @@ -52,6 +52,15 @@ spec: "off": false defaultVariant: "on" --- +apiVersion: core.openfeature.dev/v1alpha3 +kind: FlagSourceConfiguration +metadata: + name: end-to-end-test-fs-config +spec: + sources: + - source: end-to-end-test-filepath2 + provider: kubernetes +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -91,7 +100,8 @@ spec: app: open-feature-e2e-test annotations: openfeature.dev/enabled: "true" - openfeature.dev/featureflagconfiguration: "end-to-end-test-default,end-to-end-test-filepath,end-to-end-test-filepath2" + openfeature.dev/featureflagconfiguration: "end-to-end-test-default,end-to-end-test-filepath" + openfeature.dev/flagsourceconfiguration: "end-to-end-test-fs-config" spec: serviceAccountName: open-feature-e2e-test-sa volumes: diff --git a/webhooks/pod_webhook.go b/webhooks/pod_webhook.go index c82fd930f..46310cbbe 100644 --- a/webhooks/pod_webhook.go +++ b/webhooks/pod_webhook.go @@ -12,7 +12,7 @@ import ( goErr "errors" "github.com/go-logr/logr" - corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + v1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" "github.com/open-feature/open-feature-operator/pkg/utils" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" @@ -25,11 +25,16 @@ import ( // we likely want these to be configurable, eventually const ( - FlagDImagePullPolicy corev1.PullPolicy = "Always" - clusterRoleBindingName string = "open-feature-operator-flagd-kubernetes-sync" - flagdMetricPortEnvVar string = "FLAGD_METRICS_PORT" - rootFileSyncMountPath string = "/etc/flagd" - OpenFeatureEnabledAnnotationPath = "metadata.annotations.openfeature.dev/enabled" + FlagDImagePullPolicy corev1.PullPolicy = "Always" + clusterRoleBindingName string = "open-feature-operator-flagd-kubernetes-sync" + flagdMetricPortEnvVar string = "FLAGD_METRICS_PORT" + rootFileSyncMountPath string = "/etc/flagd" + OpenFeatureAnnotationPath = "metadata.annotations.openfeature.dev/openfeature.dev" + OpenFeatureAnnotationPrefix = "openfeature.dev" + AllowKubernetesSyncAnnotation = "allowkubernetessync" + FlagSourceConfigurationAnnotation = "flagsourceconfiguration" + FeatureFlagConfigurationAnnotation = "featureflagconfiguration" + EnabledAnnotation = "enabled" ) // NOTE: RBAC not needed here. @@ -56,40 +61,6 @@ func (m *PodMutator) IsReady(_ *http.Request) error { return goErr.New("pod mutator is not ready") } -// BackfillPermissions recovers the state of the flagd-kubernetes-sync role binding in the event of upgrade -func (m *PodMutator) BackfillPermissions(ctx context.Context) error { - defer func() { - m.ready = true - }() - for i := 0; i < 5; i++ { - // fetch all pods with the "openfeature.dev/enabled" annotation set to "true" - podList := &corev1.PodList{} - err := m.Client.List(ctx, podList, client.MatchingFields{OpenFeatureEnabledAnnotationPath: "true"}) - if err != nil { - if !goErr.Is(err, &cache.ErrCacheNotStarted{}) { - return err - } - time.Sleep(1 * time.Second) - continue - } - - // add each new service account to the flagd-kubernetes-sync role binding - for _, pod := range podList.Items { - m.Log.V(1).Info(fmt.Sprintf("backfilling permissions for pod %s/%s", pod.Namespace, pod.Name)) - if err := m.enableClusterRoleBinding(ctx, &pod); err != nil { - m.Log.Error( - err, - fmt.Sprintf("unable backfill permissions for pod %s/%s", pod.Namespace, pod.Name), - "webhook", - OpenFeatureEnabledAnnotationPath, - ) - } - } - return nil - } - return goErr.New("unable to backfill permissions for the flagd-kubernetes-sync role binding: timeout") -} - // Handle injects the flagd sidecar (if the prerequisites are all met) func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admission.Response { defer func() { @@ -106,7 +77,7 @@ func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admissio // Check enablement enabled := false - val, ok := pod.GetAnnotations()["openfeature.dev"] + val, ok := pod.GetAnnotations()[OpenFeatureAnnotationPrefix] if ok { m.Log.V(1).Info("DEPRECATED: The openfeature.dev annotation has been superseded by the openfeature.dev/enabled annotation. " + "Docs: https://github.com/open-feature/open-feature-operator/blob/main/docs/annotations.md") @@ -114,7 +85,7 @@ func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admissio enabled = true } } - val, ok = pod.GetAnnotations()["openfeature.dev/enabled"] + val, ok = pod.GetAnnotations()[fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, EnabledAnnotation)] if ok { if val == "true" { enabled = true @@ -127,16 +98,10 @@ func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admissio } // Check configuration - ffNames := []string{} - val, ok = pod.GetAnnotations()["openfeature.dev/featureflagconfiguration"] - if ok { - ffNames = parseList(val) - } - - fcNames := []string{} - val, ok = pod.GetAnnotations()["openfeature.dev/flagsourceconfiguration"] + fscNames := []string{} + val, ok = pod.GetAnnotations()[fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FlagSourceConfigurationAnnotation)] if ok { - fcNames = parseList(val) + fscNames = parseList(val) } // Check if the pod is static or orphaned if len(pod.GetOwnerReferences()) == 0 { @@ -149,75 +114,239 @@ func (m *PodMutator) Handle(ctx context.Context, req admission.Request) admissio } // merge any provided flagd specs - flagSourceConfigurationSpec, err := corev1alpha1.NewFlagSourceConfigurationSpec() + flagSourceConfigurationSpec, err := v1alpha1.NewFlagSourceConfigurationSpec() if err != nil { m.Log.V(1).Error(err, "unable to parse env var configuration", "webhook", "handle") return admission.Errored(http.StatusBadRequest, err) } - for _, fcName := range fcNames { - ns, name := parseAnnotation(fcName, req.Namespace) + for _, fscName := range fscNames { + ns, name := parseAnnotation(fscName, req.Namespace) if err != nil { - m.Log.V(1).Info(fmt.Sprintf("failed to parse annotation %s error: %s", fcName, err.Error())) + m.Log.V(1).Info(fmt.Sprintf("failed to parse annotation %s error: %s", fscName, err.Error())) return admission.Errored(http.StatusBadRequest, err) } - fc := m.getFlagSourceConfiguration(ctx, name, ns) - if reflect.DeepEqual(fc, corev1alpha1.FlagSourceConfiguration{}) { - m.Log.V(1).Info(fmt.Sprintf("FlagSourceConfiguration could not be found for %s", fcName)) + fc := m.getFlagSourceConfiguration(ctx, ns, name) + if reflect.DeepEqual(fc, v1alpha1.FlagSourceConfiguration{}) { + m.Log.V(1).Info(fmt.Sprintf("FlagSourceConfiguration could not be found for %s", fscName)) return admission.Errored(http.StatusBadRequest, err) } flagSourceConfigurationSpec.Merge(&fc.Spec) } - ffConfigs := []*corev1alpha1.FeatureFlagConfiguration{} - for _, ffName := range ffNames { - ns, name := parseAnnotation(ffName, req.Namespace) - if err != nil { - m.Log.V(1).Info(fmt.Sprintf("failed to parse annotation %s error: %s", ffName, err.Error())) - return admission.Errored(http.StatusBadRequest, err) + // maintain backwards compatibility of the openfeature.dev/featureflagconfiguration annotation + ffConfigAnnotation, ffConfigAnnotationOk := pod.GetAnnotations()[fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation)] + if ffConfigAnnotationOk { + m.Log.V(1).Info("DEPRECATED: The openfeature.dev/featureflagconfiguration annotation has been superseded by the openfeature.dev/flagsourceconfiguration annotation. " + + "Docs: https://github.com/open-feature/open-feature-operator/blob/main/docs/annotations.md") + if err := m.handleFeatureFlagConfigurationAnnotation(ctx, flagSourceConfigurationSpec, ffConfigAnnotation, req.Namespace); err != nil { + m.Log.Error(err, "unable to handle openfeature.dev/featureflagconfiguration annotation") + return admission.Errored(http.StatusInternalServerError, err) } - // Check to see whether the FeatureFlagConfiguration has service or sync overrides - ff := m.getFeatureFlag(ctx, name, ns) - if reflect.DeepEqual(ff, corev1alpha1.FeatureFlagConfiguration{}) { - m.Log.V(1).Info(fmt.Sprintf("FeatureFlagConfiguration could not be found for %s", ffName)) - return admission.Errored(http.StatusBadRequest, err) + } + + marshaledPod, err := m.injectSidecar(ctx, pod, flagSourceConfigurationSpec) + if err != nil { + m.Log.Error(err, "unable to inject flagd sidecar") + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +func (m *PodMutator) injectSidecar( + ctx context.Context, + pod *corev1.Pod, + flagSourceConfig *v1alpha1.FlagSourceConfigurationSpec, +) ([]byte, error) { + m.Log.V(1).Info(fmt.Sprintf("creating sidecar for pod %s/%s", pod.Namespace, pod.Name)) + sidecar := corev1.Container{ + Name: "flagd", + Image: fmt.Sprintf("%s:%s", flagSourceConfig.Image, flagSourceConfig.Tag), + Args: []string{ + "start", + }, + ImagePullPolicy: FlagDImagePullPolicy, + VolumeMounts: []corev1.VolumeMount{}, + Env: []corev1.EnvVar{}, + Ports: []corev1.ContainerPort{ + { + Name: "metrics", + ContainerPort: flagSourceConfig.MetricsPort, + }, + }, + SecurityContext: setSecurityContext(), + Resources: m.FlagDResourceRequirements, + } + + for _, source := range flagSourceConfig.Sources { + if source.Provider == "" { + source.Provider = flagSourceConfig.DefaultSyncProvider } - if ff.Spec.SyncProvider == nil || ff.Spec.SyncProvider.Name == "" { - ff.Spec.SyncProvider = &corev1alpha1.FeatureFlagSyncProvider{ - Name: flagSourceConfigurationSpec.DefaultSyncProvider, + switch { + case source.Provider.IsFilepath(): + if err := m.handleFilepathProvider(ctx, pod, &sidecar, source); err != nil { + return nil, err } - } - if !ff.Spec.SyncProvider.Name.IsKubernetes() { - // Check for ConfigMap and create it if it doesn't exist (only required if sync provider isn't kubernetes) - cm := corev1.ConfigMap{} - if err := m.Client.Get(ctx, client.ObjectKey{Name: name, Namespace: req.Namespace}, &cm); errors.IsNotFound(err) { - err := m.createConfigMap(ctx, name, req.Namespace, pod) - if err != nil { - m.Log.V(1).Info(fmt.Sprintf("failed to create config map %s error: %s", ffName, err.Error())) - return admission.Errored(http.StatusInternalServerError, err) - } + case source.Provider.IsKubernetes(): + if err := m.handleKubernetesProvider(ctx, pod, &sidecar, source); err != nil { + return nil, err } + case source.Provider.IsHttp(): + m.handleHttpProvider(&sidecar, source) + default: + return nil, fmt.Errorf("unrecognized sync provider in config: %s", source.Provider) + } + } - // Add owner reference of the pod's owner - if !podOwnerIsOwner(pod, cm) { - reference := pod.OwnerReferences[0] - reference.Controller = utils.FalseVal() - cm.OwnerReferences = append(cm.OwnerReferences, reference) - err := m.Client.Update(ctx, &cm) - if err != nil { - m.Log.V(1).Info(fmt.Sprintf("failed to update owner reference for %s error: %s", ffName, err.Error())) - } - } + sidecar.Env = append(sidecar.Env, flagSourceConfig.ToEnvVars()...) + for i := 0; i < len(pod.Spec.Containers); i++ { + cntr := pod.Spec.Containers[i] + cntr.Env = append(cntr.Env, sidecar.Env...) + } + + // append sync provider args + if flagSourceConfig.SyncProviderArgs != nil { + for _, v := range flagSourceConfig.SyncProviderArgs { + sidecar.Args = append( + sidecar.Args, + "--sync-provider-args", + v, + ) } + } - ffConfigs = append(ffConfigs, &ff) + pod.Spec.Containers = append(pod.Spec.Containers, sidecar) + + return json.Marshal(pod) +} + +func (m *PodMutator) handleHttpProvider(sidecar *corev1.Container, source v1alpha1.Source) { + // append args + sidecar.Args = append( + sidecar.Args, + "--uri", + source.Source, + ) + if source.HttpSyncBearerToken != "" { + sidecar.Args = append( + sidecar.Args, + "--bearer-token", + source.HttpSyncBearerToken, + ) } +} - marshaledPod, err := m.injectSidecar(pod, flagSourceConfigurationSpec, ffConfigs) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) +func (m *PodMutator) handleKubernetesProvider(ctx context.Context, pod *corev1.Pod, sidecar *corev1.Container, source v1alpha1.Source) error { + ns, n := parseAnnotation(source.Source, pod.Namespace) + // ensure that the FeatureFlagConfiguration exists + ff := m.getFeatureFlag(ctx, ns, n) + if ff.Name == "" { + return fmt.Errorf("feature flag configuration %s/%s not found", ns, n) } - return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) + // add permissions to pod + if err := m.enableClusterRoleBinding(ctx, pod); err != nil { + return err + } + // mark pod with annotation (required to backfill permissions if they are dropped) + pod.Annotations[fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, AllowKubernetesSyncAnnotation)] = "true" + // append args + sidecar.Args = append( + sidecar.Args, + "--uri", + fmt.Sprintf( + "core.openfeature.dev/%s/%s", + ns, + n, + ), + ) + return nil +} + +func (m *PodMutator) handleFilepathProvider(ctx context.Context, pod *corev1.Pod, sidecar *corev1.Container, source v1alpha1.Source) error { + // create config map + ns, n := parseAnnotation(source.Source, pod.Namespace) + cm := corev1.ConfigMap{} + if err := m.Client.Get(ctx, client.ObjectKey{Name: n, Namespace: ns}, &cm); errors.IsNotFound(err) { + err := m.createConfigMap(ctx, ns, n, pod) + if err != nil { + m.Log.V(1).Info(fmt.Sprintf("failed to create config map %s error: %s", n, err.Error())) + return err + } + } + + // Add owner reference of the pod's owner + if !podOwnerIsOwner(pod, cm) { + reference := pod.OwnerReferences[0] + reference.Controller = utils.FalseVal() + cm.OwnerReferences = append(cm.OwnerReferences, reference) + err := m.Client.Update(ctx, &cm) + if err != nil { + m.Log.V(1).Info(fmt.Sprintf("failed to update owner reference for %s error: %s", n, err.Error())) + } + } + // mount configmap + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: n, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: n, + }, + }, + }, + }) + mountPath := fmt.Sprintf("%s/%s", rootFileSyncMountPath, v1alpha1.FeatureFlagConfigurationId(ns, n)) + sidecar.VolumeMounts = append(sidecar.VolumeMounts, corev1.VolumeMount{ + Name: n, + // create a directory mount per featureFlag spec + // file mounts will not work + MountPath: mountPath, + }) + sidecar.Args = append( + sidecar.Args, + "--uri", + fmt.Sprintf("file:%s/%s", + mountPath, + v1alpha1.FeatureFlagConfigurationConfigMapKey(ns, n), + ), + ) + return nil +} + +// BackfillPermissions recovers the state of the flagd-kubernetes-sync role binding in the event of upgrade +func (m *PodMutator) BackfillPermissions(ctx context.Context) error { + defer func() { + m.ready = true + }() + for i := 0; i < 5; i++ { + // fetch all pods with the fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, EnabledAnnotation) annotation set to "true" + podList := &corev1.PodList{} + err := m.Client.List(ctx, podList, client.MatchingFields{ + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPath, AllowKubernetesSyncAnnotation): "true", + }) + if err != nil { + if !goErr.Is(err, &cache.ErrCacheNotStarted{}) { + return err + } + time.Sleep(1 * time.Second) + continue + } + + // add each new service account to the flagd-kubernetes-sync role binding + for _, pod := range podList.Items { + m.Log.V(1).Info(fmt.Sprintf("backfilling permissions for pod %s/%s", pod.Namespace, pod.Name)) + if err := m.enableClusterRoleBinding(ctx, &pod); err != nil { + m.Log.Error( + err, + fmt.Sprintf("unable backfill permissions for pod %s/%s", pod.Namespace, pod.Name), + "webhook", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPath, AllowKubernetesSyncAnnotation), + ) + } + } + return nil + } + return goErr.New("unable to backfill permissions for the flagd-kubernetes-sync role binding: timeout") } func parseList(s string) []string { @@ -304,166 +433,39 @@ func (m *PodMutator) enableClusterRoleBinding(ctx context.Context, pod *corev1.P return nil } -func (m *PodMutator) createConfigMap(ctx context.Context, name string, namespace string, pod *corev1.Pod) error { +func (m *PodMutator) createConfigMap(ctx context.Context, namespace string, name string, pod *corev1.Pod) error { m.Log.V(1).Info(fmt.Sprintf("Creating configmap %s", name)) references := []metav1.OwnerReference{ pod.OwnerReferences[0], } references[0].Controller = utils.FalseVal() - ff := m.getFeatureFlag(ctx, name, namespace) - if ff.Name != "" { - references = append(references, corev1alpha1.GetFfReference(&ff)) + ff := m.getFeatureFlag(ctx, namespace, name) + if ff.Name == "" { + return fmt.Errorf("feature flag configuration %s/%s not found", namespace, name) } + references = append(references, v1alpha1.GetFfReference(&ff)) - cm := corev1alpha1.GenerateFfConfigMap(name, namespace, references, ff.Spec) + cm := v1alpha1.GenerateFfConfigMap(name, namespace, references, ff.Spec) return m.Client.Create(ctx, &cm) } -func (m *PodMutator) getFeatureFlag(ctx context.Context, name string, namespace string) corev1alpha1.FeatureFlagConfiguration { - ffConfig := corev1alpha1.FeatureFlagConfiguration{} +func (m *PodMutator) getFeatureFlag(ctx context.Context, namespace string, name string) v1alpha1.FeatureFlagConfiguration { + ffConfig := v1alpha1.FeatureFlagConfiguration{} if err := m.Client.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, &ffConfig); errors.IsNotFound(err) { - return corev1alpha1.FeatureFlagConfiguration{} + return v1alpha1.FeatureFlagConfiguration{} } return ffConfig } -func (m *PodMutator) getFlagSourceConfiguration(ctx context.Context, name string, namespace string) corev1alpha1.FlagSourceConfiguration { - fcConfig := corev1alpha1.FlagSourceConfiguration{} +func (m *PodMutator) getFlagSourceConfiguration(ctx context.Context, namespace string, name string) v1alpha1.FlagSourceConfiguration { + fcConfig := v1alpha1.FlagSourceConfiguration{} if err := m.Client.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, &fcConfig); errors.IsNotFound(err) { - return corev1alpha1.FlagSourceConfiguration{} + return v1alpha1.FlagSourceConfiguration{} } return fcConfig } -func (m *PodMutator) injectSidecar( - pod *corev1.Pod, - flagdConfig *corev1alpha1.FlagSourceConfigurationSpec, - featureFlags []*corev1alpha1.FeatureFlagConfiguration, -) ([]byte, error) { - m.Log.V(1).Info(fmt.Sprintf("Creating sidecar for pod %s/%s", pod.Namespace, pod.Name)) - - commandSequence := []string{ - "start", - } - var envs []corev1.EnvVar - var volumeMounts []corev1.VolumeMount - - for _, featureFlag := range featureFlags { - if featureFlag.Spec.FlagDSpec != nil { - m.Log.V(1).Info("DEPRECATED: The FlagDSpec property of the FeatureFlagConfiguration CRD has been superseded by " + - "the FlagSourceConfiguration CRD." + - "Docs: https://github.com/open-feature/open-feature-operator/blob/main/docs/flagd_configuration.md") - if featureFlag.Spec.FlagDSpec.MetricsPort != 0 && flagdConfig.MetricsPort == 8013 { - flagdConfig.MetricsPort = featureFlag.Spec.FlagDSpec.MetricsPort - } - envs = append(envs, featureFlag.Spec.FlagDSpec.Envs...) - } - switch { - // kubernetes sync is the default state - case featureFlag.Spec.SyncProvider == nil || featureFlag.Spec.SyncProvider.Name.IsKubernetes(): - m.Log.V(1).Info(fmt.Sprintf("FeatureFlagConfiguration %s using kubernetes sync implementation", featureFlag.Name)) - commandSequence = append( - commandSequence, - "--uri", - fmt.Sprintf( - "core.openfeature.dev/%s/%s", - featureFlag.ObjectMeta.Namespace, - featureFlag.ObjectMeta.Name, - ), - ) - // if http is explicitly set - case featureFlag.Spec.SyncProvider.Name.IsHttp(): - m.Log.V(1).Info(fmt.Sprintf("FeatureFlagConfiguration %s using http sync implementation", featureFlag.Name)) - if featureFlag.Spec.SyncProvider.HttpSyncConfiguration != nil { - commandSequence = append( - commandSequence, - "--uri", - featureFlag.Spec.SyncProvider.HttpSyncConfiguration.Target, - ) - if featureFlag.Spec.SyncProvider.HttpSyncConfiguration.BearerToken != "" { - commandSequence = append( - commandSequence, - "--bearer-token", - featureFlag.Spec.SyncProvider.HttpSyncConfiguration.BearerToken, - ) - } - } else { - err := fmt.Errorf("FeatureFlagConfiguration %s is missing a httpSyncConfiguration", featureFlag.Name) - m.Log.V(1).Error(err, "unable to add http sync provider") - } - // if filepath is explicitly set - case featureFlag.Spec.SyncProvider.Name.IsFilepath(): - m.Log.V(1).Info(fmt.Sprintf("FeatureFlagConfiguration %s using filepath sync implementation", featureFlag.Name)) - commandSequence = append( - commandSequence, - "--uri", - fmt.Sprintf("file:%s/%s", - fileSyncMountPath(featureFlag), - corev1alpha1.FeatureFlagConfigurationConfigMapKey(featureFlag.Namespace, featureFlag.Name)), - ) - pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ - Name: featureFlag.Name, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: featureFlag.Name, - }, - }, - }, - }) - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: featureFlag.Name, - // create a directory mount per featureFlag spec - // file mounts will not work - MountPath: fileSyncMountPath(featureFlag), - }) - default: - err := fmt.Errorf( - "sync provider for ffconfig %s not recognized: %s", - featureFlag.Name, - featureFlag.Spec.SyncProvider.Name, - ) - m.Log.Error(err, err.Error()) - } - } - - // append sync provider args - if flagdConfig.SyncProviderArgs != nil { - for _, v := range flagdConfig.SyncProviderArgs { - commandSequence = append( - commandSequence, - "--sync-provider-args", - v, - ) - } - } - - envs = append(envs, flagdConfig.ToEnvVars()...) - for i := 0; i < len(pod.Spec.Containers); i++ { - cntr := pod.Spec.Containers[i] - cntr.Env = append(cntr.Env, envs...) - } - - pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ - Name: "flagd", - Image: fmt.Sprintf("%s:%s", flagdConfig.Image, flagdConfig.Tag), - Args: commandSequence, - ImagePullPolicy: FlagDImagePullPolicy, - VolumeMounts: volumeMounts, - Env: envs, - Ports: []corev1.ContainerPort{ - { - Name: "metrics", - ContainerPort: flagdConfig.MetricsPort, - }, - }, - SecurityContext: setSecurityContext(), - Resources: m.FlagDResourceRequirements, - }) - return json.Marshal(pod) -} - func setSecurityContext() *corev1.SecurityContext { // user and group have been set to 65532 to mirror the configuration in the Dockerfile user := int64(65532) @@ -490,10 +492,6 @@ func setSecurityContext() *corev1.SecurityContext { } } -func fileSyncMountPath(featureFlag *corev1alpha1.FeatureFlagConfiguration) string { - return fmt.Sprintf("%s/%s", rootFileSyncMountPath, corev1alpha1.FeatureFlagConfigurationId(featureFlag.Namespace, featureFlag.Name)) -} - func OpenFeatureEnabledAnnotationIndex(o client.Object) []string { pod := o.(*corev1.Pod) if pod.ObjectMeta.Annotations == nil { @@ -501,18 +499,12 @@ func OpenFeatureEnabledAnnotationIndex(o client.Object) []string { "false", } } - val, ok := pod.ObjectMeta.Annotations["openfeature.dev/enabled"] + val, ok := pod.ObjectMeta.Annotations[fmt.Sprintf("openfeature.dev/%s", AllowKubernetesSyncAnnotation)] if ok && val == "true" { return []string{ "true", } } - val, ok = pod.ObjectMeta.Annotations["openfeature.dev"] - if ok && val == "enabled" { - return []string{ - "true", - } - } return []string{ "false", } diff --git a/webhooks/pod_webhook_deprecated.go b/webhooks/pod_webhook_deprecated.go new file mode 100644 index 000000000..939b64e54 --- /dev/null +++ b/webhooks/pod_webhook_deprecated.go @@ -0,0 +1,56 @@ +package webhooks + +import ( + "context" + "fmt" + "reflect" + + v1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" +) + +func (m *PodMutator) handleFeatureFlagConfigurationAnnotation(ctx context.Context, fcConfig *v1alpha1.FlagSourceConfigurationSpec, ffconfigAnnotation string, defaultNamespace string) error { + for _, ffName := range parseList(ffconfigAnnotation) { + ns, name := parseAnnotation(ffName, defaultNamespace) + fsConfig := m.getFeatureFlag(ctx, ns, name) + if reflect.DeepEqual(fsConfig, v1alpha1.FeatureFlagConfiguration{}) { + return fmt.Errorf("FeatureFlagConfiguration %s not found", ffName) + } + if fsConfig.Spec.FlagDSpec != nil { + if len(fsConfig.Spec.FlagDSpec.Envs) > 0 { + fcConfig.EnvVars = append(fsConfig.Spec.FlagDSpec.Envs, fcConfig.EnvVars...) + } + if fsConfig.Spec.FlagDSpec.MetricsPort != 0 && fcConfig.MetricsPort == v1alpha1.DefaultMetricPort { + fcConfig.MetricsPort = fsConfig.Spec.FlagDSpec.MetricsPort + } + } + switch { + case fsConfig.Spec.SyncProvider == nil: + fcConfig.Sources = append(fcConfig.Sources, v1alpha1.Source{ + Provider: fcConfig.DefaultSyncProvider, + Source: ffName, + }) + case v1alpha1.SyncProviderType(fsConfig.Spec.SyncProvider.Name).IsKubernetes(): + fcConfig.Sources = append(fcConfig.Sources, v1alpha1.Source{ + Provider: v1alpha1.SyncProviderKubernetes, + Source: ffName, + }) + case v1alpha1.SyncProviderType(fsConfig.Spec.SyncProvider.Name).IsFilepath(): + fcConfig.Sources = append(fcConfig.Sources, v1alpha1.Source{ + Provider: v1alpha1.SyncProviderFilepath, + Source: ffName, + }) + case v1alpha1.SyncProviderType(fsConfig.Spec.SyncProvider.Name).IsHttp(): + if fsConfig.Spec.SyncProvider.HttpSyncConfiguration == nil { + return fmt.Errorf("FeatureFlagConfiguration %s is missing HttpSyncConfiguration", ffName) + } + fcConfig.Sources = append(fcConfig.Sources, v1alpha1.Source{ + Provider: v1alpha1.SyncProviderHttp, + Source: fsConfig.Spec.SyncProvider.HttpSyncConfiguration.Target, + HttpSyncBearerToken: fsConfig.Spec.SyncProvider.HttpSyncConfiguration.BearerToken, + }) + default: + return fmt.Errorf("FeatureFlagConfiguration %s configuration is unrecognized", ffName) + } + } + return nil +} diff --git a/webhooks/pod_webhook_test.go b/webhooks/pod_webhook_test.go index e27ebdc2b..28d080ea7 100644 --- a/webhooks/pod_webhook_test.go +++ b/webhooks/pod_webhook_test.go @@ -1,12 +1,15 @@ package webhooks import ( + "errors" "fmt" "os" + "reflect" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" + v1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,7 +22,10 @@ const ( defaultPodName = "test-pod" defaultPodServiceAccountName = "test-pod-service-account" featureFlagConfigurationName = "test-feature-flag-configuration" + featureFlagConfigurationName2 = "test-feature-flag-configuration-2" flagSourceConfigurationName = "test-flag-source-configuration" + flagSourceConfigurationName2 = "test-flag-source-configuration-2" + flagSourceConfigurationName3 = "test-flag-source-configuration-3" existingPod1Name = "existing-pod-1" existingPod1ServiceAccountName = "existing-pod-1-service-account" existingPod2Name = "existing-pod-2" @@ -58,15 +64,17 @@ func setupPreviouslyExistingPods() { Expect(err).ShouldNot(HaveOccurred()) existingPod := testPod(existingPod1Name, existingPod1ServiceAccountName, map[string]string{ - "openfeature.dev/enabled": "true", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, EnabledAnnotation): "true", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, AllowKubernetesSyncAnnotation): "true", }) err = k8sClient.Create(testCtx, existingPod) Expect(err).ShouldNot(HaveOccurred()) existingPod = testPod(existingPod2Name, existingPod2ServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, AllowKubernetesSyncAnnotation): "true", }) err = k8sClient.Create(testCtx, existingPod) Expect(err).ShouldNot(HaveOccurred()) @@ -79,17 +87,17 @@ func setupMutatePodResources() { err := k8sClient.Create(testCtx, svcAccount) Expect(err).ShouldNot(HaveOccurred()) - ffConfig := &corev1alpha1.FeatureFlagConfiguration{} + ffConfig := &v1alpha1.FeatureFlagConfiguration{} ffConfig.Namespace = mutatePodNamespace ffConfig.Name = featureFlagConfigurationName - ffConfig.Spec.FlagDSpec = &corev1alpha1.FlagDSpec{Envs: []corev1.EnvVar{ + ffConfig.Spec.FlagDSpec = &v1alpha1.FlagDSpec{Envs: []corev1.EnvVar{ {Name: "LOG_LEVEL", Value: "dev"}, }} ffConfig.Spec.FeatureFlagSpec = featureFlagSpec err = k8sClient.Create(testCtx, ffConfig) Expect(err).ShouldNot(HaveOccurred()) - fsConfig := &corev1alpha1.FlagSourceConfiguration{} + fsConfig := &v1alpha1.FlagSourceConfiguration{} fsConfig.Namespace = mutatePodNamespace fsConfig.Name = flagSourceConfigurationName fsConfig.Spec.Port = 8080 @@ -102,8 +110,52 @@ func setupMutatePodResources() { "key3=val3", } fsConfig.Spec.LogFormat = "console" + fsConfig.Spec.Sources = []v1alpha1.Source{ + { + Source: "not-real.com", + Provider: "http", + }, + } err = k8sClient.Create(testCtx, fsConfig) Expect(err).ShouldNot(HaveOccurred()) + + ffConfig2 := &v1alpha1.FeatureFlagConfiguration{} + ffConfig2.Namespace = mutatePodNamespace + ffConfig2.Name = featureFlagConfigurationName2 + ffConfig2.Spec.FeatureFlagSpec = featureFlagSpec + err = k8sClient.Create(testCtx, ffConfig2) + Expect(err).ShouldNot(HaveOccurred()) + + fsConfig2 := &v1alpha1.FlagSourceConfiguration{} + fsConfig2.Namespace = mutatePodNamespace + fsConfig2.Name = flagSourceConfigurationName2 + fsConfig2.Spec.Sources = []v1alpha1.Source{ + { + Source: fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + }, + { + Source: fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName2), + Provider: v1alpha1.SyncProviderFilepath, + }, + } + err = k8sClient.Create(testCtx, fsConfig2) + Expect(err).ShouldNot(HaveOccurred()) + + fsConfig3 := &v1alpha1.FlagSourceConfiguration{} + fsConfig3.Namespace = mutatePodNamespace + fsConfig3.Name = flagSourceConfigurationName3 + fsConfig3.Spec.Sources = []v1alpha1.Source{ + { + Source: fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName2), + Provider: v1alpha1.SyncProviderKubernetes, + }, + { + Source: "i don't exist", + Provider: v1alpha1.SyncProviderFilepath, + }, + } + err = k8sClient.Create(testCtx, fsConfig3) + Expect(err).ShouldNot(HaveOccurred()) } func testPod(podName string, serviceAccountName string, annotations map[string]string) *corev1.Pod { @@ -166,33 +218,61 @@ func podMutationWebhookCleanup() { var _ = Describe("pod mutation webhook", func() { It("should backfill role binding subjects when annotated pods already exist in the cluster", func() { - pod1 := getPod(existingPod1Name) - pod2 := getPod(existingPod2Name) - // Pod 1 and 2 must not have been mutated by the webhook (we want the rolebinding to be updated via BackfillPermissions) - - Expect(len(pod1.Spec.Containers)).To(Equal(1)) - Expect(len(pod2.Spec.Containers)).To(Equal(1)) - - rb := getRoleBinding(clusterRoleBindingName) - - Expect(rb.Subjects).To(ContainElement(v1.Subject{ - Kind: "ServiceAccount", - APIGroup: "", - Name: existingPod1ServiceAccountName, - Namespace: mutatePodNamespace, - })) - Expect(rb.Subjects).To(ContainElement(v1.Subject{ - Kind: "ServiceAccount", - APIGroup: "", - Name: existingPod2ServiceAccountName, - Namespace: mutatePodNamespace, - })) + // this integration test confirms the proper execution of the podMutator.BackfillPermissions method + // this method is responsible for backfilling the subjects of the open-feature-operator-flagd-kubernetes-sync + // cluster role binding, for previously existing pods on startup + // a retry is required on this test as the backfilling occurs asynchronously + var finalError error + for i := 0; i < 3; i++ { + pod1 := getPod(existingPod1Name) + pod2 := getPod(existingPod2Name) + // Pod 1 and 2 must not have been mutated by the webhook (we want the rolebinding to be updated via BackfillPermissions) + + if len(pod1.Spec.Containers) != 1 { + finalError = errors.New("pod1 has had a container injected, it should not be mutated by the webhook") + time.Sleep(1 * time.Second) + continue + } + if len(pod2.Spec.Containers) != 1 { + finalError = errors.New("pod2 has had a container injected, it should not be mutated by the webhook") + time.Sleep(1 * time.Second) + continue + } + + rb := getRoleBinding(clusterRoleBindingName) + + unexpectedServiceAccount := "" + for _, subject := range rb.Subjects { + if !reflect.DeepEqual(subject, v1.Subject{ + Kind: "ServiceAccount", + APIGroup: "", + Name: existingPod1ServiceAccountName, + Namespace: mutatePodNamespace, + }) && + !reflect.DeepEqual(subject, v1.Subject{ + Kind: "ServiceAccount", + APIGroup: "", + Name: existingPod2ServiceAccountName, + Namespace: mutatePodNamespace, + }) { + unexpectedServiceAccount = subject.Name + } + } + if unexpectedServiceAccount != "" { + finalError = fmt.Errorf("unexpected subject found in role binding, name: %s", unexpectedServiceAccount) + time.Sleep(1 * time.Second) + continue + } + finalError = nil + break + } + Expect(finalError).ShouldNot(HaveOccurred()) }) It("should update cluster role binding's subjects", func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -212,16 +292,16 @@ var _ = Describe("pod mutation webhook", func() { }) It("should create flagd sidecar", func() { - flagConfig, _ := corev1alpha1.NewFlagSourceConfigurationSpec() + flagConfig, _ := v1alpha1.NewFlagSourceConfigurationSpec() pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) pod = getPod(defaultPodName) - + Expect(pod.Annotations["openfeature.dev/allowkubernetessync"]).To(Equal("true")) Expect(len(pod.Spec.Containers)).To(Equal(2)) Expect(pod.Spec.Containers[1].Name).To(Equal("flagd")) Expect(pod.Spec.Containers[1].Image).To(Equal(fmt.Sprintf("%s:%s", flagConfig.Image, flagConfig.Tag))) @@ -230,7 +310,7 @@ var _ = Describe("pod mutation webhook", func() { })) Expect(pod.Spec.Containers[1].ImagePullPolicy).To(Equal(FlagDImagePullPolicy)) Expect(pod.Spec.Containers[1].Env).To(Equal([]corev1.EnvVar{ - {Name: "LOG_LEVEL", Value: "dev"}, + {Name: "FLAGD_LOG_LEVEL", Value: "dev"}, })) Expect(pod.Spec.Containers[1].Ports).To(Equal([]corev1.ContainerPort{ { @@ -245,7 +325,7 @@ var _ = Describe("pod mutation webhook", func() { It("should create flagd sidecar even if openfeature.dev/featureflagconfiguration annotation isn't present", func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", + OpenFeatureAnnotationPrefix: "enabled", }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -259,8 +339,8 @@ var _ = Describe("pod mutation webhook", func() { It("should not create flagd sidecar if openfeature.dev annotation is disabled", func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "disabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "disabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -274,8 +354,8 @@ var _ = Describe("pod mutation webhook", func() { It("should fail if pod has no owner references", func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) pod.OwnerReferences = nil err := k8sClient.Create(testCtx, pod) @@ -284,32 +364,32 @@ var _ = Describe("pod mutation webhook", func() { It("should fail if service account not found", func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) pod.Spec.ServiceAccountName = "foo" err := k8sClient.Create(testCtx, pod) Expect(err).Should(HaveOccurred()) }) - It("should create config map if sync provider isn't kubernetes", func() { - ffConfig := &corev1alpha1.FeatureFlagConfiguration{} + It("should create config map if sync provider is filepath", func() { + ffConfig := &v1alpha1.FeatureFlagConfiguration{} err := k8sClient.Get( testCtx, client.ObjectKey{Name: featureFlagConfigurationName, Namespace: mutatePodNamespace}, ffConfig, ) Expect(err).ShouldNot(HaveOccurred()) - ffConfig.Spec = corev1alpha1.FeatureFlagConfigurationSpec{ - SyncProvider: &corev1alpha1.FeatureFlagSyncProvider{ - Name: "not-kubernetes", + ffConfig.Spec = v1alpha1.FeatureFlagConfigurationSpec{ + SyncProvider: &v1alpha1.FeatureFlagSyncProvider{ + Name: string(v1alpha1.SyncProviderFilepath), }, } err = k8sClient.Update(testCtx, ffConfig) Expect(err).ShouldNot(HaveOccurred()) pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err = k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -323,9 +403,10 @@ var _ = Describe("pod mutation webhook", func() { Expect(cm.Name).To(Equal(featureFlagConfigurationName)) Expect(cm.Namespace).To(Equal(mutatePodNamespace)) Expect(cm.Annotations).To(Equal(map[string]string{ - "openfeature.dev/featureflagconfiguration": featureFlagConfigurationName, + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): featureFlagConfigurationName, })) Expect(len(cm.OwnerReferences)).To(Equal(2)) + Expect(cm.Data).To(Equal(map[string]string{ fmt.Sprintf("%s_%s.flagd.json", mutatePodNamespace, featureFlagConfigurationName): ffConfig.Spec.FeatureFlagSpec, })) @@ -339,7 +420,7 @@ var _ = Describe("pod mutation webhook", func() { It("should not panic if flagDSpec isn't provided", func() { ffConfigName := "feature-flag-configuration-panic-test" - ffConfig := &corev1alpha1.FeatureFlagConfiguration{} + ffConfig := &v1alpha1.FeatureFlagConfiguration{} ffConfig.Namespace = mutatePodNamespace ffConfig.Name = ffConfigName ffConfig.Spec.FeatureFlagSpec = featureFlagSpec @@ -347,8 +428,8 @@ var _ = Describe("pod mutation webhook", func() { Expect(err).ShouldNot(HaveOccurred()) pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, ffConfigName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, ffConfigName), }) err = k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -360,8 +441,8 @@ var _ = Describe("pod mutation webhook", func() { It(`should create flagd sidecar if openfeature.dev/enabled annotation is "true"`, func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev/enabled": "true", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, EnabledAnnotation): "true", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -375,9 +456,9 @@ var _ = Describe("pod mutation webhook", func() { It(`should only write non default flagsourceconfiguration env vars to the flagd container`, func() { pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), - "openfeature.dev/flagsourceconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + "openfeature.dev/flagsourceconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -395,20 +476,20 @@ var _ = Describe("pod mutation webhook", func() { }) It(`should use env var configuration to overwrite flagsourceconfiguration defaults`, func() { - os.Setenv(corev1alpha1.SidecarEnvVarPrefix, "MY_SIDECAR") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarMetricPortEnvVar), "10") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarPortEnvVar), "20") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarSocketPathEnvVar), "socket") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarEvaluatorEnvVar), "evaluator") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarImageEnvVar), "image") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarVersionEnvVar), "version") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarDefaultSyncProviderEnvVar), "filepath") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarProviderArgsEnvVar), "key=value,key2=value2") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarLogFormatEnvVar), "yaml") + os.Setenv(v1alpha1.SidecarEnvVarPrefix, "MY_SIDECAR") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarMetricPortEnvVar), "10") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarPortEnvVar), "20") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarSocketPathEnvVar), "socket") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarEvaluatorEnvVar), "evaluator") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarImageEnvVar), "image") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarVersionEnvVar), "version") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarDefaultSyncProviderEnvVar), "filepath") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarProviderArgsEnvVar), "key=value,key2=value2") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarLogFormatEnvVar), "yaml") pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", - "openfeature.dev/featureflagconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FeatureFlagConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, featureFlagConfigurationName), }) err := k8sClient.Create(testCtx, pod) Expect(err).ShouldNot(HaveOccurred()) @@ -434,20 +515,20 @@ var _ = Describe("pod mutation webhook", func() { podMutationWebhookCleanup() }) - It(`should overwrite env var configuration with flagsourceconfiguration values, sync-provider-args should be compounded`, func() { - os.Setenv(corev1alpha1.SidecarEnvVarPrefix, "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarMetricPortEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarPortEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarSocketPathEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarEvaluatorEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarImageEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarVersionEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarDefaultSyncProviderEnvVar), "") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarProviderArgsEnvVar), "key=value,key2=value2") - os.Setenv(fmt.Sprintf("%s_%s", corev1alpha1.InputConfigurationEnvVarPrefix, corev1alpha1.SidecarLogFormatEnvVar), "") + It(`should overwrite env var configuration with flagsourceconfiguration values`, func() { + os.Setenv(v1alpha1.SidecarEnvVarPrefix, "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarMetricPortEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarPortEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarSocketPathEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarEvaluatorEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarImageEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarVersionEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarDefaultSyncProviderEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarProviderArgsEnvVar), "key=value,key2=value2") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarLogFormatEnvVar), "") pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ - "openfeature.dev": "enabled", + OpenFeatureAnnotationPrefix: "enabled", "openfeature.dev/flagsourceconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName), }) err := k8sClient.Create(testCtx, pod) @@ -464,6 +545,8 @@ var _ = Describe("pod mutation webhook", func() { Expect(pod.Spec.Containers[1].Image).To(Equal("new-image:latest")) Expect(pod.Spec.Containers[1].Args).To(Equal([]string{ "start", + "--uri", + "not-real.com", "--sync-provider-args", "key=value", "--sync-provider-args", @@ -473,4 +556,85 @@ var _ = Describe("pod mutation webhook", func() { })) podMutationWebhookCleanup() }) + + It("should create flagd sidecar using flagsourceconfiguration", func() { + os.Setenv(v1alpha1.SidecarEnvVarPrefix, "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarMetricPortEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarPortEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarSocketPathEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarEvaluatorEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarImageEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarVersionEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarDefaultSyncProviderEnvVar), "") + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarProviderArgsEnvVar), "") + flagConfig, _ := v1alpha1.NewFlagSourceConfigurationSpec() + pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, EnabledAnnotation): "true", + "openfeature.dev/flagsourceconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName2), + }) + err := k8sClient.Create(testCtx, pod) + Expect(err).ShouldNot(HaveOccurred()) + + pod = getPod(defaultPodName) + Expect(pod.Annotations["openfeature.dev/allowkubernetessync"]).To(Equal("true")) + Expect(len(pod.Spec.Containers)).To(Equal(2)) + Expect(pod.Spec.Containers[1].Name).To(Equal("flagd")) + Expect(pod.Spec.Containers[1].Image).To(Equal(fmt.Sprintf("%s:%s", flagConfig.Image, flagConfig.Tag))) + Expect(pod.Spec.Containers[1].Args).To(Equal([]string{ + "start", + "--uri", + "core.openfeature.dev/test-mutate-pod/test-feature-flag-configuration", + "--uri", + "file:/etc/flagd/test-mutate-pod_test-feature-flag-configuration-2/test-mutate-pod_test-feature-flag-configuration-2.flagd.json", + })) + Expect(pod.Spec.Containers[1].ImagePullPolicy).To(Equal(FlagDImagePullPolicy)) + Expect(pod.Spec.Containers[1].Ports).To(Equal([]corev1.ContainerPort{ + { + Name: "metrics", + Protocol: "TCP", + ContainerPort: 8014, + }, + })) + + podMutationWebhookCleanup() + }) + + It("should not create flagd sidecar if flagsourceconfiguration does not exist", func() { + pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ + OpenFeatureAnnotationPrefix: "enabled", + "openfeature.dev/flagsourceconfiguration": "im-not-real", + }) + err := k8sClient.Create(testCtx, pod) + Expect(err).Should(HaveOccurred()) + }) + + It("should not create flagd sidecar if flagsourceconfiguration contains a source that does not exist", func() { + pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ + OpenFeatureAnnotationPrefix: "enabled", + "openfeature.dev/flagsourceconfiguration": fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName3), + }) + err := k8sClient.Create(testCtx, pod) + Expect(err).Should(HaveOccurred()) + }) + + It(`should use defaultSyncProvider if one isn't provided`, func() { + os.Setenv(fmt.Sprintf("%s_%s", v1alpha1.InputConfigurationEnvVarPrefix, v1alpha1.SidecarDefaultSyncProviderEnvVar), "filepath") + + pod := testPod(defaultPodName, defaultPodServiceAccountName, map[string]string{ + OpenFeatureAnnotationPrefix: "enabled", + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPrefix, FlagSourceConfigurationAnnotation): fmt.Sprintf("%s/%s", mutatePodNamespace, flagSourceConfigurationName2), + }) + err := k8sClient.Create(testCtx, pod) + Expect(err).ShouldNot(HaveOccurred()) + + pod = getPod(defaultPodName) + Expect(pod.Spec.Containers[1].Args).To(Equal([]string{ + "start", + "--uri", + "file:/etc/flagd/test-mutate-pod_test-feature-flag-configuration/test-mutate-pod_test-feature-flag-configuration.flagd.json", + "--uri", + "file:/etc/flagd/test-mutate-pod_test-feature-flag-configuration-2/test-mutate-pod_test-feature-flag-configuration-2.flagd.json", + })) + podMutationWebhookCleanup() + }) }) diff --git a/webhooks/suite_test.go b/webhooks/suite_test.go index 914e6d5cf..88086c327 100644 --- a/webhooks/suite_test.go +++ b/webhooks/suite_test.go @@ -9,10 +9,9 @@ import ( "testing" "time" - "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" corev1alpha1 "github.com/open-feature/open-feature-operator/apis/core/v1alpha1" - "github.com/open-feature/open-feature-operator/apis/core/v1alpha2" corev1alpha2 "github.com/open-feature/open-feature-operator/apis/core/v1alpha2" + corev1alpha3 "github.com/open-feature/open-feature-operator/apis/core/v1alpha3" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -140,10 +139,13 @@ var _ = BeforeSuite(func() { err := clientgoscheme.AddToScheme(scheme) Expect(err).ToNot(HaveOccurred()) - err = v1alpha1.AddToScheme(scheme) + err = corev1alpha1.AddToScheme(scheme) Expect(err).ToNot(HaveOccurred()) - err = v1alpha2.AddToScheme(scheme) + err = corev1alpha2.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + err = corev1alpha3.AddToScheme(scheme) Expect(err).ToNot(HaveOccurred()) testEnv = &envtest.Environment{ @@ -183,13 +185,22 @@ var _ = BeforeSuite(func() { err = (&corev1alpha1.FeatureFlagConfiguration{}).SetupWebhookWithManager(mgr) Expect(err).ToNot(HaveOccurred()) + err = (&corev1alpha1.FlagSourceConfiguration{}).SetupWebhookWithManager(mgr) + Expect(err).ToNot(HaveOccurred()) + err = (&corev1alpha2.FeatureFlagConfiguration{}).SetupWebhookWithManager(mgr) Expect(err).ToNot(HaveOccurred()) + err = (&corev1alpha2.FlagSourceConfiguration{}).SetupWebhookWithManager(mgr) + Expect(err).ToNot(HaveOccurred()) + + err = (&corev1alpha3.FlagSourceConfiguration{}).SetupWebhookWithManager(mgr) + Expect(err).ToNot(HaveOccurred()) + err = mgr.GetFieldIndexer().IndexField( context.Background(), &corev1.Pod{}, - OpenFeatureEnabledAnnotationPath, + fmt.Sprintf("%s/%s", OpenFeatureAnnotationPath, AllowKubernetesSyncAnnotation), OpenFeatureEnabledAnnotationIndex, ) Expect(err).ToNot(HaveOccurred())