diff --git a/cli/run.go b/cli/run.go index d62a66b6..21cb3bdf 100644 --- a/cli/run.go +++ b/cli/run.go @@ -117,13 +117,14 @@ func run(cmd *cobra.Command) (logsInitialized bool, err error) { logs.AddFlags(cmd.PersistentFlags()) // Inject logs.InitLogs after command line parsing into one of the - // PersistentPre* functions. + // PersistentPre* functions. Also inform about race detection, if enabled. switch { case cmd.PersistentPreRun != nil: pre := cmd.PersistentPreRun cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { logs.InitLogs() logsInitialized = true + logRaceDetection() pre(cmd, args) } case cmd.PersistentPreRunE != nil: @@ -131,12 +132,14 @@ func run(cmd *cobra.Command) (logsInitialized bool, err error) { cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { logs.InitLogs() logsInitialized = true + logRaceDetection() return pre(cmd, args) } default: cmd.PersistentPreRun = func(cmd *cobra.Command, args []string) { logs.InitLogs() logsInitialized = true + logRaceDetection() } } diff --git a/cli/withoutrace.go b/cli/withoutrace.go new file mode 100644 index 00000000..4ae2749a --- /dev/null +++ b/cli/withoutrace.go @@ -0,0 +1,24 @@ +//go:build !race +// +build !race + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +func logRaceDetection() { + // NOP. The variant in withrace.go prints a message if race detection is built in. +} diff --git a/cli/withrace.go b/cli/withrace.go new file mode 100644 index 00000000..a71cec99 --- /dev/null +++ b/cli/withrace.go @@ -0,0 +1,29 @@ +//go:build race +// +build race + +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cli + +import ( + "k8s.io/klog/v2" +) + +func logRaceDetection() { + // Only called if race detection is built in. + klog.Info("Data race detection enabled") +} diff --git a/compatibility/OWNERS b/compatibility/OWNERS index fab38d80..283b80fe 100644 --- a/compatibility/OWNERS +++ b/compatibility/OWNERS @@ -9,5 +9,6 @@ approvers: reviewers: - sig-api-machinery-api-reviewers - siyuanfoundation + - jefftree labels: - sig/api-machinery diff --git a/compatibility/registry.go b/compatibility/registry.go index fdadff73..78c8269e 100644 --- a/compatibility/registry.go +++ b/compatibility/registry.go @@ -46,18 +46,12 @@ type ComponentGlobals struct { effectiveVersion MutableEffectiveVersion featureGate featuregate.MutableVersionedFeatureGate - // emulationVersionMapping contains the mapping from the emulation version of this component - // to the emulation version of another component. - emulationVersionMapping map[string]VersionMapping - // dependentEmulationVersion stores whether or not this component's EmulationVersion is dependent through mapping on another component. - // If true, the emulation version cannot be set from the flag, or version mapping from another component. - dependentEmulationVersion bool - // minCompatibilityVersionMapping contains the mapping from the min compatibility version of this component - // to the min compatibility version of another component. - minCompatibilityVersionMapping map[string]VersionMapping - // dependentMinCompatibilityVersion stores whether or not this component's MinCompatibilityVersion is dependent through mapping on another component - // If true, the min compatibility version cannot be set from the flag, or version mapping from another component. - dependentMinCompatibilityVersion bool + // componentVersionMapping contains the mapping from the version of this component + // to the version of another component. + componentVersionMapping map[string]VersionMapping + // dependentComponentVersion stores whether or not this component's version is dependent through mapping on another component. + // If true, the emulation/minCompatibility version cannot be set from the flag, or version mapping from another component. + dependentComponentVersion bool } // ComponentGlobalsRegistry stores the global variables for different components for easy access, including feature gate and effective version of each component. @@ -74,7 +68,7 @@ type ComponentGlobalsRegistry interface { // ComponentGlobalsOrRegister would return the registered global variables for the component if it already exists in the registry. // Otherwise, the provided variables would be registered under the component, and the same variables would be returned. ComponentGlobalsOrRegister(component string, effectiveVersion MutableEffectiveVersion, featureGate featuregate.MutableVersionedFeatureGate) (MutableEffectiveVersion, featuregate.MutableVersionedFeatureGate) - // AddFlags adds flags of "--emulated-version" and "--feature-gates" + // AddFlags adds flags of "--emulated-version", "--min-compatibility-version" and "--feature-gates" AddFlags(fs *pflag.FlagSet) // Set sets the flags for all global variables for all components registered. // A component's feature gate and effective version would not be updated until Set() is called. @@ -85,12 +79,12 @@ type ComponentGlobalsRegistry interface { Validate() []error // Reset removes all stored ComponentGlobals, configurations, and version mappings. Reset() - // SetEmulationVersionMapping sets the mapping from the emulation version of one component - // to the emulation version of another component. - // Once set, the emulation version of the toComponent will be determined by the emulation version of the fromComponent, + // SetVersionMapping sets the mapping from the emulation/minCompatibility version of one component + // to the emulation/minCompatibility version of another component. + // Once set, the emulation/minCompatibility version of the toComponent will be determined by the emulation/minCompatibility version of the fromComponent, // and cannot be set from cmd flags anymore. - // For a given component, its emulation version can only depend on one other component, no multiple dependency is allowed. - SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error + // For a given component, its emulation/minCompatibility version can only depend on one other component, no multiple dependency is allowed. + SetVersionMapping(fromComponent, toComponent string, f VersionMapping) error // AddMetrics adds metrics for the emulation version of a component. AddMetrics() } @@ -100,11 +94,15 @@ type componentGlobalsRegistry struct { mutex sync.RWMutex // emulationVersionConfig stores the list of component name to emulation version set from the flag. // When the `--emulated-version` flag is parsed, it would not take effect until Set() is called, - // because the emulation version needs to be set before the feature gate is set. + // because we have to enforce of order of setting the emulation version first, then min compatibility version, and last the feature gate. emulationVersionConfig []string + // minCompatibilityVersionConfig stores the list of component name to min compatibility version set from the flag. + // When the `--min-compatibility-version` flag is parsed, it would not take effect until Set() is called, + // because we have to enforce of order of setting the emulation version first, then min compatibility version, and last the feature gate. + minCompatibilityVersionConfig []string // featureGatesConfig stores the map of component name to the list of feature gates set from the flag. // When the `--feature-gates` flag is parsed, it would not take effect until Set() is called, - // because the emulation version needs to be set before the feature gate is set. + // because we have to enforce of order of setting the emulation version first, then min compatibility version, and last the feature gate. featureGatesConfig map[string][]string // featureGatesConfigFlags stores a pointer to the flag value, allowing other commands // to append to the feature gates configuration rather than overwriting it @@ -115,9 +113,10 @@ type componentGlobalsRegistry struct { func NewComponentGlobalsRegistry() *componentGlobalsRegistry { return &componentGlobalsRegistry{ - componentGlobals: make(map[string]*ComponentGlobals), - emulationVersionConfig: nil, - featureGatesConfig: nil, + componentGlobals: make(map[string]*ComponentGlobals), + emulationVersionConfig: nil, + minCompatibilityVersionConfig: nil, + featureGatesConfig: nil, } } @@ -136,6 +135,7 @@ func (r *componentGlobalsRegistry) Reset() { defer r.mutex.Unlock() r.componentGlobals = make(map[string]*ComponentGlobals) r.emulationVersionConfig = nil + r.minCompatibilityVersionConfig = nil r.featureGatesConfig = nil r.featureGatesConfigFlags = nil r.set = false @@ -166,15 +166,15 @@ func (r *componentGlobalsRegistry) unsafeRegister(component string, effectiveVer return fmt.Errorf("component globals of %s already registered", component) } if featureGate != nil { - if err := featureGate.SetEmulationVersion(effectiveVersion.EmulationVersion()); err != nil { + if err := featureGate.SetEmulationVersionAndMinCompatibilityVersion( + effectiveVersion.EmulationVersion(), effectiveVersion.MinCompatibilityVersion()); err != nil { return err } } c := ComponentGlobals{ - effectiveVersion: effectiveVersion, - featureGate: featureGate, - emulationVersionMapping: make(map[string]VersionMapping), - minCompatibilityVersionMapping: make(map[string]VersionMapping), + effectiveVersion: effectiveVersion, + featureGate: featureGate, + componentVersionMapping: make(map[string]VersionMapping), } r.componentGlobals[component] = &c return nil @@ -217,15 +217,12 @@ func (r *componentGlobalsRegistry) unsafeKnownFeatures() []string { func (r *componentGlobalsRegistry) unsafeVersionFlagOptions(isEmulation bool) []string { var vs []string for component, globals := range r.componentGlobals { + if globals.dependentComponentVersion { + continue + } if isEmulation { - if globals.dependentEmulationVersion { - continue - } vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedEmulationVersionRange())) } else { - if globals.dependentMinCompatibilityVersion { - continue - } vs = append(vs, fmt.Sprintf("%s=%s", component, globals.effectiveVersion.AllowedMinCompatibilityVersionRange())) } } @@ -251,6 +248,16 @@ func (r *componentGlobalsRegistry) AddFlags(fs *pflag.FlagSet) { "Version format could only be major.minor, for example: '--emulated-version=wardle=1.2,kube=1.31'.\nOptions are: "+strings.Join(r.unsafeVersionFlagOptions(true), ",")+ "\nIf the component is not specified, defaults to \"kube\"") + // min-compatibility-version is typically the minimal binary version of all control plane components the server expects to coexist with throughout the lifecycle of this server. + // If set to smaller than the server emulated version, the newest CEL features/libraries/parameters introduced after the min compatibility version would not be turned on, + // feature stages with a MinCompatibilityVersion higher than this value will be skipped, and the resource storage version will be set based on the min compatibility version. + // It can be set to equal to the binary version after all control plane components are upgraded, after which features with compatibility implications could be enabled, the newest CEL features/libraries/parameters would turned on, but rollbacks are no longer supported. + fs.StringSliceVar(&r.minCompatibilityVersionConfig, "min-compatibility-version", r.minCompatibilityVersionConfig, ""+ + "The min version of control plane components the server should be compatible with.\n"+ + "Must be less or equal to the emulated-version. Version format could only be major.minor, for example: '--min-compatibility-version=wardle=1.2,kube=1.31'.\n"+ + "Options are: "+strings.Join(r.unsafeVersionFlagOptions(false), ",")+ + "\nIf the component is not specified, defaults to \"kube\"") + if r.featureGatesConfigFlags == nil { r.featureGatesConfigFlags = cliflag.NewColonSeparatedMultimapStringStringAllowDefaultEmptyKey(&r.featureGatesConfig) } @@ -264,9 +271,9 @@ type componentVersion struct { ver *version.Version } -// getFullEmulationVersionConfig expands the given version config with version registered version mapping, +// getFullVersionConfig expands the given version config with registered version mapping, // and returns the map of component to Version. -func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( +func (r *componentGlobalsRegistry) getFullVersionConfig( versionConfigMap map[string]*version.Version) (map[string]*version.Version, error) { result := map[string]*version.Version{} setQueue := []componentVersion{} @@ -284,7 +291,7 @@ func (r *componentGlobalsRegistry) getFullEmulationVersionConfig( } setQueue = setQueue[1:] result[cv.component] = cv.ver - for toComp, f := range r.componentGlobals[cv.component].emulationVersionMapping { + for toComp, f := range r.componentGlobals[cv.component].componentVersionMapping { toVer := f(cv.ver) if toVer == nil { return result, fmt.Errorf("got nil version from mapping of %s=%s to component:%s", cv.component, cv.ver.String(), toComp) @@ -350,24 +357,45 @@ func (r *componentGlobalsRegistry) Set() error { return fmt.Errorf("component not registered: %s", comp) } // only components without any dependencies can be set from the flag. - if r.componentGlobals[comp].dependentEmulationVersion { + if r.componentGlobals[comp].dependentComponentVersion { return fmt.Errorf("EmulationVersion of %s is set by mapping, cannot set it by flag", comp) } } - if emulationVersions, err := r.getFullEmulationVersionConfig(emulationVersionConfigMap); err != nil { + if emulationVersions, err := r.getFullVersionConfig(emulationVersionConfigMap); err != nil { return err } else { for comp, ver := range emulationVersions { r.componentGlobals[comp].effectiveVersion.SetEmulationVersion(ver) } } + + minCompatibilityVersionConfigMap, err := toVersionMap(r.minCompatibilityVersionConfig) + if err != nil { + return err + } + for comp := range minCompatibilityVersionConfigMap { + if _, ok := r.componentGlobals[comp]; !ok { + return fmt.Errorf("component not registered: %s", comp) + } + // only components without any dependencies can be set from the flag. + if r.componentGlobals[comp].dependentComponentVersion { + return fmt.Errorf("MinCompatibilityVersion of %s is set by mapping, cannot set it by flag", comp) + } + } + if minCompatibilityVersions, err := r.getFullVersionConfig(minCompatibilityVersionConfigMap); err != nil { + return err + } else { + for comp, ver := range minCompatibilityVersions { + r.componentGlobals[comp].effectiveVersion.SetMinCompatibilityVersion(ver) + } + } // Set feature gate emulation version before setting feature gate flag values. for comp, globals := range r.componentGlobals { if globals.featureGate == nil { continue } - klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s", comp, globals.effectiveVersion.EmulationVersion().String()) - if err := globals.featureGate.SetEmulationVersion(globals.effectiveVersion.EmulationVersion()); err != nil { + klog.V(klogLevel).Infof("setting %s:feature gate emulation version to %s, min compatibility version to %s", comp, globals.effectiveVersion.EmulationVersion().String(), globals.effectiveVersion.MinCompatibilityVersion().String()) + if err := globals.featureGate.SetEmulationVersionAndMinCompatibilityVersion(globals.effectiveVersion.EmulationVersion(), globals.effectiveVersion.MinCompatibilityVersion()); err != nil { return err } } @@ -426,7 +454,7 @@ func enabledAlphaFeatures(features map[featuregate.Feature]featuregate.FeatureSp return enabled } -func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toComponent string, f VersionMapping) error { +func (r *componentGlobalsRegistry) SetVersionMapping(fromComponent, toComponent string, f VersionMapping) error { if f == nil { return nil } @@ -440,19 +468,19 @@ func (r *componentGlobalsRegistry) SetEmulationVersionMapping(fromComponent, toC return fmt.Errorf("component not registered: %s", toComponent) } // check multiple dependency - if r.componentGlobals[toComponent].dependentEmulationVersion { + if r.componentGlobals[toComponent].dependentComponentVersion { return fmt.Errorf("mapping of %s already exists from another component", toComponent) } - r.componentGlobals[toComponent].dependentEmulationVersion = true + r.componentGlobals[toComponent].dependentComponentVersion = true - versionMapping := r.componentGlobals[fromComponent].emulationVersionMapping + versionMapping := r.componentGlobals[fromComponent].componentVersionMapping if _, ok := versionMapping[toComponent]; ok { return fmt.Errorf("EmulationVersion from %s to %s already exists", fromComponent, toComponent) } versionMapping[toComponent] = f klog.V(klogLevel).Infof("setting the default EmulationVersion of %s based on mapping from the default EmulationVersion of %s", toComponent, fromComponent) defaultFromVersion := r.componentGlobals[fromComponent].effectiveVersion.EmulationVersion() - emulationVersions, err := r.getFullEmulationVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) + emulationVersions, err := r.getFullVersionConfig(map[string]*version.Version{fromComponent: defaultFromVersion}) if err != nil { return err } diff --git a/compatibility/registry_test.go b/compatibility/registry_test.go index 52447abf..e24edcdd 100644 --- a/compatibility/registry_test.go +++ b/compatibility/registry_test.go @@ -17,7 +17,6 @@ limitations under the License. package compatibility import ( - "fmt" "reflect" "strings" "testing" @@ -119,7 +118,7 @@ func TestVersionFlagOptions(t *testing.T) { func TestVersionFlagOptionsWithMapping(t *testing.T) { r := testRegistry(t) - utilruntime.Must(r.SetEmulationVersionMapping(testComponent, DefaultKubeComponent, + utilruntime.Must(r.SetVersionMapping(testComponent, DefaultKubeComponent, func(from *version.Version) *version.Version { return version.MajorMinor(1, from.Minor()+23) })) emuVers := strings.Join(r.unsafeVersionFlagOptions(true), ",") expectedEmuVers := "test=2.8..2.8(default:2.8)" @@ -127,7 +126,7 @@ func TestVersionFlagOptionsWithMapping(t *testing.T) { t.Errorf("wanted emulation version flag options to be: %s, got %s", expectedEmuVers, emuVers) } minCompVers := strings.Join(r.unsafeVersionFlagOptions(false), ",") - expectedMinCompVers := "kube=1.30..1.31(default:1.30),test=2.7..2.8(default:2.7)" + expectedMinCompVers := "test=2.7..2.8(default:2.7)" if minCompVers != expectedMinCompVers { t.Errorf("wanted min compatibility version flag options to be: %s, got %s", expectedMinCompVers, minCompVers) } @@ -151,19 +150,22 @@ func TestVersionedFeatureGateFlags(t *testing.T) { func TestFlags(t *testing.T) { tests := []struct { - name string - setupRegistry func(r *componentGlobalsRegistry) error - flags []string - parseError string - expectedKubeEmulationVersion string - expectedTestEmulationVersion string - expectedKubeFeatureValues map[featuregate.Feature]bool - expectedTestFeatureValues map[featuregate.Feature]bool + name string + setupRegistry func(r *componentGlobalsRegistry) error + flags []string + parseError string + expectedKubeEmulationVersion string + expectedTestEmulationVersion string + expectedKubeMinCompatibilityVersion string + expectedTestMinCompatibilityVersion string + expectedKubeFeatureValues map[featuregate.Feature]bool + expectedTestFeatureValues map[featuregate.Feature]bool }{ { - name: "setting kube emulation version", - flags: []string{"--emulated-version=kube=1.30"}, - expectedKubeEmulationVersion: "1.30", + name: "setting kube emulation version", + flags: []string{"--emulated-version=kube=1.30", "--min-compatibility-version=kube=1.28"}, + expectedKubeEmulationVersion: "1.30", + expectedKubeMinCompatibilityVersion: "1.28", }, { name: "setting kube emulation version twice", @@ -174,9 +176,18 @@ func TestFlags(t *testing.T) { parseError: "duplicate version flag, kube=1.30 and kube=1.32", }, { - name: "prefix v ok", - flags: []string{"--emulated-version=kube=v1.30"}, - expectedKubeEmulationVersion: "1.30", + name: "setting min compatibility version twice", + flags: []string{ + "--min-compatibility-version=kube=1.30", + "--min-compatibility-version=kube=1.29", + }, + parseError: "duplicate version flag, kube=1.30 and kube=1.29", + }, + { + name: "prefix v ok", + flags: []string{"--emulated-version=kube=v1.30", "--min-compatibility-version=kube=v1.28"}, + expectedKubeEmulationVersion: "1.30", + expectedKubeMinCompatibilityVersion: "1.28", }, { name: "patch version not ok", @@ -184,15 +195,23 @@ func TestFlags(t *testing.T) { parseError: "patch version not allowed, got: kube=1.30.2", }, { - name: "setting test emulation version", - flags: []string{"--emulated-version=test=2.7"}, - expectedKubeEmulationVersion: "1.31", - expectedTestEmulationVersion: "2.7", + name: "patch min compatibility version not ok", + flags: []string{"--min-compatibility-version=kube=1.30.2"}, + parseError: "patch version not allowed, got: kube=1.30.2", }, { - name: "version missing component default to kube", - flags: []string{"--emulated-version=1.30"}, - expectedKubeEmulationVersion: "1.30", + name: "setting test emulation version", + flags: []string{"--emulated-version=test=2.7", "--min-compatibility-version=test=v2.5"}, + expectedKubeEmulationVersion: "1.31", + expectedTestEmulationVersion: "2.7", + expectedKubeMinCompatibilityVersion: "1.30", + expectedTestMinCompatibilityVersion: "2.5", + }, + { + name: "version missing component default to kube", + flags: []string{"--emulated-version=1.30", "--min-compatibility-version=v1.28"}, + expectedKubeEmulationVersion: "1.30", + expectedKubeMinCompatibilityVersion: "1.28", }, { name: "version missing component default to kube with duplicate", @@ -209,6 +228,21 @@ func TestFlags(t *testing.T) { flags: []string{"--emulated-version=test=1.foo"}, parseError: "illegal version string \"1.foo\"", }, + { + name: "min compatibility version missing component default to kube with duplicate", + flags: []string{"--min-compatibility-version=1.30", "--min-compatibility-version=kube=1.30"}, + parseError: "duplicate version flag, kube=1.30 and kube=1.30", + }, + { + name: "min compatibility version unregistered component", + flags: []string{"--min-compatibility-version=test3=1.31"}, + parseError: "component not registered: test3", + }, + { + name: "invalid min compatibility version", + flags: []string{"--min-compatibility-version=test=1.foo"}, + parseError: "illegal version string \"1.foo\"", + }, { name: "setting test feature flag", flags: []string{ @@ -332,6 +366,12 @@ func TestFlags(t *testing.T) { if len(test.expectedTestEmulationVersion) > 0 { assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).EmulationVersion(), test.expectedTestEmulationVersion) } + if len(test.expectedKubeMinCompatibilityVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(DefaultKubeComponent).MinCompatibilityVersion(), test.expectedKubeMinCompatibilityVersion) + } + if len(test.expectedTestMinCompatibilityVersion) > 0 { + assertVersionEqualTo(t, r.EffectiveVersionFor(testComponent).MinCompatibilityVersion(), test.expectedTestMinCompatibilityVersion) + } for f, v := range test.expectedKubeFeatureValues { if r.FeatureGateFor(DefaultKubeComponent).Enabled(f) != v { t.Errorf("%d: expected kube feature Enabled(%s)=%v", i, f, v) @@ -357,25 +397,31 @@ func TestVersionMapping(t *testing.T) { utilruntime.Must(r.Register("test3", ver3, nil)) assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").MinCompatibilityVersion(), "0.57") assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").MinCompatibilityVersion(), "1.27") assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").MinCompatibilityVersion(), "2.9") - utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + utilruntime.Must(r.SetVersionMapping("test2", "test3", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()+1, from.Minor()-19) })) - utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + utilruntime.Must(r.SetVersionMapping("test1", "test2", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()+1, from.Minor()-28) })) assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").MinCompatibilityVersion(), "0.57") assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.30") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").MinCompatibilityVersion(), "1.29") assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.11") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").MinCompatibilityVersion(), "2.10") fs := pflag.NewFlagSet("testflag", pflag.ContinueOnError) r.AddFlags(fs) - if err := fs.Parse([]string{fmt.Sprintf("--emulated-version=%s", "test1=0.56")}); err != nil { + if err := fs.Parse([]string{"--emulated-version=test1=0.56", "--min-compatibility-version=test1=0.54"}); err != nil { t.Fatal(err) return } @@ -384,8 +430,11 @@ func TestVersionMapping(t *testing.T) { return } assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.56") + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").MinCompatibilityVersion(), "0.54") assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") - assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.09") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").MinCompatibilityVersion(), "1.26") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.9") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").MinCompatibilityVersion(), "2.7") } func TestVersionMappingWithMultipleDependency(t *testing.T) { @@ -401,12 +450,15 @@ func TestVersionMappingWithMultipleDependency(t *testing.T) { assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").MinCompatibilityVersion(), "0.57") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").MinCompatibilityVersion(), "1.27") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").MinCompatibilityVersion(), "2.9") - utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + utilruntime.Must(r.SetVersionMapping("test1", "test2", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()+1, from.Minor()-28) })) - err := r.SetEmulationVersionMapping("test3", "test2", + err := r.SetVersionMapping("test3", "test2", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()-1, from.Minor()+19) }) @@ -428,16 +480,19 @@ func TestVersionMappingWithCyclicDependency(t *testing.T) { assertVersionEqualTo(t, r.EffectiveVersionFor("test1").EmulationVersion(), "0.58") assertVersionEqualTo(t, r.EffectiveVersionFor("test2").EmulationVersion(), "1.28") assertVersionEqualTo(t, r.EffectiveVersionFor("test3").EmulationVersion(), "2.10") + assertVersionEqualTo(t, r.EffectiveVersionFor("test1").MinCompatibilityVersion(), "0.57") + assertVersionEqualTo(t, r.EffectiveVersionFor("test2").MinCompatibilityVersion(), "1.27") + assertVersionEqualTo(t, r.EffectiveVersionFor("test3").MinCompatibilityVersion(), "2.9") - utilruntime.Must(r.SetEmulationVersionMapping("test1", "test2", + utilruntime.Must(r.SetVersionMapping("test1", "test2", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()+1, from.Minor()-28) })) - utilruntime.Must(r.SetEmulationVersionMapping("test2", "test3", + utilruntime.Must(r.SetVersionMapping("test2", "test3", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()+1, from.Minor()-19) })) - err := r.SetEmulationVersionMapping("test3", "test1", + err := r.SetVersionMapping("test3", "test1", func(from *version.Version) *version.Version { return version.MajorMinor(from.Major()-2, from.Minor()+48) }) diff --git a/compatibility/version_test.go b/compatibility/version_test.go index e7042778..0c1fc3e7 100644 --- a/compatibility/version_test.go +++ b/compatibility/version_test.go @@ -147,6 +147,67 @@ func TestSetEmulationVersion(t *testing.T) { } } +func TestSetMinCompatibilityVersion(t *testing.T) { + tests := []struct { + name string + binaryVersion string + emulationVersion string + minCompatibilityVersion string + emulationVersionFloor string + minCompatibilityVersionFloor string + }{ + { + name: "normal case", + binaryVersion: "v1.34", + emulationVersion: "v1.32", + minCompatibilityVersion: "v1.30", + emulationVersionFloor: "v1.31", + minCompatibilityVersionFloor: "v1.30", + }, + { + name: "minCompatibilityVersion equal to emulationVersion", + binaryVersion: "v1.34", + emulationVersion: "v1.34", + minCompatibilityVersion: "v1.34", + emulationVersionFloor: "v1.31", + minCompatibilityVersionFloor: "v1.31", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + effective := NewEffectiveVersionFromString(test.binaryVersion, test.emulationVersionFloor, test.minCompatibilityVersionFloor) + + emulationVersion := version.MustParseGeneric(test.emulationVersion) + effective.SetEmulationVersion(emulationVersion) + if !effective.EmulationVersion().EqualTo(emulationVersion) { + t.Errorf("expected emulationVersion %s, got %s", emulationVersion.String(), effective.EmulationVersion().String()) + } + minCompatibilityVersion := version.MustParseGeneric(test.minCompatibilityVersion) + effective.SetMinCompatibilityVersion(minCompatibilityVersion) + if !effective.MinCompatibilityVersion().EqualTo(minCompatibilityVersion) { + t.Errorf("expected minCompatibilityVersion %s, got %s", minCompatibilityVersion.String(), effective.MinCompatibilityVersion().String()) + } + // verify emulationVersion is not changed + if !effective.EmulationVersion().EqualTo(emulationVersion) { + t.Errorf("expected emulationVersion %s, got %s", emulationVersion.String(), effective.EmulationVersion().String()) + } + errs := effective.Validate() + if len(errs) > 0 { + t.Fatalf("expected no Validate errors, errors found %+v", errs) + return + } + // if SetEmulationVersion is called again, it would change the minCompatibilityVersion back to emulationVersion - 1 + effective.SetEmulationVersion(emulationVersion) + if !effective.EmulationVersion().EqualTo(emulationVersion) { + t.Errorf("expected emulationVersion %s, got %s", emulationVersion.String(), effective.EmulationVersion().String()) + } + if !effective.MinCompatibilityVersion().EqualTo(emulationVersion.SubtractMinor(1)) { + t.Errorf("expected minCompatibilityVersion %s, got %s", emulationVersion.SubtractMinor(1).String(), effective.MinCompatibilityVersion().String()) + } + }) + } +} + func TestInfo(t *testing.T) { tests := []struct { name string diff --git a/config/testing/apigroup.go b/config/testing/apigroup.go index ead01ad2..863cab16 100644 --- a/config/testing/apigroup.go +++ b/config/testing/apigroup.go @@ -22,6 +22,8 @@ import ( "regexp" "strings" + "golang.org/x/text/cases" + "golang.org/x/text/language" apinamingtest "k8s.io/apimachinery/pkg/api/apitesting/naming" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -201,7 +203,7 @@ func dashesToCapitalCase(str string) string { segments := strings.Split(str, "-") result := "" for _, segment := range segments { - result += strings.Title(segment) + result += cases.Title(language.English).String(segment) } return result } diff --git a/config/v1alpha1/doc.go b/config/v1alpha1/doc.go index 7e4d6e89..d1ef3491 100644 --- a/config/v1alpha1/doc.go +++ b/config/v1alpha1/doc.go @@ -16,5 +16,7 @@ limitations under the License. // +k8s:deepcopy-gen=package // +k8s:conversion-gen=k8s.io/component-base/config +// +k8s:openapi-gen=true +// +k8s:openapi-model-package=io.k8s.component-base.config.v1alpha1 package v1alpha1 diff --git a/config/v1alpha1/zz_generated.model_name.go b/config/v1alpha1/zz_generated.model_name.go new file mode 100644 index 00000000..587e6380 --- /dev/null +++ b/config/v1alpha1/zz_generated.model_name.go @@ -0,0 +1,37 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by openapi-gen. DO NOT EDIT. + +package v1alpha1 + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in ClientConnectionConfiguration) OpenAPIModelName() string { + return "io.k8s.component-base.config.v1alpha1.ClientConnectionConfiguration" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in DebuggingConfiguration) OpenAPIModelName() string { + return "io.k8s.component-base.config.v1alpha1.DebuggingConfiguration" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in LeaderElectionConfiguration) OpenAPIModelName() string { + return "io.k8s.component-base.config.v1alpha1.LeaderElectionConfiguration" +} diff --git a/featuregate/feature_gate.go b/featuregate/feature_gate.go index 30e430d5..7c18ef4a 100644 --- a/featuregate/feature_gate.go +++ b/featuregate/feature_gate.go @@ -19,7 +19,9 @@ package featuregate import ( "context" "fmt" + "maps" "reflect" + "slices" "sort" "strconv" "strings" @@ -63,7 +65,7 @@ var ( } // Special handling for a few gates. - specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version){ + specialFeatures = map[Feature]func(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, emuVer, minCompatVer *version.Version){ allAlphaGate: setUnsetAlphaGates, allBetaGate: setUnsetBetaGates, } @@ -80,6 +82,39 @@ type FeatureSpec struct { // If multiple FeatureSpecs exist for a Feature, the one with the highest version that is less // than or equal to the effective version of the component is used. Version *version.Version + // MinCompatibilityVersion indicates the lowest version that this feature spec is compatible with. + // The component's minimum compatibility version must be greater than or equal to this version for this spec to be used. + // This allows features with skew compatibility implications to be introduced as Beta, + // but only on by default once the minimum compatibility version is high enough. + // If unspecified, it is inherited from the previous feature stage. + // If specified, it must be preceded by another FeatureSpec with the same version and different default but without the MinCompatibilityVersion. + // i.e. MinCompatibilityVersion should only be used to specify a different default value at the same version. + // + // Version vs. MinCompatibilityVersion: + // - Version determines the stability of the feature based on the server's emulation version. + // - MinCompatibilityVersion adds a further check based on the version compatibility + // with all servers in the control plane this server expects to communicate with. + // + // Example: + // FeatureA: []FeatureSpec{ + // {Version: version.MustParse("1.35"), Default: false, PreRelease: Beta}, + // {Version: version.MustParse("1.35"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.35")}, + // } + // In a server running version 1.35: + // - With --min-compatibility-version=1.34 (default), FeatureA is Beta and disabled by default. + // - With --min-compatibility-version=1.35, FeatureA becomes enabled by default. + MinCompatibilityVersion *version.Version +} + +func (spec *FeatureSpec) HelpString(featureName Feature, includeCompatVer bool) string { + if spec.PreRelease == GA || spec.PreRelease == Deprecated || spec.PreRelease == PreAlpha { + return "" + } + s := fmt.Sprintf("%s=true|false (%s - default=%t)", featureName, spec.PreRelease, spec.Default) + if includeCompatVer && spec.MinCompatibilityVersion != nil { + return s + fmt.Sprintf(" if --min-compatibility-version>=%s", spec.MinCompatibilityVersion) + } + return s } type VersionedSpecs []FeatureSpec @@ -111,6 +146,8 @@ type FeatureGate interface { Enabled(key Feature) bool // KnownFeatures returns a slice of strings describing the FeatureGate's known features. KnownFeatures() []string + // Dependencies returns a copy of the known feature dependencies. + Dependencies() map[Feature][]Feature // DeepCopy returns a deep copy of the FeatureGate object, such that gates can be // set on the copy without mutating the original. This is useful for validating // config against potential feature gate changes before committing those changes. @@ -161,14 +198,26 @@ type MutableVersionedFeatureGate interface { // If set, the feature gate would enable/disable features based on // feature availability and pre-release at the emulated version instead of the binary version. EmulationVersion() *version.Version - // SetEmulationVersion overrides the emulationVersion of the feature gate. + // MinCompatibilityVersion returns the minimum version of all control plane components + // that the current server must maintain compatibility with. + // This is used to gate features that have cross-component compatibility concerns, for example, during cluster upgrades/rollbacks. + // If not explicitly set, it defaults to one minor version prior to EmulationVersion(). + // A feature stage is disabled if its FeatureSpec.MinCompatibilityVersion is greater than this version. + MinCompatibilityVersion() *version.Version + // SetEmulationVersion overrides the emulationVersion of the feature gate, and + // overrides the minCompatibilityVersion to 1 minor before emulationVersion.` // Otherwise, the emulationVersion will be the same as the binary version. // If set, the feature defaults and availability will be as if the binary is at the emulated version. SetEmulationVersion(emulationVersion *version.Version) error + // SetEmulationVersion overrides the emulationVersion and minCompatibilityVersion of the feature gate. + SetEmulationVersionAndMinCompatibilityVersion(emulationVersion *version.Version, minCompatibilityVersion *version.Version) error // GetAll returns a copy of the map of known feature names to versioned feature specs. GetAllVersioned() map[Feature]VersionedSpecs // AddVersioned adds versioned feature specs to the featureGate. AddVersioned(features map[Feature]VersionedSpecs) error + // AddDependencies marks features that depend on other features. Must be called after all + // referenced features have already been added (dependents & depnedencies). Cycles are forbidden. + AddDependencies(features map[Feature][]Feature) error // OverrideDefaultAtVersion sets a local override for the registered default value of a named // feature for the prerelease lifecycle the given version is at. // If the feature has not been previously registered (e.g. by a call to Add), @@ -196,14 +245,15 @@ type MutableVersionedFeatureGate interface { type featureGate struct { featureGateName string - special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version) + special map[Feature]func(map[Feature]VersionedSpecs, map[Feature]bool, bool, *version.Version, *version.Version) // lock guards writes to all below fields. lock sync.Mutex // known holds a map[Feature]FeatureSpec known atomic.Value // enabled holds a map[Feature]bool - enabled atomic.Value + enabled atomic.Value + dependencies atomic.Pointer[map[Feature][]Feature] // enabledRaw holds a raw map[string]bool of the parsed flag. // It keeps the original values of "special" features like "all alpha gates", // while enabled keeps the values of all resolved features. @@ -212,16 +262,17 @@ type featureGate struct { closed bool // queriedFeatures stores all the features that have been queried through the Enabled interface. // It is reset when SetEmulationVersion is called. - queriedFeatures atomic.Value - emulationVersion atomic.Pointer[version.Version] + queriedFeatures atomic.Value + emulationVersion atomic.Pointer[version.Version] + minCompatibilityVersion atomic.Pointer[version.Version] } -func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { +func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, emuVer, minCompatVer *version.Version) { for k, v := range known { if k == "AllAlpha" || k == "AllBeta" { continue } - featureSpec := featureSpecAtEmulationVersion(v, cVer) + featureSpec := featureSpecAtEmulationAndMinCompatVersion(v, emuVer, minCompatVer) if featureSpec.PreRelease == Alpha { if _, found := enabled[k]; !found { enabled[k] = val @@ -230,12 +281,12 @@ func setUnsetAlphaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bo } } -func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, cVer *version.Version) { +func setUnsetBetaGates(known map[Feature]VersionedSpecs, enabled map[Feature]bool, val bool, emuVer, minCompatVer *version.Version) { for k, v := range known { if k == "AllAlpha" || k == "AllBeta" { continue } - featureSpec := featureSpecAtEmulationVersion(v, cVer) + featureSpec := featureSpecAtEmulationAndMinCompatVersion(v, emuVer, minCompatVer) if featureSpec.PreRelease == Beta { if _, found := enabled[k]; !found { enabled[k] = val @@ -252,8 +303,13 @@ var _ pflag.Value = &featureGate{} var internalPackages = []string{"k8s.io/component-base/featuregate/feature_gate.go"} // NewVersionedFeatureGate creates a feature gate with the emulation version set to the provided version. +// Equivalent to calling NewVersionedFeatureGateWithMinCompatibility with emulationVersion, emulationVersion-1. // SetEmulationVersion can be called after to change emulation version to a desired value. func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { + return NewVersionedFeatureGateWithMinCompatibility(emulationVersion, emulationVersion.SubtractMinor(1)) +} + +func NewVersionedFeatureGateWithMinCompatibility(emulationVersion, minCompatibilityVersion *version.Version) *featureGate { known := map[Feature]VersionedSpecs{} for k, v := range defaultFeatures { known[k] = v @@ -264,9 +320,11 @@ func NewVersionedFeatureGate(emulationVersion *version.Version) *featureGate { special: specialFeatures, } f.known.Store(known) + f.dependencies.Store(new(map[Feature][]Feature)) f.enabled.Store(map[Feature]bool{}) f.enabledRaw.Store(map[string]bool{}) f.emulationVersion.Store(emulationVersion) + f.minCompatibilityVersion.Store(minCompatibilityVersion) f.queriedFeatures.Store(sets.Set[Feature]{}) return f } @@ -309,11 +367,11 @@ func (f *featureGate) Validate() []error { return []error{fmt.Errorf("cannot cast enabledRaw to map[string]bool")} } enabled := map[Feature]bool{} - return f.unsafeSetFromMap(enabled, m, f.EmulationVersion()) + return f.unsafeSetFromMap(enabled, m, f.EmulationVersion(), f.MinCompatibilityVersion()) } // unsafeSetFromMap stores flag gates for known features from a map[string]bool into an enabled map. -func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool, emulationVersion *version.Version) []error { +func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bool, emulationVersion, minCompatibilityVersion *version.Version) []error { var errs []error // Copy existing state known := map[Feature]VersionedSpecs{} @@ -329,19 +387,19 @@ func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bo errs = append(errs, fmt.Errorf("unrecognized feature gate: %s", k)) return errs } - featureSpec := featureSpecAtEmulationVersion(versionedSpecs, emulationVersion) + featureSpec := featureSpecAtEmulationAndMinCompatVersion(versionedSpecs, emulationVersion, minCompatibilityVersion) if featureSpec.LockToDefault && featureSpec.Default != v { errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is locked to %v", k, v, featureSpec.Default)) continue } // Handle "special" features like "all alpha gates" if fn, found := f.special[key]; found { - fn(known, enabled, v, emulationVersion) + fn(known, enabled, v, emulationVersion, minCompatibilityVersion) enabled[key] = v continue } if featureSpec.PreRelease == PreAlpha { - errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s", k, v, emulationVersion.String())) + errs = append(errs, fmt.Errorf("cannot set feature gate %v to %v, feature is PreAlpha at emulated version %s, min compatibility version %s", k, v, emulationVersion, minCompatibilityVersion)) continue } enabled[key] = v @@ -352,6 +410,29 @@ func (f *featureGate) unsafeSetFromMap(enabled map[Feature]bool, m map[string]bo klog.Warningf("Setting GA feature gate %s=%t. It will be removed in a future release.", k, v) } } + + if len(errs) > 0 { + return errs + } + + // If enabled features were set successfully, validate them against the dependencies. + dependencies := *f.dependencies.Load() + for feature, deps := range dependencies { + if !featureEnabled(feature, enabled, known, emulationVersion, minCompatibilityVersion) { + continue + } + + var disabledDeps []Feature + for _, dep := range deps { + if !featureEnabled(dep, enabled, known, emulationVersion, minCompatibilityVersion) { + disabledDeps = append(disabledDeps, dep) + } + } + if len(disabledDeps) > 0 { + errs = append(errs, fmt.Errorf("%s is enabled, but depends on features that are disabled: %v", feature, disabledDeps)) + } + } + return errs } @@ -378,7 +459,7 @@ func (f *featureGate) SetFromMap(m map[string]bool) error { } f.enabledRaw.Store(enabledRaw) - errs := f.unsafeSetFromMap(enabled, enabledRaw, f.EmulationVersion()) + errs := f.unsafeSetFromMap(enabled, enabledRaw, f.EmulationVersion(), f.MinCompatibilityVersion()) if len(errs) == 0 { // Persist changes f.enabled.Store(enabled) @@ -424,31 +505,56 @@ func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { known := f.GetAllVersioned() for name, specs := range features { - if existingSpec, found := known[name]; found { - if reflect.DeepEqual(existingSpec, specs) { - continue - } - return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) - } + var completedSpecs VersionedSpecs // Validate new specs are well-formed - var lastVersion *version.Version - var wasBeta, wasGA, wasDeprecated bool + var lastSpec FeatureSpec + var wasBeta, wasGA, wasDeprecated, wasLockedToDefault bool for i, spec := range specs { if spec.Version == nil { return fmt.Errorf("feature %q did not provide a version", name) } if len(spec.Version.Components()) != 2 { - return fmt.Errorf("feature %q specified patch version: %s", name, spec.Version.String()) - + return fmt.Errorf("feature %q specified patch version: %s", name, spec.Version) + } + if spec.MinCompatibilityVersion != nil { + if i == 0 || !spec.Version.EqualTo(lastSpec.Version) { + return fmt.Errorf("feature %q specified MinCompatibilityVersion: %s without a preceding entry at the same Version without MinCompatibilityVersion set", name, spec.MinCompatibilityVersion) + } + if len(spec.MinCompatibilityVersion.Components()) != 2 { + return fmt.Errorf("feature %q specified patch MinCompatibilityVersion: %s", name, spec.MinCompatibilityVersion) + } + if spec.MinCompatibilityVersion.GreaterThan(spec.Version) { + return fmt.Errorf("feature %q MinCompatibilityVersion %s greater than Version %s", name, spec.MinCompatibilityVersion, spec.Version) + } } // gates that begin as deprecated must indicate their prior state if i == 0 && spec.PreRelease == Deprecated && spec.Version.Minor() != 0 { return fmt.Errorf("feature %q introduced as deprecated must provide a 1.0 entry indicating initial state", name) } if i > 0 { - // versions must strictly increase - if !lastVersion.LessThan(spec.Version) { - return fmt.Errorf("feature %q lists version transitions in non-increasing order (%s <= %s)", name, spec.Version, lastVersion) + // either versions or minCompatibilityVersions have to increase + if spec.Version.LessThan(lastSpec.Version) { + return fmt.Errorf("feature %q lists version transitions in decreasing order (%s < %s)", name, spec.Version, lastSpec.Version) + } + if spec.MinCompatibilityVersion != nil && lastSpec.MinCompatibilityVersion != nil && spec.MinCompatibilityVersion.LessThan(lastSpec.MinCompatibilityVersion) { + return fmt.Errorf("feature %q lists min compatibility version transitions in decreasing order (%s < %s)", name, spec.MinCompatibilityVersion, lastSpec.MinCompatibilityVersion) + } + if spec.Version.EqualTo(lastSpec.Version) { + if spec.MinCompatibilityVersion.EqualTo(lastSpec.MinCompatibilityVersion) { + return fmt.Errorf("feature %q lists duplicate entries with the same versions (%s = %s)", name, spec.Version, lastSpec.Version) + } + if spec.PreRelease != lastSpec.PreRelease { + return fmt.Errorf("feature %q lists stability changes with the same version (%s -> %s)", name, lastSpec.PreRelease, spec.PreRelease) + } + if spec.LockToDefault != lastSpec.LockToDefault { + return fmt.Errorf("feature %q lists locked to default changes with the same version (%v -> %v)", name, lastSpec.LockToDefault, spec.LockToDefault) + } + } + if spec.PreRelease == lastSpec.PreRelease && spec.Default == lastSpec.Default && spec.LockToDefault == lastSpec.LockToDefault { + return fmt.Errorf("feature %q lists transition without stability change (%s -> %s)", name, lastSpec.Version, spec.Version) + } + if wasLockedToDefault && !spec.LockToDefault { + return fmt.Errorf("feature %q must not unlock after locking to default", name) } // stability must not regress from ga --> {beta,alpha} or beta --> alpha, and // Deprecated state must be the terminal state @@ -461,25 +567,160 @@ func (f *featureGate) AddVersioned(features map[Feature]VersionedSpecs) error { return fmt.Errorf("feature %q regresses stability from more stable level to %s in %s", name, spec.PreRelease, spec.Version) } } - lastVersion = spec.Version + completedSpec := spec + // set MinCompatibilityVersion to the last MinCompatibilityVersion if unspecified. + // this makes sure MinCompatibilityVersion does not decrease. + // For example: + // FeatureA: { + // {Version: version.MustParse("1.35"), Default: false, PreRelease: Beta}, + // {Version: version.MustParse("1.35"), Default: true, PreRelease: Beta, MinCompatibilityVersion: 1.35}, + // {Version: version.MustParse("1.37"), Default: true, PreRelease: GA}, + // }, + // FeatureA should have MinCompatibilityVersion at least 1.35 at GA. + if completedSpec.MinCompatibilityVersion == nil && i > 0 { + completedSpec.MinCompatibilityVersion = lastSpec.MinCompatibilityVersion + } + completedSpecs = append(completedSpecs, completedSpec) + lastSpec = completedSpec wasBeta = wasBeta || spec.PreRelease == Beta wasGA = wasGA || spec.PreRelease == GA wasDeprecated = wasDeprecated || spec.PreRelease == Deprecated + wasLockedToDefault = wasLockedToDefault || spec.LockToDefault + } + if existingSpec, found := known[name]; found { + if reflect.DeepEqual(existingSpec, completedSpecs) { + continue + } + return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec) } - known[name] = specs + known[name] = completedSpecs } - // Persist updated state f.known.Store(known) return nil } +// AddDependencies adds feature gate dependencies. +func (f *featureGate) AddDependencies(dependencies map[Feature][]Feature) error { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return fmt.Errorf("cannot add a feature gate dependency after adding it to the flag set") + } + // Copy existing state + known := f.GetAllVersioned() + + // Merge existing dependencies in. + existing := *f.dependencies.Load() + for k, v := range existing { + dependencies[k] = append(dependencies[k], v...) + } + + // Sort and compact all dependency lists. + for k, v := range dependencies { + slices.Sort(v) + dependencies[k] = slices.Compact(v) + } + + // Validate dependencies for each emulated version: + // 1. Features & dependencies must be known + // 2. Features cannot depend on features with a lower prerelease level + // 3. Enabled features cannot depend on disabled features + // 4. Locked-to-default featurs cannot depend on unlocked features + for feature, deps := range dependencies { + versionedFeature, ok := known[feature] + if !ok { + return fmt.Errorf("cannot add dependency for unknown feature %s", feature) + } + + for _, dep := range deps { + versionedDep, ok := known[dep] + if !ok { + return fmt.Errorf("cannot add dependency from %s to unknown feature %s", feature, dep) + } + + // Check versions starting with the most recent for more intuitive error messages. + for _, spec := range slices.Backward(versionedFeature) { + // depSpec is the effective FeatureSpec for the dependency at the version declared by the dependent FeatureSpec. + depSpec := featureSpecAtEmulationAndMinCompatVersion(versionedDep, spec.Version, spec.MinCompatibilityVersion) + + if stabilityOrder(spec.PreRelease) > stabilityOrder(depSpec.PreRelease) { + return fmt.Errorf("%s feature %s cannot depend on %s feature %s at version %s", spec.PreRelease, feature, depSpec.PreRelease, dep, spec.Version) + } + if spec.Default && !depSpec.Default { + return fmt.Errorf("default-enabled feature %s cannot depend on default-disabled feature %s at version %s", feature, dep, spec.Version) + } + if spec.LockToDefault && !depSpec.LockToDefault { + return fmt.Errorf("locked-to-default feature %s cannot depend on unlocked feature %s at version %s", feature, dep, spec.Version) + } + } + } + } + + // Check for cycles + visited := map[Feature]bool{} + finished := map[Feature]bool{} + var detectCycles func(Feature) error + detectCycles = func(feature Feature) error { + if finished[feature] { + return nil + } + if visited[feature] { + return fmt.Errorf("cycle detected with feature %s", feature) + } + visited[feature] = true + for _, dep := range dependencies[feature] { + if err := detectCycles(dep); err != nil { + return err + } + } + finished[feature] = true + return nil + } + for feature := range dependencies { + if err := detectCycles(feature); err != nil { + return err + } + } + + // Persist updated state + f.dependencies.Store(&dependencies) + + return nil +} + +// stabilityOrder converts prereleases to a numerical value for dependency comparison. +// Features cannot depend on features with a lower prerelease level. +func stabilityOrder(p prerelease) int { + switch p { + case Deprecated: + return 0 // Non-deprecated features cannot depend on deprecated features. + case PreAlpha, Alpha: // Alpha features are allowed to depend on pre-alpha features. + return 1 + case Beta: + return 2 + case GA: + return 3 + default: + return -1 // Unknown prerelease + } +} + +func (f *featureGate) Dependencies() map[Feature][]Feature { + return maps.Clone(*f.dependencies.Load()) +} + func (f *featureGate) OverrideDefault(name Feature, override bool) error { - return f.OverrideDefaultAtVersion(name, override, f.EmulationVersion()) + return f.overrideDefaultAtEmulationAndMinCompatVersion(name, override, f.EmulationVersion(), f.MinCompatibilityVersion()) } func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver *version.Version) error { + return f.overrideDefaultAtEmulationAndMinCompatVersion(name, override, ver, nil) +} + +func (f *featureGate) overrideDefaultAtEmulationAndMinCompatVersion(name Feature, override bool, emuVer, minCompatVer *version.Version) error { f.lock.Lock() defer f.lock.Unlock() @@ -494,12 +735,12 @@ func (f *featureGate) OverrideDefaultAtVersion(name Feature, override bool, ver if !ok { return fmt.Errorf("cannot override default: feature %q is not registered", name) } - spec := featureSpecAtEmulationVersion(specs, ver) + spec := featureSpecAtEmulationAndMinCompatVersion(specs, emuVer, minCompatVer) switch { case spec.LockToDefault: return fmt.Errorf("cannot override default: feature %q default is locked to %t", name, spec.Default) case spec.PreRelease == PreAlpha: - return fmt.Errorf("cannot override default: feature %q is not available before version %s", name, ver.String()) + return fmt.Errorf("cannot override default: feature %q is not available before version %s", name, emuVer) case spec.PreRelease == Deprecated: klog.Warningf("Overriding default of deprecated feature gate %s=%t. It will be removed in a future release.", name, override) case spec.PreRelease == GA: @@ -519,9 +760,10 @@ func (f *featureGate) GetAll() map[Feature]FeatureSpec { f.lock.Lock() versionedSpecs := f.GetAllVersioned() emuVer := f.EmulationVersion() + minCompatVer := f.MinCompatibilityVersion() f.lock.Unlock() for k, v := range versionedSpecs { - spec := featureSpecAtEmulationVersion(v, emuVer) + spec := featureSpecAtEmulationAndMinCompatVersion(v, emuVer, minCompatVer) if spec.PreRelease == PreAlpha { // The feature is not available at the emulation version. continue @@ -543,12 +785,16 @@ func (f *featureGate) GetAllVersioned() map[Feature]VersionedSpecs { } func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) error { - if emulationVersion.EqualTo(f.EmulationVersion()) { + return f.SetEmulationVersionAndMinCompatibilityVersion(emulationVersion, emulationVersion.SubtractMinor(1)) +} + +func (f *featureGate) SetEmulationVersionAndMinCompatibilityVersion(emulationVersion *version.Version, minCompatibilityVersion *version.Version) error { + if emulationVersion.EqualTo(f.EmulationVersion()) && minCompatibilityVersion.EqualTo(f.MinCompatibilityVersion()) { return nil } f.lock.Lock() defer f.lock.Unlock() - klog.V(1).Infof("set feature gate emulationVersion to %s", emulationVersion.String()) + klog.V(1).Infof("set feature gate emulationVersion to %s, minCompatibilityVersion to %s", emulationVersion, minCompatibilityVersion) // Copy existing state enabledRaw := map[string]bool{} @@ -557,15 +803,15 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err } // enabled map should be reset whenever emulationVersion is changed. enabled := map[Feature]bool{} - errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion) + errs := f.unsafeSetFromMap(enabled, enabledRaw, emulationVersion, minCompatibilityVersion) queriedFeatures := f.queriedFeatures.Load().(sets.Set[Feature]) known := f.known.Load().(map[Feature]VersionedSpecs) for feature := range queriedFeatures { - newVal := featureEnabled(feature, enabled, known, emulationVersion) - oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion()) + newVal := featureEnabled(feature, enabled, known, emulationVersion, minCompatibilityVersion) + oldVal := featureEnabled(feature, f.enabled.Load().(map[Feature]bool), known, f.EmulationVersion(), f.MinCompatibilityVersion()) if newVal != oldVal { - klog.Warningf("SetEmulationVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal) + klog.Warningf("SetEmulationVersionAndMinCompatibilityVersion will change already queried feature:%s from %v to %v", feature, oldVal, newVal) } } @@ -573,6 +819,7 @@ func (f *featureGate) SetEmulationVersion(emulationVersion *version.Version) err // Persist changes f.enabled.Store(enabled) f.emulationVersion.Store(emulationVersion) + f.minCompatibilityVersion.Store(minCompatibilityVersion) f.queriedFeatures.Store(sets.Set[Feature]{}) } return utilerrors.NewAggregate(errs) @@ -582,11 +829,15 @@ func (f *featureGate) EmulationVersion() *version.Version { return f.emulationVersion.Load() } +func (f *featureGate) MinCompatibilityVersion() *version.Version { + return f.minCompatibilityVersion.Load() +} + // featureSpec returns the featureSpec at the EmulationVersion if the key exists, an error otherwise. // This is useful to keep multiple implementations of a feature based on the PreRelease or Version info. func (f *featureGate) featureSpec(key Feature) (FeatureSpec, error) { if v, ok := f.known.Load().(map[Feature]VersionedSpecs)[key]; ok { - featureSpec := f.featureSpecAtEmulationVersion(v) + featureSpec := f.featureSpecAtEmulationAndMinCompatVersion(v) return *featureSpec, nil } return FeatureSpec{}, fmt.Errorf("feature %q is not registered in FeatureGate %q", key, f.featureGateName) @@ -603,13 +854,14 @@ func (f *featureGate) unsafeRecordQueried(key Feature) { f.queriedFeatures.Store(newQueriedFeatures) } -func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion *version.Version) bool { +func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]VersionedSpecs, emulationVersion, minCompatibilityVersion *version.Version) bool { // check explicitly set enabled list if v, ok := enabled[key]; ok { return v } if v, ok := known[key]; ok { - return featureSpecAtEmulationVersion(v, emulationVersion).Default + featureSpec := featureSpecAtEmulationAndMinCompatVersion(v, emulationVersion, minCompatibilityVersion) + return featureSpec.Default } panic(fmt.Errorf("feature %q is not registered in FeatureGate", key)) @@ -618,21 +870,28 @@ func featureEnabled(key Feature, enabled map[Feature]bool, known map[Feature]Ver // Enabled returns true if the key is enabled. If the key is not known, this call will panic. func (f *featureGate) Enabled(key Feature) bool { // TODO: ideally we should lock the feature gate in this call to be safe, need to evaluate how much performance impact locking would have. - v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion()) + v := featureEnabled(key, f.enabled.Load().(map[Feature]bool), f.known.Load().(map[Feature]VersionedSpecs), f.EmulationVersion(), f.MinCompatibilityVersion()) f.unsafeRecordQueried(key) return v } -func (f *featureGate) featureSpecAtEmulationVersion(v VersionedSpecs) *FeatureSpec { - return featureSpecAtEmulationVersion(v, f.EmulationVersion()) +func (f *featureGate) featureSpecAtEmulationAndMinCompatVersion(v VersionedSpecs) *FeatureSpec { + return featureSpecAtEmulationAndMinCompatVersion(v, f.EmulationVersion(), f.MinCompatibilityVersion()) } -func featureSpecAtEmulationVersion(v VersionedSpecs, emulationVersion *version.Version) *FeatureSpec { +func featureSpecAtEmulationAndMinCompatVersion(v VersionedSpecs, emulationVersion, minCompatibilityVersion *version.Version) *FeatureSpec { i := len(v) - 1 + if minCompatibilityVersion == nil { + minCompatibilityVersion = emulationVersion.SubtractMinor(1) + } for ; i >= 0; i-- { if v[i].Version.GreaterThan(emulationVersion) { continue } + specMinCompatibilityVersion := v[i].MinCompatibilityVersion + if specMinCompatibilityVersion != nil && !minCompatibilityVersion.AtLeast(specMinCompatibilityVersion) { + continue + } return &v[i] } return &FeatureSpec{ @@ -678,11 +937,26 @@ func (f *featureGate) KnownFeatures() []string { known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, v[0].PreRelease, v[0].Default)) continue } - featureSpec := f.featureSpecAtEmulationVersion(v) - if featureSpec.PreRelease == GA || featureSpec.PreRelease == Deprecated || featureSpec.PreRelease == PreAlpha { + featureSpec := f.featureSpecAtEmulationAndMinCompatVersion(v) + featureSpecStr := featureSpec.HelpString(k, false) + featureSpecMinCompatMode := featureSpecAtEmulationAndMinCompatVersion(v, f.EmulationVersion(), f.EmulationVersion()) + featureSpecMinCompatModeStr := featureSpecMinCompatMode.HelpString(k, true) + if reflect.DeepEqual(featureSpec, featureSpecMinCompatMode) { + if featureSpecStr != "" { + known = append(known, featureSpecStr) + } continue } - known = append(known, fmt.Sprintf("%s=true|false (%s - default=%t)", k, featureSpec.PreRelease, featureSpec.Default)) + components := []string{} + if featureSpecStr != "" { + components = append(components, featureSpecStr) + } + if featureSpecMinCompatModeStr != "" { + components = append(components, featureSpecMinCompatModeStr) + } + if len(components) > 0 { + known = append(known, strings.Join(components, ", or ")) + } } sort.Strings(known) return known @@ -692,7 +966,7 @@ func (f *featureGate) KnownFeatures() []string { // and resets all the enabled status of the new feature gate. // This is useful for creating a new instance of feature gate without inheriting all the enabled configurations of the base feature gate. func (f *featureGate) DeepCopyAndReset() MutableVersionedFeatureGate { - fg := NewVersionedFeatureGate(f.EmulationVersion()) + fg := NewVersionedFeatureGateWithMinCompatibility(f.EmulationVersion(), f.MinCompatibilityVersion()) known := f.GetAllVersioned() fg.known.Store(known) return fg @@ -714,6 +988,10 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { for k, v := range f.enabledRaw.Load().(map[string]bool) { enabledRaw[k] = v } + dependencies := map[Feature][]Feature{} + for k, v := range *f.dependencies.Load() { + dependencies[k] = append([]Feature{}, v...) + } // Construct a new featureGate around the copied state. // Note that specialFeatures is treated as immutable by convention, @@ -723,8 +1001,10 @@ func (f *featureGate) DeepCopy() MutableVersionedFeatureGate { closed: f.closed, } fg.emulationVersion.Store(f.EmulationVersion()) + fg.minCompatibilityVersion.Store(f.MinCompatibilityVersion()) fg.known.Store(known) fg.enabled.Store(enabled) + fg.dependencies.Store(&dependencies) fg.enabledRaw.Store(enabledRaw) fg.queriedFeatures.Store(sets.Set[Feature]{}) return fg diff --git a/featuregate/feature_gate_test.go b/featuregate/feature_gate_test.go index ba626435..2dce1504 100644 --- a/featuregate/feature_gate_test.go +++ b/featuregate/feature_gate_test.go @@ -721,6 +721,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { const testLockedFalseGate Feature = "TestLockedFalse" const testAlphaGateNoVersion Feature = "TestAlphaNoVersion" const testBetaGateNoVersion Feature = "TestBetaNoVersion" + const testCompatibilityGate Feature = "TestCompatibility" tests := []struct { arg string @@ -738,6 +739,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { @@ -750,20 +752,11 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: true, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { - arg: "fooBarBaz=true", - expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: false, - testLockedFalseGate: false, - testAlphaGateNoVersion: false, - testBetaGateNoVersion: false, - }, + arg: "fooBarBaz=true", parseError: "unrecognized feature gate: fooBarBaz", }, { @@ -777,6 +770,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { @@ -790,6 +784,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: true, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { @@ -803,21 +798,12 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, parseError: "invalid value of AllAlpha", }, { - arg: "AllAlpha=false,TestAlpha=true,TestAlphaNoVersion=true", - expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: false, - testLockedFalseGate: false, - testAlphaGateNoVersion: true, - testBetaGateNoVersion: false, - }, + arg: "AllAlpha=false,TestAlpha=true,TestAlphaNoVersion=true", parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", }, { @@ -831,34 +817,15 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: true, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { - arg: "TestAlpha=true,TestAlphaNoVersion=true,AllAlpha=false", - expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: false, - testLockedFalseGate: false, - testAlphaGateNoVersion: true, - testBetaGateNoVersion: false, - }, + arg: "TestAlpha=true,TestAlphaNoVersion=true,AllAlpha=false", parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", }, { - arg: "AllAlpha=true,TestAlpha=false,TestAlphaNoVersion=false", - expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: true, - testLockedFalseGate: false, - testAlphaGateNoVersion: false, - testBetaGateNoVersion: false, - }, + arg: "AllAlpha=true,TestAlpha=false,TestAlphaNoVersion=false", parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", }, { @@ -872,20 +839,11 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { - arg: "TestAlpha=false,TestAlphaNoVersion=false,AllAlpha=true", - expect: map[Feature]bool{ - allAlphaGate: true, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: true, - testLockedFalseGate: false, - testAlphaGateNoVersion: false, - testBetaGateNoVersion: false, - }, + arg: "TestAlpha=false,TestAlphaNoVersion=false,AllAlpha=true", parseError: "cannot set feature gate TestAlpha to false, feature is PreAlpha at emulated version 1.28", }, { @@ -899,6 +857,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: true, + testCompatibilityGate: false, }, }, @@ -913,6 +872,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: false, }, }, { @@ -926,19 +886,11 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: true, + testCompatibilityGate: true, }, }, { - arg: "AllBeta=banana", - expect: map[Feature]bool{ - allAlphaGate: false, - allBetaGate: false, - testGAGate: false, - testAlphaGate: false, - testBetaGate: false, - testAlphaGateNoVersion: false, - testBetaGateNoVersion: false, - }, + arg: "AllBeta=banana", parseError: "invalid value of AllBeta", }, { @@ -952,6 +904,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: true, + testCompatibilityGate: false, }, }, { @@ -965,6 +918,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: true, + testCompatibilityGate: false, }, }, { @@ -978,6 +932,7 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: true, }, }, { @@ -991,21 +946,26 @@ func TestVersionedFeatureGateFlag(t *testing.T) { testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: true, }, }, { - arg: "TestAlpha=true,AllBeta=false", + arg: "TestAlpha=true,AllBeta=false", + parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", + }, + { + arg: "TestCompatibility=true", expect: map[Feature]bool{ allAlphaGate: false, allBetaGate: false, testGAGate: false, - testAlphaGate: true, + testAlphaGate: false, testBetaGate: false, testLockedFalseGate: false, testAlphaGateNoVersion: false, testBetaGateNoVersion: false, + testCompatibilityGate: true, }, - parseError: "cannot set feature gate TestAlpha to true, feature is PreAlpha at emulated version 1.28", }, } for i, test := range tests { @@ -1032,6 +992,11 @@ func TestVersionedFeatureGateFlag(t *testing.T) { {Version: version.MustParse("1.28"), Default: false, PreRelease: GA}, {Version: version.MustParse("1.29"), Default: false, PreRelease: GA, LockToDefault: true}, }, + testCompatibilityGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.28")}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA, LockToDefault: true}, + }, }) require.NoError(t, err) err = f.Add(map[Feature]FeatureSpec{ @@ -1181,6 +1146,8 @@ func TestVersionedFeatureGateKnownFeatures(t *testing.T) { testPreAlphaGate Feature = "TestPreAlpha" testAlphaGate Feature = "TestAlpha" testBetaGate Feature = "TestBeta" + testFastBetaGate Feature = "TestFastBeta" + testLatestFastBetaGate Feature = "TestLatestFastBeta" testGAGate Feature = "TestGA" testDeprecatedGate Feature = "TestDeprecated" testGAGateNoVersion Feature = "TestGANoVersion" @@ -1206,6 +1173,14 @@ func TestVersionedFeatureGateKnownFeatures(t *testing.T) { testBetaGate: { {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, }, + testFastBetaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.28")}, + }, + testLatestFastBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + }, testDeprecatedGate: { {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, {Version: version.MustParse("1.28"), Default: true, PreRelease: Deprecated}, @@ -1225,6 +1200,8 @@ func TestVersionedFeatureGateKnownFeatures(t *testing.T) { assert.NotContains(t, known, testPreAlphaGate) assert.Contains(t, known, testAlphaGate) assert.Contains(t, known, testBetaGate) + assert.Contains(t, known, "TestFastBeta=true|false (BETA - default=true) if --min-compatibility-version>=1.28") + assert.NotContains(t, known, testLatestFastBetaGate) assert.NotContains(t, known, testGAGate) assert.NotContains(t, known, testDeprecatedGate) assert.Contains(t, known, testAlphaGateNoVersion) @@ -1233,6 +1210,98 @@ func TestVersionedFeatureGateKnownFeatures(t *testing.T) { assert.NotContains(t, known, testDeprecatedGateNoVersion) } +func TestVersionedFeatureGateKnownFeaturesWithMinCompatVersion(t *testing.T) { + // gates for testing + const ( + testPreAlphaGate Feature = "TestPreAlpha" + testAlphaGate Feature = "TestAlpha" + testBetaGate Feature = "TestBeta" + testFastBetaGate Feature = "TestFastBeta" + testLatestFastBetaGate Feature = "TestLatestFastBeta" + testGAGate Feature = "TestGA" + testDeprecatedGate Feature = "TestDeprecated" + testGAGateNoVersion Feature = "TestGANoVersion" + testAlphaGateNoVersion Feature = "TestAlphaNoVersion" + testBetaGateNoVersion Feature = "TestBetaNoVersion" + testDeprecatedGateNoVersion Feature = "TestDeprecatedNoVersion" + ) + // Don't parse the flag, assert defaults are used. + f := NewVersionedFeatureGate(version.MustParse("1.29")) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testGAGate: { + {Version: version.MustParse("1.26"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.26"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.26")}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: GA}, + }, + testPreAlphaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + }, + testAlphaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Alpha}, + }, + testBetaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + }, + testFastBetaGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.28")}, + }, + testLatestFastBetaGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + testDeprecatedGate: { + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Deprecated}, + }, + }) + require.NoError(t, err) + err = f.Add(map[Feature]FeatureSpec{ + testAlphaGateNoVersion: {Default: false, PreRelease: Alpha}, + testBetaGateNoVersion: {Default: false, PreRelease: Beta}, + testGAGateNoVersion: {Default: false, PreRelease: GA}, + testDeprecatedGateNoVersion: {Default: false, PreRelease: Deprecated}, + }) + require.NoError(t, err) + + knownExpected := []string{ + "AllAlpha=true|false (ALPHA - default=false)", + "AllBeta=true|false (BETA - default=false)", + "TestAlpha=true|false (ALPHA - default=false)", + "TestAlphaNoVersion=true|false (ALPHA - default=false)", + "TestBeta=true|false (BETA - default=false)", + "TestBetaNoVersion=true|false (BETA - default=false)", + "TestFastBeta=true|false (BETA - default=true)", + "TestLatestFastBeta=true|false (BETA - default=false), or TestLatestFastBeta=true|false (BETA - default=true) if --min-compatibility-version>=1.29", + "TestPreAlpha=true|false (ALPHA - default=false)", + } + assert.ElementsMatch(t, f.KnownFeatures(), knownExpected) + + require.NoError(t, f.SetEmulationVersionAndMinCompatibilityVersion(version.MustParse("1.28"), version.MustParse("1.28"))) + knownExpected = []string{ + "AllAlpha=true|false (ALPHA - default=false)", + "AllBeta=true|false (BETA - default=false)", + "TestAlpha=true|false (ALPHA - default=false)", + "TestAlphaNoVersion=true|false (ALPHA - default=false)", + "TestBeta=true|false (BETA - default=false)", + "TestBetaNoVersion=true|false (BETA - default=false)", + "TestFastBeta=true|false (BETA - default=true)", + } + assert.ElementsMatch(t, f.KnownFeatures(), knownExpected) + + require.NoError(t, f.SetEmulationVersionAndMinCompatibilityVersion(version.MustParse("1.28"), version.MustParse("1.26"))) + knownExpected = []string{ + "AllAlpha=true|false (ALPHA - default=false)", + "AllBeta=true|false (BETA - default=false)", + "TestAlpha=true|false (ALPHA - default=false)", + "TestAlphaNoVersion=true|false (ALPHA - default=false)", + "TestBeta=true|false (BETA - default=false)", + "TestBetaNoVersion=true|false (BETA - default=false)", + "TestFastBeta=true|false (BETA - default=false), or TestFastBeta=true|false (BETA - default=true) if --min-compatibility-version>=1.28", + } + assert.ElementsMatch(t, f.KnownFeatures(), knownExpected) +} + func TestVersionedFeatureGateMetrics(t *testing.T) { // gates for testing featuremetrics.ResetFeatureInfoMetric() @@ -1311,11 +1380,11 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { require.NoError(t, f.SetEmulationVersion(version.MustParse("1.28"))) if err := f.AddVersioned(map[Feature]VersionedSpecs{ "TestFeature1": { - {Version: version.MustParse("1.28"), Default: true}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta}, }, "TestFeature2": { - {Version: version.MustParse("1.26"), Default: false}, - {Version: version.MustParse("1.29"), Default: false}, + {Version: version.MustParse("1.26"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, }, }); err != nil { t.Fatal(err) @@ -1527,47 +1596,148 @@ func TestVersionedFeatureGateOverrideDefault(t *testing.T) { }) } -func TestFeatureSpecAtEmulationVersion(t *testing.T) { +func TestFeatureGateMinCompatibilityVersion(t *testing.T) { + const testCompatGateA Feature = "TestCompatA" + const testCompatGateB Feature = "TestCompatB" + const testCompatGateC Feature = "TestCompatC" + + f := NewVersionedFeatureGateWithMinCompatibility(version.MustParse("1.30"), version.MustParse("1.27")) + err := f.AddVersioned(map[Feature]VersionedSpecs{ + testCompatGateA: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.30")}, + }, + testCompatGateB: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.30")}, + }, + testCompatGateC: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.28"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.28")}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA, LockToDefault: true}, + }, + }) + require.NoError(t, err) + + // KnownFeatures should show min-compatibility-version requirement. + knownExpected := []string{ + "AllAlpha=true|false (ALPHA - default=false)", + "AllBeta=true|false (BETA - default=false)", + "TestCompatA=true|false (BETA - default=false), or TestCompatA=true|false (BETA - default=true) if --min-compatibility-version>=1.30", + "TestCompatB=true|false (BETA - default=false), or TestCompatB=true|false (BETA - default=true) if --min-compatibility-version>=1.30", + "TestCompatC=true|false (BETA - default=false)", + } + assert.ElementsMatch(t, f.KnownFeatures(), knownExpected) + + // Gate is Alpha min compat version is 1.29, feature's is 1.30. Feature should be disabled. + assert.False(t, f.Enabled(testCompatGateA), "TestCompatA should be disabled at Alpha stage") + assert.False(t, f.Enabled(testCompatGateB), "TestCompatB should be disabled when its min compat version is too high") + assert.False(t, f.Enabled(testCompatGateC), "TestCompatC should be disabled when its min compat version is too high") + // Trying to override TestCompatA default should work. + require.NoError(t, f.OverrideDefault(testCompatGateA, true)) + assert.True(t, f.Enabled(testCompatGateA), "TestCompatA should be enabled after overriding default") + // Trying to enable TestCompatA should work. + require.NoError(t, f.Set(string(testCompatGateA)+"=false")) + assert.False(t, f.Enabled(testCompatGateA), "TestCompatA should be enabled after setting it explicitly") + // Trying to override TestCompatB, TestCompatC default should fail. + require.NoError(t, f.OverrideDefault(testCompatGateB, false)) + require.NoError(t, f.OverrideDefault(testCompatGateC, false)) + // Trying to enable TestCompatB, TestCompatC should work. + require.NoError(t, f.Set(string(testCompatGateB)+"=true")) + require.NoError(t, f.Set(string(testCompatGateC)+"=true")) + + // reset the feature gate first so no overrides or raw flags are carried over. + f = f.DeepCopyAndReset().(*featureGate) + // Now, update the gate's min compat version. + require.NoError(t, f.SetEmulationVersionAndMinCompatibilityVersion(version.MustParse("1.30"), version.MustParse("1.30"))) + // KnownFeatures should not show the min-compatibility-version requirement anymore. + knownExpected = []string{ + "AllAlpha=true|false (ALPHA - default=false)", + "AllBeta=true|false (BETA - default=false)", + "TestCompatA=true|false (BETA - default=true)", + "TestCompatB=true|false (BETA - default=true)", + } + assert.ElementsMatch(t, f.KnownFeatures(), knownExpected) + + // Feature should now be enabled by default. + assert.True(t, f.Enabled(testCompatGateA), "TestCompatA should be Beta and enabled when its min compat version is met") + assert.True(t, f.Enabled(testCompatGateB), "TestCompatB should be enabled when its min compat version is met") + assert.True(t, f.Enabled(testCompatGateC), "TestCompatC should be enabled when its min compat version is met") + + // Trying to disable feature should succeed. + require.NoError(t, f.Set(string(testCompatGateA)+"=false")) + assert.False(t, f.Enabled(testCompatGateA), "feature should be disabled") + require.NoError(t, f.Set(string(testCompatGateB)+"=false")) + assert.False(t, f.Enabled(testCompatGateB), "feature should be disabled") + + require.NoError(t, f.ResetFeatureValueToDefault(testCompatGateB)) + assert.True(t, f.Enabled(testCompatGateB), "feature should be enabled after resetting to default") + // Trying to override default should now succeed. + require.NoError(t, f.OverrideDefault(testCompatGateB, false)) + assert.False(t, f.Enabled(testCompatGateB), "feature should be disabled by default after override default to false") + // Trying to override GA default should fail. + require.Error(t, f.OverrideDefault(testCompatGateC, false)) + require.Error(t, f.Set(string(testCompatGateC)+"=false")) +} + +func TestFeatureSpecAtEmulationAndMinCompatVersion(t *testing.T) { specs := VersionedSpecs{ {Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, - {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, - {Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + {Version: version.MustParse("1.25"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.25")}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA, MinCompatibilityVersion: version.MustParse("1.25")}, } sort.Sort(specs) tests := []struct { - cVersion string - expect FeatureSpec + emuVer *version.Version + minCompatVer *version.Version + expect FeatureSpec }{ { - cVersion: "1.30", - expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + emuVer: version.MustParse("1.30"), + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA, MinCompatibilityVersion: version.MustParse("1.25")}, }, { - cVersion: "1.29", - expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA}, + emuVer: version.MustParse("1.29"), + expect: FeatureSpec{Version: version.MustParse("1.29"), Default: true, PreRelease: GA, MinCompatibilityVersion: version.MustParse("1.25")}, }, { - cVersion: "1.28", - expect: FeatureSpec{Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + emuVer: version.MustParse("1.28"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.25")}, }, { - cVersion: "1.27", - expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + emuVer: version.MustParse("1.26"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.25")}, }, { - cVersion: "1.25", - expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + emuVer: version.MustParse("1.25"), + minCompatVer: version.MustParse("1.25"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.25")}, }, { - cVersion: "1.24", - expect: FeatureSpec{Version: version.MajorMinor(0, 0), Default: false, PreRelease: PreAlpha}, + emuVer: version.MustParse("1.25"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + emuVer: version.MustParse("1.24"), + expect: FeatureSpec{Version: version.MajorMinor(0, 0), Default: false, PreRelease: PreAlpha}, + }, + { + emuVer: version.MustParse("1.26"), + minCompatVer: version.MustParse("1.23"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, + }, + { + emuVer: version.MustParse("1.29"), + minCompatVer: version.MustParse("1.23"), + expect: FeatureSpec{Version: version.MustParse("1.25"), Default: false, PreRelease: Alpha}, }, } for i, test := range tests { - t.Run(fmt.Sprintf("featureSpecAtEmulationVersion for emulationVersion %s", test.cVersion), func(t *testing.T) { - result := featureSpecAtEmulationVersion(specs, version.MustParse(test.cVersion)) + t.Run(fmt.Sprintf("featureSpecAtEmulationVersion for emulationVersion %s", test.emuVer), func(t *testing.T) { + result := featureSpecAtEmulationAndMinCompatVersion(specs, test.emuVer, test.minCompatVer) if !reflect.DeepEqual(*result, test.expect) { - t.Errorf("%d: featureSpecAtEmulationVersion(, %s) Expected %v, Got %v", i, test.cVersion, test.expect, result) + t.Errorf("%d: featureSpecAtEmulationVersion(, %s) Expected %v, Got %v", i, test.emuVer, test.expect, result) } }) } @@ -1884,6 +2054,191 @@ func TestAddVersioned(t *testing.T) { }, }, }, + { + name: "patch min compat version", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.30"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.30.1")}, + }, + }, + }, + { + name: "Alpha to GA", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA}, + }, + }, + }, + { + name: "GA with default false", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: GA}, + }, + }, + }, + { + name: "Beta to Deprecated", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: Deprecated}, + }, + }, + }, + { + name: "Alpha to Deprecated", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: Deprecated}, + }, + }, + }, + { + name: "MinCompatibilityVersion > Version", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.30")}, + }, + }, + }, + { + name: "decreasing minCompatibilityVersion", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Alpha, MinCompatibilityVersion: version.MustParse("1.28")}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.27")}, + }, + }, + }, + { + name: "transition without stability change", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.30"), Default: false, PreRelease: Alpha}, + }, + }, + }, + { + name: "resurrect from locked", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: GA, LockToDefault: true}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA, LockToDefault: false}, + }, + }, + }, + { + name: "no version provided", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Default: false, PreRelease: Alpha}, + }, + }, + }, + { + name: "direct to Beta with minCompatibilityVersion, no preceding entry", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + }, + }, + { + name: "direct to Beta with minCompatibilityVersion", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + }, + }, + { + name: "minCompatibilityVersion greater than version", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.30")}, + }, + }, + }, + { + name: "same version with different minCompatibilityVersion", + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA}, + }, + }, + }, + { + name: "different version with different minCompatibilityVersion", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.28"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA}, + }, + }, + }, + { + name: "same version with different stability", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + {Version: version.MustParse("1.30"), Default: true, PreRelease: GA}, + }, + }, + }, + { + name: "same version with different lockToDefault", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Beta}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, LockToDefault: true, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + }, + }, + { + name: "only minCompatibilityVersion different", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha}, + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + }, + }, + { + name: "multiple entries for the same version and minCompatibilityVersion", + expectError: true, + features: map[Feature]VersionedSpecs{ + testAGate: { + {Version: version.MustParse("1.29"), Default: false, PreRelease: Alpha, MinCompatibilityVersion: version.MustParse("1.29")}, + {Version: version.MustParse("1.29"), Default: true, PreRelease: Beta, MinCompatibilityVersion: version.MustParse("1.29")}, + }, + }, + }, } for _, test := range tests { t.Run(fmt.Sprintf("AddVersioned-%s", test.name), func(t *testing.T) { @@ -1905,3 +2260,321 @@ func TestAddVersioned(t *testing.T) { }) } } + +func TestAddDependencies(t *testing.T) { + const ( + // Test features + fA Feature = "FeatureA" + fB Feature = "FeatureB" + fC Feature = "FeatureC" + fUnknown Feature = "FeatureUnknown" + ) + + var ( + v129 = version.MustParse("1.29") + v130 = version.MustParse("1.30") + v131 = version.MustParse("1.31") + v132 = version.MustParse("1.32") + ) + + testCases := []struct { + name string + features map[Feature]VersionedSpecs + initialDeps map[Feature][]Feature + newDeps map[Feature][]Feature + expectedErr string + finalDeps map[Feature][]Feature + closeGate bool + }{ + { + name: "dependency on unknown feature", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{fA: {fUnknown}}, + expectedErr: "cannot add dependency from FeatureA to unknown feature FeatureUnknown", + }, + { + name: "unknown feature has dependency", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{fUnknown: {fA}}, + expectedErr: "cannot add dependency for unknown feature FeatureUnknown", + }, + { + name: "cycle", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + fB: {{Version: v129}}, + fC: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{ + fA: {fB}, + fB: {fC}, + fC: {fA}, + }, + expectedErr: "cycle detected with feature", + }, + { + name: "valid: no cycle with overlapping dependencies", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + fB: {{Version: v129}}, + fC: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{ + fA: {fB, fC}, + fB: {fC}, + }, + }, + { + name: "self cycle", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{fA: {fA}}, + expectedErr: "cycle detected with feature FeatureA", + }, + { + name: "merge dependencies", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + fB: {{Version: v129}}, + fC: {{Version: v129}}, + }, + initialDeps: map[Feature][]Feature{fA: {fB}}, + newDeps: map[Feature][]Feature{fA: {fC}}, + finalDeps: map[Feature][]Feature{fA: {fB, fC}}, + }, + { + name: "add after close", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129}}, + fB: {{Version: v129}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + closeGate: true, + expectedErr: "cannot add a feature gate dependency after adding it to the flag set", + }, + { + name: "valid: dependency is valid across all versions", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta}, {Version: v130, PreRelease: GA}}, + fB: {{Version: v129, PreRelease: Beta}, {Version: v130, PreRelease: GA}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + }, + { + name: "invalid: stability inversion", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta}}, + fB: {{Version: v129, PreRelease: Alpha}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "BETA feature FeatureA cannot depend on ALPHA feature FeatureB at version 1.29", + }, + { + name: "invalid: stability inversion at later version", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Alpha}, {Version: v130, PreRelease: Beta}}, + fB: {{Version: v129, PreRelease: Alpha}, {Version: v130, PreRelease: Alpha, Default: true}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "BETA feature FeatureA cannot depend on ALPHA feature FeatureB at version 1.30", + }, + { + name: "invalid: stability inversion at earlier version", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Alpha}, {Version: v130, PreRelease: Beta}, {Version: v132, PreRelease: GA}}, + fB: {{Version: v129, PreRelease: Alpha}, {Version: v131, PreRelease: Beta}, {Version: v132, PreRelease: GA}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "BETA feature FeatureA cannot depend on ALPHA feature FeatureB at version 1.30", + }, + { + name: "valid: alpha feature depending on pre-alpha", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Alpha}, {Version: v131, PreRelease: Beta}}, + fB: {{Version: v130, PreRelease: Alpha}, {Version: v131, PreRelease: Beta}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + }, + { + name: "valid: default-enabled depends on default-enabled", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta, Default: true}}, + fB: {{Version: v129, PreRelease: Beta, Default: true}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + }, + { + name: "valid: default-disabled depends on default-enabled", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta, Default: false}}, + fB: {{Version: v129, PreRelease: Beta, Default: true}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + }, + { + name: "invalid: default-enabled depends on default-disabled", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta, Default: true}}, + fB: {{Version: v129, PreRelease: Beta, Default: false}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "default-enabled feature FeatureA cannot depend on default-disabled feature FeatureB at version 1.29", + }, + { + name: "invalid: default-enabled depends on default-disabled at a later version", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Alpha, Default: false}, {Version: v130, PreRelease: Beta, Default: true}}, + fB: {{Version: v129, PreRelease: Alpha, Default: false}, {Version: v130, PreRelease: Beta, Default: false}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "default-enabled feature FeatureA cannot depend on default-disabled feature FeatureB at version 1.30", + }, + { + name: "invalid: default-enabled depends on default-disabled at an earlier version", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: Beta, Default: true}}, + fB: {{Version: v129, PreRelease: Beta, Default: false}, {Version: v130, PreRelease: Beta, Default: true}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "default-enabled feature FeatureA cannot depend on default-disabled feature FeatureB at version 1.29", + }, + { + name: "valid: locked depends on locked", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: true}}, + fB: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: true}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + }, + { + name: "invalid: locked depends on unlocked", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: true}}, + fB: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: false}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "locked-to-default feature FeatureA cannot depend on unlocked feature FeatureB at version 1.29", + }, + { + name: "invalid: locked depends on unlocked at a later version", + features: map[Feature]VersionedSpecs{ + fA: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: false}, {Version: v130, PreRelease: GA, Default: true, LockToDefault: true}}, + fB: {{Version: v129, PreRelease: GA, Default: true, LockToDefault: false}}, + }, + newDeps: map[Feature][]Feature{fA: {fB}}, + expectedErr: "locked-to-default feature FeatureA cannot depend on unlocked feature FeatureB at version 1.30", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.AddVersioned(tc.features)) + if tc.initialDeps != nil { + require.NoError(t, f.AddDependencies(tc.initialDeps)) + } + if tc.closeGate { + f.Close() + } + + err := f.AddDependencies(tc.newDeps) + + if tc.expectedErr != "" { + assert.ErrorContains(t, err, tc.expectedErr) + } else { + require.NoError(t, err) + + deps := f.Dependencies() + finalDeps := tc.finalDeps + if finalDeps == nil { + finalDeps = tc.newDeps + } + assert.Equal(t, finalDeps, deps) + + // Verify that DeepCopy clones dependencies. + clone := f.DeepCopy().Dependencies() + assert.Equal(t, deps, clone, "dependencies should identical after DeepCopy") + } + }) + } +} + +func TestValidateDependencies(t *testing.T) { + const ( + featureA Feature = "FeatureA" + featureB Feature = "FeatureB" + featureC Feature = "FeatureC" + featureD Feature = "FeatureD" + ) + + features := map[Feature]FeatureSpec{ + featureA: {Default: false, PreRelease: Alpha}, + featureB: {Default: false, PreRelease: Alpha}, + featureC: {Default: false, PreRelease: Alpha}, + featureD: {Default: false, PreRelease: Alpha}, + } + + dependencies := map[Feature][]Feature{ + featureA: {featureB, featureC}, + featureB: {featureD}, + } + + testCases := []struct { + name string + set string + expectedErr string + }{ + { + name: "all enabled", + set: "FeatureA=true,FeatureB=true,FeatureC=true,FeatureD=true", + }, + { + name: "one dependency disabled", + set: "FeatureA=true,FeatureB=false,FeatureC=true,FeatureD=true", + expectedErr: "FeatureA is enabled, but depends on features that are disabled: [FeatureB]", + }, + { + name: "another dependency disabled", + set: "FeatureA=true,FeatureB=true,FeatureC=false,FeatureD=true", + expectedErr: "FeatureA is enabled, but depends on features that are disabled: [FeatureC]", + }, + { + name: "multiple dependencies disabled", + set: "FeatureA=true,FeatureB=false,FeatureC=false,FeatureD=true", + expectedErr: "FeatureA is enabled, but depends on features that are disabled: [FeatureB FeatureC]", + }, + { + name: "transitive dependency disabled", + set: "FeatureA=true,FeatureB=true,FeatureC=true,FeatureD=false", + expectedErr: "FeatureB is enabled, but depends on features that are disabled: [FeatureD]", + }, + { + name: "feature disabled", + set: "FeatureA=false,FeatureB=false,FeatureC=true,FeatureD=true", + }, + { + name: "all disabled", + set: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + f := NewFeatureGate() + require.NoError(t, f.Add(features)) + require.NoError(t, f.AddDependencies(dependencies)) + + err := f.Set(tc.set) + if tc.expectedErr == "" { + require.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} diff --git a/featuregate/testing/feature_gate.go b/featuregate/testing/feature_gate.go index 1d7fc467..fa72773f 100644 --- a/featuregate/testing/feature_gate.go +++ b/featuregate/testing/feature_gate.go @@ -26,18 +26,21 @@ import ( ) var ( - overrideLock sync.Mutex - featureFlagOverride map[featuregate.Feature]string - emulationVersionOverride string - emulationVersionOverrideValue *version.Version + overrideLock sync.Mutex + featureFlagOverride map[featuregate.Feature]string + versionsOverride string + versionsOverrideValue string ) func init() { featureFlagOverride = map[featuregate.Feature]string{} } +type FeatureOverrides = map[featuregate.Feature]bool + // SetFeatureGateDuringTest sets the specified gate to the specified value for duration of the test. // Fails when it detects second call to the same flag or is unable to set or restore feature flag. +// When disabling a feature, this automatically disables all dependents. // // WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set the same feature flag will cause fatal. // @@ -46,52 +49,120 @@ func init() { // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true) func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { tb.Helper() - detectParallelOverrideCleanup := detectParallelOverride(tb, f) - originalValue := gate.Enabled(f) + SetFeatureGatesDuringTest(tb, gate, FeatureOverrides{f: value}) +} + +// SetFeatureGatesDuringTest sets the specified map of feature gate values for duration of the test. +// Fails when it detects second call to the same flag or is unable to set or restore feature flag. +// When disabling a feature, this automatically disables all dependents that weren't explicitly set. +// +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set the same feature flag will cause fatal. +// +// Example use: +// +// featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ +// features.: true, +// features.: false, +// }) +func SetFeatureGatesDuringTest(tb TB, gate featuregate.FeatureGate, features FeatureOverrides) { + tb.Helper() originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() - originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) + originalValues := map[string]bool{} + var originalUnset []featuregate.Feature + overrides := FeatureOverrides{} // Specially handle AllAlpha and AllBeta - if f == "AllAlpha" || f == "AllBeta" { + allAlphaValue, allAlpha := features["AllAlpha"] + allBetaValue, allBeta := features["AllBeta"] + if allAlpha || allBeta { // Iterate over individual gates so their individual values get restored for k, v := range gate.(featuregate.MutableFeatureGate).GetAll() { - if k == "AllAlpha" || k == "AllBeta" { - continue - } - if (f == "AllAlpha" && v.PreRelease == featuregate.Alpha) || (f == "AllBeta" && v.PreRelease == featuregate.Beta) { - SetFeatureGateDuringTest(tb, gate, k, value) + if (allAlpha && v.PreRelease == featuregate.Alpha) || (allBeta && v.PreRelease == featuregate.Beta) { + // Setting AllAlpha or AllBeta on their own only sets unset features, but for + // testing we want to override ALL alpha/beta features. So we explicitly set each + // alpha/beta feature in addition to AllAlpha/AllBeta + if v.PreRelease == featuregate.Alpha { + overrides[k] = allAlphaValue + } else { + overrides[k] = allBetaValue + } } } } - if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, value)); err != nil { - if s := suggestChangeEmulationVersion(tb, gate, f, value); s != "" { - tb.Errorf("error setting %s=%v: %v. %s", f, value, err, s) - } else { - tb.Errorf("error setting %s=%v: %v", f, value, err) + // Explicit features take precedence, so merge them in now. + for f, v := range features { + overrides[f] = v + } + + // Automatically disable dependents when disabling a dependency. + dependencies := gate.Dependencies() + for f := range dependencies { + if _, overridden := overrides[f]; overridden || !gate.Enabled(f) { + continue // Don't automatically disable features that have been explicitly set. + } + // If the feature gate was default-enabled and has an explicitly + // disabled dependency, then automatically disable it. + if disabled, disabledDep := hasDisabledDependency(f, dependencies, features); disabled { + tb.Logf("Disabling feature %s since it depends on disabled feature %s", f, disabledDep) + overrides[f] = false + } + } + + for f := range overrides { + originalValues[string(f)] = gate.Enabled(f) + if !gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) { + originalUnset = append(originalUnset, f) + } + tb.Cleanup(detectParallelOverride(tb, featuregate.Feature(f))) + } + + m := map[string]bool{} + for f, v := range overrides { + m[string(f)] = v + } + if err := gate.(featuregate.MutableFeatureGate).SetFromMap(m); err != nil { + tb.Errorf("Failed to set feature gates: %v", err) + for f, v := range features { + if s := suggestChangeEmulationVersion(tb, gate, f, v); s != "" { + tb.Errorf("error setting %s=%v: %v. %s", f, v, err, s) + } } } tb.Cleanup(func() { tb.Helper() - detectParallelOverrideCleanup() emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() if !emuVer.EqualTo(originalEmuVer) { tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests", originalEmuVer.String(), emuVer.String()) } - if originalExplicitlySet { - if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { - tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) - } - } else { + // To avoid violating feature dependencies, first atomicaly restore all original values, + // then reset features that were unset (the value should be unchanged). + if err := gate.(featuregate.MutableVersionedFeatureGate).SetFromMap(originalValues); err != nil { + tb.Errorf("error restoring features %v: %v", originalValues, err) + } + for _, f := range originalUnset { if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil { - tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + tb.Errorf("error resetting %s: %v", f, err) } } }) } +// hasDisabledDependency recursively walks the dependencies for feature f, and checks whether any are explicitly disabled in the features map. +func hasDisabledDependency(f featuregate.Feature, dependencies map[featuregate.Feature][]featuregate.Feature, features map[featuregate.Feature]bool) (bool, featuregate.Feature) { + if enabled, set := features[f]; set { + return !enabled, f + } + for _, dep := range dependencies[f] { + if disabled, disabledDep := hasDisabledDependency(dep, dependencies, features); disabled { + return disabled, disabledDep + } + } + return false, "" +} + func suggestChangeEmulationVersion(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) string { mutableVersionedFeatureGate, ok := gate.(featuregate.MutableVersionedFeatureGate) if !ok { @@ -114,28 +185,45 @@ func suggestChangeEmulationVersion(tb TB, gate featuregate.FeatureGate, f featur return "" } -// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. -// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. -// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// SetFeatureGateVersionsDuringTest sets the specified gate to the specified emulation version and min compatibility version for duration of the test. +// Fails when it detects second call to set a different emulation version or min compatibility version, or is unable to set or restore emulation version and min compatibility version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version or min compatibility version will cause fatal. // Example use: -// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) -func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { +// featuregatetesting.SetFeatureGateVersionsDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31"), version.MustParse("1.31")) +func SetFeatureGateVersionsDuringTest(tb TB, gate featuregate.FeatureGate, emuVer, minCompatVer *version.Version) { tb.Helper() - detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver) - originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() - if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil { - tb.Fatalf("failed to set emulation version to %s during test: %v", ver.String(), err) + versions := fmt.Sprintf("emu=%s,min=%s", emuVer.String(), minCompatVer.String()) + detectParallelOverrideCleanup := detectParallelOverrideVersions(tb, versions) + + mutableGate := gate.(featuregate.MutableVersionedFeatureGate) + originalEmuVer := mutableGate.EmulationVersion() + originalMinCompatVer := mutableGate.MinCompatibilityVersion() + + if err := mutableGate.SetEmulationVersionAndMinCompatibilityVersion(emuVer, minCompatVer); err != nil { + tb.Fatalf("failed to set versions (emu=%s, min=%s) during test: %v", emuVer.String(), minCompatVer.String(), err) } + tb.Cleanup(func() { tb.Helper() detectParallelOverrideCleanup() - if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil { - tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String()) + if err := mutableGate.SetEmulationVersionAndMinCompatibilityVersion(originalEmuVer, originalMinCompatVer); err != nil { + tb.Fatalf("failed to restore versions (emu=%s, min=%s) during test: %v", originalEmuVer.String(), originalMinCompatVer.String(), err) } }) } +// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. +// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// Example use: + +// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) +func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { + tb.Helper() + SetFeatureGateVersionsDuringTest(tb, gate, ver, ver.SubtractMinor(1)) +} + func detectParallelOverride(tb TB, f featuregate.Feature) func() { tb.Helper() overrideLock.Lock() @@ -157,30 +245,30 @@ func detectParallelOverride(tb TB, f featuregate.Feature) func() { } } -func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() { +func detectParallelOverrideVersions(tb TB, vers string) func() { tb.Helper() overrideLock.Lock() defer overrideLock.Unlock() - beforeOverrideTestName := emulationVersionOverride - beforeOverrideValue := emulationVersionOverrideValue - if ver.EqualTo(beforeOverrideValue) { + beforeOverrideTestName := versionsOverride + beforeOverrideValue := versionsOverrideValue + if vers == beforeOverrideValue { return func() {} } if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { - tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name()) + tb.Fatalf("Detected parallel setting of feature gate versions by both %q and %q", beforeOverrideTestName, tb.Name()) } - emulationVersionOverride = tb.Name() - emulationVersionOverrideValue = ver + versionsOverride = tb.Name() + versionsOverrideValue = vers return func() { tb.Helper() overrideLock.Lock() defer overrideLock.Unlock() - if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() { - tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name()) + if afterOverrideTestName := versionsOverride; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of feature gate versions between both %q and %q", afterOverrideTestName, tb.Name()) } - emulationVersionOverride = beforeOverrideTestName - emulationVersionOverrideValue = beforeOverrideValue + versionsOverride = beforeOverrideTestName + versionsOverrideValue = beforeOverrideValue } } @@ -191,6 +279,7 @@ func sameTestOrSubtest(tb TB, testName string) bool { type TB interface { Cleanup(func()) + Logf(format string, args ...any) Error(args ...any) Errorf(format string, args ...any) Fatal(args ...any) diff --git a/featuregate/testing/feature_gate_test.go b/featuregate/testing/feature_gate_test.go index 376d7213..627e86da 100644 --- a/featuregate/testing/feature_gate_test.go +++ b/featuregate/testing/feature_gate_test.go @@ -17,6 +17,7 @@ limitations under the License. package testing import ( + "maps" gotest "testing" "github.com/stretchr/testify/assert" @@ -159,6 +160,134 @@ func TestSetFeatureGateInTest(t *gotest.T) { assert.True(t, gate.Enabled("feature")) } +func TestSetFeatureGatesInTestWithDependencies(t *gotest.T) { + const ( + alphaFeature = "AlphaFeature" + alphaFeatureDep = "AlphaFeatureDep" + betaOffFeature = "BetaOffFeature" + betaOffFeatureDep = "BetaOffFeatureDep" + betaOnFeature = "BetaOnFeature" + betaOnFeatureDep = "BetaOnFeatureDep" + gaFeature = "GAFeature" + ) + gate := featuregate.NewFeatureGate() + require.NoError(t, gate.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + alphaFeature: {PreRelease: featuregate.Alpha, Default: false}, + alphaFeatureDep: {PreRelease: featuregate.Alpha, Default: false}, + betaOffFeature: {PreRelease: featuregate.Beta, Default: false}, + betaOffFeatureDep: {PreRelease: featuregate.Beta, Default: false}, + betaOnFeature: {PreRelease: featuregate.Beta, Default: true}, + betaOnFeatureDep: {PreRelease: featuregate.Beta, Default: true}, + gaFeature: {PreRelease: featuregate.GA, Default: true}, + })) + require.NoError(t, gate.AddDependencies(map[featuregate.Feature][]featuregate.Feature{ + alphaFeature: {betaOnFeatureDep, betaOffFeatureDep, alphaFeatureDep}, + betaOffFeature: {betaOnFeatureDep, betaOffFeatureDep}, + betaOnFeature: {betaOnFeatureDep}, + gaFeature: {}, + })) + + initialState := map[featuregate.Feature]bool{ + "AllAlpha": false, + "AllBeta": false, + + alphaFeature: false, + alphaFeatureDep: false, + betaOffFeature: false, + betaOffFeatureDep: false, + betaOnFeature: true, + betaOnFeatureDep: true, + gaFeature: true, + } + expect(t, gate, initialState) + + tests := []struct { + name string + overrides FeatureOverrides + expectedOverrides FeatureOverrides + expectError bool + }{{ + name: "AllBeta", + overrides: FeatureOverrides{"AllBeta": true}, + expectedOverrides: FeatureOverrides{ + "AllBeta": true, + betaOffFeature: true, + betaOffFeatureDep: true, + }, + }, { + name: "AllBeta=false", + overrides: FeatureOverrides{"AllBeta": false}, + expectedOverrides: FeatureOverrides{ + "AllBeta": false, + betaOnFeature: false, + betaOnFeatureDep: false, + }, + }, { + name: "AllBeta+AllAlpha", + overrides: FeatureOverrides{"AllBeta": true, "AllAlpha": true}, + expectedOverrides: FeatureOverrides{ + "AllAlpha": true, + "AllBeta": true, + alphaFeature: true, + alphaFeatureDep: true, + betaOffFeature: true, + betaOffFeatureDep: true, + }, + }, { + name: "AllAlpha", + overrides: FeatureOverrides{"AllAlpha": true}, + expectError: true, + }, { + name: "Automatically disable deps", + overrides: FeatureOverrides{betaOnFeatureDep: false}, + expectedOverrides: FeatureOverrides{ + betaOnFeature: false, + betaOnFeatureDep: false, + }, + }, { + name: "Don't automatically enable deps", + overrides: FeatureOverrides{betaOffFeatureDep: true}, + expectedOverrides: FeatureOverrides{ + betaOffFeatureDep: true, + betaOffFeature: false, + }, + }, { + name: "Error when disabling dependency", + overrides: FeatureOverrides{ + betaOnFeature: true, // Explicitly enabled so it's not automatically disabled. + betaOnFeatureDep: false, + }, + expectError: true, + }, { + name: "Error when enabling dependent", + overrides: FeatureOverrides{ + betaOffFeature: true, + }, + expectError: true, + }} + + for _, test := range tests { + t.Run(test.name, func(t *gotest.T) { + // Separate inner test so we can verify cleanup in the outer test. + t.Run("Set", func(t *gotest.T) { + if test.expectError { + fakeT := &ignoreErrorT{ignoreFatalT: &ignoreFatalT{T: t}} + SetFeatureGatesDuringTest(fakeT, gate, test.overrides) + require.True(t, fakeT.errorRecorded, "should capture error") + expect(t, gate, initialState) + } else { + SetFeatureGatesDuringTest(t, gate, test.overrides) + expectedState := maps.Clone(initialState) + maps.Copy(expectedState, test.expectedOverrides) + expect(t, gate, expectedState) + } + }) + // Verify revert in cleanup. + expect(t, gate, initialState) + }) + } +} + func TestSpecialGatesVersioned(t *gotest.T) { originalEmulationVersion := version.MustParse("1.31") gate := featuregate.NewVersionedFeatureGate(originalEmulationVersion) @@ -427,6 +556,130 @@ func TestDetectEmulationVersionLeakToOtherSubtest(t *gotest.T) { }) } +func TestSetFeatureGateVersionsInTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmuVer := version.MustParse("1.31") + originalMinCompatVer := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGateWithMinCompatibility(originalEmuVer, originalMinCompatVer) + + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + + newEmuVer := version.MustParse("1.29") + newMinCompatVer := version.MustParse("1.27") + + SetFeatureGateVersionsDuringTest(t, gate, newEmuVer, newMinCompatVer) + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + + t.Run("Subtest", func(t *gotest.T) { + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + newerEmuVer := version.MustParse("1.27") + newerMinCompatVer := version.MustParse("1.26") + SetFeatureGateVersionsDuringTest(t, gate, newerEmuVer, newerMinCompatVer) + assert.True(t, gate.EmulationVersion().EqualTo(newerEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newerMinCompatVer)) + }) + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + + t.Run("ParallelSubtest", func(t *gotest.T) { + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + t.Parallel() + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + }) + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) +} + +func TestDetectVersionsLeakToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmuVer := version.MustParse("1.31") + originalMinCompatVer := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGateWithMinCompatibility(originalEmuVer, originalMinCompatVer) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + + newEmuVer := version.MustParse("1.29") + newMinCompatVer := version.MustParse("1.28") + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateVersionsDuringTest(fakeT, gate, newEmuVer, newMinCompatVer) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + fakeT := &ignoreFatalT{T: t} + SetFeatureGateVersionsDuringTest(fakeT, gate, originalEmuVer, originalMinCompatVer) + assert.True(t, fakeT.fatalRecorded) +} + +func TestNoLeakFromSameVersionsToMainTest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmuVer := version.MustParse("1.31") + originalMinCompatVer := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGateWithMinCompatibility(originalEmuVer, originalMinCompatVer) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + + newEmuVer := version.MustParse("1.31") + newMinCompatVer := version.MustParse("1.30") + + // Subtest setting feature gate and calling parallel will leak it out + t.Run("LeakingSubtest", func(t *gotest.T) { + SetFeatureGateVersionsDuringTest(t, gate, newEmuVer, newMinCompatVer) + // Calling t.Parallel in subtest will resume the main test body + t.Parallel() + // Leaked from main test + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + }) + // Leaked from subtest + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + SetFeatureGateVersionsDuringTest(t, gate, originalEmuVer, originalMinCompatVer) +} + +func TestDetectVersionsLeakToOtherSubtest(t *gotest.T) { + t.Cleanup(cleanup) + originalEmuVer := version.MustParse("1.31") + originalMinCompatVer := version.MustParse("1.30") + gate := featuregate.NewVersionedFeatureGateWithMinCompatibility(originalEmuVer, originalMinCompatVer) + assert.True(t, gate.EmulationVersion().EqualTo(originalEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(originalMinCompatVer)) + + subtestName := "Subtest" + newEmuVer := version.MustParse("1.29") + newMinCompatVer := version.MustParse("1.28") + + // Subtest setting feature gate and calling parallel will leak it out + t.Run(subtestName, func(t *gotest.T) { + fakeT := &ignoreFatalT{T: t} + SetFeatureGateVersionsDuringTest(fakeT, gate, newEmuVer, newMinCompatVer) + t.Parallel() + }) + // Add suffix to name to prevent tests with the same prefix. + t.Run(subtestName+"Suffix", func(t *gotest.T) { + // Leaked newEmulationVersion and newMinCompatibilityVersion + assert.True(t, gate.EmulationVersion().EqualTo(newEmuVer)) + assert.True(t, gate.MinCompatibilityVersion().EqualTo(newMinCompatVer)) + + fakeT := &ignoreFatalT{T: t} + SetFeatureGateVersionsDuringTest(fakeT, gate, originalEmuVer, originalMinCompatVer) + assert.True(t, fakeT.fatalRecorded) + }) +} + type ignoreFatalT struct { *gotest.T fatalRecorded bool @@ -435,7 +688,7 @@ type ignoreFatalT struct { func (f *ignoreFatalT) Fatal(args ...any) { f.T.Helper() f.fatalRecorded = true - newArgs := []any{"[IGNORED]"} + newArgs := []any{"[IGNORED Fatal]"} newArgs = append(newArgs, args...) f.T.Log(newArgs...) } @@ -443,11 +696,30 @@ func (f *ignoreFatalT) Fatal(args ...any) { func (f *ignoreFatalT) Fatalf(format string, args ...any) { f.T.Helper() f.fatalRecorded = true - f.T.Logf("[IGNORED] "+format, args...) + f.T.Logf("[IGNORED Fatalf] "+format, args...) +} + +type ignoreErrorT struct { + *ignoreFatalT + errorRecorded bool +} + +func (f *ignoreErrorT) Error(args ...any) { + f.T.Helper() + f.errorRecorded = true + newArgs := []any{"[IGNORED Error]"} + newArgs = append(newArgs, args...) + f.T.Log(newArgs...) +} + +func (f *ignoreErrorT) Errorf(format string, args ...any) { + f.T.Helper() + f.errorRecorded = true + f.T.Logf("[IGNORED Errorf] "+format, args...) } func cleanup() { featureFlagOverride = map[featuregate.Feature]string{} - emulationVersionOverride = "" - emulationVersionOverrideValue = nil + versionsOverride = "" + versionsOverrideValue = "" } diff --git a/go.mod b/go.mod index 139c76ca..72e603c6 100644 --- a/go.mod +++ b/go.mod @@ -2,37 +2,37 @@ module k8s.io/component-base -go 1.24.0 +go 1.25.0 -godebug default=go1.24 +godebug default=go1.25 require ( github.com/blang/semver/v4 v4.0.0 - github.com/go-logr/logr v1.4.2 + github.com/go-logr/logr v1.4.3 github.com/go-logr/zapr v1.3.0 github.com/google/go-cmp v0.7.0 github.com/moby/term v0.5.0 - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.62.0 - github.com/prometheus/procfs v0.15.1 - github.com/spf13/cobra v1.9.1 - github.com/spf13/pflag v1.0.6 - github.com/stretchr/testify v1.10.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 - go.opentelemetry.io/otel v1.35.0 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + github.com/prometheus/common v0.66.1 + github.com/prometheus/procfs v0.16.1 + github.com/spf13/cobra v1.10.0 + github.com/spf13/pflag v1.0.9 + github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 + go.opentelemetry.io/otel v1.36.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 - go.opentelemetry.io/otel/sdk v1.34.0 - go.opentelemetry.io/otel/trace v1.35.0 + go.opentelemetry.io/otel/sdk v1.36.0 + go.opentelemetry.io/otel/trace v1.36.0 go.uber.org/zap v1.27.0 - go.yaml.in/yaml/v2 v2.4.2 - golang.org/x/sys v0.31.0 - k8s.io/apimachinery v0.0.0-20250725024258-04507a37f6a4 - k8s.io/client-go v0.0.0-20250725024918-f78361a6474d + go.yaml.in/yaml/v2 v2.4.3 + golang.org/x/sys v0.38.0 + golang.org/x/text v0.31.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 k8s.io/klog/v2 v2.130.1 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 ) require ( @@ -48,7 +48,6 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect @@ -59,29 +58,28 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/term v0.37.0 // indirect golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/grpc v1.72.1 // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/grpc v1.72.2 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.0.0-20250725024535-b95b43d5b95d // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/api v0.35.0 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/go.sum b/go.sum index 07406699..732f469a 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -22,8 +24,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= @@ -38,8 +40,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= @@ -47,8 +47,8 @@ github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7O github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= @@ -59,8 +59,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -84,29 +82,28 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -116,30 +113,28 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -148,82 +143,61 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= -google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.0.0-20250725024535-b95b43d5b95d h1:OdC1L69BYvh/4HX6Wgg7DeL0A++gD3gadKfaWT4IdbA= -k8s.io/api v0.0.0-20250725024535-b95b43d5b95d/go.mod h1:wKZv1VB6nzJ6L449TteVelrBfRsawhrthiOsEylKo8U= -k8s.io/apimachinery v0.0.0-20250725024258-04507a37f6a4 h1:N25HX4lRPTvLHSUPoCMFP+B/oEcOmPESB+BRkYMD8Io= -k8s.io/apimachinery v0.0.0-20250725024258-04507a37f6a4/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= -k8s.io/client-go v0.0.0-20250725024918-f78361a6474d h1:8i9q3fd352G8FsurDLj4++5oMS4gWqZ5O2lErveG9Ok= -k8s.io/client-go v0.0.0-20250725024918-f78361a6474d/go.mod h1:j4aKw1XACZSFUnRBqWNU0d3TXbXzaBhAbazCrGTYHdg= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= diff --git a/logs/api/v1/doc.go b/logs/api/v1/doc.go index dee5335f..fe5367f9 100644 --- a/logs/api/v1/doc.go +++ b/logs/api/v1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +k8s:deepcopy-gen=package +// +k8s:openapi-model-package=io.k8s.component-base.logs.api.v1 // Package v1 contains the configuration API for logging. // diff --git a/logs/api/v1/options.go b/logs/api/v1/options.go index 4c8a0d2c..a61d142c 100644 --- a/logs/api/v1/options.go +++ b/logs/api/v1/options.go @@ -138,6 +138,9 @@ func validateAndApply(c *LoggingConfiguration, options *LoggingOptions, featureG // can be passed when the struct is not embedded in some larger struct. func Validate(c *LoggingConfiguration, featureGate featuregate.FeatureGate, fldPath *field.Path) field.ErrorList { errs := field.ErrorList{} + if c.FlushFrequency.Duration.Duration <= 0 { + errs = append(errs, field.Invalid(fldPath.Child("flushFrequency"), c.FlushFrequency, "Must be greater than zero")) + } if c.Format != DefaultLogFormat { // WordSepNormalizeFunc is just a guess. Commands should use it, // but we cannot know for sure. diff --git a/logs/api/v1/validate_test.go b/logs/api/v1/validate_test.go index 956f225c..2ca4588c 100644 --- a/logs/api/v1/validate_test.go +++ b/logs/api/v1/validate_test.go @@ -19,9 +19,11 @@ package v1 import ( "math" "testing" + "time" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/component-base/featuregate" ) @@ -145,6 +147,15 @@ func TestValidation(t *testing.T) { config: jsonOptionsEnabled, featureGate: enabledFeatureGate, }, + "Invalid flush frequency": { + config: LoggingConfiguration{ + Format: "text", + FlushFrequency: TimeOrMetaDuration{ + Duration: metav1.Duration{Duration: -1 * time.Second}, + }, + }, + expectErrors: `flushFrequency: Invalid value: -1000000000: Must be greater than zero`, + }, } for name, test := range testcases { @@ -153,6 +164,7 @@ func TestValidation(t *testing.T) { if featureGate == nil { featureGate = defaultFeatureGate } + SetRecommendedLoggingConfiguration(&test.config) err := Validate(&test.config, featureGate, test.path) if len(err) == 0 { if test.expectErrors != "" { diff --git a/logs/api/v1/zz_generated.model_name.go b/logs/api/v1/zz_generated.model_name.go new file mode 100644 index 00000000..e5dd5c69 --- /dev/null +++ b/logs/api/v1/zz_generated.model_name.go @@ -0,0 +1,67 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by openapi-gen. DO NOT EDIT. + +package v1 + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in FormatOptions) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.FormatOptions" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in JSONOptions) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.JSONOptions" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in LoggingConfiguration) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.LoggingConfiguration" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in LoggingOptions) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.LoggingOptions" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in OutputRoutingOptions) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.OutputRoutingOptions" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in RuntimeControl) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.RuntimeControl" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in TextOptions) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.TextOptions" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in TimeOrMetaDuration) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.TimeOrMetaDuration" +} + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in VModuleItem) OpenAPIModelName() string { + return "io.k8s.component-base.logs.api.v1.VModuleItem" +} diff --git a/logs/logreduction/logreduction.go b/logs/logreduction/logreduction.go index 6534a5a6..71465358 100644 --- a/logs/logreduction/logreduction.go +++ b/logs/logreduction/logreduction.go @@ -21,7 +21,7 @@ import ( "time" ) -var nowfunc = func() time.Time { return time.Now() } +var nowfunc = time.Now // LogReduction provides a filter for consecutive identical log messages; // a message will be printed no more than once per interval. diff --git a/metrics/collector_test.go b/metrics/collector_test.go index e946171c..3f779014 100644 --- a/metrics/collector_test.go +++ b/metrics/collector_test.go @@ -70,7 +70,7 @@ func TestBaseCustomCollector(t *testing.T) { deprecatedDesc = NewDesc("metric_deprecated", "stable deprecated metrics", []string{"name"}, nil, STABLE, "1.17.0") hiddenDesc = NewDesc("metric_hidden", "stable hidden metrics", []string{"name"}, nil, - STABLE, "1.16.0") + STABLE, "1.14.0") ) registry := newKubeRegistry(currentVersion) diff --git a/metrics/counter.go b/metrics/counter.go index e41d5383..6805bf94 100644 --- a/metrics/counter.go +++ b/metrics/counter.go @@ -30,7 +30,6 @@ import ( // Counter is our internal representation for our wrapping struct around prometheus // counters. Counter implements both kubeCollector and CounterMetric. type Counter struct { - ctx context.Context CounterMetric *CounterOpts lazyMetric @@ -40,12 +39,10 @@ type Counter struct { // The implementation of the Metric interface is expected by testutil.GetCounterMetricValue. var _ Metric = &Counter{} -// All supported exemplar metric types implement the metricWithExemplar interface. -var _ metricWithExemplar = &Counter{} - // exemplarCounterMetric holds a context to extract exemplar labels from, and a counter metric to attach them to. It implements the metricWithExemplar interface. type exemplarCounterMetric struct { - *Counter + ctx context.Context + delegate CounterMetric } // NewCounter returns an object which satisfies the kubeCollector and CounterMetric interfaces. @@ -107,26 +104,12 @@ func (c *Counter) initializeDeprecatedMetric() { // WithContext allows the normal Counter metric to pass in context. func (c *Counter) WithContext(ctx context.Context) CounterMetric { - c.ctx = ctx - return c.CounterMetric -} - -// withExemplar initializes the exemplarMetric object and sets the exemplar value. -func (c *Counter) withExemplar(v float64) { - (&exemplarCounterMetric{c}).withExemplar(v) -} - -func (c *Counter) Add(v float64) { - c.withExemplar(v) -} - -func (c *Counter) Inc() { - c.withExemplar(1) + return &exemplarCounterMetric{ctx: ctx, delegate: c.CounterMetric} } -// withExemplar attaches an exemplar to the metric. -func (e *exemplarCounterMetric) withExemplar(v float64) { - if m, ok := e.CounterMetric.(prometheus.ExemplarAdder); ok { +// Add attaches an exemplar to the metric and then calls the delegate. +func (e *exemplarCounterMetric) Add(v float64) { + if m, ok := e.delegate.(prometheus.ExemplarAdder); ok { maybeSpanCtx := trace.SpanContextFromContext(e.ctx) if maybeSpanCtx.IsValid() && maybeSpanCtx.IsSampled() { exemplarLabels := prometheus.Labels{ @@ -138,7 +121,12 @@ func (e *exemplarCounterMetric) withExemplar(v float64) { } } - e.CounterMetric.Add(v) + e.delegate.Add(v) +} + +// Inc attaches an exemplar to the metric and then calls the delegate. +func (e *exemplarCounterMetric) Inc() { + e.Add(1) } // CounterVec is the internal representation of our wrapping struct around prometheus diff --git a/metrics/counter_test.go b/metrics/counter_test.go index 2afcc4d6..66162c38 100644 --- a/metrics/counter_test.go +++ b/metrics/counter_test.go @@ -19,27 +19,36 @@ package metrics import ( "bytes" "context" + "sync" "testing" - "github.com/blang/semver/v4" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" apimachineryversion "k8s.io/apimachinery/pkg/version" ) func TestCounter(t *testing.T) { + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *CounterOpts + currentVersion apimachineryversion.Info expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", CounterOpts: &CounterOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -51,7 +60,32 @@ func TestCounter(t *testing.T) { expectedHelp: "[ALPHA] counter help", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "counter help", + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] counter help", + }, + { + desc: "STABLE metric non deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "counter help", + }, + expectedMetricCount: 1, + expectedHelp: "[STABLE] counter help", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", CounterOpts: &CounterOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -60,30 +94,78 @@ func TestCounter(t *testing.T) { StabilityLevel: ALPHA, DeprecatedVersion: "1.15.0", }, + expectedMetricCount: 0, + expectedHelp: "counter help", + }, + { + desc: "BETA metric deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "counter help", + StabilityLevel: BETA, + DeprecatedVersion: "1.15.0", + }, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) counter help", + expectedHelp: "[BETA] (Deprecated since 1.15.0) counter help", }, { - desc: "Test hidden", + desc: "STABLE metric deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "counter help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.14.0", + }, + expectedMetricCount: 1, + expectedHelp: "[STABLE] (Deprecated since 1.14.0) counter help", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", CounterOpts: &CounterOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", Help: "counter help", StabilityLevel: ALPHA, + DeprecatedVersion: "1.15.0", + }, + expectedMetricCount: 0, + expectedHelp: "counter help", + }, + { + desc: "BETA metric hidden", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "counter help", + StabilityLevel: BETA, DeprecatedVersion: "1.14.0", }, + expectedMetricCount: 0}, + { + desc: "STABLE metric hidden", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "counter help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.12.0", + }, expectedMetricCount: 0, + expectedHelp: "counter help", }, } for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) // c is a pointer to a Counter c := NewCounter(test.CounterOpts) registry.MustRegister(c) @@ -118,45 +200,110 @@ func TestCounter(t *testing.T) { } func TestCounterVec(t *testing.T) { + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *CounterOpts labels []string - registryVersion *semver.Version expectedMetricFamilyCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", CounterOpts: &CounterOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "counter help", + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "counter help", }, labels: []string{"label_a", "label_b"}, expectedMetricFamilyCount: 1, expectedHelp: "[ALPHA] counter help", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "counter help", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricFamilyCount: 1, + expectedHelp: "[BETA] counter help", + }, + { + desc: "STABLE metric non deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "counter help", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricFamilyCount: 1, + expectedHelp: "[STABLE] counter help", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "counter help", + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricFamilyCount: 0, + expectedHelp: "counter help", + }, + { + desc: "BETA metric deprecated", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "counter help", + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricFamilyCount: 1, + expectedHelp: "[BETA] (Deprecated since 1.15.0) counter help", + }, + { + desc: "STABLE metric deprecated", CounterOpts: &CounterOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: STABLE, Help: "counter help", DeprecatedVersion: "1.15.0", }, labels: []string{"label_a", "label_b"}, expectedMetricFamilyCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) counter help", + expectedHelp: "[STABLE] (Deprecated since 1.15.0) counter help", }, + // Hidden metrics { - desc: "Test hidden", + desc: "ALPHA metric hidden", CounterOpts: &CounterOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "counter help", DeprecatedVersion: "1.14.0", }, @@ -165,27 +312,38 @@ func TestCounterVec(t *testing.T) { expectedHelp: "counter help", }, { - desc: "Test alpha", + desc: "BETA metric hidden", CounterOpts: &CounterOpts{ - StabilityLevel: ALPHA, - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "counter help", + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "counter help", + DeprecatedVersion: "1.14.0", }, labels: []string{"label_a", "label_b"}, - expectedMetricFamilyCount: 1, - expectedHelp: "[ALPHA] counter help", + expectedMetricFamilyCount: 0, + expectedHelp: "counter help", + }, + { + desc: "STABLE metric hidden", + CounterOpts: &CounterOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "counter help", + DeprecatedVersion: "1.12.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricFamilyCount: 0, + expectedHelp: "counter help", }, } for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewCounterVec(test.CounterOpts, test.labels) registry.MustRegister(c) c.WithLabelValues("1", "2").Inc() @@ -314,7 +472,6 @@ func TestCounterWithExemplar(t *testing.T) { Name: "metric_exemplar_test", Help: "helpless", }) - _ = counter.WithContext(ctxForSpanCtx) // Register counter. registry := newKubeRegistry(apimachineryversion.Info{ @@ -325,9 +482,9 @@ func TestCounterWithExemplar(t *testing.T) { registry.MustRegister(counter) // Call underlying exemplar methods. - counter.Add(toAdd) - counter.Inc() - counter.Inc() + counter.WithContext(ctxForSpanCtx).Add(toAdd) + counter.WithContext(ctxForSpanCtx).Inc() + counter.WithContext(ctxForSpanCtx).Inc() // Gather. mfs, err := registry.Gather() @@ -382,3 +539,118 @@ func TestCounterWithExemplar(t *testing.T) { } } } + +// TestCounterConcurrentWithContextRace reproduces the race condition in Counter.WithContext +// where c.ctx is written concurrently with reads in withExemplar method. +// This test simulates the real authentication flow that triggers the race: +// x509.AuthenticateRequest -> union.AuthenticateRequest -> group.AuthenticateRequest +func TestCounterConcurrentWithContextRace(t *testing.T) { + opts := &CounterOpts{ + Namespace: "apiserver", + Subsystem: "authentication", + Name: "requests_total", + Help: "Authentication requests counter for race condition testing", + } + + c := NewCounter(opts) + + // Force initialization by calling initializeMetric directly + c.initializeMetric() + + // Create contexts with trace spans to trigger exemplar code path + ctx1, span1 := createContextWithSpanCounter("x509-auth", "authenticate-request") + defer span1.End() + ctx2, span2 := createContextWithSpanCounter("union-auth", "union-authenticate") + defer span2.End() + + var wg sync.WaitGroup + iterations := 10000 // Increase iterations to make race more likely + + // Goroutine 1: Simulate x509 Authenticator calling WithContext + // This matches: x509.(*Authenticator).AuthenticateRequest() calling WithContext + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate authentication request processing + simulateX509AuthenticateRequestCounter(c, ctx1, i) + } + }() + + // Goroutine 2: Simulate concurrent Add/Inc calls (metrics collection) + // This matches the withExemplar/Add path that reads c.ctx + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + c.Add(float64(i % 10)) + if i%2 == 0 { + c.Inc() + } + } + }() + + // Goroutine 3: Simulate union AuthenticateRequest calling WithContext + // This matches: union.(*unionAuthRequestHandler).AuthenticateRequest() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate union authentication processing + simulateUnionAuthenticateRequestCounter(c, ctx2, i) + } + }() + + // Goroutine 4: Simulate group AuthenticateRequest calling WithContext + // This matches: group.(*AuthenticatedGroupAdder).AuthenticateRequest() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate group authentication processing + simulateGroupAuthenticateRequestCounter(c, ctx1, ctx2, i) + } + }() + + wg.Wait() +} + +// simulateX509AuthenticateRequestCounter simulates the call path from x509 authenticator +func simulateX509AuthenticateRequestCounter(c CounterMetric, ctx context.Context, iteration int) { + // Simulate the authentication request processing that calls WithContext + if cWithCtx, ok := c.(*Counter); ok { + cWithCtx.WithContext(ctx) + } +} + +// simulateUnionAuthenticateRequestCounter simulates the call path from union authenticator +func simulateUnionAuthenticateRequestCounter(c CounterMetric, ctx context.Context, iteration int) { + // Simulate union authentication processing + if cWithCtx, ok := c.(*Counter); ok { + cWithCtx.WithContext(ctx) + } +} + +// simulateGroupAuthenticateRequestCounter simulates the call path from group authenticator +func simulateGroupAuthenticateRequestCounter(c CounterMetric, ctx1, ctx2 context.Context, iteration int) { + // Alternate between contexts to simulate different authentication scenarios + ctx := ctx1 + if iteration%3 == 0 { + ctx = ctx2 + } + + if cWithCtx, ok := c.(*Counter); ok { + cWithCtx.WithContext(ctx) + } +} + +// Helper function to create a context with a valid trace span +func createContextWithSpanCounter(traceID, spanID string) (context.Context, trace.Span) { + ctx := context.Background() + + // Create a noop tracer and span for testing + tracer := tracenoop.NewTracerProvider().Tracer("test") + ctx, span := tracer.Start(ctx, "test-span") + + return ctx, span +} diff --git a/metrics/desc.go b/metrics/desc.go index 2ca9cfa7..f921be61 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -107,20 +107,18 @@ func (d *Desc) DeprecatedVersion() *semver.Version { } -func (d *Desc) determineDeprecationStatus(version semver.Version) { - selfVersion := d.DeprecatedVersion() - if selfVersion == nil { +func (d *Desc) determineDeprecationStatus(currentVersion semver.Version) { + deprecatedVersion := d.DeprecatedVersion() + if deprecatedVersion == nil { return } d.markDeprecationOnce.Do(func() { - if selfVersion.LTE(version) { - d.isDeprecated = true - } - if ShouldShowHidden() { - klog.Warningf("Hidden metrics(%s) have been manually overridden, showing this very deprecated metric.", d.fqName) - return - } - if shouldHide(&version, selfVersion) { + d.isDeprecated = isDeprecated(currentVersion, *deprecatedVersion) + if shouldHide(d.stabilityLevel, ¤tVersion, deprecatedVersion) { + if shouldShowHidden() { + klog.Warningf("Hidden metrics(%s) have been manually overridden, showing this very deprecated metric.", d.fqName) + return + } // TODO(RainbowMango): Remove this log temporarily. https://github.com/kubernetes/kubernetes/issues/85369 // klog.Warningf("This metric(%s) has been deprecated for more than one release, hiding.", d.fqName) d.isHidden = true diff --git a/metrics/desc_test.go b/metrics/desc_test.go index b96791bc..17102a13 100644 --- a/metrics/desc_test.go +++ b/metrics/desc_test.go @@ -72,7 +72,7 @@ func TestDescCreate(t *testing.T) { fqName: "hidden_stable_descriptor", help: "this is a hidden descriptor", stabilityLevel: STABLE, - deprecatedVersion: "1.16.0", + deprecatedVersion: "1.14.0", shouldCreate: false, expectedAnnotatedHelp: "this is a hidden descriptor", // hidden descriptor shall not be annotated. }, diff --git a/metrics/features/kube_features.go b/metrics/features/kube_features.go index 5f41802c..b2f8c652 100644 --- a/metrics/features/kube_features.go +++ b/metrics/features/kube_features.go @@ -17,26 +17,11 @@ limitations under the License. package features import ( - "k8s.io/apimachinery/pkg/util/version" "k8s.io/component-base/featuregate" ) -const ( - // owner: @logicalhan - // kep: https://kep.k8s.io/3466 - ComponentSLIs featuregate.Feature = "ComponentSLIs" -) - func featureGates() map[featuregate.Feature]featuregate.VersionedSpecs { - return map[featuregate.Feature]featuregate.VersionedSpecs{ - ComponentSLIs: { - {Version: version.MustParse("1.26"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta}, - // ComponentSLIs officially graduated to GA in v1.29 but the gate was not updated until v1.32. - // To support emulated versions, keep the gate until v1.35. - {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, - }, - } + return map[featuregate.Feature]featuregate.VersionedSpecs{} } // AddFeatureGates adds all feature gates used by this package. diff --git a/metrics/gauge.go b/metrics/gauge.go index 0d6c8b7f..20982f67 100644 --- a/metrics/gauge.go +++ b/metrics/gauge.go @@ -161,6 +161,17 @@ func (v *GaugeVec) WithLabelValuesChecked(lvs ...string) (GaugeMetric, error) { return v.GetMetricWithLabelValues(lvs...) } +func (v *GaugeVec) DeleteLabelValuesChecked(lvs ...string) (bool, error) { + if !v.IsCreated() { + if v.IsHidden() { + return false, nil + } + return false, errNotRegistered + } + + return v.GaugeVec.DeleteLabelValues(lvs...), nil +} + // Default Prometheus Vec behavior is that member extraction results in creation of a new element // if one with the unique label values is not found in the underlying stored metricMap. // This means that if this function is called but the underlying metric is not registered @@ -184,6 +195,14 @@ func (v *GaugeVec) WithLabelValues(lvs ...string) GaugeMetric { panic(err) } +func (v *GaugeVec) DeleteLabelValues(lvs ...string) bool { + ans, err := v.DeleteLabelValuesChecked(lvs...) + if err == nil || ErrIsNotRegistered(err) { + return ans + } + panic(err) +} + func (v *GaugeVec) WithChecked(labels map[string]string) (GaugeMetric, error) { if !v.IsCreated() { if v.IsHidden() { diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index d41baebb..f20be352 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -20,7 +20,6 @@ import ( "strings" "testing" - "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,49 +28,132 @@ import ( ) func TestGauge(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *GaugeOpts - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", GaugeOpts: &GaugeOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "gauge help", + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "gauge help", }, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] gauge help", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] gauge help", + }, + { + desc: "STABLE metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + }, + expectedMetricCount: 1, + expectedHelp: "[STABLE] gauge help", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "gauge help", + DeprecatedVersion: "1.15.0", + }, + expectedMetricCount: 0, + expectedHelp: "gauge help", + }, + { + desc: "BETA metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "gauge help", + DeprecatedVersion: "1.15.0", + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] (Deprecated since 1.15.0) gauge help", + }, + { + desc: "STABLE metric deprecated", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", Help: "gauge help", + StabilityLevel: STABLE, DeprecatedVersion: "1.15.0", }, - registryVersion: &v115, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) gauge help", + expectedHelp: "[STABLE] (Deprecated since 1.15.0) gauge help", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "gauge help", + DeprecatedVersion: "1.14.0", + }, + expectedMetricCount: 0, + expectedHelp: "gauge help", }, { - desc: "Test hidden", + desc: "BETA metric hidden", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: BETA, Help: "gauge help", DeprecatedVersion: "1.14.0", }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "gauge help", + }, + { + desc: "STABLE metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.12.0", + }, expectedMetricCount: 0, expectedHelp: "gauge help", }, @@ -79,11 +161,7 @@ func TestGauge(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewGauge(test.GaugeOpts) registry.MustRegister(c) @@ -113,17 +191,22 @@ func TestGauge(t *testing.T) { } func TestGaugeVec(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *GaugeOpts labels []string - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -131,12 +214,38 @@ func TestGaugeVec(t *testing.T) { Help: "gauge help", }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] gauge help", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[BETA] gauge help", + }, + { + desc: "STABLE metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[STABLE] gauge help", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -145,21 +254,76 @@ func TestGaugeVec(t *testing.T) { DeprecatedVersion: "1.15.0", }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "gauge help", + }, + { + desc: "BETA metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[BETA] (Deprecated since 1.15.0) gauge help", + }, + { + desc: "STABLE metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) gauge help", + expectedHelp: "[STABLE] (Deprecated since 1.15.0) gauge help", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 0, + expectedHelp: "gauge help", }, { - desc: "Test hidden", + desc: "BETA metric hidden", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", Help: "gauge help", + StabilityLevel: BETA, DeprecatedVersion: "1.14.0", }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "gauge help", + }, + { + desc: "STABLE metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.12.0", + }, + labels: []string{"label_a", "label_b"}, expectedMetricCount: 0, expectedHelp: "gauge help", }, @@ -167,11 +331,7 @@ func TestGaugeVec(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewGaugeVec(test.GaugeOpts, test.labels) registry.MustRegister(c) c.WithLabelValues("1", "2").Set(1.0) @@ -196,7 +356,7 @@ func TestGaugeVec(t *testing.T) { } func TestGaugeFunc(t *testing.T) { - currentVersion := apimachineryversion.Info{ + version1_15Alpha1 := apimachineryversion.Info{ Major: "1", Minor: "17", GitVersion: "v1.17.0-alpha-1.12345", @@ -211,13 +371,15 @@ func TestGaugeFunc(t *testing.T) { *GaugeOpts expectedMetrics string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", GaugeOpts: &GaugeOpts{ - Namespace: "namespace", - Subsystem: "subsystem", - Name: "metric_non_deprecated", - Help: "gauge help", + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Name: "metric_non_deprecated", + Help: "gauge help", }, expectedMetrics: ` # HELP namespace_subsystem_metric_non_deprecated [ALPHA] gauge help @@ -226,38 +388,122 @@ namespace_subsystem_metric_non_deprecated 1 `, }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: BETA, + Name: "metric_non_deprecated", + Help: "gauge help", + }, + expectedMetrics: ` +# HELP namespace_subsystem_metric_non_deprecated [BETA] gauge help +# TYPE namespace_subsystem_metric_non_deprecated gauge +namespace_subsystem_metric_non_deprecated 1 + `, + }, + { + desc: "STABLE metric non deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Name: "metric_non_deprecated", + Help: "gauge help", + }, + expectedMetrics: ` +# HELP namespace_subsystem_metric_non_deprecated [STABLE] gauge help +# TYPE namespace_subsystem_metric_non_deprecated gauge +namespace_subsystem_metric_non_deprecated 1 + `, + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Subsystem: "subsystem", Name: "metric_deprecated", Help: "gauge help", + StabilityLevel: ALPHA, DeprecatedVersion: "1.17.0", }, - expectedMetrics: ` -# HELP namespace_subsystem_metric_deprecated [ALPHA] (Deprecated since 1.17.0) gauge help + expectedMetrics: "", + }, + { + desc: "BETA metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + Name: "metric_deprecated", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.17.0", + }, + expectedMetrics: `# HELP namespace_subsystem_metric_deprecated [BETA] (Deprecated since 1.17.0) gauge help # TYPE namespace_subsystem_metric_deprecated gauge namespace_subsystem_metric_deprecated 1 -`, + `, }, { - desc: "Test hidden", + desc: "STABLE metric deprecated", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + Name: "metric_deprecated", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.17.0", + }, + expectedMetrics: `# HELP namespace_subsystem_metric_deprecated [STABLE] (Deprecated since 1.17.0) gauge help +# TYPE namespace_subsystem_metric_deprecated gauge +namespace_subsystem_metric_deprecated 1 + `, + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", GaugeOpts: &GaugeOpts{ Namespace: "namespace", Subsystem: "subsystem", Name: "metric_hidden", Help: "gauge help", + StabilityLevel: ALPHA, + DeprecatedVersion: "1.17.0", + }, + expectedMetrics: "", + }, + { + desc: "BETA metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + Name: "metric_hidden", + Help: "gauge help", + StabilityLevel: BETA, DeprecatedVersion: "1.16.0", }, expectedMetrics: "", }, + { + desc: "STABLE metric hidden", + GaugeOpts: &GaugeOpts{ + Namespace: "namespace", + Subsystem: "subsystem", + Name: "metric_hidden", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.14.0", + }, + expectedMetrics: "", + }, } for _, test := range tests { tc := test t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(currentVersion) - gauge := newGaugeFunc(tc.GaugeOpts, function, parseVersion(currentVersion)) + registry := newKubeRegistry(version1_15Alpha1) + gauge := newGaugeFunc(tc.GaugeOpts, function, parseVersion(version1_15Alpha1)) if gauge != nil { // hidden metrics will not be initialize, register is not allowed registry.RawMustRegister(gauge) } @@ -347,3 +593,378 @@ func TestGaugeWithLabelValueAllowList(t *testing.T) { }) } } + +func TestGaugeVecDeleteLabelValues(t *testing.T) { + version := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { + desc string + opts *GaugeOpts + labels []string + expectMetricExists bool + expectDelete bool + }{ + // Non-deprecated metrics + { + desc: "ALPHA metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "BETA metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "STABLE metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "BETA metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "STABLE metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "BETA metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "STABLE metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.12.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + registry := newKubeRegistry(version) + gv := NewGaugeVec(test.opts, test.labels) + registry.MustRegister(gv) + gv.WithLabelValues("foo", "bar").Set(42) + + ms, err := registry.Gather() + require.NoError(t, err) + found := false + for _, mf := range ms { + if *mf.Name == BuildFQName(test.opts.Namespace, test.opts.Subsystem, test.opts.Name) { + for _, m := range mf.GetMetric() { + for _, l := range m.Label { + if *l.Name == "label_a" && *l.Value == "foo" { + found = true + } + } + } + } + } + assert.Equal(t, test.expectMetricExists, found, "Metric existence mismatch before deletion") + + deleted := gv.DeleteLabelValues("foo", "bar") + assert.Equal(t, test.expectDelete, deleted, "DeleteLabelValues return mismatch") + + // Confirm it no longer exists + ms, err = registry.Gather() + require.NoError(t, err) + found = false + for _, mf := range ms { + if *mf.Name == BuildFQName(test.opts.Namespace, test.opts.Subsystem, test.opts.Name) { + for _, m := range mf.GetMetric() { + for _, l := range m.Label { + if *l.Name == "label_a" && *l.Value == "foo" { + found = true + } + } + } + } + } + assert.False(t, found, "Metric with label values should not exist after deletion") + }) + } +} + +func TestGaugeVecDeleteLabelValuesChecked(t *testing.T) { + version := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { + desc string + opts *GaugeOpts + labels []string + expectMetricExists bool + expectDelete bool + }{ + // Non-deprecated metrics + { + desc: "ALPHA metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "BETA metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "STABLE metric non deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "BETA metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + { + desc: "STABLE metric deprecated", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: true, + expectDelete: true, + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: ALPHA, + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "BETA metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: BETA, + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + { + desc: "STABLE metric hidden", + opts: &GaugeOpts{ + Namespace: "namespace", + Name: "metric_delete_checked_table", + Subsystem: "subsystem", + Help: "gauge help", + StabilityLevel: STABLE, + DeprecatedVersion: "1.12.0", + }, + labels: []string{"label_a", "label_b"}, + expectMetricExists: false, + expectDelete: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + registry := newKubeRegistry(version) + gv := NewGaugeVec(test.opts, test.labels) + registry.MustRegister(gv) + gv.WithLabelValues("foo", "bar").Set(42) + + ms, err := registry.Gather() + require.NoError(t, err) + found := false + for _, mf := range ms { + if *mf.Name == BuildFQName(test.opts.Namespace, test.opts.Subsystem, test.opts.Name) { + for _, m := range mf.GetMetric() { + for _, l := range m.Label { + if *l.Name == "label_a" && *l.Value == "foo" { + found = true + } + } + } + } + } + assert.Equal(t, test.expectMetricExists, found, "Metric existence mismatch before deletion") + + deleted, err := gv.DeleteLabelValuesChecked("foo", "bar") + assert.Equal(t, test.expectDelete, deleted, "DeleteLabelValuesChecked return mismatch") + require.NoError(t, err, "DeleteLabelValuesChecked should not return error") + + // Confirm it no longer exists + ms, err = registry.Gather() + require.NoError(t, err) + found = false + for _, mf := range ms { + if *mf.Name == BuildFQName(test.opts.Namespace, test.opts.Subsystem, test.opts.Name) { + for _, m := range mf.GetMetric() { + for _, l := range m.Label { + if *l.Name == "label_a" && *l.Value == "foo" { + found = true + } + } + } + } + } + assert.False(t, found, "Metric with label values should not exist after deletion") + }) + } +} diff --git a/metrics/histogram.go b/metrics/histogram.go index b410951b..73b56d1b 100644 --- a/metrics/histogram.go +++ b/metrics/histogram.go @@ -28,7 +28,6 @@ import ( // Histogram is our internal representation for our wrapping struct around prometheus // histograms. Summary implements both kubeCollector and ObserverMetric type Histogram struct { - ctx context.Context ObserverMetric *HistogramOpts lazyMetric @@ -37,7 +36,8 @@ type Histogram struct { // exemplarHistogramMetric holds a context to extract exemplar labels from, and a historgram metric to attach them to. It implements the metricWithExemplar interface. type exemplarHistogramMetric struct { - *Histogram + ctx context.Context + delegate ObserverMetric } type exemplarHistogramVec struct { @@ -45,18 +45,9 @@ type exemplarHistogramVec struct { observer prometheus.Observer } -func (h *Histogram) Observe(v float64) { - h.withExemplar(v) -} - -// withExemplar initializes the exemplarMetric object and sets the exemplar value. -func (h *Histogram) withExemplar(v float64) { - (&exemplarHistogramMetric{h}).withExemplar(v) -} - -// withExemplar attaches an exemplar to the metric. -func (e *exemplarHistogramMetric) withExemplar(v float64) { - if m, ok := e.Histogram.ObserverMetric.(prometheus.ExemplarObserver); ok { +// Observe attaches an exemplar to the metric and then calls the delegate. +func (e *exemplarHistogramMetric) Observe(v float64) { + if m, ok := e.delegate.(prometheus.ExemplarObserver); ok { maybeSpanCtx := trace.SpanContextFromContext(e.ctx) if maybeSpanCtx.IsValid() && maybeSpanCtx.IsSampled() { exemplarLabels := prometheus.Labels{ @@ -68,7 +59,7 @@ func (e *exemplarHistogramMetric) withExemplar(v float64) { } } - e.ObserverMetric.Observe(v) + e.delegate.Observe(v) } // NewHistogram returns an object which is Histogram-like. However, nothing @@ -113,8 +104,7 @@ func (h *Histogram) initializeDeprecatedMetric() { // WithContext allows the normal Histogram metric to pass in context. The context is no-op now. func (h *Histogram) WithContext(ctx context.Context) ObserverMetric { - h.ctx = ctx - return h.ObserverMetric + return &exemplarHistogramMetric{ctx: ctx, delegate: h.ObserverMetric} } // HistogramVec is the internal representation of our wrapping struct around prometheus diff --git a/metrics/histogram_test.go b/metrics/histogram_test.go index 5efbfb6e..44df0f1f 100644 --- a/metrics/histogram_test.go +++ b/metrics/histogram_test.go @@ -18,65 +18,155 @@ package metrics import ( "context" + "sync" "testing" - "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/trace" + tracenoop "go.opentelemetry.io/otel/trace/noop" apimachineryversion "k8s.io/apimachinery/pkg/version" ) func TestHistogram(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *HistogramOpts - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", HistogramOpts: &HistogramOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "histogram help message", - Buckets: prometheus.DefBuckets, + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + Buckets: prometheus.DefBuckets, }, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] histogram help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + Buckets: prometheus.DefBuckets, + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] histogram help message", + }, + { + desc: "STABLE metric non deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + Buckets: prometheus.DefBuckets, + }, + expectedMetricCount: 1, + expectedHelp: "[STABLE] histogram help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: prometheus.DefBuckets, + }, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "BETA metric deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: prometheus.DefBuckets, + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] (Deprecated since 1.15.0) histogram help message", + }, + { + desc: "STABLE metric deprecated", HistogramOpts: &HistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: STABLE, Help: "histogram help message", DeprecatedVersion: "1.15.0", Buckets: prometheus.DefBuckets, }, - registryVersion: &v115, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) histogram help message", + expectedHelp: "[STABLE] (Deprecated since 1.15.0) histogram help message", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: prometheus.DefBuckets, + }, + expectedMetricCount: 0, + expectedHelp: "histogram help message", }, { - desc: "Test hidden", + desc: "BETA metric hidden", HistogramOpts: &HistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: BETA, Help: "histogram help message", DeprecatedVersion: "1.14.0", Buckets: prometheus.DefBuckets, }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "STABLE metric hidden", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + DeprecatedVersion: "1.12.0", + Buckets: prometheus.DefBuckets, + }, expectedMetricCount: 0, expectedHelp: "histogram help message", }, @@ -84,11 +174,7 @@ func TestHistogram(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewHistogram(test.HistogramOpts) registry.MustRegister(c) cm := c.ObserverMetric.(prometheus.Metric) @@ -132,17 +218,22 @@ func TestHistogram(t *testing.T) { } func TestHistogramVec(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *HistogramOpts labels []string - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", HistogramOpts: &HistogramOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -151,37 +242,115 @@ func TestHistogramVec(t *testing.T) { Buckets: prometheus.DefBuckets, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] histogram help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + Buckets: prometheus.DefBuckets, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[BETA] histogram help message", + }, + { + desc: "STABLE metric non deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + Buckets: prometheus.DefBuckets, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[STABLE] histogram help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: prometheus.DefBuckets, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "BETA metric deprecated", HistogramOpts: &HistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: BETA, Help: "histogram help message", DeprecatedVersion: "1.15.0", Buckets: prometheus.DefBuckets, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) histogram help message", + expectedHelp: "[BETA] (Deprecated since 1.15.0) histogram help message", + }, + { + desc: "STABLE metric deprecated", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + + DeprecatedVersion: "1.15.0", + Buckets: prometheus.DefBuckets, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[STABLE] (Deprecated since 1.15.0) histogram help message", }, + // Hidden metrics { - desc: "Test hidden", + desc: "ALPHA metric hidden", HistogramOpts: &HistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "histogram help message", DeprecatedVersion: "1.14.0", Buckets: prometheus.DefBuckets, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "BETA metric hidden", + HistogramOpts: &HistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + + Subsystem: "subsystem", + StabilityLevel: BETA, + + Help: "histogram help message", + DeprecatedVersion: "1.14.0", + Buckets: prometheus.DefBuckets, + }, + + labels: []string{"label_a", "label_b"}, expectedMetricCount: 0, expectedHelp: "histogram help message", }, @@ -189,11 +358,7 @@ func TestHistogramVec(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewHistogramVec(test.HistogramOpts, test.labels) registry.MustRegister(c) ov12 := c.WithLabelValues("1", "2") @@ -333,7 +498,6 @@ func TestHistogramWithExemplar(t *testing.T) { Help: "helpless", Buckets: []float64{100}, }) - _ = histogram.WithContext(ctxForSpanCtx) registry := newKubeRegistry(apimachineryversion.Info{ Major: "1", @@ -343,7 +507,7 @@ func TestHistogramWithExemplar(t *testing.T) { registry.MustRegister(histogram) // Act. - histogram.Observe(value) + histogram.WithContext(ctxForSpanCtx).Observe(value) // Assert. mfs, err := registry.Gather() @@ -492,3 +656,116 @@ func TestHistogramVecWithExemplar(t *testing.T) { } } } + +// TestHistogramConcurrentWithContextRace reproduces the race condition in Histogram.WithContext +// where h.ctx is written concurrently with reads in withExemplar method. +// This test simulates the real authentication flow that triggers the race: +// x509.AuthenticateRequest -> union.AuthenticateRequest -> group.AuthenticateRequest +func TestHistogramConcurrentWithContextRace(t *testing.T) { + opts := &HistogramOpts{ + Namespace: "apiserver", + Subsystem: "authentication", + Name: "requests_total", + Help: "Authentication requests histogram for race condition testing", + Buckets: prometheus.DefBuckets, + } + + h := NewHistogram(opts) + + // Force initialization by calling initializeMetric directly + h.initializeMetric() + + // Create contexts with trace spans to trigger exemplar code path + ctx1, span1 := createContextWithSpan("x509-auth", "authenticate-request") + defer span1.End() + ctx2, span2 := createContextWithSpan("union-auth", "union-authenticate") + defer span2.End() + + var wg sync.WaitGroup + iterations := 10000 // Increase iterations to make race more likely + + // Goroutine 1: Simulate x509 Authenticator calling WithContext + // This matches: x509.(*Authenticator).AuthenticateRequest() calling WithContext + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate authentication request processing + simulateX509AuthenticateRequest(h, ctx1, i) + } + }() + + // Goroutine 2: Simulate concurrent Observe calls (metrics collection) + // This matches the withExemplar/Observe path that reads h.ctx + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + h.Observe(float64(i % 10)) + } + }() + + // Goroutine 3: Simulate union AuthenticateRequest calling WithContext + // This matches: union.(*unionAuthRequestHandler).AuthenticateRequest() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate union authentication processing + simulateUnionAuthenticateRequest(h, ctx2, i) + } + }() + + // Goroutine 4: Simulate group AuthenticateRequest calling WithContext + // This matches: group.(*AuthenticatedGroupAdder).AuthenticateRequest() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + // Simulate group authentication processing + simulateGroupAuthenticateRequest(h, ctx1, ctx2, i) + } + }() + + wg.Wait() +} + +// simulateX509AuthenticateRequest simulates the call path from x509 authenticator +func simulateX509AuthenticateRequest(h ObserverMetric, ctx context.Context, iteration int) { + // Simulate the authentication request processing that calls WithContext + if hWithCtx, ok := h.(*Histogram); ok { + hWithCtx.WithContext(ctx) + } +} + +// simulateUnionAuthenticateRequest simulates the call path from union authenticator +func simulateUnionAuthenticateRequest(h ObserverMetric, ctx context.Context, iteration int) { + // Simulate union authentication processing + if hWithCtx, ok := h.(*Histogram); ok { + hWithCtx.WithContext(ctx) + } +} + +// simulateGroupAuthenticateRequest simulates the call path from group authenticator +func simulateGroupAuthenticateRequest(h ObserverMetric, ctx1, ctx2 context.Context, iteration int) { + // Alternate between contexts to simulate different authentication scenarios + ctx := ctx1 + if iteration%3 == 0 { + ctx = ctx2 + } + + if hWithCtx, ok := h.(*Histogram); ok { + hWithCtx.WithContext(ctx) + } +} + +// Helper function to create a context with a valid trace span +func createContextWithSpan(traceID, spanID string) (context.Context, trace.Span) { + ctx := context.Background() + + // Create a noop tracer and span for testing + tracer := tracenoop.NewTracerProvider().Tracer("test") + ctx, span := tracer.Start(ctx, "test-span") + + return ctx, span +} diff --git a/metrics/legacyregistry/registry.go b/metrics/legacyregistry/registry.go index 64a430b7..c8c0bc08 100644 --- a/metrics/legacyregistry/registry.go +++ b/metrics/legacyregistry/registry.go @@ -90,3 +90,8 @@ func CustomMustRegister(cs ...metrics.StableCollector) { prometheus.MustRegister(c) } } + +// GetProcessStart return processStart value +func GetProcessStart() time.Time { + return processStart +} diff --git a/metrics/legacyregistry/registry_test.go b/metrics/legacyregistry/registry_test.go index fa279c35..a17d571d 100644 --- a/metrics/legacyregistry/registry_test.go +++ b/metrics/legacyregistry/registry_test.go @@ -21,18 +21,21 @@ import ( "net/http/httptest" "strconv" "testing" - "time" ) const ( processStartTimeHeader = "Process-Start-Time-Unix" ) +var ( + processStartTestCopy = &processStart +) + func TestProcessStartTimeHeader(t *testing.T) { - now := time.Now() + now := GetProcessStart() handler := Handler() - request, _ := http.NewRequest("GET", "/", nil) + request, _ := http.NewRequest(http.MethodGet, "/", nil) writer := httptest.NewRecorder() handler.ServeHTTP(writer, request) got := writer.Header().Get(processStartTimeHeader) @@ -41,3 +44,11 @@ func TestProcessStartTimeHeader(t *testing.T) { t.Errorf("got %d, wanted %d", gotInt, now.Unix()) } } + +// processStart must never be reassigned after init. +// This test ensures processStart doesn't change across calls +func TestProcessStartImmutable(t *testing.T) { + if *processStartTestCopy != processStart { + t.Errorf("processStart test values differ: %v != %v", processStartTestCopy, processStart) + } +} diff --git a/metrics/metric.go b/metrics/metric.go index c8b08399..cb40a856 100644 --- a/metrics/metric.go +++ b/metrics/metric.go @@ -18,6 +18,7 @@ package metrics import ( "sync" + "sync/atomic" "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" @@ -65,8 +66,8 @@ with the kubeCollector itself as an argument. */ type lazyMetric struct { fqName string - isDeprecated bool - isHidden bool + isDeprecated atomic.Bool + isHidden atomic.Bool isCreated bool createLock sync.RWMutex markDeprecationOnce sync.Once @@ -99,41 +100,39 @@ func (r *lazyMetric) lazyInit(self kubeCollector, fqName string) { // Disclaimer: disabling a metric via a CLI flag has higher precedence than // deprecation and will override show-hidden-metrics for the explicitly // disabled metric. -func (r *lazyMetric) preprocessMetric(version semver.Version) { +func (r *lazyMetric) preprocessMetric(currentVersion semver.Version) { disabledMetricsLock.RLock() defer disabledMetricsLock.RUnlock() // disabling metrics is higher in precedence than showing hidden metrics if _, ok := disabledMetrics[r.fqName]; ok { - r.isHidden = true + r.isHidden.Store(true) return } - selfVersion := r.self.DeprecatedVersion() - if selfVersion == nil { + deprecatedVersion := r.self.DeprecatedVersion() + if deprecatedVersion == nil { return } r.markDeprecationOnce.Do(func() { - if selfVersion.LTE(version) { - r.isDeprecated = true - } + r.isDeprecated.Store(isDeprecated(currentVersion, *deprecatedVersion)) - if ShouldShowHidden() { - klog.Warningf("Hidden metrics (%s) have been manually overridden, showing this very deprecated metric.", r.fqName) - return - } - if shouldHide(&version, selfVersion) { + if shouldHide(r.stabilityLevel, ¤tVersion, deprecatedVersion) { + if shouldShowHidden() { + klog.Warningf("Hidden metrics (%s) have been manually overridden, showing this very deprecated metric.", r.fqName) + return + } // TODO(RainbowMango): Remove this log temporarily. https://github.com/kubernetes/kubernetes/issues/85369 // klog.Warningf("This metric has been deprecated for more than one release, hiding.") - r.isHidden = true + r.isHidden.Store(true) } }) } func (r *lazyMetric) IsHidden() bool { - return r.isHidden + return r.isHidden.Load() } func (r *lazyMetric) IsDeprecated() bool { - return r.isDeprecated + return r.isDeprecated.Load() } // Create forces the initialization of metric which has been deferred until @@ -176,8 +175,8 @@ func (r *lazyMetric) ClearState() { r.createLock.Lock() defer r.createLock.Unlock() - r.isDeprecated = false - r.isHidden = false + r.isDeprecated.Store(false) + r.isHidden.Store(false) r.isCreated = false r.markDeprecationOnce = sync.Once{} r.createOnce = sync.Once{} @@ -210,11 +209,6 @@ func (c *selfCollector) Collect(ch chan<- prometheus.Metric) { ch <- c.metric } -// metricWithExemplar is an interface that knows how to attach an exemplar to certain supported metric types. -type metricWithExemplar interface { - withExemplar(v float64) -} - // no-op vecs for convenience var noopCounterVec = &prometheus.CounterVec{} var noopHistogramVec = &prometheus.HistogramVec{} diff --git a/metrics/prometheus/restclient/metrics.go b/metrics/prometheus/restclient/metrics.go index d0c80de0..fd2c5f75 100644 --- a/metrics/prometheus/restclient/metrics.go +++ b/metrics/prometheus/restclient/metrics.go @@ -165,6 +165,17 @@ var ( []string{"code", "call_status"}, ) + execPluginPolicyCalls = k8smetrics.NewCounterVec( + &k8smetrics.CounterOpts{ + StabilityLevel: k8smetrics.ALPHA, + Name: "rest_client_exec_plugin_policy_call_total", + Help: "Number of comparisons of an exec plugin to the plugin policy " + + "and allowlist (if any), partitioned by whether or not the policy " + + "permits the plugin", + }, + []string{"allowed", "denied"}, + ) + transportCacheEntries = k8smetrics.NewGauge( &k8smetrics.GaugeOpts{ Name: "rest_client_transport_cache_entries", @@ -208,6 +219,7 @@ func init() { RequestResult: &resultAdapter{requestResult}, RequestRetry: &retryAdapter{requestRetry}, ExecPluginCalls: &callsAdapter{m: execPluginCalls}, + ExecPluginPolicyCalls: &policyAdapter{m: execPluginPolicyCalls}, TransportCacheEntries: &transportCacheAdapter{m: transportCacheEntries}, TransportCreateCalls: &transportCacheCallsAdapter{m: transportCacheCalls}, }) @@ -269,6 +281,14 @@ func (r *callsAdapter) Increment(code int, callStatus string) { r.m.WithLabelValues(fmt.Sprintf("%d", code), callStatus).Inc() } +type policyAdapter struct { + m *k8smetrics.CounterVec +} + +func (r *policyAdapter) Increment(status string) { + r.m.WithLabelValues(status).Inc() +} + type retryAdapter struct { m *k8smetrics.CounterVec } diff --git a/metrics/registry.go b/metrics/registry.go index 203813e8..aa12f2e2 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -17,7 +17,7 @@ limitations under the License. package metrics import ( - "fmt" + "strings" "sync" "sync/atomic" @@ -71,19 +71,58 @@ var ( ) ) -// shouldHide be used to check if a specific metric with deprecated version should be hidden +// shouldHide is used to check if a specific metric with deprecated version should be hidden // according to metrics deprecation lifecycle. -func shouldHide(currentVersion *semver.Version, deprecatedVersion *semver.Version) bool { - guardVersion, err := semver.Make(fmt.Sprintf("%d.%d.0", currentVersion.Major, currentVersion.Minor)) - if err != nil { - panic("failed to make version from current version") +func shouldHide(stabilityLevel StabilityLevel, currentVersion *semver.Version, deprecatedVersion *semver.Version) bool { + hiddenMinor := deprecatedVersion.Minor + deprecationPeriodMinorVersions(stabilityLevel) + + switch { + case deprecatedVersion.Major < currentVersion.Major: + return true + case deprecatedVersion.Major > currentVersion.Major: + return false + + // deprecatedVersion.Major == currentVersion.Major + case hiddenMinor < currentVersion.Minor: + return true + case hiddenMinor > currentVersion.Minor: + return false + + // deprecatedVersion.Minor == currentVersion.Minor + case strings.Contains(currentVersion.String(), "alpha.0"): + // Wait until we're past the alpha.0 period of a minor development cycle to hide metrics whose deprecation period ends in that minor version. + // See discussion in https://github.com/kubernetes/kubernetes/issues/133429#issuecomment-3165551443 + return false + default: + return true } +} + +// getDeprecationReleaseWindow returns the number of minor releases a metric should be served +// after its deprecated version, based on its stability level. +func deprecationPeriodMinorVersions(stabilityLevel StabilityLevel) uint64 { + switch stabilityLevel { + case STABLE: + return 3 + case BETA: + return 1 + default: // ALPHA, INTERNAL + return 0 + } +} - if deprecatedVersion.LT(guardVersion) { +// isDeprecated returns true if the current version, ignoring pre-release tags, +// is greater than or equal to the deprecated version. +func isDeprecated(currentVersion, deprecatedVersion semver.Version) bool { + switch { + case currentVersion.Major < deprecatedVersion.Major: + return false + case currentVersion.Major > deprecatedVersion.Major: return true } - return false + // currentVersion.Major == deprecatedVersion.Major + return currentVersion.Minor >= deprecatedVersion.Minor } // ValidateShowHiddenMetricsVersion checks invalid version for which show hidden metrics. @@ -117,10 +156,10 @@ func SetShowHidden() { }) } -// ShouldShowHidden returns whether showing hidden deprecated metrics +// shouldShowHidden returns whether showing hidden deprecated metrics // is enabled. While the primary usecase for this is internal (to determine // registration behavior) this can also be used to introspect -func ShouldShowHidden() bool { +func shouldShowHidden() bool { return showHidden.Load() } diff --git a/metrics/registry_test.go b/metrics/registry_test.go index ff607dd8..9be044e1 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -51,6 +51,16 @@ var ( DeprecatedVersion: "1.15.0", }, ) + betaDeprecatedCounter = NewCounter( + &CounterOpts{ + Namespace: "some_namespace", + Name: "test_beta_dep_counter", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "counter help", + DeprecatedVersion: "1.15.0", + }, + ) alphaHiddenCounter = NewCounter( &CounterOpts{ Namespace: "some_namespace", @@ -64,33 +74,194 @@ var ( ) func TestShouldHide(t *testing.T) { - currentVersion := parseVersion(apimachineryversion.Info{ - Major: "1", - Minor: "17", - GitVersion: "v1.17.1-alpha-1.12345", - }) + currentV1_17 := parseSemver("1.17.0") + currentV1_18Alpha0 := parseSemver("1.18.0-alpha.0") + currentV1_18Alpha1 := parseSemver("1.18.0-alpha.1") var tests = []struct { desc string + currentVersion *semver.Version + stabilityLevel StabilityLevel deprecatedVersion string shouldHide bool }{ { - desc: "current minor release should not be hidden", + desc: "INTERNAL metric deprecated in the current release - should be hidden", + currentVersion: currentV1_17, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.17.0", + shouldHide: true, + }, + { + desc: "INTERNAL metric deprecated 1 release ago - should be hidden", + currentVersion: currentV1_17, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.16.0", + shouldHide: true, + }, + { + desc: "INTERNAL metric to be deprecated 1 release later - should not be hidden", + currentVersion: currentV1_17, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "BETA metric deprecated in the current release - should not be hidden", + currentVersion: currentV1_17, + stabilityLevel: BETA, deprecatedVersion: "1.17.0", shouldHide: false, }, { - desc: "older minor release should be hidden", + desc: "BETA metric deprecated 1 release ago - should be hidden", + currentVersion: currentV1_17, + stabilityLevel: BETA, deprecatedVersion: "1.16.0", shouldHide: true, }, + { + desc: "BETA metric to be deprecated 1 release later - should not be hidden", + currentVersion: currentV1_17, + stabilityLevel: BETA, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated in the current release - should not be hidden", + currentVersion: parseSemver("1.17.0"), + stabilityLevel: STABLE, + deprecatedVersion: "1.17.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated 3 releases ago - should be hidden", + currentVersion: currentV1_17, + stabilityLevel: STABLE, + deprecatedVersion: "1.14.0", + shouldHide: true, + }, + { + desc: "STABLE metric deprecated 2 releases ago - should not be hidden", + currentVersion: currentV1_17, + stabilityLevel: STABLE, + deprecatedVersion: "1.15.0", + shouldHide: false, + }, + { + desc: "STABLE metric to be deprecated 1 release later - should not be hidden", + currentVersion: currentV1_17, + stabilityLevel: STABLE, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + // --- Pre-Alpha.0 Tests --- + { + desc: "INTERNAL metric deprecated in the current minor - should not be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "INTERNAL metric deprecated 1 release ago - should be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.17.0", + shouldHide: true, + }, + { + desc: "BETA metric deprecated in the current minor - should not be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: BETA, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "BETA metric deprecated 1 release ago - should not be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: BETA, + deprecatedVersion: "1.17.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated in the current minor - should be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: STABLE, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated 1 release ago - should not be hidden", + currentVersion: currentV1_18Alpha0, + stabilityLevel: STABLE, + deprecatedVersion: "1.17.0", + shouldHide: false, + }, + // --- Pre-Alpha.1 Tests --- + { + + desc: "INTERNAL metric in the current minor - should be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.18.0", + shouldHide: true, + }, + { + desc: "INTERNAL metric deprecated in prior patch - should be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: INTERNAL, + deprecatedVersion: "1.18.0", + shouldHide: true, + }, + { + desc: "BETA metric deprecated in the current minor - should not be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: BETA, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "BETA metric deprecated in prior patch - should not be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: BETA, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "BETA metric deprecated 1 release ago - should be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: BETA, + deprecatedVersion: "1.17.0", + shouldHide: true, + }, + { + desc: "STABLE metric deprecated in the current minor - should not be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: STABLE, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated in prior patch - should not be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: STABLE, + deprecatedVersion: "1.18.0", + shouldHide: false, + }, + { + desc: "STABLE metric deprecated 3 minors ago - should be hidden", + currentVersion: currentV1_18Alpha1, + stabilityLevel: STABLE, + deprecatedVersion: "1.15.0", + shouldHide: true, + }, } for _, test := range tests { tc := test t.Run(tc.desc, func(t *testing.T) { - result := shouldHide(¤tVersion, parseSemver(tc.deprecatedVersion)) + result := shouldHide(tc.stabilityLevel, tc.currentVersion, parseSemver(tc.deprecatedVersion)) assert.Equalf(t, tc.shouldHide, result, "expected should hide %v, but got %v", tc.shouldHide, result) }) } @@ -125,9 +296,9 @@ func TestRegister(t *testing.T) { desc: "test alpha deprecated metric", metrics: []*Counter{alphaDeprecatedCounter}, expectedErrors: []error{nil}, - expectedIsCreatedValues: []bool{true}, + expectedIsCreatedValues: []bool{false}, expectedIsDeprecated: []bool{true}, - expectedIsHidden: []bool{false}, + expectedIsHidden: []bool{true}, }, { desc: "test alpha hidden metric", @@ -192,7 +363,7 @@ func TestMustRegister(t *testing.T) { }, { desc: "test must registering same deprecated metric", - metrics: []*Counter{alphaDeprecatedCounter, alphaDeprecatedCounter}, + metrics: []*Counter{betaDeprecatedCounter, betaDeprecatedCounter}, registryVersion: &v115, expectedPanics: []bool{false, true}, }, @@ -331,11 +502,11 @@ func TestEnableHiddenMetrics(t *testing.T) { Name: "hidden_metric_register", Help: "counter help", StabilityLevel: STABLE, - DeprecatedVersion: "1.16.0", + DeprecatedVersion: "1.14.0", }), mustRegister: false, expectedMetric: ` - # HELP hidden_metric_register [STABLE] (Deprecated since 1.16.0) counter help + # HELP hidden_metric_register [STABLE] (Deprecated since 1.14.0) counter help # TYPE hidden_metric_register counter hidden_metric_register 1 `, @@ -347,11 +518,11 @@ func TestEnableHiddenMetrics(t *testing.T) { Name: "hidden_metric_must_register", Help: "counter help", StabilityLevel: STABLE, - DeprecatedVersion: "1.16.0", + DeprecatedVersion: "1.14.0", }), mustRegister: true, expectedMetric: ` - # HELP hidden_metric_must_register [STABLE] (Deprecated since 1.16.0) counter help + # HELP hidden_metric_must_register [STABLE] (Deprecated since 1.14.0) counter help # TYPE hidden_metric_must_register counter hidden_metric_must_register 1 `, @@ -394,8 +565,8 @@ func TestEnableHiddenStableCollector(t *testing.T) { GitVersion: "v1.17.0-alpha-1.12345", } var normal = NewDesc("test_enable_hidden_custom_metric_normal", "this is a normal metric", []string{"name"}, nil, STABLE, "") - var hiddenA = NewDesc("test_enable_hidden_custom_metric_hidden_a", "this is the hidden metric A", []string{"name"}, nil, STABLE, "1.16.0") - var hiddenB = NewDesc("test_enable_hidden_custom_metric_hidden_b", "this is the hidden metric B", []string{"name"}, nil, STABLE, "1.16.0") + var hiddenA = NewDesc("test_enable_hidden_custom_metric_hidden_a", "this is the hidden metric A", []string{"name"}, nil, STABLE, "1.14.0") + var hiddenB = NewDesc("test_enable_hidden_custom_metric_hidden_b", "this is the hidden metric B", []string{"name"}, nil, STABLE, "1.14.0") var tests = []struct { name string @@ -411,10 +582,10 @@ func TestEnableHiddenStableCollector(t *testing.T) { "test_enable_hidden_custom_metric_hidden_b"}, expectMetricsBeforeEnable: "", expectMetricsAfterEnable: ` - # HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.16.0) this is the hidden metric A + # HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A # TYPE test_enable_hidden_custom_metric_hidden_a gauge test_enable_hidden_custom_metric_hidden_a{name="value"} 1 - # HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.16.0) this is the hidden metric B + # HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B # TYPE test_enable_hidden_custom_metric_hidden_b gauge test_enable_hidden_custom_metric_hidden_b{name="value"} 1 `, @@ -434,10 +605,10 @@ func TestEnableHiddenStableCollector(t *testing.T) { # HELP test_enable_hidden_custom_metric_normal [STABLE] this is a normal metric # TYPE test_enable_hidden_custom_metric_normal gauge test_enable_hidden_custom_metric_normal{name="value"} 1 - # HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.16.0) this is the hidden metric A + # HELP test_enable_hidden_custom_metric_hidden_a [STABLE] (Deprecated since 1.14.0) this is the hidden metric A # TYPE test_enable_hidden_custom_metric_hidden_a gauge test_enable_hidden_custom_metric_hidden_a{name="value"} 1 - # HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.16.0) this is the hidden metric B + # HELP test_enable_hidden_custom_metric_hidden_b [STABLE] (Deprecated since 1.14.0) this is the hidden metric B # TYPE test_enable_hidden_custom_metric_hidden_b gauge test_enable_hidden_custom_metric_hidden_b{name="value"} 1 `, diff --git a/metrics/summary.go b/metrics/summary.go index 3654e4ea..82898e60 100644 --- a/metrics/summary.go +++ b/metrics/summary.go @@ -33,7 +33,7 @@ const ( // Summary is our internal representation for our wrapping struct around prometheus // summaries. Summary implements both kubeCollector and ObserverMetric // -// DEPRECATED: as per the metrics overhaul KEP +// Deprecated: as per the metrics overhaul KEP type Summary struct { ObserverMetric *SummaryOpts @@ -44,7 +44,7 @@ type Summary struct { // NewSummary returns an object which is Summary-like. However, nothing // will be measured until the summary is registered somewhere. // -// DEPRECATED: as per the metrics overhaul KEP +// Deprecated: as per the metrics overhaul KEP func NewSummary(opts *SummaryOpts) *Summary { opts.StabilityLevel.setDefaults() @@ -91,7 +91,7 @@ func (s *Summary) WithContext(ctx context.Context) ObserverMetric { // SummaryVec is the internal representation of our wrapping struct around prometheus // summaryVecs. // -// DEPRECATED: as per the metrics overhaul KEP +// Deprecated: as per the metrics overhaul KEP type SummaryVec struct { *prometheus.SummaryVec *SummaryOpts @@ -105,7 +105,7 @@ type SummaryVec struct { // and only members extracted after // registration will actually measure anything. // -// DEPRECATED: as per the metrics overhaul KEP +// Deprecated: as per the metrics overhaul KEP func NewSummaryVec(opts *SummaryOpts, labels []string) *SummaryVec { opts.StabilityLevel.setDefaults() diff --git a/metrics/summary_test.go b/metrics/summary_test.go index 02fa895e..ae3848a9 100644 --- a/metrics/summary_test.go +++ b/metrics/summary_test.go @@ -19,7 +19,6 @@ package metrics import ( "testing" - "github.com/blang/semver/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,16 +26,21 @@ import ( ) func TestSummary(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *SummaryOpts - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", SummaryOpts: &SummaryOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -44,12 +48,36 @@ func TestSummary(t *testing.T) { Help: "summary help message", StabilityLevel: ALPHA, }, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] summary help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "summary help message", + StabilityLevel: BETA, + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] summary help message", + }, + { + desc: "STABLE metric non deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "summary help message", + StabilityLevel: STABLE, + }, + expectedMetricCount: 1, + expectedHelp: "[STABLE] summary help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", SummaryOpts: &SummaryOpts{ Namespace: "namespace", Name: "metric_test_name", @@ -58,20 +86,72 @@ func TestSummary(t *testing.T) { DeprecatedVersion: "1.15.0", StabilityLevel: ALPHA, }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "BETA metric deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "summary help message", + DeprecatedVersion: "1.15.0", + StabilityLevel: BETA, + }, + expectedMetricCount: 1, + expectedHelp: "[BETA] (Deprecated since 1.15.0) summary help message", + }, + { + desc: "STABLE metric deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "summary help message", + DeprecatedVersion: "1.15.0", + StabilityLevel: STABLE, + }, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) summary help message", + expectedHelp: "[STABLE] (Deprecated since 1.15.0) summary help message", }, + // Hidden metrics { - desc: "Test hidden", + desc: "ALPHA metric hidden", SummaryOpts: &SummaryOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "summary help message", DeprecatedVersion: "1.14.0", }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "BETA metric hidden", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "summary help message", + DeprecatedVersion: "1.14.0", + }, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "STABLE metric hidden", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "summary help message", + DeprecatedVersion: "1.12.0", + }, expectedMetricCount: 0, expectedHelp: "summary help message", }, @@ -79,11 +159,7 @@ func TestSummary(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewSummary(test.SummaryOpts) registry.MustRegister(c) @@ -114,53 +190,141 @@ func TestSummary(t *testing.T) { } func TestSummaryVec(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *SummaryOpts labels []string - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", SummaryOpts: &SummaryOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "summary help message", + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "summary help message", }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "[ALPHA] summary help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "summary help message", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[BETA] summary help message", + }, + { + desc: "STABLE metric non deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "summary help message", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[STABLE] summary help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + Help: "summary help message", + DeprecatedVersion: "1.15.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "BETA metric deprecated", SummaryOpts: &SummaryOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", Help: "summary help message", DeprecatedVersion: "1.15.0", + StabilityLevel: BETA, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, - expectedHelp: "[ALPHA] (Deprecated since 1.15.0) summary help message", + expectedHelp: "[BETA] (Deprecated since 1.15.0) summary help message", }, { - desc: "Test hidden", + desc: "STABLE metric deprecated", SummaryOpts: &SummaryOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", Help: "summary help message", + DeprecatedVersion: "1.15.0", + StabilityLevel: STABLE, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "[STABLE] (Deprecated since 1.15.0) summary help message", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "summary help message", + DeprecatedVersion: "1.14.0", + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "BETA metric hidden", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "summary help message", DeprecatedVersion: "1.14.0", }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "summary help message", + }, + { + desc: "STABLE metric hidden", + SummaryOpts: &SummaryOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "summary help message", + DeprecatedVersion: "1.12.0", + }, + labels: []string{"label_a", "label_b"}, expectedMetricCount: 0, expectedHelp: "summary help message", }, @@ -168,11 +332,7 @@ func TestSummaryVec(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) c := NewSummaryVec(test.SummaryOpts, test.labels) registry.MustRegister(c) c.WithLabelValues("1", "2").Observe(1.0) diff --git a/metrics/testutil/metrics.go b/metrics/testutil/metrics.go index c729ac49..a8d710af 100644 --- a/metrics/testutil/metrics.go +++ b/metrics/testutil/metrics.go @@ -109,7 +109,7 @@ func ParseMetrics(data string, output *Metrics) error { // proto messages in a map where the metric names are the keys, along with any // error encountered. func TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) { - var textParser expfmt.TextParser + textParser := expfmt.NewTextParser(model.UTF8Validation) return textParser.TextToMetricFamilies(in) } diff --git a/metrics/testutil/testutil_test.go b/metrics/testutil/testutil_test.go index 61af603a..083b978e 100644 --- a/metrics/testutil/testutil_test.go +++ b/metrics/testutil/testutil_test.go @@ -31,11 +31,12 @@ func TestNewFakeKubeRegistry(t *testing.T) { Help: "counter help", }, ) - deprecatedCounter := metrics.NewCounter( + deprecatedBetaCounter := metrics.NewCounter( &metrics.CounterOpts{ Name: "test_deprecated_total", Help: "counter help", DeprecatedVersion: "1.18.0", + StabilityLevel: metrics.BETA, }, ) hiddenCounter := metrics.NewCounter( @@ -62,9 +63,9 @@ func TestNewFakeKubeRegistry(t *testing.T) { }, { name: "deprecated", - metric: deprecatedCounter, + metric: deprecatedBetaCounter, expected: ` - # HELP test_deprecated_total [ALPHA] (Deprecated since 1.18.0) counter help + # HELP test_deprecated_total [BETA] (Deprecated since 1.18.0) counter help # TYPE test_deprecated_total counter test_deprecated_total 0 `, diff --git a/metrics/timing_histogram_test.go b/metrics/timing_histogram_test.go index 80fcb634..0b88810f 100644 --- a/metrics/timing_histogram_test.go +++ b/metrics/timing_histogram_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,55 +29,150 @@ import ( ) func TestTimingHistogram(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *TimingHistogramOpts - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", TimingHistogramOpts: &TimingHistogramOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "histogram help message", - Buckets: DefBuckets, - InitialValue: 13, + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 13, }, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "EXPERIMENTAL: [ALPHA] histogram help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 17, + }, + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [BETA] histogram help message", + }, + { + desc: "STABLE metric non deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 19, + }, + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [STABLE] histogram help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", TimingHistogramOpts: &TimingHistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "histogram help message", DeprecatedVersion: "1.15.0", Buckets: DefBuckets, InitialValue: 3, }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "BETA metric deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: DefBuckets, + InitialValue: 11, + }, expectedMetricCount: 1, - expectedHelp: "EXPERIMENTAL: [ALPHA] (Deprecated since 1.15.0) histogram help message", + expectedHelp: "EXPERIMENTAL: [BETA] (Deprecated since 1.15.0) histogram help message", }, { - desc: "Test hidden", + desc: "STABLE metric deprecated", TimingHistogramOpts: &TimingHistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: DefBuckets, + InitialValue: 23, + }, + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [STABLE] (Deprecated since 1.15.0) histogram help message", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "histogram help message", DeprecatedVersion: "1.14.0", Buckets: DefBuckets, InitialValue: 5, }, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "EXPERIMENTAL: histogram help message", + }, + { + desc: "BETA metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + DeprecatedVersion: "1.14.0", + Buckets: DefBuckets, + InitialValue: 7, + }, + expectedMetricCount: 0, + expectedHelp: "EXPERIMENTAL: histogram help message", + }, + { + desc: "STABLE metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + DeprecatedVersion: "1.12.0", + Buckets: DefBuckets, + InitialValue: 9, + }, expectedMetricCount: 0, expectedHelp: "EXPERIMENTAL: histogram help message", }, @@ -86,11 +180,7 @@ func TestTimingHistogram(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) t0 := time.Now() clk := testclock.NewFakePassiveClock(t0) c := NewTestableTimingHistogram(clk.Now, test.TimingHistogramOpts) @@ -152,59 +242,161 @@ func TestTimingHistogram(t *testing.T) { } func TestTimingHistogramVec(t *testing.T) { - v115 := semver.MustParse("1.15.0") + version1_15Alpha1 := apimachineryversion.Info{ + Major: "1", + Minor: "15", + GitVersion: "v1.15.0-alpha-1.12345", + } + var tests = []struct { desc string *TimingHistogramOpts labels []string - registryVersion *semver.Version expectedMetricCount int expectedHelp string }{ + // Non-deprecated metrics { - desc: "Test non deprecated", + desc: "ALPHA metric non deprecated", TimingHistogramOpts: &TimingHistogramOpts{ - Namespace: "namespace", - Name: "metric_test_name", - Subsystem: "subsystem", - Help: "histogram help message", - Buckets: DefBuckets, - InitialValue: 5, + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 5, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, expectedMetricCount: 1, expectedHelp: "EXPERIMENTAL: [ALPHA] histogram help message", }, { - desc: "Test deprecated", + desc: "BETA metric non deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 7, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [BETA] histogram help message", + }, + { + desc: "STABLE metric non deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + Buckets: DefBuckets, + InitialValue: 9, + }, + labels: []string{"label_a", "label_b"}, + + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [STABLE] histogram help message", + }, + // Deprecated metrics + { + desc: "ALPHA metric deprecated", TimingHistogramOpts: &TimingHistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "histogram help message", DeprecatedVersion: "1.15.0", Buckets: DefBuckets, InitialValue: 13, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "histogram help message", + }, + { + desc: "BETA metric deprecated", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: DefBuckets, + InitialValue: 11, + }, + labels: []string{"label_a", "label_b"}, expectedMetricCount: 1, - expectedHelp: "EXPERIMENTAL: [ALPHA] (Deprecated since 1.15.0) histogram help message", + expectedHelp: "EXPERIMENTAL: [BETA] (Deprecated since 1.15.0) histogram help message", }, { - desc: "Test hidden", + desc: "STABLE metric deprecated", TimingHistogramOpts: &TimingHistogramOpts{ Namespace: "namespace", Name: "metric_test_name", Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + DeprecatedVersion: "1.15.0", + Buckets: DefBuckets, + InitialValue: 17, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 1, + expectedHelp: "EXPERIMENTAL: [STABLE] (Deprecated since 1.15.0) histogram help message", + }, + // Hidden metrics + { + desc: "ALPHA metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: ALPHA, Help: "histogram help message", DeprecatedVersion: "1.14.0", Buckets: DefBuckets, InitialValue: 42, }, labels: []string{"label_a", "label_b"}, - registryVersion: &v115, + expectedMetricCount: 0, + expectedHelp: "EXPERIMENTAL: histogram help message", + }, + { + desc: "BETA metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: BETA, + Help: "histogram help message", + DeprecatedVersion: "1.14.0", + Buckets: DefBuckets, + InitialValue: 19, + }, + labels: []string{"label_a", "label_b"}, + expectedMetricCount: 0, + expectedHelp: "EXPERIMENTAL: histogram help message", + }, + { + desc: "STABLE metric hidden", + TimingHistogramOpts: &TimingHistogramOpts{ + Namespace: "namespace", + Name: "metric_test_name", + Subsystem: "subsystem", + StabilityLevel: STABLE, + Help: "histogram help message", + DeprecatedVersion: "1.12.0", + Buckets: DefBuckets, + InitialValue: 23, + }, + labels: []string{"label_a", "label_b"}, expectedMetricCount: 0, expectedHelp: "EXPERIMENTAL: histogram help message", }, @@ -212,11 +404,7 @@ func TestTimingHistogramVec(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - registry := newKubeRegistry(apimachineryversion.Info{ - Major: "1", - Minor: "15", - GitVersion: "v1.15.0-alpha-1.12345", - }) + registry := newKubeRegistry(version1_15Alpha1) t0 := time.Now() clk := testclock.NewFakePassiveClock(t0) c := NewTestableTimingHistogramVec(clk.Now, test.TimingHistogramOpts, test.labels) diff --git a/tracing/api/v1/doc.go b/tracing/api/v1/doc.go index 48e6e20f..29f9451d 100644 --- a/tracing/api/v1/doc.go +++ b/tracing/api/v1/doc.go @@ -15,6 +15,8 @@ limitations under the License. */ // +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:openapi-model-package=io.k8s.component-base.tracing.api.v1 // Package v1 contains the configuration API for tracing. // diff --git a/tracing/api/v1/zz_generated.model_name.go b/tracing/api/v1/zz_generated.model_name.go new file mode 100644 index 00000000..3efcbf1a --- /dev/null +++ b/tracing/api/v1/zz_generated.model_name.go @@ -0,0 +1,27 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by openapi-gen. DO NOT EDIT. + +package v1 + +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in TracingConfiguration) OpenAPIModelName() string { + return "io.k8s.component-base.tracing.api.v1.TracingConfiguration" +} diff --git a/version/base.go b/version/base.go index b5e88901..94da79c0 100644 --- a/version/base.go +++ b/version/base.go @@ -60,5 +60,5 @@ const ( // DefaultKubeBinaryVersion is the hard coded k8 binary version based on the latest K8s release. // It is supposed to be consistent with gitMajor and gitMinor, except for local tests, where gitMajor and gitMinor are "". // Should update for each minor release! - DefaultKubeBinaryVersion = "1.34" + DefaultKubeBinaryVersion = "1.35" ) diff --git a/zpages/flagz/flagreader.go b/zpages/flagz/flagreader.go deleted file mode 100644 index 40fe09b4..00000000 --- a/zpages/flagz/flagreader.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagz - -import ( - "github.com/spf13/pflag" - cliflag "k8s.io/component-base/cli/flag" -) - -type Reader interface { - GetFlagz() map[string]string -} - -// NamedFlagSetsReader implements Reader for cliflag.NamedFlagSets -type NamedFlagSetsReader struct { - FlagSets cliflag.NamedFlagSets -} - -func (n NamedFlagSetsReader) GetFlagz() map[string]string { - return convertNamedFlagSetToFlags(&n.FlagSets) -} - -func convertNamedFlagSetToFlags(flagSets *cliflag.NamedFlagSets) map[string]string { - flags := make(map[string]string) - for _, fs := range flagSets.FlagSets { - fs.VisitAll(func(flag *pflag.Flag) { - if flag.Value != nil { - value := flag.Value.String() - if set, ok := flag.Annotations["classified"]; ok && len(set) > 0 { - value = "CLASSIFIED" - } - flags[flag.Name] = value - } - }) - } - - return flags -} diff --git a/zpages/flagz/flagreader_test.go b/zpages/flagz/flagreader_test.go deleted file mode 100644 index 3d634d60..00000000 --- a/zpages/flagz/flagreader_test.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagz - -import ( - "reflect" - "testing" - - "github.com/spf13/pflag" - - "k8s.io/component-base/cli/flag" -) - -func TestConvertNamedFlagSetToFlags(t *testing.T) { - tests := []struct { - name string - flagSets *flag.NamedFlagSets - want map[string]string - }{ - { - name: "basic flags", - flagSets: &flag.NamedFlagSets{ - FlagSets: map[string]*pflag.FlagSet{ - "test": flagSet(t, map[string]flagValue{ - "flag1": {value: "value1", sensitive: false}, - "flag2": {value: "value2", sensitive: false}, - }), - }, - }, - want: map[string]string{ - "flag1": "value1", - "flag2": "value2", - }, - }, - { - name: "classified flags", - flagSets: &flag.NamedFlagSets{ - FlagSets: map[string]*pflag.FlagSet{ - "test": flagSet(t, map[string]flagValue{ - "secret1": {value: "value1", sensitive: true}, - "flag2": {value: "value2", sensitive: false}, - }), - }, - }, - want: map[string]string{ - "flag2": "value2", - "secret1": "CLASSIFIED", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := convertNamedFlagSetToFlags(tt.flagSets) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ConvertNamedFlagSetToFlags() = %v, want %v", got, tt.want) - } - }) - } -} - -type flagValue struct { - value string - sensitive bool -} - -func flagSet(t *testing.T, flags map[string]flagValue) *pflag.FlagSet { - fs := pflag.NewFlagSet("test-set", pflag.ContinueOnError) - for flagName, flagVal := range flags { - flagValue := "" - fs.StringVar(&flagValue, flagName, flagVal.value, "test-usage") - if flagVal.sensitive { - err := fs.SetAnnotation(flagName, "classified", []string{"true"}) - if err != nil { - t.Fatalf("unexpected error when setting flag annotation: %v", err) - } - } - } - - return fs -} diff --git a/zpages/flagz/flagz.go b/zpages/flagz/flagz.go deleted file mode 100644 index 99a2f448..00000000 --- a/zpages/flagz/flagz.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagz - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "net/http" - "sort" - "sync" - - "k8s.io/component-base/zpages/httputil" - "k8s.io/klog/v2" -) - -const ( - DefaultFlagzPath = "/flagz" - - flagzHeaderFmt = ` -%s flags -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. - -` -) - -var ( - delimiters = []string{":", ": ", "=", " "} -) - -type registry struct { - response bytes.Buffer - once sync.Once -} - -type mux interface { - Handle(path string, handler http.Handler) -} - -func Install(m mux, componentName string, flagReader Reader) { - var reg registry - reg.installHandler(m, componentName, flagReader) -} - -func (reg *registry) installHandler(m mux, componentName string, flagReader Reader) { - m.Handle(DefaultFlagzPath, reg.handleFlags(componentName, flagReader)) -} - -func (reg *registry) handleFlags(componentName string, flagReader Reader) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if !httputil.AcceptableMediaType(r) { - http.Error(w, httputil.ErrUnsupportedMediaType.Error(), http.StatusNotAcceptable) - return - } - - reg.once.Do(func() { - fmt.Fprintf(®.response, flagzHeaderFmt, componentName) - if flagReader == nil { - klog.Error("received nil flagReader") - return - } - - randomIndex := rand.Intn(len(delimiters)) - separator := delimiters[randomIndex] - // Randomize the delimiter for printing to prevent scraping of the response. - printSortedFlags(®.response, flagReader.GetFlagz(), separator) - }) - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - _, err := w.Write(reg.response.Bytes()) - if err != nil { - klog.Errorf("error writing response: %v", err) - http.Error(w, "error writing response", http.StatusInternalServerError) - } - } -} - -func printSortedFlags(w io.Writer, flags map[string]string, separator string) { - var sortedKeys []string - for key := range flags { - sortedKeys = append(sortedKeys, key) - } - - sort.Strings(sortedKeys) - for _, key := range sortedKeys { - fmt.Fprintf(w, "%s%s%s\n", key, separator, flags[key]) - } -} diff --git a/zpages/flagz/flagz_test.go b/zpages/flagz/flagz_test.go deleted file mode 100644 index c8568c8b..00000000 --- a/zpages/flagz/flagz_test.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package flagz - -import ( - "fmt" - "net/http" - "net/http/httptest" - "sort" - "strings" - "testing" - - "github.com/spf13/pflag" - "github.com/stretchr/testify/assert" - cliflag "k8s.io/component-base/cli/flag" -) - -const wantTmpl = `%s flags -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. -` - -func TestFlagz(t *testing.T) { - componentName := "test-server" - delimiters = []string{"="} - wantHeaderLines := strings.Split(fmt.Sprintf(wantTmpl, componentName), "\n") - tests := []struct { - name string - header string - flagzReader Reader - wantStatus int - wantResp []string - }{ - { - name: "nil flags", - wantStatus: http.StatusOK, - wantResp: wantHeaderLines, - }, - { - name: "unaccepted header", - header: "some header", - wantStatus: http.StatusNotAcceptable, - }, - { - name: "test flags", - flagzReader: NamedFlagSetsReader{ - FlagSets: cliflag.NamedFlagSets{ - FlagSets: map[string]*pflag.FlagSet{ - "test": flagSet(t, map[string]flagValue{ - "test-flag-bar": { - value: "test-value-bar", - sensitive: false, - }, - "test-flag-foo": { - value: "test-value-foo", - sensitive: false, - }, - }), - }, - }, - }, - wantStatus: http.StatusOK, - wantResp: append(wantHeaderLines, - "test-flag-bar=test-value-bar", - "test-flag-foo=test-value-foo", - ), - }, - } - - for i, test := range tests { - t.Run(test.name, func(t *testing.T) { - mux := http.NewServeMux() - Install(mux, componentName, test.flagzReader) - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com%s", DefaultFlagzPath), nil) - if err != nil { - t.Fatalf("case[%d] Unexpected error: %v", i, err) - } - - req.Header.Set("Accept", "text/plain; charset=utf-8") - if test.header != "" { - req.Header.Set("Accept", test.header) - } - - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - assert.Equal(t, test.wantStatus, w.Code, "case[%s] Expected status code %d, got %d", test.name, test.wantStatus, w.Code) - - if test.wantStatus == http.StatusOK { - assert.Equal(t, "text/plain; charset=utf-8", w.Header().Get("Content-Type"), "case[%s] Incorrect Content-Type header", test.name) - - gotLines := strings.Split(w.Body.String(), "\n") - gotLines = trimEmptyLines(gotLines) - sort.Strings(gotLines) - - sort.Strings(test.wantResp) - wantLines := trimEmptyLines(test.wantResp) - - assert.Equal(t, wantLines, gotLines, "case[%s] Response body mismatch", test.name) - } - }) - } -} - -func trimEmptyLines(lines []string) []string { - var result []string - for _, line := range lines { - if line != "" { - result = append(result, line) - } - } - return result -} diff --git a/zpages/httputil/httputil.go b/zpages/httputil/httputil.go deleted file mode 100644 index da49474b..00000000 --- a/zpages/httputil/httputil.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httputil - -import ( - "fmt" - "net/http" - "strings" - - "github.com/munnerz/goautoneg" -) - -// ErrUnsupportedMediaType is the error returned when the request's -// Accept header does not contain "text/plain". -var ErrUnsupportedMediaType = fmt.Errorf("media type not acceptable, must be: text/plain") - -// AcceptableMediaType checks if the request's Accept header contains -// a supported media type with optional "charset=utf-8" parameter. -func AcceptableMediaType(r *http.Request) bool { - accepts := goautoneg.ParseAccept(r.Header.Get("Accept")) - for _, accept := range accepts { - if !mediaTypeMatches(accept) { - continue - } - if len(accept.Params) == 0 { - return true - } - if len(accept.Params) == 1 { - if charset, ok := accept.Params["charset"]; ok && strings.EqualFold(charset, "utf-8") { - return true - } - } - } - return false -} - -func mediaTypeMatches(a goautoneg.Accept) bool { - return (a.Type == "text" || a.Type == "*") && - (a.SubType == "plain" || a.SubType == "*") -} diff --git a/zpages/httputil/httputil_test.go b/zpages/httputil/httputil_test.go deleted file mode 100644 index ab15cd27..00000000 --- a/zpages/httputil/httputil_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package httputil - -import ( - "net/http" - "testing" -) - -func TestAcceptableMediaTypes(t *testing.T) { - tests := []struct { - name string - reqHeader string - want bool - }{ - { - name: "valid text/plain header", - reqHeader: "text/plain", - want: true, - }, - { - name: "valid text/* header", - reqHeader: "text/*", - want: true, - }, - { - name: "valid */plain header", - reqHeader: "*/plain", - want: true, - }, - { - name: "valid accept args", - reqHeader: "text/plain; charset=utf-8", - want: true, - }, - { - name: "invalid text/foo header", - reqHeader: "text/foo", - want: false, - }, - { - name: "invalid text/plain params", - reqHeader: "text/plain; foo=bar", - want: false, - }, - } - for _, tt := range tests { - req, err := http.NewRequest(http.MethodGet, "http://example.com/statusz", nil) - if err != nil { - t.Fatalf("Unexpected error while creating request: %v", err) - } - - req.Header.Set("Accept", tt.reqHeader) - got := AcceptableMediaType(req) - - if got != tt.want { - t.Errorf("Unexpected response from AcceptableMediaType(), want %v, got = %v", tt.want, got) - } - } -} diff --git a/zpages/statusz/registry.go b/zpages/statusz/registry.go deleted file mode 100644 index 92f468e5..00000000 --- a/zpages/statusz/registry.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package statusz - -import ( - "time" - - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/klog/v2" - - "k8s.io/component-base/compatibility" - compbasemetrics "k8s.io/component-base/metrics" - utilversion "k8s.io/component-base/version" -) - -type statuszRegistry interface { - processStartTime() time.Time - goVersion() string - binaryVersion() *version.Version - emulationVersion() *version.Version - paths() []string -} - -type registry struct { - // componentGlobalsRegistry compatibility.ComponentGlobalsRegistry - effectiveVersion compatibility.EffectiveVersion - // listedPaths is an alphabetically sorted list of paths to be reported at /. - listedPaths []string -} - -// Option is a function to configure registry. -type Option func(reg *registry) - -// WithListedPaths returns an Option to configure the ListedPaths. -func WithListedPaths(listedPaths []string) Option { - cpyListedPaths := make([]string, len(listedPaths)) - copy(cpyListedPaths, listedPaths) - - return func(reg *registry) { reg.listedPaths = cpyListedPaths } -} - -func (*registry) processStartTime() time.Time { - start, err := compbasemetrics.GetProcessStart() - if err != nil { - klog.Errorf("Could not get process start time, %v", err) - } - - return time.Unix(int64(start), 0) -} - -func (*registry) goVersion() string { - return utilversion.Get().GoVersion -} - -func (r *registry) binaryVersion() *version.Version { - if r.effectiveVersion != nil { - return r.effectiveVersion.BinaryVersion() - } - return version.MustParse(utilversion.Get().String()) -} - -func (r *registry) emulationVersion() *version.Version { - if r.effectiveVersion != nil { - return r.effectiveVersion.EmulationVersion() - } - - return nil -} - -func (r *registry) paths() []string { - if r.listedPaths != nil { - return r.listedPaths - } - - return nil -} diff --git a/zpages/statusz/registry_test.go b/zpages/statusz/registry_test.go deleted file mode 100644 index 3d866953..00000000 --- a/zpages/statusz/registry_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package statusz - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/component-base/compatibility" - utilversion "k8s.io/component-base/version" -) - -func TestBinaryVersion(t *testing.T) { - tests := []struct { - name string - setFakeEffectiveVersion bool - fakeVersion string - wantBinaryVersion *version.Version - }{ - { - name: "binaryVersion with effective version", - wantBinaryVersion: version.MustParseSemantic("v1.2.3"), - setFakeEffectiveVersion: true, - fakeVersion: "1.2.3", - }, - { - name: "binaryVersion without effective version", - wantBinaryVersion: version.MustParse(utilversion.Get().String()), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - registry := ®istry{} - if tt.setFakeEffectiveVersion { - verKube := compatibility.NewEffectiveVersionFromString(tt.fakeVersion, "", "") - registry.effectiveVersion = verKube - } - - got := registry.binaryVersion() - assert.Equal(t, tt.wantBinaryVersion, got) - }) - } -} - -func TestEmulationVersion(t *testing.T) { - tests := []struct { - name string - setFakeEffectiveVersion bool - fakeEmulVer string - wantEmul *version.Version - }{ - { - name: "emulationVersion with effective version", - fakeEmulVer: "2.3.4", - setFakeEffectiveVersion: true, - wantEmul: version.MustParseSemantic("2.3.4"), - }, - { - name: "emulationVersion without effective version", - wantEmul: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - registry := ®istry{} - if tt.setFakeEffectiveVersion { - verKube := compatibility.NewEffectiveVersionFromString("0.0.0", "", "") - verKube.SetEmulationVersion(version.MustParse(tt.fakeEmulVer)) - registry.effectiveVersion = verKube - } - - got := registry.emulationVersion() - if tt.wantEmul != nil && got != nil { - assert.Equal(t, tt.wantEmul.Major(), got.Major()) - assert.Equal(t, tt.wantEmul.Minor(), got.Minor()) - } else { - assert.Equal(t, tt.wantEmul, got) - } - }) - } -} diff --git a/zpages/statusz/statusz.go b/zpages/statusz/statusz.go deleted file mode 100644 index f001a63c..00000000 --- a/zpages/statusz/statusz.go +++ /dev/null @@ -1,146 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package statusz - -import ( - "fmt" - "html" - "math/rand" - "net/http" - "sort" - "strings" - "time" - - "k8s.io/component-base/compatibility" - "k8s.io/component-base/zpages/httputil" - "k8s.io/klog/v2" -) - -var ( - delimiters = []string{":", ": ", "=", " "} - nonDebuggingEndpoints = map[string]bool{ - "/apis": true, - "/api": true, - "/openid": true, - "/openapi": true, - "/.well-known": true, - } -) - -const DefaultStatuszPath = "/statusz" - -const headerFmt = ` -%s statusz -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. -` - -type mux interface { - Handle(path string, handler http.Handler) -} - -type ListedPathsOption []string - -func NewRegistry(effectiveVersion compatibility.EffectiveVersion, opts ...func(*registry)) statuszRegistry { - r := ®istry{effectiveVersion: effectiveVersion} - for _, opt := range opts { - opt(r) - } - - return r -} - -func Install(m mux, componentName string, reg statuszRegistry) { - m.Handle(DefaultStatuszPath, handleStatusz(componentName, reg)) -} - -func handleStatusz(componentName string, reg statuszRegistry) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if !httputil.AcceptableMediaType(r) { - http.Error(w, httputil.ErrUnsupportedMediaType.Error(), http.StatusNotAcceptable) - return - } - - fmt.Fprintf(w, headerFmt, componentName) - data, err := populateStatuszData(reg, componentName) - if err != nil { - klog.Errorf("error while populating statusz data: %v", err) - http.Error(w, "error while populating statusz data", http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - fmt.Fprint(w, data) - } -} - -func populateStatuszData(reg statuszRegistry, componentName string) (string, error) { - randomIndex := rand.Intn(len(delimiters)) - delim := html.EscapeString(delimiters[randomIndex]) - startTime := html.EscapeString(reg.processStartTime().Format(time.UnixDate)) - uptime := html.EscapeString(uptime(reg.processStartTime())) - goVersion := html.EscapeString(reg.goVersion()) - binaryVersion := html.EscapeString(reg.binaryVersion().String()) - - var emulationVersion string - if reg.emulationVersion() != nil { - emulationVersion = fmt.Sprintf(`Emulation version%s %s`, delim, html.EscapeString(reg.emulationVersion().String())) - } - paths := aggregatePaths(reg.paths()) - if paths != "" { - paths = fmt.Sprintf(`Paths%s %s`, delim, html.EscapeString(paths)) - } - - status := fmt.Sprintf(` -Started%[1]s %[2]s -Up%[1]s %[3]s -Go version%[1]s %[4]s -Binary version%[1]s %[5]s -%[6]s -%[7]s -`, delim, startTime, uptime, goVersion, binaryVersion, emulationVersion, paths) - - return status, nil -} - -func uptime(t time.Time) string { - upSince := int64(time.Since(t).Seconds()) - return fmt.Sprintf("%d hr %02d min %02d sec", - upSince/3600, (upSince/60)%60, upSince%60) -} - -func aggregatePaths(listedPaths []string) string { - paths := make(map[string]bool) - for _, listedPath := range listedPaths { - folder := "/" + strings.Split(listedPath, "/")[1] - if !paths[folder] && !nonDebuggingEndpoints[folder] { - paths[folder] = true - } - } - - var sortedPaths []string - for p := range paths { - sortedPaths = append(sortedPaths, p) - } - sort.Strings(sortedPaths) - - var path string - for _, p := range sortedPaths { - path += " " + p - } - - return path -} diff --git a/zpages/statusz/statusz_test.go b/zpages/statusz/statusz_test.go deleted file mode 100644 index 8d9f5d0e..00000000 --- a/zpages/statusz/statusz_test.go +++ /dev/null @@ -1,228 +0,0 @@ -/* -Copyright 2024 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package statusz - -import ( - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "k8s.io/apimachinery/pkg/util/version" -) - -const wantTmpl = ` -%s statusz -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. - -Started: %v -Up: %s -Go version: %s -Binary version: %v -Emulation version: %v -Paths: /livez /readyz -` - -const wantTmplWithoutEmulation = ` -%s statusz -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. - -Started: %v -Up: %s -Go version: %s -Binary version: %v - -Paths: /livez /readyz -` - -const wantTmplWithKubeApiserverComp = ` -%s statusz -Warning: This endpoint is not meant to be machine parseable, has no formatting compatibility guarantees and is for debugging purposes only. - -Started: %v -Up: %s -Go version: %s -Binary version: %v - -Paths: /livez /readyz -` - -func TestStatusz(t *testing.T) { - delimiters = []string{":"} - fakeStartTime := time.Now() - fakeUptime := uptime(fakeStartTime) - fakeGoVersion := "1.21" - fakeBvStr := "1.31" - fakeEvStr := "1.30" - fakeBinaryVersion := parseVersion(t, fakeBvStr) - fakeEmulationVersion := parseVersion(t, fakeEvStr) - fakeListedPaths := []string{"/livez/poststarthook/peer-discovery-cache-sync", "/livez/post", "/readyz/informer-sync", "/readyz/log", "/readyz/ping"} - tests := []struct { - name string - componentName string - reqHeader string - registry fakeRegistry - wantStatusCode int - wantBody string - }{ - { - name: "invalid header", - reqHeader: "some header", - wantStatusCode: http.StatusNotAcceptable, - }, - { - name: "valid request", - componentName: "test-server", - reqHeader: "text/plain; charset=utf-8", - registry: fakeRegistry{ - startTime: fakeStartTime, - goVer: fakeGoVersion, - binaryVer: fakeBinaryVersion, - emulationVer: fakeEmulationVersion, - listedPaths: fakeListedPaths, - }, - wantStatusCode: http.StatusOK, - wantBody: fmt.Sprintf( - wantTmpl, - "test-server", - fakeStartTime.Format(time.UnixDate), - fakeUptime, - fakeGoVersion, - fakeBinaryVersion, - fakeEmulationVersion, - ), - }, - { - name: "missing emulation version", - componentName: "test-server", - reqHeader: "text/plain; charset=utf-8", - registry: fakeRegistry{ - startTime: fakeStartTime, - goVer: fakeGoVersion, - binaryVer: fakeBinaryVersion, - emulationVer: nil, - listedPaths: fakeListedPaths, - }, - wantStatusCode: http.StatusOK, - wantBody: fmt.Sprintf( - wantTmplWithoutEmulation, - "test-server", - fakeStartTime.Format(time.UnixDate), - fakeUptime, - fakeGoVersion, - fakeBinaryVersion, - ), - }, - { - name: "valid request for kube-apiserver", - componentName: "kube-apiserver", - reqHeader: "text/plain; charset=utf-8", - registry: fakeRegistry{ - startTime: fakeStartTime, - goVer: fakeGoVersion, - binaryVer: fakeBinaryVersion, - emulationVer: nil, - listedPaths: fakeListedPaths, - }, - wantStatusCode: http.StatusOK, - wantBody: fmt.Sprintf( - wantTmplWithKubeApiserverComp, - "kube-apiserver", - fakeStartTime.Format(time.UnixDate), - fakeUptime, - fakeGoVersion, - fakeBinaryVersion, - ), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mux := http.NewServeMux() - - Install(mux, tt.componentName, tt.registry) - - path := "/statusz" - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://example.com%s", path), nil) - if err != nil { - t.Fatalf("unexpected error while creating request: %v", err) - } - - req.Header.Set("Accept", "text/plain; charset=utf-8") - if tt.reqHeader != "" { - req.Header.Set("Accept", tt.reqHeader) - } - - w := httptest.NewRecorder() - mux.ServeHTTP(w, req) - - if w.Code != tt.wantStatusCode { - t.Fatalf("want status code: %v, got: %v", tt.wantStatusCode, w.Code) - } - - if tt.wantStatusCode == http.StatusOK { - c := w.Header().Get("Content-Type") - if c != "text/plain; charset=utf-8" { - t.Fatalf("want header: %v, got: %v", "text/plain", c) - } - - if diff := cmp.Diff(tt.wantBody, string(w.Body.String())); diff != "" { - t.Errorf("Unexpected diff on response (-want,+got):\n%s", diff) - } - } - }) - } -} - -func parseVersion(t *testing.T, v string) *version.Version { - parsed, err := version.ParseMajorMinor(v) - if err != nil { - t.Fatalf("error parsing binary version: %s", v) - } - - return parsed -} - -type fakeRegistry struct { - startTime time.Time - goVer string - binaryVer *version.Version - emulationVer *version.Version - listedPaths []string -} - -func (f fakeRegistry) processStartTime() time.Time { - return f.startTime -} - -func (f fakeRegistry) goVersion() string { - return f.goVer -} - -func (f fakeRegistry) binaryVersion() *version.Version { - return f.binaryVer -} - -func (f fakeRegistry) emulationVersion() *version.Version { - return f.emulationVer -} - -func (f fakeRegistry) paths() []string { - return f.listedPaths -}