Skip to content

Commit 2ed9388

Browse files
committed
(feat) block cluster upgrade when installed incompatible operators
1 parent 8d692e3 commit 2ed9388

File tree

7 files changed

+287
-11
lines changed

7 files changed

+287
-11
lines changed

cmd/cluster-olm-operator/main.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"k8s.io/component-base/cli"
2121
utilflag "k8s.io/component-base/cli/flag"
2222

23+
"github.com/openshift/cluster-olm-operator/internal/utils"
2324
"github.com/openshift/cluster-olm-operator/pkg/clients"
2425
"github.com/openshift/cluster-olm-operator/pkg/controller"
2526
"github.com/openshift/cluster-olm-operator/pkg/version"
@@ -105,13 +106,28 @@ func runOperator(ctx context.Context, cc *controllercmd.ControllerContext) error
105106
deploymentControllerList = append(deploymentControllerList, controller)
106107
}
107108

109+
operatorImageVersion := status.VersionForOperatorFromEnv()
110+
nextOCPMinorVersion, err := utils.GetNextOCPMinorVersion(operatorImageVersion)
111+
if err != nil {
112+
return err
113+
}
114+
108115
upgradeableConditionController := controller.NewStaticUpgradeableConditionController(
109116
"OLMStaticUpgradeableConditionController",
110117
cl.OperatorClient,
111118
cc.EventRecorder.ForComponent("OLMStaticUpgradeableConditionController"),
112119
controllerNames,
113120
)
114121

122+
incompatibleOperatorController := controller.NewIncompatibleOperatorController(
123+
"OLMIncompatibleOperatorController",
124+
nextOCPMinorVersion,
125+
cl.KubeClient,
126+
cl.ClusterExtensionClient,
127+
cl.OperatorClient,
128+
cc.EventRecorder.ForComponent("OLMIncompatibleOperatorController"),
129+
)
130+
115131
versionGetter := status.NewVersionGetter()
116132
versionGetter.SetVersion("operator", status.VersionForOperatorFromEnv())
117133

@@ -129,7 +145,7 @@ func runOperator(ctx context.Context, cc *controllercmd.ControllerContext) error
129145

130146
cl.StartInformers(ctx)
131147

132-
for _, c := range append(staticResourceControllerList, upgradeableConditionController, clusterOperatorController, operatorLoggingController) {
148+
for _, c := range append(staticResourceControllerList, upgradeableConditionController, incompatibleOperatorController, clusterOperatorController, operatorLoggingController) {
133149
go func(c factory.Controller) {
134150
defer runtime.HandleCrash()
135151
c.Run(ctx, 1)

internal/utils/utils.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package utils
2+
3+
import (
4+
"fmt"
5+
"strconv"
6+
"strings"
7+
8+
semver "github.com/blang/semver/v4"
9+
)
10+
11+
func GetNextOCPMinorVersion(versionString string) (*semver.Version, error) {
12+
v, err := semver.Parse(versionString)
13+
if err != nil {
14+
return &v, err
15+
}
16+
v.Build = nil // Builds are irrelevant
17+
v.Pre = nil // Next Y release
18+
return &v, v.IncrementMinor() // Sets Y=Y+1 and Z=0
19+
}
20+
21+
func ToAllowedSemver(value string) (*semver.Version, error) {
22+
if value == "" {
23+
// Handle "" separately, so parse doesn't treat it as a zero
24+
return nil, fmt.Errorf(`value cannot be "" (an empty string)`)
25+
}
26+
27+
if value != strings.TrimPrefix(value, "v") {
28+
return nil, fmt.Errorf(`version string cannot start with 'v'`)
29+
}
30+
31+
parts := strings.Split(value, ".")
32+
if len(parts) == 3 {
33+
return nil, fmt.Errorf(`version string should not include patch version`)
34+
}
35+
36+
if len(parts) != 2 {
37+
return nil, fmt.Errorf(`invalid version string. String must be in the form: Major.Minor`)
38+
}
39+
major, err := strconv.ParseUint(parts[0], 10, 64)
40+
if err != nil {
41+
return nil, err
42+
}
43+
minor, err := strconv.ParseUint(parts[1], 10, 64)
44+
if err != nil {
45+
return nil, err
46+
}
47+
return &semver.Version{Major: major, Minor: minor}, err
48+
}

manifests/0000_51_olm_02_operator_clusterrole.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,14 @@ rules:
7272
- get
7373
- list
7474
- watch
75+
- apiGroups:
76+
- ""
77+
resources:
78+
- secrets
79+
verbs:
80+
- get
81+
- list
82+
- watch
7583
- apiGroups:
7684
- ""
7785
resources:
@@ -210,6 +218,7 @@ rules:
210218
verbs:
211219
- get
212220
- list
221+
- watch
213222
- apiGroups:
214223
- ""
215224
resources:

manifests/0000_51_olm_06_deployment.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ spec:
5858
- /cluster-olm-operator
5959
args:
6060
- start
61-
- -v=2
6261
imagePullPolicy: IfNotPresent
6362
env:
6463
- name: OPERATOR_NAME

pkg/clients/clients.go

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/openshift/library-go/pkg/controller/controllercmd"
1818
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
1919
"github.com/openshift/library-go/pkg/operator/v1helpers"
20+
ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1"
2021
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
2122
"k8s.io/apimachinery/pkg/api/meta"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,6 +27,7 @@ import (
2627
"k8s.io/apimachinery/pkg/types"
2728
"k8s.io/apimachinery/pkg/util/sets"
2829
"k8s.io/client-go/dynamic"
30+
"k8s.io/client-go/dynamic/dynamicinformer"
2931
"k8s.io/client-go/informers"
3032
"k8s.io/client-go/kubernetes"
3133
"k8s.io/client-go/rest"
@@ -43,6 +45,7 @@ type Clients struct {
4345
RESTMapper meta.RESTMapper
4446
OperatorClient *OperatorClient
4547
OperatorInformers operatorinformers.SharedInformerFactory
48+
ClusterExtensionClient *ClusterExtensionClient
4649
ConfigClient configclient.Interface
4750
KubeInformerFactory informers.SharedInformerFactory
4851
ConfigInformerFactory configinformer.SharedInformerFactory
@@ -91,23 +94,34 @@ func New(cc *controllercmd.ControllerContext) (*Clients, error) {
9194
return nil, err
9295
}
9396

97+
infFact := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, defaultResyncPeriod)
98+
clusterExtensionGVR := ocv1alpha1.GroupVersion.WithResource("clusterextensions")
99+
inf := infFact.ForResource(clusterExtensionGVR)
100+
101+
ceClient := &ClusterExtensionClient{
102+
factory: infFact,
103+
informer: inf,
104+
}
105+
94106
return &Clients{
95-
KubeClient: kubeClient,
96-
APIExtensionsClient: apiExtensionsClient,
97-
DynamicClient: dynClient,
98-
RESTMapper: rm,
99-
OperatorClient: opClient,
100-
OperatorInformers: operatorInformersFactory,
101-
ConfigClient: configClient,
102-
KubeInformerFactory: informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod),
103-
ConfigInformerFactory: configinformer.NewSharedInformerFactory(configClient, defaultResyncPeriod),
107+
KubeClient: kubeClient,
108+
APIExtensionsClient: apiExtensionsClient,
109+
DynamicClient: dynClient,
110+
RESTMapper: rm,
111+
OperatorClient: opClient,
112+
OperatorInformers: operatorInformersFactory,
113+
ClusterExtensionClient: ceClient,
114+
ConfigClient: configClient,
115+
KubeInformerFactory: informers.NewSharedInformerFactory(kubeClient, defaultResyncPeriod),
116+
ConfigInformerFactory: configinformer.NewSharedInformerFactory(configClient, defaultResyncPeriod),
104117
}, nil
105118
}
106119

107120
func (c *Clients) StartInformers(ctx context.Context) {
108121
c.KubeInformerFactory.Start(ctx.Done())
109122
c.ConfigInformerFactory.Start(ctx.Done())
110123
c.OperatorInformers.Start(ctx.Done())
124+
c.ClusterExtensionClient.factory.Start(ctx.Done())
111125
if c.KubeInformersForNamespaces != nil {
112126
c.KubeInformersForNamespaces.Start(ctx.Done())
113127
}
@@ -131,6 +145,15 @@ const (
131145
fieldManager = "cluster-olm-operator"
132146
)
133147

148+
type ClusterExtensionClient struct {
149+
factory dynamicinformer.DynamicSharedInformerFactory
150+
informer informers.GenericInformer
151+
}
152+
153+
func (ce ClusterExtensionClient) Informer() informers.GenericInformer {
154+
return ce.informer
155+
}
156+
134157
type OperatorClient struct {
135158
clientset operatorclient.Interface
136159
informers operatorinformers.SharedInformerFactory
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package controller
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"sort"
9+
"strings"
10+
11+
semver "github.com/blang/semver/v4"
12+
"github.com/go-logr/logr"
13+
operatorv1 "github.com/openshift/api/operator/v1"
14+
"github.com/openshift/cluster-olm-operator/internal/utils"
15+
"github.com/openshift/cluster-olm-operator/pkg/clients"
16+
"github.com/openshift/library-go/pkg/controller/factory"
17+
"github.com/openshift/library-go/pkg/operator/events"
18+
"github.com/openshift/library-go/pkg/operator/v1helpers"
19+
storage "github.com/operator-framework/helm-operator-plugins/pkg/storage"
20+
"github.com/operator-framework/operator-registry/alpha/property"
21+
helm "helm.sh/helm/v3/pkg/storage"
22+
"helm.sh/helm/v3/pkg/storage/driver"
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/labels"
25+
"k8s.io/client-go/kubernetes"
26+
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
27+
"k8s.io/klog/v2"
28+
)
29+
30+
const (
31+
reasonIncompatibleOperatorsInstalled = "IncompatibleOperatorsInstalled"
32+
typeIncompatibelOperatorsUpgradeable = "InstalledOLMOperatorsUpgradeable"
33+
reasonInvalidBundleProperties = "InvalidBundleProperties"
34+
maxOpenShiftVersionProperty = "olm.maxOpenShiftVersion"
35+
ownerKindKey = "olm.operatorframework.io/owner-kind"
36+
ownerNameKey = "olm.operatorframework.io/owner-name"
37+
packageNameKey = "olm.operatorframework.io/package-name"
38+
bundleNameKey = "olm.operatorframework.io/bundle-name"
39+
bundleVersionKey = "olm.operatorframework.io/bundle-version"
40+
)
41+
42+
type incompatibleOperatorController struct {
43+
name string
44+
nextOCPMinorVersion *semver.Version
45+
kubeclient kubernetes.Interface
46+
clusterExtensionClient *clients.ClusterExtensionClient
47+
operatorClient *clients.OperatorClient
48+
logger logr.Logger
49+
}
50+
51+
func NewIncompatibleOperatorController(name string, nextOCPMinorVersion *semver.Version, kubeclient kubernetes.Interface, clusterExtensionClient *clients.ClusterExtensionClient, operatorClient *clients.OperatorClient, eventRecorder events.Recorder) factory.Controller {
52+
c := &incompatibleOperatorController{
53+
name: name,
54+
nextOCPMinorVersion: nextOCPMinorVersion,
55+
kubeclient: kubeclient,
56+
clusterExtensionClient: clusterExtensionClient,
57+
operatorClient: operatorClient,
58+
logger: klog.NewKlogr().WithName(name),
59+
}
60+
61+
return factory.New().WithSync(c.sync).WithSyncDegradedOnError(operatorClient).WithInformers(operatorClient.Informer(), clusterExtensionClient.Informer().Informer()).ToController(name, eventRecorder)
62+
}
63+
64+
func (c *incompatibleOperatorController) sync(ctx context.Context, _ factory.SyncContext) error {
65+
c.logger.Info("sync started")
66+
defer c.logger.Info("sync finished")
67+
68+
var updateStatusFn v1helpers.UpdateStatusFunc
69+
incompatibleOperators, err := c.getIncompatibleOperators()
70+
if len(incompatibleOperators) > 0 {
71+
updateStatusFn = v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{
72+
Type: typeIncompatibelOperatorsUpgradeable,
73+
Status: operatorv1.ConditionFalse,
74+
Reason: reasonIncompatibleOperatorsInstalled,
75+
Message: strings.Join(incompatibleOperators, ","),
76+
})
77+
} else {
78+
if err != nil {
79+
updateStatusFn = v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{
80+
Type: typeIncompatibelOperatorsUpgradeable,
81+
Status: operatorv1.ConditionFalse,
82+
Reason: reasonInvalidBundleProperties,
83+
Message: err.Error(),
84+
})
85+
} else {
86+
updateStatusFn = v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{
87+
Type: typeIncompatibelOperatorsUpgradeable,
88+
Status: operatorv1.ConditionTrue,
89+
})
90+
}
91+
}
92+
93+
if _, _, updateErr := v1helpers.UpdateStatus(ctx, c.operatorClient, updateStatusFn); updateErr != nil {
94+
c.logger.V(4).Info(fmt.Sprintf("Error updating operator condition status: %v", updateErr))
95+
return updateErr
96+
}
97+
return err
98+
}
99+
100+
func (c *incompatibleOperatorController) getIncompatibleOperators() ([]string, error) {
101+
var incompatibleOperators []string
102+
103+
ceList, err := c.clusterExtensionClient.Informer().Lister().List(labels.NewSelector())
104+
if err != nil {
105+
c.logger.V(4).Error(err, "Error listing cluster extensions")
106+
return nil, err
107+
}
108+
109+
store := c.buildHelmStore(c.kubeclient.CoreV1().Secrets("openshift-operator-controller"))
110+
111+
var errs []error
112+
// Get all ClusterExtensions incompatible with next Y-stream
113+
for _, obj := range ceList {
114+
metaObj, ok := obj.(metav1.Object)
115+
if !ok {
116+
errs = append(errs, fmt.Errorf("metav1.Object type assertion failed for object %v", obj))
117+
continue
118+
}
119+
name := metaObj.GetName()
120+
rel, err := store.Deployed(name)
121+
if errors.Is(err, driver.ErrNoDeployedReleases) {
122+
c.logger.Info("Cluster Extension not yet deployed - will check again later")
123+
continue
124+
}
125+
if err != nil {
126+
c.logger.V(4).Error(err, "Error returning the last deployed release")
127+
return nil, err
128+
}
129+
130+
if rel.Chart == nil || rel.Chart.Metadata == nil {
131+
continue
132+
}
133+
props, err := propertyListFromPropertiesAnnotation(rel.Chart.Metadata.Annotations["olm.properties"])
134+
if err != nil {
135+
c.logger.V(4).Error(err, "Error converting olm.properties")
136+
return nil, err
137+
}
138+
numMaxOCPProps := 0
139+
for _, p := range props {
140+
if p.Type == maxOpenShiftVersionProperty {
141+
numMaxOCPProps++
142+
maxOCPVersion, err := utils.ToAllowedSemver(string(p.Value))
143+
if err != nil {
144+
errs = append(errs, fmt.Errorf(fmt.Sprintf("Error converting to semver for version %s: %v", string(p.Value), err)))
145+
continue
146+
}
147+
if numMaxOCPProps > 1 {
148+
errs = append(errs, fmt.Errorf("%s has more than one %s", name, maxOpenShiftVersionProperty))
149+
continue
150+
}
151+
if maxOCPVersion != nil && !maxOCPVersion.GTE(*c.nextOCPMinorVersion) {
152+
// Incompatible
153+
incompatibleOperators = append(incompatibleOperators, rel.Labels[bundleNameKey])
154+
}
155+
}
156+
}
157+
}
158+
159+
// deterministic ordering
160+
sort.Strings(incompatibleOperators)
161+
162+
return incompatibleOperators, errors.Join(errs...)
163+
}
164+
165+
func propertyListFromPropertiesAnnotation(raw string) ([]property.Property, error) {
166+
var props []property.Property
167+
if err := json.Unmarshal([]byte(raw), &props); err != nil {
168+
return nil, fmt.Errorf("failed to unmarshal properties annotation: %w", err)
169+
}
170+
return props, nil
171+
}
172+
173+
func (c *incompatibleOperatorController) buildHelmStore(secretClient v1.SecretInterface) helm.Storage {
174+
log := func(s string, args ...interface{}) { c.logger.V(4).Info(fmt.Sprintf(s, args...)) }
175+
csConfig := storage.ChunkedSecretsConfig{Log: log}
176+
177+
return helm.Storage{
178+
Driver: storage.NewChunkedSecrets(secretClient, "operator-controller", csConfig),
179+
Log: log,
180+
}
181+
}

0 commit comments

Comments
 (0)