From 852b8c7bd7b05fb4450e6f3490c28e4bc7baaaa0 Mon Sep 17 00:00:00 2001 From: priyanka19-98 Date: Thu, 1 Jul 2021 18:00:09 +0530 Subject: [PATCH] Refactored logs for storagecluster controller Signed off by: Priyanka Jiandani --- controllers/storagecluster/cephblockpools.go | 24 +++++---- controllers/storagecluster/cephcluster.go | 45 +++++++++++------ controllers/storagecluster/cephfilesystem.go | 25 ++++++---- .../storagecluster/cephobjectstores.go | 33 +++++++------ .../storagecluster/cephobjectstoreusers.go | 33 ++++++++----- .../storagecluster/external_resources.go | 38 +++++++------- controllers/storagecluster/kms_resources.go | 9 +++- .../noobaa_system_reconciler.go | 21 ++++---- controllers/storagecluster/prometheus.go | 4 +- controllers/storagecluster/quickstarts.go | 27 +++++----- controllers/storagecluster/reconcile.go | 46 +++++++++-------- controllers/storagecluster/routes.go | 28 ++++++----- controllers/storagecluster/storageclasses.go | 23 +++++---- .../storagecluster_controller.go | 10 ++-- controllers/storagecluster/topology.go | 6 +-- .../storagecluster/uninstall_reconciler.go | 49 ++++++++++--------- .../volumesnapshotterclasses.go | 23 ++++----- go.mod | 1 + vendor/modules.txt | 1 + 19 files changed, 254 insertions(+), 192 deletions(-) diff --git a/controllers/storagecluster/cephblockpools.go b/controllers/storagecluster/cephblockpools.go index 26dff892f7..e87ed9daf6 100644 --- a/controllers/storagecluster/cephblockpools.go +++ b/controllers/storagecluster/cephblockpools.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -33,6 +34,7 @@ func (r *StorageClusterReconciler) newCephBlockPoolInstances(initData *ocsv1.Sto for _, obj := range ret { err := controllerutil.SetControllerReference(initData, obj, r.Scheme) if err != nil { + r.Log.Error(err, "Unable to set controller reference for CephBlockPool.", "CephBlockPool", klog.KRef(obj.Namespace, obj.Name)) return nil, err } } @@ -61,21 +63,23 @@ func (obj *ocsCephBlockPools) ensureCreated(r *StorageClusterReconciler, instanc return nil } if existing.DeletionTimestamp != nil { - r.Log.Info(fmt.Sprintf("Unable to restore init object because %s is marked for deletion", existing.Name)) + r.Log.Info("Unable to restore CephBlockPool because it is marked for deletion.", "CephBlockPool", klog.KRef(existing.Namespace, existing.Name)) return fmt.Errorf("failed to restore initialization object %s because it is marked for deletion", existing.Name) } - r.Log.Info(fmt.Sprintf("Restoring original cephBlockPool %s", cephBlockPool.Name)) + r.Log.Info("Restoring original CephBlockPool.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) existing.ObjectMeta.OwnerReferences = cephBlockPool.ObjectMeta.OwnerReferences cephBlockPool.ObjectMeta = existing.ObjectMeta err = r.Client.Update(context.TODO(), cephBlockPool) if err != nil { + r.Log.Error(err, "Failed to update CephBlockPool.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) return err } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("Creating cephBlockPool %s", cephBlockPool.Name)) + r.Log.Info("Creating CephBlockPool.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) err = r.Client.Create(context.TODO(), cephBlockPool) if err != nil { + r.Log.Error(err, "Failed to create CephBlockPool.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) return err } } @@ -96,28 +100,30 @@ func (obj *ocsCephBlockPools) ensureDeleted(r *StorageClusterReconciler, sc *ocs err := r.Client.Get(context.TODO(), types.NamespacedName{Name: cephBlockPool.Name, Namespace: sc.Namespace}, foundCephBlockPool) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephBlockPool not found", "CephBlockPool Name", cephBlockPool.Name) + r.Log.Info("Uninstall: CephBlockPool not found.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) continue } - return fmt.Errorf("Uninstall: Unable to retrieve cephBlockPool %v: %v", cephBlockPool.Name, err) + return fmt.Errorf("uninstall: unable to retrieve CephBlockPool %v: %v", cephBlockPool.Name, err) } if cephBlockPool.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting cephBlockPool", "CephBlockPool Name", cephBlockPool.Name) + r.Log.Info("Uninstall: Deleting CephBlockPool.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) err = r.Client.Delete(context.TODO(), foundCephBlockPool) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete cephBlockPool %v: %v", foundCephBlockPool.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete CephBlockPool.", "CephBlockPool", klog.KRef(foundCephBlockPool.Namespace, foundCephBlockPool.Name)) + return fmt.Errorf("uninstall: Failed to delete CephBlockPool %v: %v", foundCephBlockPool.Name, err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: cephBlockPool.Name, Namespace: sc.Namespace}, foundCephBlockPool) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephBlockPool is deleted", "CephBlockPool Name", cephBlockPool.Name) + r.Log.Info("Uninstall: CephBlockPool is deleted.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) continue } } - return fmt.Errorf("Uninstall: Waiting for cephBlockPool %v to be deleted", cephBlockPool.Name) + r.Log.Error(err, "Uninstall: Waiting for CephBlockPool to be deleted.", "CephBlockPool", klog.KRef(cephBlockPool.Namespace, cephBlockPool.Name)) + return fmt.Errorf("uninstall: Waiting for CephBlockPool %v to be deleted", cephBlockPool.Name) } return nil diff --git a/controllers/storagecluster/cephcluster.go b/controllers/storagecluster/cephcluster.go index c1bfa6be1c..0c7d7d63e2 100644 --- a/controllers/storagecluster/cephcluster.go +++ b/controllers/storagecluster/cephcluster.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/tools/reference" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -96,6 +97,7 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. if isMultus(sc.Spec.Network) { err := validateMultusSelectors(sc.Spec.Network.Selectors) if err != nil { + r.Log.Error(err, "Failed to validate Multus Selectors specified in StorageCluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) return err } } @@ -107,11 +109,12 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. } else { kmsConfigMap, err := getKMSConfigMap(KMSConfigMapName, sc, r.Client) if err != nil { - r.Log.Error(err, "failed to procure KMS config") + r.Log.Error(err, "Failed to procure KMS ConfigMap.", "KMSConfigMap", klog.KRef(sc.Namespace, KMSConfigMapName)) return err } if kmsConfigMap != nil { if err = reachKMSProvider(kmsConfigMap); err != nil { + r.Log.Error(err, "Address provided in KMS ConfigMap is not reachable.", "KMSConfigMap", klog.KRef(kmsConfigMap.Namespace, kmsConfigMap.Name)) return err } } @@ -120,14 +123,15 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. // Set StorageCluster instance as the owner and controller if err := controllerutil.SetControllerReference(sc, cephCluster, r.Scheme); err != nil { + r.Log.Error(err, "Unable to set controller reference for CephCluster.", "CephCluster", klog.KRef(cephCluster.Namespace, cephCluster.Name)) return err } platform, err := r.platform.GetPlatform(r.Client) if err != nil { - r.Log.Error(err, "Failed to get platform") + r.Log.Error(err, "Failed to get Platform.", "Platform", platform) } else if platform == v1.IBMCloudPlatformType || platform == IBMCloudCosPlatformType { - r.Log.Info(fmt.Sprintf("Increasing Mon failover timeout to 15m for %s", platform)) + r.Log.Info("Increasing Mon failover timeout to 15m.", "Platform", platform) cephCluster.Spec.HealthCheck.DaemonHealth.Monitor.Timeout = "15m" } @@ -137,11 +141,12 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. if err != nil { if errors.IsNotFound(err) { if sc.Spec.ExternalStorage.Enable { - r.Log.Info("Creating external CephCluster") + r.Log.Info("Creating external CephCluster.", "CephCluster", klog.KRef(cephCluster.Namespace, cephCluster.Name)) } else { - r.Log.Info("Creating CephCluster") + r.Log.Info("Creating CephCluster.", "CephCluster", klog.KRef(cephCluster.Namespace, cephCluster.Name)) } if err := r.Client.Create(context.TODO(), cephCluster); err != nil { + r.Log.Error(err, "Unable to create CephCluster.", "CephCluster", klog.KRef(cephCluster.Namespace, cephCluster.Name)) return err } // Need to happen after the ceph cluster CR creation was confirmed @@ -152,6 +157,7 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. statusutil.MapCephClusterNoConditions(&r.conditions, reason, message) return nil } + r.Log.Error(err, "Unable to fetch CephCluster.", "CephCluster", klog.KRef(cephCluster.Namespace, cephCluster.Name)) return err } @@ -161,16 +167,18 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. // Add it to the list of RelatedObjects if found objectRef, err := reference.GetReference(r.Scheme, found) if err != nil { + r.Log.Error(err, "Unable to get CephCluster ObjectReference.", "CephCluster", klog.KRef(found.Namespace, found.Name)) return err } err = objectreferencesv1.SetObjectReference(&sc.Status.RelatedObjects, *objectRef) if err != nil { + r.Log.Error(err, "Unable to add CephCluster to the list of Related Objects in StorageCluster.", "CephCluster", klog.KRef(objectRef.Namespace, objectRef.Name), "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) return err } // Handle CephCluster resource status if found.Status.State == "" { - r.Log.Info("CephCluster resource is not reporting status.") + r.Log.Info("CephCluster resource is not reporting status.", "CephCluster", klog.KRef(found.Namespace, found.Name)) // What does this mean to OCS status? Assuming progress. reason := "CephClusterStatus" message := "CephCluster resource is not reporting status" @@ -204,7 +212,7 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. // Update the CephCluster if it is not in the desired state if !reflect.DeepEqual(cephCluster.Spec, found.Spec) { - r.Log.Info("Updating spec for CephCluster") + r.Log.Info("Updating spec for CephCluster.", "CephCluster", klog.KRef(found.Namespace, found.Name)) if !sc.Spec.ExternalStorage.Enable { // Check if Cluster is Expanding if len(found.Spec.Storage.StorageClassDeviceSets) < len(cephCluster.Spec.Storage.StorageClassDeviceSets) { @@ -225,6 +233,7 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. } found.Spec = cephCluster.Spec if err := r.Client.Update(context.TODO(), found); err != nil { + r.Log.Error(err, "Unable to update CephCluster.", "CephCluster", klog.KRef(found.Namespace, found.Name)) return err } } @@ -235,31 +244,35 @@ func (obj *ocsCephCluster) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. // ensureDeleted deletes the CephCluster owned by the StorageCluster func (obj *ocsCephCluster) ensureDeleted(r *StorageClusterReconciler, sc *ocsv1.StorageCluster) error { cephCluster := &cephv1.CephCluster{} - err := r.Client.Get(context.TODO(), types.NamespacedName{Name: generateNameForCephCluster(sc), Namespace: sc.Namespace}, cephCluster) + cephClusterName := generateNameForCephCluster(sc) + err := r.Client.Get(context.TODO(), types.NamespacedName{Name: cephClusterName, Namespace: sc.Namespace}, cephCluster) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephCluster not found") + r.Log.Info("Uninstall: CephCluster not found.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) return nil } + r.Log.Error(err, "Uninstall: Unable to retrieve CephCluster.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) return fmt.Errorf("Uninstall: Unable to retrieve cephCluster: %v", err) } if cephCluster.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting cephCluster") + r.Log.Info("Uninstall: Deleting CephCluster.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) err = r.Client.Delete(context.TODO(), cephCluster) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete cephCluster: %v", err) + r.Log.Error(err, "Uninstall: Failed to delete CephCluster.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) + return fmt.Errorf("uninstall: Failed to delete CephCluster: %v", err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: generateNameForCephCluster(sc), Namespace: sc.Namespace}, cephCluster) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephCluster is deleted") + r.Log.Info("Uninstall: CephCluster is deleted.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) return nil } } - return fmt.Errorf("Uninstall: Waiting for cephCluster to be deleted") + r.Log.Error(err, "Uninstall: Waiting for CephCluster to be deleted.", "CephCluster", klog.KRef(sc.Namespace, cephClusterName)) + return fmt.Errorf("uninstall: Waiting for CephCluster to be deleted") } @@ -345,7 +358,7 @@ func newCephCluster(sc *ocsv1.StorageCluster, cephImage string, nodeCount int, s }, } } else { - reqLogger.Info(fmt.Sprintf("No monDataDirHostPath, monPVCTemplate or storageDeviceSets configured for storageCluster %s", sc.GetName())) + reqLogger.Info("No monDataDirHostPath, monPVCTemplate or storageDeviceSets configured for StorageCluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) } if isMultus(sc.Spec.Network) { cephCluster.Spec.Network.NetworkSpec = *sc.Spec.Network @@ -372,7 +385,7 @@ func validateMultusSelectors(selectors map[string]string) error { return fmt.Errorf("invalid value of the keys for the network selectors. keys should be public and cluster only") } if publicNetwork == "" && clusterNetwork == "" { - return fmt.Errorf("Both public and cluster network selector values can't be empty") + return fmt.Errorf("both public and cluster network selector values can't be empty") } if publicNetwork == "" { return fmt.Errorf("public network selector values can't be empty") @@ -489,7 +502,7 @@ func getMonCount(nodeCount int, arbiter bool) int { if override != "" { count, err := strconv.Atoi(override) if err != nil { - log.Error(err, "could not decode env var %s", monCountOverrideEnvVar) + log.Error(err, "Could not decode env var.", "monCountOverrideEnvVar", monCountOverrideEnvVar) } else { return count } diff --git a/controllers/storagecluster/cephfilesystem.go b/controllers/storagecluster/cephfilesystem.go index fa6b85e94a..d30be6ad61 100644 --- a/controllers/storagecluster/cephfilesystem.go +++ b/controllers/storagecluster/cephfilesystem.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -49,6 +50,7 @@ func (r *StorageClusterReconciler) newCephFilesystemInstances(initData *ocsv1.St for _, obj := range ret { err := controllerutil.SetControllerReference(initData, obj, r.Scheme) if err != nil { + r.Log.Error(err, "Unable to set Controller Reference for CephFileSystem.", "CephFileSystem", klog.KRef(obj.Namespace, obj.Name)) return nil, err } } @@ -76,21 +78,23 @@ func (obj *ocsCephFilesystems) ensureCreated(r *StorageClusterReconciler, instan return nil } if existing.DeletionTimestamp != nil { - r.Log.Info(fmt.Sprintf("Unable to restore init object because %s is marked for deletion", existing.Name)) + r.Log.Info("Unable to restore CephFileSystem because it is marked for deletion.", "CephFileSystem", klog.KRef(existing.Namespace, existing.Name)) return fmt.Errorf("failed to restore initialization object %s because it is marked for deletion", existing.Name) } - r.Log.Info(fmt.Sprintf("Restoring original cephFilesystem %s", cephFilesystem.Name)) + r.Log.Info("Restoring original CephFilesystem.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) existing.ObjectMeta.OwnerReferences = cephFilesystem.ObjectMeta.OwnerReferences cephFilesystem.ObjectMeta = existing.ObjectMeta err = r.Client.Update(context.TODO(), cephFilesystem) if err != nil { + r.Log.Error(err, "Unable to update CephFileSystem.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) return err } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("Creating cephFilesystem %s", cephFilesystem.Name)) + r.Log.Info("Creating CephFileSystem.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) err = r.Client.Create(context.TODO(), cephFilesystem) if err != nil { + r.Log.Error(err, "Unable to create CephFileSystem.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) return err } } @@ -111,28 +115,31 @@ func (obj *ocsCephFilesystems) ensureDeleted(r *StorageClusterReconciler, sc *oc err := r.Client.Get(context.TODO(), types.NamespacedName{Name: cephFilesystem.Name, Namespace: sc.Namespace}, foundCephFilesystem) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephFilesystem not found", "CephFilesystem Name", cephFilesystem.Name) + r.Log.Info("Uninstall: CephFileSystem not found.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) continue } - return fmt.Errorf("Uninstall: Unable to retrieve cephFilesystem %v: %v", cephFilesystem.Name, err) + r.Log.Error(err, "Uninstall: Unable to retrieve CephFileSystem.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) + return fmt.Errorf("uninstall: Unable to retrieve CephFileSystem %v: %v", cephFilesystem.Name, err) } if cephFilesystem.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting cephFilesystem", "CephFilesystem Name", cephFilesystem.Name) + r.Log.Info("Uninstall: Deleting cephFilesystem.", "CephFileSystem", klog.KRef(foundCephFilesystem.Namespace, foundCephFilesystem.Name)) err = r.Client.Delete(context.TODO(), foundCephFilesystem) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete cephFilesystem %v: %v", foundCephFilesystem.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete CephFileSystem.", "CephFileSystem", klog.KRef(foundCephFilesystem.Namespace, foundCephFilesystem.Name)) + return fmt.Errorf("uninstall: Failed to delete CephFileSystem %v: %v", foundCephFilesystem.Name, err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: cephFilesystem.Name, Namespace: sc.Namespace}, foundCephFilesystem) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephFilesystem is deleted", "CephFilesystem Name", cephFilesystem.Name) + r.Log.Info("Uninstall: CephFilesystem is deleted.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) continue } } - return fmt.Errorf("Uninstall: Waiting for cephFilesystem %v to be deleted", cephFilesystem.Name) + r.Log.Error(err, "Uninstall: Waiting for CephFileSystem to be deleted.", "CephFileSystem", klog.KRef(cephFilesystem.Namespace, cephFilesystem.Name)) + return fmt.Errorf("uninstall: Waiting for CephFileSystem %v to be deleted", cephFilesystem.Name) } return nil diff --git a/controllers/storagecluster/cephobjectstores.go b/controllers/storagecluster/cephobjectstores.go index 6fdacb22d2..b2c4037081 100644 --- a/controllers/storagecluster/cephobjectstores.go +++ b/controllers/storagecluster/cephobjectstores.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -33,7 +34,7 @@ func (obj *ocsCephObjectStores) ensureCreated(r *StorageClusterReconciler, insta if err != nil { return err } - r.Log.Info(fmt.Sprintf("not creating a CephObjectStore because the platform is '%s'", platform)) + r.Log.Info("Platform is set to avoid object store. Not creating a CephObjectStore.", "Platform", platform) return nil } @@ -43,7 +44,7 @@ func (obj *ocsCephObjectStores) ensureCreated(r *StorageClusterReconciler, insta } err = r.createCephObjectStores(cephObjectStores, instance) if err != nil { - r.Log.Error(err, "could not create CephObjectStores") + r.Log.Error(err, "Unable to create CephObjectStores for StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return err } @@ -62,28 +63,30 @@ func (obj *ocsCephObjectStores) ensureDeleted(r *StorageClusterReconciler, sc *o err := r.Client.Get(context.TODO(), types.NamespacedName{Name: cephObjectStore.Name, Namespace: sc.Namespace}, foundCephObjectStore) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephObjectStore not found", "CephObjectStore Name", cephObjectStore.Name) + r.Log.Info("Uninstall: CephObjectStore not found.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) continue } - return fmt.Errorf("Uninstall: Unable to retrieve cephObjectStore %v: %v", cephObjectStore.Name, err) + return fmt.Errorf("Uninstall: Unable to retrieve CephObjectStore %v: %v", cephObjectStore.Name, err) } if cephObjectStore.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting cephObjectStore", "CephObjectStore Name", cephObjectStore.Name) + r.Log.Info("Uninstall: Deleting CephObjectStore.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) err = r.Client.Delete(context.TODO(), foundCephObjectStore) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete cephObjectStore %v: %v", foundCephObjectStore.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete CephObjectStore.", klog.KRef(foundCephObjectStore.Namespace, foundCephObjectStore.Name)) + return fmt.Errorf("uninstall: Failed to delete CephObjectStore %v: %v", foundCephObjectStore.Name, err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: cephObjectStore.Name, Namespace: sc.Namespace}, foundCephObjectStore) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephObjectStore is deleted", "CephObjectStore Name", cephObjectStore.Name) + r.Log.Info("Uninstall: CephObjectStore is deleted.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) continue } } - return fmt.Errorf("Uninstall: Waiting for cephObjectStore %v to be deleted", cephObjectStore.Name) + r.Log.Error(err, "Uninstall: Waiting for CephObjectStore to be deleted.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) + return fmt.Errorf("uninstall: Waiting for CephObjectStore %v to be deleted", cephObjectStore.Name) } return nil @@ -101,24 +104,24 @@ func (r *StorageClusterReconciler) createCephObjectStores(cephObjectStores []*ce return nil } if existing.DeletionTimestamp != nil { - err := fmt.Errorf("failed to restore cephobjectstore object %s because it is marked for deletion", existing.Name) - r.Log.Info("cephobjectstore restore failed") + err := fmt.Errorf("failed to restore CephObjectStore object %s because it is marked for deletion", existing.Name) + r.Log.Info("Failed to restore CephObjectStore because it is marked for deletion.", "CephObjectStore", klog.KRef(existing.Namespace, existing.Name)) return err } - r.Log.Info(fmt.Sprintf("Restoring original cephObjectStore %s", cephObjectStore.Name)) + r.Log.Info("Restoring original CephObjectStore.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) existing.ObjectMeta.OwnerReferences = cephObjectStore.ObjectMeta.OwnerReferences cephObjectStore.ObjectMeta = existing.ObjectMeta err = r.Client.Update(context.TODO(), cephObjectStore) if err != nil { - r.Log.Error(err, fmt.Sprintf("failed to update CephObjectStore Object: %s", cephObjectStore.Name)) + r.Log.Error(err, "Failed to update CephObjectStore.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) return err } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("creating CephObjectStore %s", cephObjectStore.Name)) + r.Log.Info("Creating CephObjectStore.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) err = r.Client.Create(context.TODO(), cephObjectStore) if err != nil { - r.Log.Error(err, fmt.Sprintf("failed to create CephObjectStore object: %s", cephObjectStore.Name)) + r.Log.Error(err, "Failed to create CephObjectStore.", "CephObjectStore", klog.KRef(cephObjectStore.Namespace, cephObjectStore.Name)) return err } } @@ -163,7 +166,7 @@ func (r *StorageClusterReconciler) newCephObjectStoreInstances(initData *ocsv1.S for _, obj := range ret { err := controllerutil.SetControllerReference(initData, obj, r.Scheme) if err != nil { - r.Log.Error(err, fmt.Sprintf("Failed to set ControllerReference to %s", obj.Name)) + r.Log.Error(err, "Failed to set ControllerReference for CephObjectStore.", "CephObjectStore", klog.KRef(obj.Namespace, obj.Name)) return nil, err } } diff --git a/controllers/storagecluster/cephobjectstoreusers.go b/controllers/storagecluster/cephobjectstoreusers.go index 77ea55cd54..186d42bff2 100644 --- a/controllers/storagecluster/cephobjectstoreusers.go +++ b/controllers/storagecluster/cephobjectstoreusers.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -32,6 +33,7 @@ func (r *StorageClusterReconciler) newCephObjectStoreUserInstances(initData *ocs for _, obj := range ret { err := controllerutil.SetControllerReference(initData, obj, r.Scheme) if err != nil { + r.Log.Error(err, "Unable to set Controller Reference for CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(initData.Namespace, generateNameForCephObjectStore(initData))) return nil, err } } @@ -55,17 +57,18 @@ func (obj *ocsCephObjectStoreUsers) ensureCreated(r *StorageClusterReconciler, i if err != nil { return err } - r.Log.Info(fmt.Sprintf("not creating a CephObjectStoreUsers because the platform is '%s'", platform)) + r.Log.Info("Platform is set to avoid object store. Not creating a CephObjectStore.", "Platform", platform) return nil } cephObjectStoreUsers, err := r.newCephObjectStoreUserInstances(instance) if err != nil { + r.Log.Error(err, "Unable to create instances for CephObjectStoreUsers.") return err } err = r.createCephObjectStoreUsers(cephObjectStoreUsers, instance) if err != nil { - r.Log.Error(err, "could not create CephObjectStoresUsers") + r.Log.Error(err, "Could not create CephObjectStoresUsers.") return err } @@ -77,6 +80,7 @@ func (obj *ocsCephObjectStoreUsers) ensureDeleted(r *StorageClusterReconciler, s foundCephObjectStoreUser := &cephv1.CephObjectStoreUser{} cephObjectStoreUsers, err := r.newCephObjectStoreUserInstances(sc) if err != nil { + r.Log.Error(err, "Cannot create CephObjectStoreUser instances.") return err } @@ -84,28 +88,31 @@ func (obj *ocsCephObjectStoreUsers) ensureDeleted(r *StorageClusterReconciler, s err := r.Client.Get(context.TODO(), types.NamespacedName{Name: cephObjectStoreUser.Name, Namespace: sc.Namespace}, foundCephObjectStoreUser) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephObjectStoreUser not found", "CephObjectStoreUser Name", cephObjectStoreUser.Name) + r.Log.Info("Uninstall: CephObjectStoreUser not found.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) continue } - return fmt.Errorf("Uninstall: Unable to retrieve cephObjectStoreUser %v: %v", cephObjectStoreUser.Name, err) + r.Log.Error(err, "Uninstall: Unable to retrieve CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) + return fmt.Errorf("uninstall: Unable to retrieve CephObjectStoreUser %v: %v", cephObjectStoreUser.Name, err) } if cephObjectStoreUser.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting cephObjectStoreUser", "CephObjectStoreUser Name", cephObjectStoreUser.Name) + r.Log.Info("Uninstall: Deleting CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) err = r.Client.Delete(context.TODO(), foundCephObjectStoreUser) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete cephObjectStoreUser %v: %v", foundCephObjectStoreUser.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) + return fmt.Errorf("uninstall: Failed to delete CephObjectStoreUser %v: %v", foundCephObjectStoreUser.Name, err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: cephObjectStoreUser.Name, Namespace: sc.Namespace}, foundCephObjectStoreUser) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephObjectStoreUser is deleted", "CephObjectStoreUser Name", cephObjectStoreUser.Name) + r.Log.Info("Uninstall: CephObjectStoreUser is deleted.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) continue } } - return fmt.Errorf("Uninstall: Waiting for cephObjectStoreUser %v to be deleted", cephObjectStoreUser.Name) + r.Log.Error(err, "Uninstall: Waiting for CephObjectStoreUser to be deleted.", "CephObjectStoreUser", klog.KRef(sc.Namespace, cephObjectStoreUser.Name)) + return fmt.Errorf("uninstall: Waiting for CephObjectStoreUser %v to be deleted", cephObjectStoreUser.Name) } return nil @@ -123,21 +130,23 @@ func (r *StorageClusterReconciler) createCephObjectStoreUsers(cephObjectStoreUse return nil } if existing.DeletionTimestamp != nil { - r.Log.Info(fmt.Sprintf("Unable to restore init object because %s is marked for deletion", existing.Name)) - return fmt.Errorf("failed to restore initialization object %s because it is marked for deletion", existing.Name) + r.Log.Info("Unable to restore CephObjectStoreUser because it is marked for deletion.", "CephObjectStoreUser", klog.KRef(cephObjectStoreUser.Namespace, existing.Name)) + return fmt.Errorf("failed to restore CephObjectStoreUser %s because it is marked for deletion", existing.Name) } - r.Log.Info(fmt.Sprintf("Restoring original cephObjectStoreUser %s", cephObjectStoreUser.Name)) + r.Log.Info("Restoring original CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(cephObjectStoreUser.Namespace, cephObjectStoreUser.Name)) existing.ObjectMeta.OwnerReferences = cephObjectStoreUser.ObjectMeta.OwnerReferences cephObjectStoreUser.ObjectMeta = existing.ObjectMeta err = r.Client.Update(context.TODO(), cephObjectStoreUser) if err != nil { + r.Log.Error(err, "Unable to update CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(cephObjectStoreUser.Namespace, cephObjectStoreUser.Name)) return err } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("Creating cephObjectStoreUser %s", cephObjectStoreUser.Name)) + r.Log.Info("Creating CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(cephObjectStoreUser.Namespace, cephObjectStoreUser.Name)) err = r.Client.Create(context.TODO(), cephObjectStoreUser) if err != nil { + r.Log.Error(err, "Could not create CephObjectStoreUser.", "CephObjectStoreUser", klog.KRef(cephObjectStoreUser.Namespace, cephObjectStoreUser.Name)) return err } } diff --git a/controllers/storagecluster/external_resources.go b/controllers/storagecluster/external_resources.go index 9b32073684..5f6fb5ef3c 100644 --- a/controllers/storagecluster/external_resources.go +++ b/controllers/storagecluster/external_resources.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" ) const ( @@ -51,7 +52,7 @@ func (r *StorageClusterReconciler) setRookCSICephFS( types.NamespacedName{Name: rookCephOperatorConfigName, Namespace: instance.ObjectMeta.Namespace}, rookCephOperatorConfig) if err != nil { - r.Log.Error(err, fmt.Sprintf("Unable to get '%s' config", rookCephOperatorConfigName)) + r.Log.Error(err, "Unable to get RookCeph ConfigMap.", "RookCephConfigMap", klog.KRef(instance.Namespace, rookCephOperatorConfigName)) return err } enableDisableFlagStr := fmt.Sprintf("%v", enableDisableFlag) @@ -126,13 +127,13 @@ func (r *StorageClusterReconciler) retrieveExternalSecretData( instance *ocsv1.StorageCluster) ([]ExternalResource, error) { found, err := r.retrieveSecret(externalClusterDetailsSecret, instance) if err != nil { - r.Log.Error(err, "could not find the external secret resource") + r.Log.Error(err, "Could not find the RookCeph external secret resource.") return nil, err } var data []ExternalResource err = json.Unmarshal(found.Data[externalClusterDetailsKey], &data) if err != nil { - r.Log.Error(err, "could not parse json blob") + r.Log.Error(err, "Could not parse json blob.") return nil, err } return data, nil @@ -171,7 +172,7 @@ func (r *StorageClusterReconciler) newExternalCephObjectStoreInstances( initData *ocsv1.StorageCluster, rgwEndpoint string) ([]*cephv1.CephObjectStore, error) { // check whether the provided rgw endpoint is empty if rgwEndpoint = strings.TrimSpace(rgwEndpoint); rgwEndpoint == "" { - r.Log.Info("WARNING: Empty RGW Endpoint specified, external CephObjectStore won't be created") + r.Log.Info("Empty RGW Endpoint specified, external CephObjectStore won't be created.") return nil, nil } gatewaySpec, err := newExternalGatewaySpec(rgwEndpoint, r.Log) @@ -209,7 +210,7 @@ func (obj *ocsExternalResources) ensureCreated(r *StorageClusterReconciler, inst } err := r.createExternalStorageClusterResources(instance) if err != nil { - r.Log.Error(err, "could not create ExternalStorageClusterResource") + r.Log.Error(err, "Could not create ExternalStorageClusterResource.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return err } return nil @@ -234,7 +235,7 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc availableSCCs := []StorageClassConfiguration{} data, err := r.retrieveExternalSecretData(instance) if err != nil { - r.Log.Error(err, "failed to retrieve external resources") + r.Log.Error(err, "Failed to retrieve external secret resources.") return err } var extCephObjectStores []*cephv1.CephObjectStore @@ -275,7 +276,7 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc } r.monitoringPort = monitoringPort } - r.Log.Info("Monitoring Information found. Monitoring will be enabled on the external cluster") + r.Log.Info("Monitoring Information found. Monitoring will be enabled on the external cluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) r.monitoringIP = monitoringIP case "ConfigMap": cm := &corev1.ConfigMap{ @@ -285,7 +286,7 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc found := &corev1.ConfigMap{ObjectMeta: objectMeta} err := r.createExternalStorageClusterConfigMap(cm, found, objectKey) if err != nil { - r.Log.Error(err, "could not create ExternalStorageClusterConfigMap") + r.Log.Error(err, "Could not create ExternalStorageClusterConfigMap.", "ConfigMap", klog.KRef(cm.Namespace, cm.Name)) return err } case "Secret": @@ -299,7 +300,7 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc found := &corev1.Secret{ObjectMeta: objectMeta} err := r.createExternalStorageClusterSecret(sec, found, objectKey) if err != nil { - r.Log.Error(err, "could not create ExternalStorageClusterSecret") + r.Log.Error(err, "Could not create ExternalStorageClusterSecret.", "Secret", klog.KRef(sec.Namespace, sec.Name)) return err } case "StorageClass": @@ -319,7 +320,7 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc } else if d.Name == cephRgwStorageClassName { rgwEndpoint := d.Data[externalCephRgwEndpointKey] if err := checkEndpointReachable(rgwEndpoint, 5*time.Second); err != nil { - r.Log.Error(err, fmt.Sprintf("RGW endpoint, %q, is not reachable", rgwEndpoint)) + r.Log.Error(err, "RGW endpoint is not reachable.", "RGWEndpoint", rgwEndpoint) return err } extCephObjectStores, err = r.newExternalCephObjectStoreInstances(instance, rgwEndpoint) @@ -352,12 +353,11 @@ func (r *StorageClusterReconciler) createExternalStorageClusterResources(instanc // creating only the available storageClasses err = r.createStorageClasses(availableSCCs) if err != nil { - r.Log.Error(err, "failed to create needed StorageClasses") + r.Log.Error(err, "Failed to create needed StorageClasses.") return err } if err = r.setRookCSICephFS(enableRookCSICephFS, instance); err != nil { - r.Log.Error(err, - fmt.Sprintf("failed to set '%s' to %v", rookEnableCephFSCSIKey, enableRookCSICephFS)) + r.Log.Error(err, "Failed to set RookEnableCephFSCSIKey to EnableRookCSICephFS.", "RookEnableCephFSCSIKey", rookEnableCephFSCSIKey, "EnableRookCSICephFS", enableRookCSICephFS) return err } if extCephObjectStores != nil { @@ -373,14 +373,14 @@ func (r *StorageClusterReconciler) createExternalStorageClusterConfigMap(cm *cor err := r.Client.Get(context.TODO(), objectKey, found) if err != nil { if errors.IsNotFound(err) { - r.Log.Info(fmt.Sprintf("creating configmap: %s", cm.Name)) + r.Log.Info("Creating External StorageCluster ConfigMap.", "ConfigMap", klog.KRef(objectKey.Namespace, cm.Name)) err = r.Client.Create(context.TODO(), cm) if err != nil { - r.Log.Error(err, "creation of configmap failed") + r.Log.Error(err, "Creation of External StorageCluster ConfigMap failed.", "ConfigMap", klog.KRef(objectKey.Namespace, cm.Name)) return err } } else { - r.Log.Error(err, "unable the get the configmap") + r.Log.Error(err, "Unable the get the External StorageCluster ConfigMap.", "ConfigMap", klog.KRef(objectKey.Namespace, cm.Name)) return err } } @@ -392,14 +392,14 @@ func (r *StorageClusterReconciler) createExternalStorageClusterSecret(sec *corev err := r.Client.Get(context.TODO(), objectKey, found) if err != nil { if errors.IsNotFound(err) { - r.Log.Info(fmt.Sprintf("creating secret: %s", sec.Name)) + r.Log.Info("Creating External StorageCluster Secret.", "Secret", klog.KRef(sec.Name, objectKey.Namespace)) err = r.Client.Create(context.TODO(), sec) if err != nil { - r.Log.Error(err, "creation of secret failed") + r.Log.Error(err, "Creation of External StorageCluster Secret failed.", "Secret", klog.KRef(sec.Name, objectKey.Namespace)) return err } } else { - r.Log.Error(err, "unable the get the secret") + r.Log.Error(err, "Unable the get External StorageCluster Secret", "Secret", klog.KRef(sec.Name, objectKey.Namespace)) return err } } diff --git a/controllers/storagecluster/kms_resources.go b/controllers/storagecluster/kms_resources.go index cc3ebae90c..77681d82d6 100644 --- a/controllers/storagecluster/kms_resources.go +++ b/controllers/storagecluster/kms_resources.go @@ -7,6 +7,7 @@ import ( ocsv1 "github.com/openshift/ocs-operator/api/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -83,7 +84,11 @@ func deleteKMSResources(r *StorageClusterReconciler, sc *ocsv1.StorageCluster) e continue } if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Error occurred %v the kms resource: %v", errLog, kmsResourceName)) + if errLog == "while retrieving" { + r.Log.Error(err, "Uninstall: Error occurred while retrieving the KMS resource.", "KMSResource", klog.KRef(sc.Namespace, kmsResourceName)) + } else { + r.Log.Error(err, "Uninstall: Error occurred while deleting the KMS resource.", "KMSResource", klog.KRef(sc.Namespace, kmsResourceName)) + } } // collect the error into the return error if err != nil { @@ -97,7 +102,7 @@ func deleteKMSResources(r *StorageClusterReconciler, sc *ocsv1.StorageCluster) e } if returnError == nil { - r.Log.Info("Uninstall: All KMS resources removed successfully") + r.Log.Info("Uninstall: All KMS resources removed successfully.") } return returnError diff --git a/controllers/storagecluster/noobaa_system_reconciler.go b/controllers/storagecluster/noobaa_system_reconciler.go index f7d0eb3671..4a99c6770d 100644 --- a/controllers/storagecluster/noobaa_system_reconciler.go +++ b/controllers/storagecluster/noobaa_system_reconciler.go @@ -15,6 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/reference" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -35,19 +36,20 @@ func (obj *ocsNoobaaSystem) ensureCreated(r *StorageClusterReconciler, sc *ocsv1 err := r.Client.Get(context.TODO(), types.NamespacedName{Name: generateNameForCephCluster(sc), Namespace: sc.Namespace}, foundCeph) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Waiting on ceph cluster to be created before starting noobaa") + r.Log.Info("Waiting on Ceph Cluster to be created before starting Noobaa.", "CephCluster", klog.KRef(sc.Namespace, generateNameForCephCluster(sc))) return nil } + r.Log.Error(err, "Failed to retrieve Ceph Cluster.", "CephCluster", klog.KRef(sc.Namespace, generateNameForCephCluster(sc))) return err } if !sc.Spec.ExternalStorage.Enable { if foundCeph.Status.State != cephv1.ClusterStateCreated { - r.Log.Info("Waiting on ceph cluster to initialize before starting noobaa") + r.Log.Info("Waiting on Ceph Cluster to initialize before starting Noobaa.", "CephCluster", klog.KRef(sc.Namespace, generateNameForCephCluster(sc))) return nil } } else { if foundCeph.Status.State != cephv1.ClusterStateConnected { - r.Log.Info("Waiting for the external ceph cluster to be connected before starting noobaa") + r.Log.Info("Waiting for the External Ceph Cluster to be connected before starting Noobaa.", "CephCluster", klog.KRef(sc.Namespace, generateNameForCephCluster(sc))) return nil } } @@ -65,6 +67,7 @@ func (obj *ocsNoobaaSystem) ensureCreated(r *StorageClusterReconciler, sc *ocsv1 } err = controllerutil.SetControllerReference(sc, nb, r.Scheme) if err != nil { + r.Log.Error(err, "Unable to set controller reference for Noobaa.", "Noobaa", klog.KRef(nb.Namespace, nb.Name)) return err } @@ -73,7 +76,7 @@ func (obj *ocsNoobaaSystem) ensureCreated(r *StorageClusterReconciler, sc *ocsv1 return r.setNooBaaDesiredState(nb, sc) }) if err != nil { - r.Log.Error(err, "Failed to create or update NooBaa system") + r.Log.Error(err, "Failed to create or update NooBaa system.", "Noobaa", klog.KRef(nb.Namespace, nb.Name)) return err } // Need to happen after the noobaa CR update was confirmed @@ -198,17 +201,17 @@ func (obj *ocsNoobaaSystem) ensureDeleted(r *StorageClusterReconciler, sc *ocsv1 } if !isOwned { // if the noobaa found is not owned by our storagecluster, we skip it from deletion. - r.Log.Info("Uninstall: NooBaa object found, but ownerReference not set to storagecluster. Skipping", "Object", noobaa.ObjectMeta.Name) + r.Log.Info("Uninstall: NooBaa object found, but ownerReference not set to storagecluster. Skipping Deletion.", "Noobaa", klog.KRef(noobaa.Namespace, noobaa.Name)) return nil } if noobaa.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting NooBaa system", "Object", noobaa.ObjectMeta.Name) + r.Log.Info("Uninstall: Deleting NooBaa system.", "Nooba", klog.KRef(noobaa.Namespace, noobaa.Name)) err = r.Client.Delete(context.TODO(), noobaa) if err != nil { - r.Log.Error(err, "Uninstall: Failed to delete NooBaa system", "Object", noobaa.ObjectMeta.Name) - return fmt.Errorf("Uninstall: Failed to delete NooBaa system %v : %v", noobaa.ObjectMeta.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete NooBaa system.", "Noobaa", klog.KRef(noobaa.Namespace, noobaa.Name)) + return fmt.Errorf("uninstall: Failed to delete NooBaa system %v : %v", noobaa.ObjectMeta.Name, err) } } - return fmt.Errorf("Uninstall: Waiting on NooBaa system %v to be deleted", noobaa.ObjectMeta.Name) + return fmt.Errorf("uninstall: Waiting on NooBaa system %v to be deleted", noobaa.ObjectMeta.Name) } diff --git a/controllers/storagecluster/prometheus.go b/controllers/storagecluster/prometheus.go index 1b30543b4e..142b6eab51 100644 --- a/controllers/storagecluster/prometheus.go +++ b/controllers/storagecluster/prometheus.go @@ -28,7 +28,7 @@ const ( func (r *StorageClusterReconciler) enablePrometheusRules(instance *ocsv1.StorageCluster) error { rule, err := getPrometheusRules(instance.Spec.ExternalStorage.Enable) if err != nil { - r.Log.Error(err, "prometheus rules file not found") + r.Log.Error(err, "Prometheus rules file not found.") return err } err = mergo.Merge(&rule.ObjectMeta.Labels, instance.Spec.Monitoring.Labels, mergo.WithOverride) @@ -37,7 +37,7 @@ func (r *StorageClusterReconciler) enablePrometheusRules(instance *ocsv1.Storage } err = r.CreateOrUpdatePrometheusRules(rule) if err != nil { - r.Log.Error(err, "unable to deploy Prometheus rules") + r.Log.Error(err, "Unable to deploy Prometheus rules.") return err } return nil diff --git a/controllers/storagecluster/quickstarts.go b/controllers/storagecluster/quickstarts.go index 76fef0983a..aa09b201ca 100644 --- a/controllers/storagecluster/quickstarts.go +++ b/controllers/storagecluster/quickstarts.go @@ -9,6 +9,7 @@ import ( extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" ) type ocsQuickStarts struct{} @@ -18,20 +19,20 @@ func (obj *ocsQuickStarts) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. err := r.Client.Get(context.TODO(), types.NamespacedName{Name: "consolequickstarts.console.openshift.io", Namespace: ""}, &qscrd) if err != nil { if errors.IsNotFound(err) { - r.Log.V(2).Info("No custom resource definition found for consolequickstart. Skipping quickstart initialization") + r.Log.V(2).Info("No custom resource definition found for consolequickstart. Skipping quickstart initialization.") return nil } return err } if len(AllQuickStarts) == 0 { - r.Log.Info("No quickstarts found") + r.Log.Info("No quickstarts found.") return nil } for _, qs := range AllQuickStarts { cqs := consolev1.ConsoleQuickStart{} err := yaml.Unmarshal(qs, &cqs) if err != nil { - r.Log.Error(err, "Failed to unmarshal ConsoleQuickStart", "ConsoleQuickStartString", string(qs)) + r.Log.Error(err, "Failed to unmarshal ConsoleQuickStart.", "ConsoleQuickStartString", string(qs)) continue } found := consolev1.ConsoleQuickStart{} @@ -40,36 +41,36 @@ func (obj *ocsQuickStarts) ensureCreated(r *StorageClusterReconciler, sc *ocsv1. if errors.IsNotFound(err) { err = r.Client.Create(context.TODO(), &cqs) if err != nil { - r.Log.Error(err, "Failed to create quickstart", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Error(err, "Failed to create QuickStart.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) return nil } - r.Log.Info("Creating quickstarts", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Info("Creating Quickstarts.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) continue } - r.Log.Error(err, "Error has occurred when fetching quickstarts") + r.Log.Error(err, "Error has occurred when fetching QuickStarts.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) return nil } found.Spec = cqs.Spec err = r.Client.Update(context.TODO(), &found) if err != nil { - r.Log.Error(err, "Failed to update quickstart", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Error(err, "Failed to update QuickStart.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) return nil } - r.Log.Info("Updating quickstarts", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Info("Updating QuickStarts.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) } return nil } func (obj *ocsQuickStarts) ensureDeleted(r *StorageClusterReconciler, sc *ocsv1.StorageCluster) error { if len(AllQuickStarts) == 0 { - r.Log.Info("No quickstarts found") + r.Log.Info("No quickstarts found.") return nil } for _, qs := range AllQuickStarts { cqs := consolev1.ConsoleQuickStart{} err := yaml.Unmarshal(qs, &cqs) if err != nil { - r.Log.Error(err, "Failed to unmarshal ConsoleQuickStart", "ConsoleQuickStartString", string(qs)) + r.Log.Error(err, "Failed to unmarshal ConsoleQuickStart.", "ConsoleQuickStartString", string(qs)) continue } found := consolev1.ConsoleQuickStart{} @@ -78,15 +79,15 @@ func (obj *ocsQuickStarts) ensureDeleted(r *StorageClusterReconciler, sc *ocsv1. if errors.IsNotFound(err) { continue } - r.Log.Error(err, "Uninstall: Failed to get QuickStart %s", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Error(err, "Uninstall: Failed to get QuickStart.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) return nil } err = r.Client.Delete(context.TODO(), &found) if err != nil { - r.Log.Error(err, "Uninstall: Failed to delete QuickStart %s", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Error(err, "Uninstall: Failed to delete QuickStart.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) return nil } - r.Log.Info("Uninstall: Deleting QuickStart", "Name", cqs.Name, "Namespace", cqs.Namespace) + r.Log.Info("Uninstall: Deleting QuickStart.", "QuickStart", klog.KRef(cqs.Namespace, cqs.Name)) } return nil } diff --git a/controllers/storagecluster/reconcile.go b/controllers/storagecluster/reconcile.go index 59d035570b..7fbe06c102 100644 --- a/controllers/storagecluster/reconcile.go +++ b/controllers/storagecluster/reconcile.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -145,13 +146,14 @@ func (r *StorageClusterReconciler) Reconcile(ctx context.Context, request reconc sc := &ocsv1.StorageCluster{} if err := r.Client.Get(ctx, request.NamespacedName, sc); err != nil { if errors.IsNotFound(err) { - r.Log.Info("No StorageCluster resource") + r.Log.Info("No StorageCluster resource.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) // Request object not found, could have been deleted after reconcile request. // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. // Return and don't requeue return reconcile.Result{}, nil } // Error reading the object - requeue the request. + r.Log.Error(err, "Failed to retrieve StorageCluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) return reconcile.Result{}, err } @@ -165,7 +167,7 @@ func (r *StorageClusterReconciler) Reconcile(ctx context.Context, request reconc // Apply status changes to the storagecluster statusError := r.Client.Status().Update(ctx, sc) if statusError != nil { - r.Log.Info("Status Update Error", "StatusUpdateErr", "Could not update storagecluster status") + r.Log.Info("Could not update StorageCluster status.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) } // Reconcile errors have higher priority than status update errors @@ -199,10 +201,11 @@ func (r *StorageClusterReconciler) initializeImagesStatus(sc *ocsv1.StorageClust // validateStorageClusterSpec must be called before reconciling. Any syntactic and sematic errors in the CR must be caught here. func (r *StorageClusterReconciler) validateStorageClusterSpec(instance *ocsv1.StorageCluster, request reconcile.Request) error { if err := versionCheck(instance, r.Log); err != nil { - r.Log.Error(err, "Failed to validate version") + r.Log.Error(err, "Failed to validate StorageCluster version.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, statusutil.EventReasonValidationFailed, err.Error()) instance.Status.Phase = statusutil.PhaseError if updateErr := r.Client.Status().Update(context.TODO(), instance); updateErr != nil { + r.Log.Error(updateErr, "Failed to update StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return updateErr } return err @@ -210,10 +213,11 @@ func (r *StorageClusterReconciler) validateStorageClusterSpec(instance *ocsv1.St if !instance.Spec.ExternalStorage.Enable { if err := r.validateStorageDeviceSets(instance); err != nil { - r.Log.Error(err, "Failed to validate StorageDeviceSets") + r.Log.Error(err, "Failed to validate StorageDeviceSets.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, statusutil.EventReasonValidationFailed, err.Error()) instance.Status.Phase = statusutil.PhaseError if updateErr := r.Client.Status().Update(context.TODO(), instance); updateErr != nil { + r.Log.Error(updateErr, "Failed to update StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return updateErr } return err @@ -221,10 +225,11 @@ func (r *StorageClusterReconciler) validateStorageClusterSpec(instance *ocsv1.St } if err := validateArbiterSpec(instance, r.Log); err != nil { - r.Log.Error(err, "Failed to validate ArbiterSpec") + r.Log.Error(err, "Failed to validate ArbiterSpec.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, statusutil.EventReasonValidationFailed, err.Error()) instance.Status.Phase = statusutil.PhaseError if updateErr := r.Client.Status().Update(context.TODO(), instance); updateErr != nil { + r.Log.Error(updateErr, "Could not update StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return updateErr } return err @@ -237,9 +242,9 @@ func (r *StorageClusterReconciler) reconcilePhases( request reconcile.Request) (reconcile.Result, error) { if instance.Spec.ExternalStorage.Enable { - r.Log.Info("Reconciling external StorageCluster") + r.Log.Info("Reconciling external StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) } else { - r.Log.Info("Reconciling StorageCluster") + r.Log.Info("Reconciling StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) } // Initialize the StatusImages section of the storageclsuter CR @@ -252,7 +257,7 @@ func (r *StorageClusterReconciler) reconcilePhases( if instance.Status.Phase == "" { isActive, err := r.isActiveStorageCluster(instance) if err != nil { - r.Log.Error(err, "StorageCluster could not be reconciled. Retrying") + r.Log.Error(err, "StorageCluster could not be reconciled. Retrying.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return reconcile.Result{}, err } if !isActive { @@ -280,10 +285,10 @@ func (r *StorageClusterReconciler) reconcilePhases( // Check GetDeletionTimestamp to determine if the object is under deletion if instance.GetDeletionTimestamp().IsZero() { if !contains(instance.GetFinalizers(), storageClusterFinalizer) { - r.Log.Info("Finalizer not found for storagecluster. Adding finalizer") + r.Log.Info("Finalizer not found for StorageCluster. Adding finalizer.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, storageClusterFinalizer) if err := r.Client.Update(context.TODO(), instance); err != nil { - r.Log.Info("Update Error", "MetaUpdateErr", "Failed to update storagecluster with finalizer") + r.Log.Info("Failed to update StorageCluster with finalizer.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return reconcile.Result{}, err } } @@ -298,19 +303,19 @@ func (r *StorageClusterReconciler) reconcilePhases( if contains(instance.GetFinalizers(), storageClusterFinalizer) { if err := r.deleteResources(instance); err != nil { - r.Log.Info("Uninstall in progress", "Status", err) + r.Log.Info("Uninstall in progress.", "Status", err) r.recorder.ReportIfNotPresent(instance, corev1.EventTypeWarning, statusutil.EventReasonUninstallPending, err.Error()) return reconcile.Result{RequeueAfter: time.Second * time.Duration(1)}, nil } - r.Log.Info("Removing finalizer") + r.Log.Info("Removing finalizer from StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) // Once all finalizers have been removed, the object will be deleted instance.ObjectMeta.Finalizers = remove(instance.ObjectMeta.Finalizers, storageClusterFinalizer) if err := r.Client.Update(context.TODO(), instance); err != nil { - r.Log.Info("Update Error", "MetaUpdateErr", "Failed to remove finalizer from storagecluster") + r.Log.Info("Failed to remove finalizer from StorageCluster", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return reconcile.Result{}, err } } - r.Log.Info("Object is terminated, skipping reconciliation") + r.Log.Info("StorageCluster is terminated, skipping reconciliation.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) ReadinessSet() return reconcile.Result{}, nil } @@ -318,7 +323,7 @@ func (r *StorageClusterReconciler) reconcilePhases( if !instance.Spec.ExternalStorage.Enable { // Get storage node topology labels if err := r.reconcileNodeTopologyMap(instance); err != nil { - r.Log.Error(err, "Failed to set node topology map") + r.Log.Error(err, "Failed to set node Topology Map for StorageCluster.", "StorageCluster", klog.KRef(instance.Namespace, instance.Name)) return reconcile.Result{}, err } } @@ -376,7 +381,7 @@ func (r *StorageClusterReconciler) reconcilePhases( } // All component operators are in a happy state. if r.conditions == nil { - r.Log.Info("No component operator reported negatively") + r.Log.Info("No component operator reported negatively.") reason := ocsv1.ReconcileCompleted message := ocsv1.ReconcileCompletedMessage statusutil.SetCompleteCondition(&instance.Status.Conditions, reason, message) @@ -436,12 +441,12 @@ func (r *StorageClusterReconciler) reconcilePhases( } } if err := r.enableMetricsExporter(instance); err != nil { - r.Log.Error(err, "failed to reconcile metrics exporter") + r.Log.Error(err, "Failed to reconcile metrics exporter.") return reconcile.Result{}, err } if err := r.enablePrometheusRules(instance); err != nil { - r.Log.Error(err, "failed to reconcile prometheus rules") + r.Log.Error(err, "Failed to reconcile prometheus rules.") return reconcile.Result{}, err } } @@ -540,9 +545,10 @@ func (obj *ocsCephConfig) ensureCreated(r *StorageClusterReconciler, sc *ocsv1.S if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Creating Ceph ConfigMap") + r.Log.Info("Creating Ceph ConfigMap.", "ConfigMap", klog.KRef(sc.Namespace, rookConfigMapName)) err = r.Client.Create(context.TODO(), cm) if err != nil { + r.Log.Error(err, "Failed to create Ceph ConfigMap.", "ConfigMap", klog.KRef(sc.Namespace, rookConfigMapName)) return err } } @@ -557,7 +563,7 @@ func (obj *ocsCephConfig) ensureCreated(r *StorageClusterReconciler, sc *ocsv1.S } val, ok := found.Data["config"] if !ok || val != defaultRookConfigData || !ownerRefFound { - r.Log.Info("Updating Ceph ConfigMap") + r.Log.Info("Updating Ceph ConfigMap.", "ConfigMap", klog.KRef(sc.Namespace, cm.Name)) return r.Client.Update(context.TODO(), cm) } return nil diff --git a/controllers/storagecluster/routes.go b/controllers/storagecluster/routes.go index 3859e53c6c..5ecea4e5c4 100644 --- a/controllers/storagecluster/routes.go +++ b/controllers/storagecluster/routes.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -30,7 +31,7 @@ func (obj *ocsCephRGWRoutes) ensureCreated(r *StorageClusterReconciler, instance if err != nil { return err } - r.Log.Info(fmt.Sprintf("not creating a Ceph RGW route because the platform is '%s'", platform)) + r.Log.Info("Platform is set to avoid Ceph RGW Route. Not creating a Ceph RGW Route.", "platform", platform) return nil } @@ -40,7 +41,7 @@ func (obj *ocsCephRGWRoutes) ensureCreated(r *StorageClusterReconciler, instance } err = r.createCephRGWRoutes(ocsCephRoutes, instance) if err != nil { - r.Log.Error(err, "could not create Routes") + r.Log.Error(err, "Could not create Ceph RGW Routes.") return err } @@ -59,28 +60,29 @@ func (obj *ocsCephRGWRoutes) ensureDeleted(r *StorageClusterReconciler, sc *ocsv err := r.Client.Get(context.TODO(), types.NamespacedName{Name: route.Name, Namespace: sc.Namespace}, foundRoute) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: Route not found", "Route Name", route.Name) + r.Log.Info("Uninstall: Ceph RGW Route not found.", "CephRGWRoute", klog.KRef(sc.Namespace, route.Name)) continue } return fmt.Errorf("Uninstall: Unable to retrieve route %v: %v", route.Name, err) } if route.GetDeletionTimestamp().IsZero() { - r.Log.Info("Uninstall: Deleting route", "Route Name", route.Name) + r.Log.Info("Uninstall: Deleting Ceph RGW Route.", "CephRGWRoute", klog.KRef(sc.Namespace, route.Name)) err = r.Client.Delete(context.TODO(), foundRoute) if err != nil { - return fmt.Errorf("Uninstall: Failed to delete route %v: %v", route.Name, err) + r.Log.Error(err, "Uninstall: Failed to delete Ceph RGW Route.", "CephRGWRoute", klog.KRef(sc.Namespace, route.Name)) + return fmt.Errorf("Uninstall: Failed to delete Route %v: %v", route.Name, err) } } err = r.Client.Get(context.TODO(), types.NamespacedName{Name: route.Name, Namespace: sc.Namespace}, foundRoute) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: Route is deleted", "Route Name", route.Name) + r.Log.Info("Uninstall: Ceph RGW Route is deleted.", "CephRGWRoute", klog.KRef(sc.Namespace, route.Name)) continue } } - return fmt.Errorf("Uninstall: Waiting for route %v to be deleted", route.Name) + return fmt.Errorf("Uninstall: Waiting for Ceph RGW Route %v to be deleted", route.Name) } return nil @@ -99,23 +101,23 @@ func (r *StorageClusterReconciler) createCephRGWRoutes(routes []*routev1.Route, } if existing.DeletionTimestamp != nil { err := fmt.Errorf("failed to restore route object %s because it is marked for deletion", existing.Name) - r.Log.Info("route restore failed") + r.Log.Info("Ceph RGW Route restore failed.", "CephRGWRoute", klog.KRef(route.Namespace, route.Name)) return err } - r.Log.Info(fmt.Sprintf("Restoring original route %s", route.Name)) + r.Log.Info("Restoring original Ceph RGW Route.", "CephRGWRoute", klog.KRef(route.Namespace, route.Name)) existing.ObjectMeta.OwnerReferences = route.ObjectMeta.OwnerReferences route.ObjectMeta = existing.ObjectMeta err = r.Client.Update(context.TODO(), route) if err != nil { - r.Log.Error(err, fmt.Sprintf("failed to update route Object: %s", route.Name)) + r.Log.Error(err, "Failed to update Ceph RGW Route Object.", "CephRGWRoute", klog.KRef(route.Namespace, route.Name)) return err } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("creating route %s", route.Name)) + r.Log.Info("Creating Ceph RGW Route.", "CephRGWRoute", klog.KRef(route.Namespace, route.Name)) err = r.Client.Create(context.TODO(), route) if err != nil { - r.Log.Error(err, fmt.Sprintf("failed to create CephObjectStore object: %s", route.Name)) + r.Log.Error(err, "Failed to create Ceph RGW Route.", "CephRGWRoute", klog.KRef(route.Namespace, route.Name)) return err } } @@ -145,7 +147,7 @@ func (r *StorageClusterReconciler) newCephRGWRoutes(initData *ocsv1.StorageClust for _, obj := range ret { err := controllerutil.SetControllerReference(initData, obj, r.Scheme) if err != nil { - r.Log.Error(err, fmt.Sprintf("Failed to set ControllerReference to %s", obj.Name)) + r.Log.Error(err, "Failed to set ControllerReference for Ceph RGW Route", "CephRGWRoute", klog.KRef(obj.Namespace, obj.Name)) return nil, err } } diff --git a/controllers/storagecluster/storageclasses.go b/controllers/storagecluster/storageclasses.go index 61b8579f5e..130c07bc8b 100644 --- a/controllers/storagecluster/storageclasses.go +++ b/controllers/storagecluster/storageclasses.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" ) // StorageClassConfiguration provides configuration options for a StorageClass. @@ -43,7 +44,7 @@ func (obj *ocsStorageClass) ensureDeleted(r *StorageClusterReconciler, instance sccs, err := r.newStorageClassConfigurations(instance) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to determine the StorageClass names")) //nolint:gosimple + r.Log.Error(err, "Uninstall: Unable to determine the StorageClass names.") //nolint:gosimple return nil } for _, scc := range sccs { @@ -54,22 +55,22 @@ func (obj *ocsStorageClass) ensureDeleted(r *StorageClusterReconciler, instance switch { case err == nil: if existing.DeletionTimestamp != nil { - r.Log.Info(fmt.Sprintf("Uninstall: StorageClass %s is already marked for deletion", existing.Name)) + r.Log.Info("Uninstall: StorageClass is already marked for deletion.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) break } - r.Log.Info(fmt.Sprintf("Uninstall: Deleting StorageClass %s", sc.Name)) + r.Log.Info("Uninstall: Deleting StorageClass.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) existing.ObjectMeta.OwnerReferences = sc.ObjectMeta.OwnerReferences sc.ObjectMeta = existing.ObjectMeta err = r.Client.Delete(context.TODO(), sc) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Ignoring error deleting the StorageClass %s", existing.Name)) + r.Log.Error(err, "Uninstall: Ignoring error deleting the StorageClass.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("Uninstall: StorageClass %s not found, nothing to do", sc.Name)) + r.Log.Info("Uninstall: StorageClass not found, nothing to do.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) default: - r.Log.Info(fmt.Sprintf("Uninstall: Error while getting StorageClass %s: %v", sc.Name, err)) + r.Log.Error(err, "Uninstall: Error while getting StorageClass.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) } } return nil @@ -86,7 +87,7 @@ func (r *StorageClusterReconciler) createStorageClasses(sccs []StorageClassConfi if errors.IsNotFound(err) { // Since the StorageClass is not found, we will create a new one - r.Log.Info(fmt.Sprintf("Creating StorageClass %s", sc.Name)) + r.Log.Info("Creating StorageClass.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) err = r.Client.Create(context.TODO(), sc) if err != nil { return err @@ -98,19 +99,21 @@ func (r *StorageClusterReconciler) createStorageClasses(sccs []StorageClassConfi continue } if existing.DeletionTimestamp != nil { - return fmt.Errorf("failed to restore storageclass %s because it is marked for deletion", existing.Name) + return fmt.Errorf("failed to restore StorageClass %s because it is marked for deletion", existing.Name) } if !reflect.DeepEqual(sc.Parameters, existing.Parameters) { // Since we have to update the existing StorageClass // So, we will delete the existing storageclass and create a new one - r.Log.Info(fmt.Sprintf("StorageClass %s needs to be updated, deleting it", existing.Name)) + r.Log.Info("StorageClass needs to be updated, deleting it.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) err = r.Client.Delete(context.TODO(), existing) if err != nil { + r.Log.Error(err, "Failed to delete StorageClass.", "StorageClass", klog.KRef(sc.Namespace, existing.Name)) return err } - r.Log.Info(fmt.Sprintf("Creating StorageClass %s", sc.Name)) + r.Log.Info("Creating StorageClass.", "StorageClass", klog.KRef(sc.Namespace, sc.Name)) err = r.Client.Create(context.TODO(), sc) if err != nil { + r.Log.Info("Failed to craete StorageClass.", "StorageClass", klog.KRef(sc.Namespace, sc.Name)) return err } } diff --git a/controllers/storagecluster/storagecluster_controller.go b/controllers/storagecluster/storagecluster_controller.go index 41f9335a7f..d3513f3139 100644 --- a/controllers/storagecluster/storagecluster_controller.go +++ b/controllers/storagecluster/storagecluster_controller.go @@ -33,15 +33,15 @@ func (r *StorageClusterReconciler) initializeImageVars() error { if r.images.Ceph == "" { err := fmt.Errorf("CEPH_IMAGE environment variable not found") - r.Log.Error(err, "missing required environment variable for ocs initialization") + r.Log.Error(err, "Missing CEPH_IMAGE environment variable for ocs initialization.") return err } else if r.images.NooBaaCore == "" { err := fmt.Errorf("NOOBAA_CORE_IMAGE environment variable not found") - r.Log.Error(err, "missing required environment variable for ocs initialization") + r.Log.Error(err, "Missing NOOBAA_CORE_IMAGE environment variable for ocs initialization.") return err } else if r.images.NooBaaDB == "" { err := fmt.Errorf("NOOBAA_DB_IMAGE environment variable not found") - r.Log.Error(err, "missing required environment variable for ocs initialization") + r.Log.Error(err, "Missing NOOBAA_DB_IMAGE environment variable for ocs initialization.") return err } return nil @@ -50,12 +50,12 @@ func (r *StorageClusterReconciler) initializeImageVars() error { func (r *StorageClusterReconciler) initializeServerVersion() error { clientset, err := kubernetes.NewForConfig(config.GetConfigOrDie()) if err != nil { - r.Log.Error(err, "failed creation of clientset for determining serverversion") + r.Log.Error(err, "Failed creation of clientset for determining serverversion.") return err } r.serverVersion, err = clientset.Discovery().ServerVersion() if err != nil { - r.Log.Error(err, "failed getting the serverversion") + r.Log.Error(err, "Failed getting the serverversion.") return err } return nil diff --git a/controllers/storagecluster/topology.go b/controllers/storagecluster/topology.go index 7b7fab8578..06db6696b1 100644 --- a/controllers/storagecluster/topology.go +++ b/controllers/storagecluster/topology.go @@ -226,11 +226,11 @@ func (r *StorageClusterReconciler) ensureNodeRacks( rack := determinePlacementRack(nodes, node, minRacks, nodeRacks) nodeRacks.Add(rack, node.Name) if !topologyMap.Contains(defaults.RackTopologyKey, rack) { - r.Log.Info("Adding rack label from node", "Node", node.Name, "Label", defaults.RackTopologyKey, "Value", rack) + r.Log.Info("Adding rack label from Node.", "Node", node.Name, "Label", defaults.RackTopologyKey, "Value", rack) topologyMap.Add(defaults.RackTopologyKey, rack) } - r.Log.Info("Labeling node with rack label", "Node", node.Name, "Label", defaults.RackTopologyKey, "Value", rack) + r.Log.Info("Labeling node with rack label.", "Node", node.Name, "Label", defaults.RackTopologyKey, "Value", rack) newNode := node.DeepCopy() newNode.Labels[defaults.RackTopologyKey] = rack patch, err := generateStrategicPatch(node, newNode) @@ -275,7 +275,7 @@ func (r *StorageClusterReconciler) reconcileNodeTopologyMap(sc *ocsv1.StorageClu for _, key := range validTopologyLabelKeys { if strings.Contains(label, key) { if !topologyMap.Contains(label, value) { - r.Log.Info("Adding topology label from node", "Node", node.Name, "Label", label, "Value", value) + r.Log.Info("Adding topology label from Node.", "Node", node.Name, "Label", label, "Value", value) topologyMap.Add(label, value) } } diff --git a/controllers/storagecluster/uninstall_reconciler.go b/controllers/storagecluster/uninstall_reconciler.go index 092a6ba763..7e4d772bc2 100644 --- a/controllers/storagecluster/uninstall_reconciler.go +++ b/controllers/storagecluster/uninstall_reconciler.go @@ -14,6 +14,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -46,35 +47,35 @@ func (r *StorageClusterReconciler) deleteNodeAffinityKeyFromNodes(sc *ocsv1.Stor if sc.Spec.LabelSelector == nil { nodes, err := r.getStorageClusterEligibleNodes(sc) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster")) //nolint:gosimple + r.Log.Error(err, "Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) //nolint:gosimple return nil } for _, node := range nodes.Items { - r.Log.Info(fmt.Sprintf("Uninstall: Deleting OCS label from node %s", node.Name)) + r.Log.Info("Uninstall: Deleting OCS label from Node.", "Node", node.Name) new := node.DeepCopy() delete(new.ObjectMeta.Labels, defaults.NodeAffinityKey) oldJSON, err := json.Marshal(node) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeAffinityKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name) continue } newJSON, err := json.Marshal(new) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeAffinityKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name) continue } patch, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, node) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeAffinityKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name) continue } err = r.Client.Patch(context.TODO(), &node, client.RawPatch(types.StrategicMergePatchType, patch)) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeAffinityKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeAffinityKey from the Node.", "Node", node.Name) continue } @@ -89,11 +90,11 @@ func (r *StorageClusterReconciler) deleteNodeTaint(sc *ocsv1.StorageCluster) (er nodes, err := r.getStorageClusterEligibleNodes(sc) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster")) //nolint:gosimple + r.Log.Error(err, "Uninstall: Unable to obtain the list of nodes eligible for the Storage Cluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) //nolint:gosimple return nil } for _, node := range nodes.Items { - r.Log.Info(fmt.Sprintf("Uninstall: Deleting OCS NodeTolerationKey from the node %s", node.Name)) + r.Log.Info("Uninstall: Deleting OCS NodeTolerationKey from the Node.", "Node", node.Name) new := node.DeepCopy() new.Spec.Taints = make([]corev1.Taint, 0) for _, taint := range node.Spec.Taints { @@ -105,25 +106,25 @@ func (r *StorageClusterReconciler) deleteNodeTaint(sc *ocsv1.StorageCluster) (er oldJSON, err := json.Marshal(node) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeTolerationKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name) continue } newJSON, err := json.Marshal(new) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeTolerationKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name) continue } patch, err := strategicpatch.CreateTwoWayMergePatch(oldJSON, newJSON, node) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeTolerationKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name) continue } err = r.Client.Patch(context.TODO(), &node, client.RawPatch(types.StrategicMergePatchType, patch)) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Unable to remove the NodeTolerationKey from the node %s", node.Name)) + r.Log.Error(err, "Uninstall: Unable to remove the NodeTolerationKey from the Node.", "Node", node.Name) continue } @@ -141,10 +142,10 @@ func (r *StorageClusterReconciler) setRookUninstallandCleanupPolicy(instance *oc err = r.Client.Get(context.TODO(), types.NamespacedName{Name: generateNameForCephCluster(instance), Namespace: instance.Namespace}, cephCluster) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: CephCluster not found, can't set the cleanup policy and uninstall mode") + r.Log.Info("Uninstall: CephCluster not found, can't set the cleanup policy and uninstall mode.", "CephCluster", klog.KRef(instance.Namespace, generateNameForCephCluster(instance))) return nil } - return fmt.Errorf("Uninstall: Unable to retrieve the cephCluster: %v", err) + return fmt.Errorf("Uninstall: Unable to retrieve the CephCluster: %v", err) } if v, found := instance.ObjectMeta.Annotations[CleanupPolicyAnnotation]; found { @@ -172,7 +173,7 @@ func (r *StorageClusterReconciler) setRookUninstallandCleanupPolicy(instance *oc if err != nil { return fmt.Errorf("Uninstall: Unable to update the cephCluster to set uninstall mode and/or cleanup policy: %v", err) } - r.Log.Info("Uninstall: CephCluster uninstall mode and cleanup policy has been set") + r.Log.Info("Uninstall: CephCluster uninstall mode and cleanup policy has been set.", "CephCluser", klog.KRef(cephCluster.Namespace, cephCluster.Name)) } return nil @@ -193,7 +194,7 @@ func (r *StorageClusterReconciler) setNoobaaUninstallMode(sc *ocsv1.StorageClust err := r.Client.Get(context.TODO(), types.NamespacedName{Name: "noobaa", Namespace: sc.Namespace}, noobaa) if err != nil { if errors.IsNotFound(err) { - r.Log.Info("Uninstall: NooBaa not found, can't set uninstall mode") + r.Log.Info("Uninstall: NooBaa not found, can't set uninstall mode.", "Noobaa", klog.KRef(sc.Namespace, "noobaa")) return nil } return fmt.Errorf("Uninstall: Error while getting NooBaa %v", err) @@ -216,7 +217,7 @@ func (r *StorageClusterReconciler) setNoobaaUninstallMode(sc *ocsv1.StorageClust if err != nil { return fmt.Errorf("Uninstall: Unable to update NooBaa uninstall mode: %v", err) } - r.Log.Info("Uninstall: NooBaa uninstall mode has been set") + r.Log.Info("Uninstall: NooBaa uninstall mode has been set.", "NooBaa", klog.KRef(noobaa.Namespace, noobaa.Name)) } return nil @@ -228,24 +229,24 @@ func (r *StorageClusterReconciler) reconcileUninstallAnnotations(sc *ocsv1.Stora if v, found := sc.ObjectMeta.Annotations[UninstallModeAnnotation]; !found { metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(UninstallModeAnnotation), string(UninstallModeGraceful)) - r.Log.Info("Uninstall: Setting uninstall mode annotation to default", "UninstallMode", UninstallModeGraceful) + r.Log.Info("Uninstall: Setting uninstall mode annotation to default.", "UninstallMode", UninstallModeGraceful) updateRequired = true } else if found && v != string(UninstallModeGraceful) && v != string(UninstallModeForced) { // if wrong value found metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(UninstallModeAnnotation), string(UninstallModeGraceful)) - r.Log.Info("Uninstall: Found unrecognized uninstall mode annotation. Changing it to default", + r.Log.Info("Uninstall: Found unrecognized uninstall mode annotation. Changing it to default.", "CurrentUninstallMode", v, "DefaultUninstallMode", UninstallModeGraceful) updateRequired = true } if v, found := sc.ObjectMeta.Annotations[CleanupPolicyAnnotation]; !found { metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(CleanupPolicyAnnotation), string(CleanupPolicyDelete)) - r.Log.Info("Uninstall: Setting uninstall cleanup policy annotation to default", "CleanupPolicy", CleanupPolicyDelete) + r.Log.Info("Uninstall: Setting uninstall cleanup policy annotation to default.", "CleanupPolicy", CleanupPolicyDelete) updateRequired = true } else if found && v != string(CleanupPolicyDelete) && v != string(CleanupPolicyRetain) { // if wrong value found metav1.SetMetaDataAnnotation(&sc.ObjectMeta, string(CleanupPolicyAnnotation), string(CleanupPolicyDelete)) - r.Log.Info("Uninstall: Found unrecognized uninstall cleanup policy annotation.Changing it to default", + r.Log.Info("Uninstall: Found unrecognized uninstall cleanup policy annotation.Changing it to default.", "CurrentCleanupPolicy", v, "DefaultCleanupPolicy", CleanupPolicyDelete) updateRequired = true } @@ -254,15 +255,15 @@ func (r *StorageClusterReconciler) reconcileUninstallAnnotations(sc *ocsv1.Stora oldSc := ocsv1.StorageCluster{} err := r.Client.Get(context.TODO(), types.NamespacedName{Name: sc.Name, Namespace: sc.Namespace}, &oldSc) if err != nil { - r.Log.Error(err, "Uninstall: Failed to get storagecluster") + r.Log.Error(err, "Uninstall: Failed to get StorageCluster.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) return err } sc.ObjectMeta.ResourceVersion = oldSc.ObjectMeta.ResourceVersion if err := r.Client.Update(context.TODO(), sc); err != nil { - r.Log.Error(err, "Uninstall: Failed to update the storagecluster with uninstall defaults") + r.Log.Error(err, "Uninstall: Failed to update the StorageCluster with uninstall defaults.", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) return err } - r.Log.Info("Uninstall: Default uninstall annotations has been set on storagecluster") + r.Log.Info("Uninstall: Default uninstall annotations has been set on StorageCluster", "StorageCluster", klog.KRef(sc.Namespace, sc.Name)) } return nil } diff --git a/controllers/storagecluster/volumesnapshotterclasses.go b/controllers/storagecluster/volumesnapshotterclasses.go index 3ebd6421d6..975f79055b 100644 --- a/controllers/storagecluster/volumesnapshotterclasses.go +++ b/controllers/storagecluster/volumesnapshotterclasses.go @@ -7,6 +7,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" ocsv1 "github.com/openshift/ocs-operator/api/v1" @@ -92,16 +93,16 @@ func (r *StorageClusterReconciler) createSnapshotClasses(vsccs []SnapshotClassCo if err != nil { if errors.IsNotFound(err) { // Since the SnapshotClass is not found, we will create a new one - r.Log.Info(fmt.Sprintf("creating SnapshotClass %q", vsc.Name)) + r.Log.Info("Creating SnapshotClass.", "SnapshotClass", klog.KRef(vsc.Namespace, vsc.Name)) err = r.Client.Create(context.TODO(), vsc) if err != nil { - r.Log.Error(err, fmt.Sprintf("failed to create SnapshotClass %q", vsc.Name)) + r.Log.Error(err, "Failed to create SnapshotClass.", "SnapshotClass", klog.KRef(vsc.Namespace, vsc.Name)) return err } // no error, continue with the next iteration continue } else { - r.Log.Error(err, fmt.Sprintf("failed to 'Get' SnapshotClass %q", vsc.Name)) + r.Log.Error(err, "Failed to 'Get' SnapshotClass.", "SnapshotClass", klog.KRef(vsc.Namespace, vsc.Name)) return err } } @@ -109,16 +110,16 @@ func (r *StorageClusterReconciler) createSnapshotClasses(vsccs []SnapshotClassCo return nil } if existing.DeletionTimestamp != nil { - return fmt.Errorf("failed to restore snapshotclass %q because it is marked for deletion", existing.Name) + return fmt.Errorf("failed to restore SnapshotClass %q because it is marked for deletion", existing.Name) } // if there is a mis-match in the parameters of existing vs created resources, if !reflect.DeepEqual(vsc.Parameters, existing.Parameters) { // we have to update the existing SnapshotClass - r.Log.Info(fmt.Sprintf("SnapshotClass %q needs to be updated", existing.Name)) + r.Log.Info("SnapshotClass needs to be updated", "SnapshotClass", klog.KRef(existing.Namespace, existing.Name)) existing.ObjectMeta.OwnerReferences = vsc.ObjectMeta.OwnerReferences vsc.ObjectMeta = existing.ObjectMeta if err := r.Client.Update(context.TODO(), vsc); err != nil { - r.Log.Error(err, fmt.Sprintf("SnapshotClass %q updation failed", existing.Name)) + r.Log.Error(err, "SnapshotClass updation failed.", "SnapshotClass", klog.KRef(existing.Namespace, existing.Name)) return err } } @@ -150,22 +151,22 @@ func (obj *ocsSnapshotClass) ensureDeleted(r *StorageClusterReconciler, instance switch { case err == nil: if existing.DeletionTimestamp != nil { - r.Log.Info(fmt.Sprintf("Uninstall: SnapshotClass %s is already marked for deletion", existing.Name)) + r.Log.Info("Uninstall: SnapshotClass is already marked for deletion.", "SnapshotClass", klog.KRef(existing.Namespace, existing.Name)) break } - r.Log.Info(fmt.Sprintf("Uninstall: Deleting SnapshotClass %s", sc.Name)) + r.Log.Info("Uninstall: Deleting SnapshotClass.", "SnapshotClass", klog.KRef(existing.Namespace, existing.Name)) existing.ObjectMeta.OwnerReferences = sc.ObjectMeta.OwnerReferences sc.ObjectMeta = existing.ObjectMeta err = r.Client.Delete(context.TODO(), sc) if err != nil { - r.Log.Error(err, fmt.Sprintf("Uninstall: Ignoring error deleting the SnapshotClass %s", existing.Name)) + r.Log.Error(err, "Uninstall: Ignoring error deleting the SnapshotClass.", "SnapshotClass", klog.KRef(existing.Namespace, existing.Name)) } case errors.IsNotFound(err): - r.Log.Info(fmt.Sprintf("Uninstall: SnapshotClass %s not found, nothing to do", sc.Name)) + r.Log.Info("Uninstall: SnapshotClass not found, nothing to do.", "SnapshotClass", klog.KRef(sc.Namespace, sc.Name)) default: - r.Log.Info(fmt.Sprintf("Uninstall: Error while getting SnapshotClass %s: %v", sc.Name, err)) + r.Log.Error(err, "Uninstall: Error while getting SnapshotClass.", "SnapshotClass", klog.KRef(sc.Namespace, sc.Name)) } } return nil diff --git a/go.mod b/go.mod index f80bcd377e..e4b4e24fde 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( k8s.io/apimachinery v0.20.2 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 + k8s.io/klog/v2 v2.4.0 sigs.k8s.io/controller-runtime v0.8.0 ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 56032746ff..2ab0d7bf72 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -593,6 +593,7 @@ k8s.io/component-base/config/v1alpha1 ## explicit k8s.io/klog # k8s.io/klog/v2 v2.4.0 +## explicit k8s.io/klog/v2 # k8s.io/kube-aggregator v0.18.9 => k8s.io/kube-aggregator v0.20.2 k8s.io/kube-aggregator/pkg/apis/apiregistration