HIVE-2302: Save metadata.json opaquely

Well, mostly.

Previously any time installer added a field to metadata.json, we would
need to evaluate and possibly add a bespoke field and code path for it
to make sure it was supplied to the destroyer at deprovision time.

With this change, we're offloading metadata.json verbatim (except in
some cases we have to scrub/replace credentials fields -- see HIVE-2804
/ #2612) to a new Secret in the ClusterDeployment's namespace,
referenced from a new field:
ClusterDeployment.Spec.ClusterMetadata.MetadataJSONSecretRef.

For legacy clusters -- those created before this change -- we attempt to
retrofit the new Secret based on the legacy fields. This is best effort
and may not always work.

In the future (but not here!) instead of building the installer's
ClusterMetadata structure for the destroyer with individual fields from
the CD's ClusterMetadata, we'll unmarshal it directly from the contents
of this Secret.

Author: 2uasimojo

apis/hive/v1/aws/metadata.go

@@ -4,5 +4,7 @@ package aws
 type Metadata struct {
 	// HostedZoneRole is the role to assume when performing operations
 	// on a hosted zone owned by another account.
+	// Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+	// may stop populating this section in the future.
 	HostedZoneRole *string `json:"hostedZoneRole,omitempty"`
 }

apis/hive/v1/azure/metadata.go

@@ -3,5 +3,7 @@ package azure
 // Metadata contains Azure metadata (e.g. for uninstalling the cluster).
 type Metadata struct {
 	// ResourceGroupName is the name of the resource group in which the cluster resources were created.
+	// Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+	// may stop populating this section in the future.
 	ResourceGroupName *string `json:"resourceGroupName"`
 }

apis/hive/v1/clusterdeployment_types.go

@@ -308,7 +308,13 @@ type ClusterMetadata struct {
 	// +optional
 	AdminPasswordSecretRef *corev1.LocalObjectReference `json:"adminPasswordSecretRef,omitempty"`
 
-	// Platform holds platform-specific cluster metadata
+	// MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
+	// installer, potentially scrubbed for sensitive data.
+	MetadataJSONSecretRef *corev1.LocalObjectReference `json:"metadataJSONSecretRef,omitempty"`
+
+	// Platform holds platform-specific cluster metadata.
+	// Deprecated. Use the Secret referenced by MetadataJSONSecretRef instead. We may stop
+	// populating this section in the future.
 	// +optional
 	Platform *ClusterPlatformMetadata `json:"platform,omitempty"`
 }

apis/hive/v1/gcp/metadata.go

@@ -3,6 +3,8 @@ package gcp
 // Metadata contains GCP metadata (e.g. for uninstalling the cluster).
 type Metadata struct {
 	// NetworkProjectID is used for shared VPC setups
+	// Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+	// may stop populating this section in the future.
 	// +optional
 	NetworkProjectID *string `json:"networkProjectID,omitempty"`
 }

apis/hive/v1/zz_generated.deepcopy.go

@@ -192,8 +192,27 @@ spec:
                       during installation and used for tagging/naming resources in
                       cloud providers.
                     type: string
+                  metadataJSONSecretRef:
+                    description: |-
+                      MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
+                      installer, potentially scrubbed for sensitive data.
+                    properties:
+                      name:
+                        default: ""
+                        description: |-
+                          Name of the referent.
+                          This field is effectively required, but due to backwards compatibility is
+                          allowed to be empty. Instances of this type with an empty value here are
+                          almost certainly wrong.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   platform:
-                    description: Platform holds platform-specific cluster metadata
+                    description: |-
+                      Platform holds platform-specific cluster metadata.
+                      Deprecated. Use the Secret referenced by MetadataJSONSecretRef instead. We may stop
+                      populating this section in the future.
                     properties:
                       aws:
                         description: AWS holds AWS-specific cluster metadata
@@ -202,14 +221,18 @@ spec:
                             description: |-
                               HostedZoneRole is the role to assume when performing operations
                               on a hosted zone owned by another account.
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         type: object
                       azure:
                         description: Azure holds azure-specific cluster metadata
                         properties:
                           resourceGroupName:
-                            description: ResourceGroupName is the name of the resource
-                              group in which the cluster resources were created.
+                            description: |-
+                              ResourceGroupName is the name of the resource group in which the cluster resources were created.
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         required:
                         - resourceGroupName
@@ -218,7 +241,10 @@ spec:
                         description: GCP holds GCP-specific cluster metadata
                         properties:
                           networkProjectID:
-                            description: NetworkProjectID is used for shared VPC setups
+                            description: |-
+                              NetworkProjectID is used for shared VPC setups
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         type: object
                     type: object

config/crds/hive.openshift.io_clusterdeployments.yaml

@@ -104,8 +104,27 @@ spec:
                       during installation and used for tagging/naming resources in
                       cloud providers.
                     type: string
+                  metadataJSONSecretRef:
+                    description: |-
+                      MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
+                      installer, potentially scrubbed for sensitive data.
+                    properties:
+                      name:
+                        default: ""
+                        description: |-
+                          Name of the referent.
+                          This field is effectively required, but due to backwards compatibility is
+                          allowed to be empty. Instances of this type with an empty value here are
+                          almost certainly wrong.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   platform:
-                    description: Platform holds platform-specific cluster metadata
+                    description: |-
+                      Platform holds platform-specific cluster metadata.
+                      Deprecated. Use the Secret referenced by MetadataJSONSecretRef instead. We may stop
+                      populating this section in the future.
                     properties:
                       aws:
                         description: AWS holds AWS-specific cluster metadata
@@ -114,14 +133,18 @@ spec:
                             description: |-
                               HostedZoneRole is the role to assume when performing operations
                               on a hosted zone owned by another account.
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         type: object
                       azure:
                         description: Azure holds azure-specific cluster metadata
                         properties:
                           resourceGroupName:
-                            description: ResourceGroupName is the name of the resource
-                              group in which the cluster resources were created.
+                            description: |-
+                              ResourceGroupName is the name of the resource group in which the cluster resources were created.
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         required:
                         - resourceGroupName
@@ -130,7 +153,10 @@ spec:
                         description: GCP holds GCP-specific cluster metadata
                         properties:
                           networkProjectID:
-                            description: NetworkProjectID is used for shared VPC setups
+                            description: |-
+                              NetworkProjectID is used for shared VPC setups
+                              Deprecated. Use the Secret referenced by ClusterMetadata.MetadataJSONSecretRef instead. We
+                              may stop populating this section in the future.
                             type: string
                         type: object
                     type: object

config/crds/hiveinternal.openshift.io_fakeclusterinstalls.yaml

@@ -161,6 +161,7 @@ type Options struct {
 	AdoptAdminKubeConfig              string
 	AdoptInfraID                      string
 	AdoptClusterID                    string
+	AdoptMetadataJSON                 string
 	AdoptAdminUsername                string
 	AdoptAdminPassword                string
 	MachineNetwork                    string
@@ -349,6 +350,7 @@ OpenShift Installer publishes all the services of the cluster like API server an
 	flags.StringVar(&opt.AdoptAdminKubeConfig, "adopt-admin-kubeconfig", "", "Path to a cluster admin kubeconfig file for a cluster being adopted. (required if using --adopt)")
 	flags.StringVar(&opt.AdoptInfraID, "adopt-infra-id", "", "Infrastructure ID for this cluster's cloud provider. (required if using --adopt)")
 	flags.StringVar(&opt.AdoptClusterID, "adopt-cluster-id", "", "Cluster UUID used for telemetry. (required if using --adopt)")
+	flags.StringVar(&opt.AdoptMetadataJSON, "adopt-metadata-json", "", "Path to a metadata.json file for a cluster being adopted. (optional)")
 	flags.StringVar(&opt.AdoptAdminUsername, "adopt-admin-username", "", "Username for cluster web console administrator. (optional)")
 	flags.StringVar(&opt.AdoptAdminPassword, "adopt-admin-password", "", "Password for cluster web console administrator. (optional)")
 
@@ -506,13 +508,19 @@ func (o *Options) Validate(cmd *cobra.Command) error {
 			return fmt.Errorf("--adopt-admin-kubeconfig does not exist: %s", o.AdoptAdminKubeConfig)
 		}
 
+		if o.AdoptMetadataJSON != "" {
+			if _, err := os.Stat(o.AdoptMetadataJSON); os.IsNotExist(err) {
+				return fmt.Errorf("--adopt-metadata-json does not exist: %s", o.AdoptMetadataJSON)
+			}
+		}
+
 		// Admin username and password must both be specified if either are.
 		if (o.AdoptAdminUsername != "" || o.AdoptAdminPassword != "") && !(o.AdoptAdminUsername != "" && o.AdoptAdminPassword != "") {
 			return fmt.Errorf("--adopt-admin-username and --adopt-admin-password must be used together")
 		}
 	} else {
-		if o.AdoptAdminKubeConfig != "" || o.AdoptInfraID != "" || o.AdoptClusterID != "" || o.AdoptAdminUsername != "" || o.AdoptAdminPassword != "" {
-			return fmt.Errorf("cannot use adoption options without --adopt: --adopt-admin-kube-config, --adopt-infra-id, --adopt-cluster-id, --adopt-admin-username, --adopt-admin-password")
+		if o.AdoptAdminKubeConfig != "" || o.AdoptInfraID != "" || o.AdoptClusterID != "" || o.AdoptMetadataJSON != "" || o.AdoptAdminUsername != "" || o.AdoptAdminPassword != "" {
+			return fmt.Errorf("cannot use adoption options without --adopt: --adopt-admin-kube-config, --adopt-infra-id, --adopt-cluster-id, --adopt-metadata-json, --adopt-admin-username, --adopt-admin-password")
 		}
 	}
 
@@ -658,6 +666,13 @@ func (o *Options) GenerateObjects() ([]runtime.Object, error) {
 		if err != nil {
 			return nil, err
 		}
+		if o.AdoptMetadataJSON != "" {
+			metadataJSONBytes, err := os.ReadFile(o.AdoptMetadataJSON)
+			if err != nil {
+				return nil, err
+			}
+			builder.AdoptMetadataJSON = metadataJSONBytes
+		}
 		builder.Adopt = o.Adopt
 		builder.AdoptInfraID = o.AdoptInfraID
 		builder.AdoptClusterID = o.AdoptClusterID