Skip to content

Commit bce1629

Browse files
2uasimojo2uasimojo
committed
HIVE-2302, HIVE-2644: Destroyers use metadata.json
An earlier commit ensures that ClusterDeployments have an associated Secret containing the metadata.json emitted by the installer. This change adds a new generic destroyer via the (existing) `hiveutil deprovision` command that consumes this metadata.json to deprovision the cluster. This new behavior is the default, but we also include an escape hatch to run the platform-specific legacy destroyer by setting the following annotation on the ClusterDeployment: `hive.openshift.io/legacy-deprovision: "true"`
1 parent 3c7b8a3 commit bce1629

21 files changed

+425
-132
lines changed

‎apis/hive/v1/clusterdeprovision_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ type ClusterDeprovisionSpec struct {
2424
// BaseDomain is the DNS base domain.
2525
BaseDomain string `json:"baseDomain,omitempty"`
2626

27+
// MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
28+
// installer, potentially scrubbed for sensitive data.
29+
MetadataJSONSecretRef *corev1.LocalObjectReference `json:"metadataJSONSecretRef,omitempty"`
30+
2731
// Platform contains platform-specific configuration for a ClusterDeprovision
2832
Platform ClusterDeprovisionPlatform `json:"platform,omitempty"`
2933
}

‎apis/hive/v1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crds/hive.openshift.io_clusterdeprovisions.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,22 @@ spec:
6868
infraID:
6969
description: InfraID is the identifier generated during installation for a cluster. It is used for tagging/naming resources in cloud providers.
7070
type: string
71+
metadataJSONSecretRef:
72+
description: |-
73+
MetaddataJSONSecretRef references the secret containing the metadata.json emitted by the
74+
installer, potentially scrubbed for sensitive data.
75+
properties:
76+
name:
77+
default: ""
78+
description: |-
79+
Name of the referent.
80+
This field is effectively required, but due to backwards compatibility is
81+
allowed to be empty. Instances of this type with an empty value here are
82+
almost certainly wrong.
83+
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
84+
type: string
85+
type: object
86+
x-kubernetes-map-type: atomic
7187
platform:
7288
description: Platform contains platform-specific configuration for a ClusterDeprovision
7389
properties:

‎contrib/pkg/deprovision/awstagdeprovision.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
)
1414

1515
// NewDeprovisionAWSWithTagsCommand is the entrypoint to create the 'aws-tag-deprovision' subcommand
16-
// TODO: Port to a sub-command of deprovision.
1716
func NewDeprovisionAWSWithTagsCommand() *cobra.Command {
1817
opt := &aws.ClusterUninstaller{}
1918
var credsDir string

‎contrib/pkg/deprovision/azure.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ type AzureOptions struct {
2323
}
2424

2525
// NewDeprovisionAzureCommand is the entrypoint to create the azure deprovision subcommand
26-
func NewDeprovisionAzureCommand() *cobra.Command {
27-
opt := &AzureOptions{}
26+
func NewDeprovisionAzureCommand(logLevel string) *cobra.Command {
27+
opt := &AzureOptions{
28+
logLevel: logLevel,
29+
}
2830
cmd := &cobra.Command{
2931
Use: "azure INFRAID [--azure-cloud-name CLOUDNAME] [--azure-resource-group-name RG] [--azure-base-domain-resource-group-name BDRG]",
3032
Short: "Deprovision Azure assets (as created by openshift-installer)",
@@ -46,7 +48,6 @@ func NewDeprovisionAzureCommand() *cobra.Command {
4648
},
4749
}
4850
flags := cmd.Flags()
49-
flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
5051
flags.StringVar(&opt.cloudName, "azure-cloud-name", installertypesazure.PublicCloud.Name(), "The name of the Azure cloud environment used to configure the Azure SDK")
5152
flags.StringVar(&opt.resourceGroupName, "azure-resource-group-name", "", "The name of the custom Azure resource group in which the cluster was created when not using the default installer-created resource group")
5253
flags.StringVar(&opt.baseDomainResourceGroupName, "azure-base-domain-resource-group-name", "", "The name of the custom Azure resource group in which the cluster's DNS records were created when not using the default installer-created resource group or custom resource group")

‎contrib/pkg/deprovision/deprovision.go‎

Lines changed: 145 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,164 @@
11
package deprovision
22

33
import (
4+
"encoding/json"
5+
"log"
6+
"os"
7+
8+
"sigs.k8s.io/controller-runtime/pkg/client"
9+
10+
"github.com/openshift/installer/pkg/destroy/providers"
11+
"github.com/openshift/installer/pkg/types"
12+
"github.com/openshift/installer/pkg/types/aws"
13+
"github.com/openshift/installer/pkg/types/azure"
14+
"github.com/openshift/installer/pkg/types/gcp"
15+
"github.com/openshift/installer/pkg/types/ibmcloud"
16+
"github.com/openshift/installer/pkg/types/nutanix"
17+
"github.com/openshift/installer/pkg/types/openstack"
18+
"github.com/openshift/installer/pkg/types/vsphere"
19+
20+
"github.com/openshift/hive/contrib/pkg/utils"
21+
awsutil "github.com/openshift/hive/contrib/pkg/utils/aws"
22+
azureutil "github.com/openshift/hive/contrib/pkg/utils/azure"
23+
gcputil "github.com/openshift/hive/contrib/pkg/utils/gcp"
24+
ibmcloudutil "github.com/openshift/hive/contrib/pkg/utils/ibmcloud"
25+
nutanixutil "github.com/openshift/hive/contrib/pkg/utils/nutanix"
26+
openstackutil "github.com/openshift/hive/contrib/pkg/utils/openstack"
27+
vsphereutil "github.com/openshift/hive/contrib/pkg/utils/vsphere"
28+
"github.com/openshift/hive/pkg/constants"
29+
430
"github.com/spf13/cobra"
531
)
632

733
// NewDeprovisionCommand is the entrypoint to create the 'deprovision' subcommand
834
func NewDeprovisionCommand() *cobra.Command {
935
var credsDir string
36+
var mjSecretName string
37+
var logLevel string
1038
cmd := &cobra.Command{
1139
Use: "deprovision",
1240
Short: "Deprovision clusters in supported cloud providers",
41+
Long: `Platform subcommands use a legacy code path and are deprecated. \
42+
To run the generic destroyer, use the --metadata-json-secret-name parameter.`,
1343
Run: func(cmd *cobra.Command, args []string) {
14-
cmd.Usage()
44+
if mjSecretName == "" {
45+
cmd.Usage()
46+
return
47+
}
48+
49+
// Generic deprovision flow using metadata.json
50+
logger, err := utils.NewLogger(logLevel)
51+
if err != nil {
52+
log.Fatalf("failed to create logger: %s", err)
53+
}
54+
55+
c, err := utils.GetClient("hiveutil-deprovision-generic")
56+
if err != nil {
57+
logger.WithError(err).Fatal("failed to create kube client")
58+
}
59+
60+
// TODO: Refactor LoadSecretOrDie to avoid this setenv/getenv cycle
61+
k := "METADATA_JSON_SECRET_NAME"
62+
os.Setenv(k, mjSecretName)
63+
mjSecret := utils.LoadSecretOrDie(c, k)
64+
if mjSecret == nil {
65+
// This should not be reachable -- we should have Fatal()ed in LoadSecretOrDie()
66+
logger.WithField("secretName", mjSecretName).Fatal("failed to load metadata.json Secret")
67+
}
68+
69+
mjBytes, ok := mjSecret.Data[constants.MetadataJSONSecretKey]
70+
if !ok {
71+
logger.Fatalf("metadata.json Secret did not contain %q key", constants.MetadataJSONSecretKey)
72+
}
73+
74+
var metadata *types.ClusterMetadata
75+
if err = json.Unmarshal(mjBytes, &metadata); err != nil {
76+
logger.WithError(err).Fatal("failed to unmarshal metadata.json")
77+
}
78+
79+
platform := metadata.Platform()
80+
if platform == "" {
81+
logger.Fatal("no platform configured in metadata.json")
82+
}
83+
84+
// TODO: Make a registry or interface for this
85+
var ConfigureCreds func(client.Client)
86+
switch platform {
87+
case aws.Name:
88+
ConfigureCreds = awsutil.ConfigureCreds
89+
case azure.Name:
90+
ConfigureCreds = azureutil.ConfigureCreds
91+
case gcp.Name:
92+
ConfigureCreds = gcputil.ConfigureCreds
93+
case ibmcloud.Name:
94+
ConfigureCreds = ibmcloudutil.ConfigureCreds
95+
case nutanix.Name:
96+
// Snowflake! We need to inject the creds into the metadata.
97+
// If env vars are unset, the destroyer will fail organically.
98+
ConfigureCreds = func(c client.Client) {
99+
nutanixutil.ConfigureCreds(c)
100+
metadata.Nutanix.Username = os.Getenv(constants.NutanixUsernameEnvVar)
101+
metadata.Nutanix.Password = os.Getenv(constants.NutanixPasswordEnvVar)
102+
}
103+
case openstack.Name:
104+
ConfigureCreds = openstackutil.ConfigureCreds
105+
case vsphere.Name:
106+
// Snowflake! We need to (re)inject the creds into the metadata.
107+
// (They were there originally, but we scrubbed them for security.)
108+
// If env vars are unset, the destroyer will fail organically.
109+
ConfigureCreds = func(c client.Client) {
110+
vsphereutil.ConfigureCreds(c)
111+
username, password := os.Getenv(constants.VSphereUsernameEnvVar), os.Getenv(constants.VSpherePasswordEnvVar)
112+
// Accommodate both pre- and post-zonal formats
113+
if metadata.VSphere.Username != "" {
114+
metadata.VSphere.Username = username
115+
}
116+
if metadata.VSphere.Password != "" {
117+
metadata.VSphere.Password = password
118+
}
119+
for i := range metadata.VSphere.VCenters {
120+
if metadata.VSphere.VCenters[i].Username != "" {
121+
metadata.VSphere.VCenters[i].Username = username
122+
}
123+
if metadata.VSphere.VCenters[i].Password != "" {
124+
metadata.VSphere.VCenters[i].Password = password
125+
}
126+
}
127+
}
128+
}
129+
130+
ConfigureCreds(c)
131+
132+
destroyerBuilder, ok := providers.Registry[platform]
133+
if !ok {
134+
logger.WithField("platform", platform).Fatal("no destroyers registered for platform")
135+
}
136+
137+
destroyer, err := destroyerBuilder(logger, metadata)
138+
if err != nil {
139+
logger.WithError(err).Fatal("failed to create destroyer")
140+
}
141+
142+
// Ignore quota return
143+
_, err = destroyer.Run()
144+
if err != nil {
145+
logger.WithError(err).Fatal("destroyer returned an error")
146+
}
15147
},
16148
}
17149
flags := cmd.PersistentFlags()
150+
// TODO: Unused -- remove from here and generate.go
18151
flags.StringVar(&credsDir, "creds-dir", "", "directory of the creds. Changes in the creds will cause the program to terminate")
19-
cmd.AddCommand(NewDeprovisionAzureCommand())
20-
cmd.AddCommand(NewDeprovisionGCPCommand())
21-
cmd.AddCommand(NewDeprovisionIBMCloudCommand())
22-
cmd.AddCommand(NewDeprovisionOpenStackCommand())
23-
cmd.AddCommand(NewDeprovisionvSphereCommand())
24-
cmd.AddCommand(NewDeprovisionNutanixCommand())
152+
// TODO: Make this more useful to CLI users by accepting a path to a metadata.json file in the file system
153+
flags.StringVar(&mjSecretName, "metadata-json-secret-name", "", "name of a Secret in the current namespace containing `metadata.json` from the installer")
154+
flags.StringVar(&logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
155+
156+
// Legacy destroyers
157+
cmd.AddCommand(NewDeprovisionAzureCommand(logLevel))
158+
cmd.AddCommand(NewDeprovisionGCPCommand(logLevel))
159+
cmd.AddCommand(NewDeprovisionIBMCloudCommand(logLevel))
160+
cmd.AddCommand(NewDeprovisionOpenStackCommand(logLevel))
161+
cmd.AddCommand(NewDeprovisionvSphereCommand(logLevel))
162+
cmd.AddCommand(NewDeprovisionNutanixCommand(logLevel))
25163
return cmd
26164
}

‎contrib/pkg/deprovision/gcp.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@ type gcpOptions struct {
2626
}
2727

2828
// NewDeprovisionGCPCommand is the entrypoint to create the GCP deprovision subcommand
29-
func NewDeprovisionGCPCommand() *cobra.Command {
30-
opt := &gcpOptions{}
29+
func NewDeprovisionGCPCommand(logLevel string) *cobra.Command {
30+
opt := &gcpOptions{
31+
logLevel: logLevel,
32+
}
3133
cmd := &cobra.Command{
3234
Use: "gcp INFRAID --region=REGION",
3335
Short: "Deprovision GCP assets (as created by openshift-installer)",
@@ -45,7 +47,6 @@ func NewDeprovisionGCPCommand() *cobra.Command {
4547
},
4648
}
4749
flags := cmd.Flags()
48-
flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
4950
flags.StringVar(&opt.region, "region", "", "GCP region where the cluster is installed")
5051
flags.StringVar(&opt.networkProjectID, "network-project-id", "", "For shared VPC setups")
5152
return cmd

‎contrib/pkg/deprovision/ibmcloud.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@ type ibmCloudDeprovisionOptions struct {
3030
}
3131

3232
// NewDeprovisionIBMCloudCommand is the entrypoint to create the IBM Cloud deprovision subcommand
33-
func NewDeprovisionIBMCloudCommand() *cobra.Command {
34-
opt := &ibmCloudDeprovisionOptions{}
33+
func NewDeprovisionIBMCloudCommand(logLevel string) *cobra.Command {
34+
opt := &ibmCloudDeprovisionOptions{
35+
logLevel: logLevel,
36+
}
3537
cmd := &cobra.Command{
3638
Use: "ibmcloud INFRAID --region=us-east --base-domain=BASE_DOMAIN --cluster-name=CLUSTERNAME",
3739
Short: "Deprovision IBM Cloud assets (as created by openshift-installer)",
@@ -50,7 +52,6 @@ func NewDeprovisionIBMCloudCommand() *cobra.Command {
5052
}
5153

5254
flags := cmd.Flags()
53-
flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
5455

5556
// Required flags
5657
flags.StringVar(&opt.baseDomain, "base-domain", "", "cluster's base domain")

‎contrib/pkg/deprovision/nutanix.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@ type nutanixOptions struct {
2626
password string
2727
}
2828

29-
func NewDeprovisionNutanixCommand() *cobra.Command {
30-
opt := &nutanixOptions{}
29+
func NewDeprovisionNutanixCommand(logLevel string) *cobra.Command {
30+
opt := &nutanixOptions{
31+
logLevel: logLevel,
32+
}
3133

3234
cmd := &cobra.Command{
3335
Use: "nutanix INFRAID",
@@ -46,7 +48,6 @@ func NewDeprovisionNutanixCommand() *cobra.Command {
4648
},
4749
}
4850
flags := cmd.Flags()
49-
flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
5051
flags.StringVar(&opt.endpoint, constants.CliNutanixPcAddressOpt, "", "Domain name or IP address of the Nutanix Prism Central endpoint")
5152
flags.StringVar(&opt.port, constants.CliNutanixPcPortOpt, "", "Port of the Nutanix Prism Central endpoint")
5253
return cmd

‎contrib/pkg/deprovision/openstack.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ type openStackOptions struct {
2323
}
2424

2525
// NewDeprovisionOpenStackCommand is the entrypoint to create the OpenStack deprovision subcommand
26-
func NewDeprovisionOpenStackCommand() *cobra.Command {
27-
opt := &openStackOptions{}
26+
func NewDeprovisionOpenStackCommand(logLevel string) *cobra.Command {
27+
opt := &openStackOptions{
28+
logLevel: logLevel,
29+
}
2830
cmd := &cobra.Command{
2931
Use: "openstack INFRAID --cloud=OS_CLOUD",
3032
Short: "Deprovision OpenStack assets (as created by openshift-installer)",
@@ -42,7 +44,6 @@ func NewDeprovisionOpenStackCommand() *cobra.Command {
4244
},
4345
}
4446
flags := cmd.Flags()
45-
flags.StringVar(&opt.logLevel, "loglevel", "info", "log level, one of: debug, info, warn, error, fatal, panic")
4647
flags.StringVar(&opt.cloud, "cloud", "", "OpenStack cloud entry name from clouds.yaml for access/authentication")
4748
return cmd
4849
}

0 commit comments

Comments
 (0)