[release-4.16] delete COS bucket during teardown of cluster

ocs_ci/deployment/ibmcloud.py

@@ -19,6 +19,7 @@
     LeftoversExistError,
     VolumesExistError,
 )
+from ocs_ci.ocs.resources.backingstore import get_backingstore
 from ocs_ci.ocs.resources.pvc import (
     scale_down_pods_and_remove_pvcs,
 )
@@ -203,6 +204,7 @@ def destroy_cluster(self, log_level="DEBUG"):
         resource_group = self.get_resource_group()
         if resource_group:
             try:
+                self.delete_bucket()
                 scale_down_pods_and_remove_pvcs(self.DEFAULT_STORAGECLASS)
             except Exception as err:
                 logger.warning(
@@ -235,6 +237,25 @@ def destroy_cluster(self, log_level="DEBUG"):
             logger.info("Force cleaning up Service IDs and Account Policies leftovers")
             ibmcloud.cleanup_policies_and_service_ids(self.cluster_name)
 
+    def delete_bucket(self):
+        """
+        Deletes the COS bucket
+        """
+        api_key = config.AUTH["ibmcloud"]["api_key"]
+        service_instance_id = config.AUTH["ibmcloud"]["cos_instance_crn"]
+        endpoint_url = constants.IBM_COS_GEO_ENDPOINT_TEMPLATE.format(
+            config.ENV_DATA.get("region", "us-east").lower()
+        )
+        backingstore = get_backingstore()
+        bucket_name = backingstore["spec"]["ibmCos"]["targetBucket"]
+        logger.debug(f"bucket name from backingstore: {bucket_name}")
+        cos = ibmcloud.IBMCloudObjectStorage(
+            api_key=api_key,
+            service_instance_id=service_instance_id,
+            endpoint_url=endpoint_url,
+        )
+        cos.delete_bucket(bucket_name=bucket_name)
+
     def manually_create_iam_for_vpc(self):
         """
         Manually specify the IAM secrets for the cloud provider

ocs_ci/ocs/resources/backingstore.py

@@ -489,3 +489,17 @@ def clone_bs_dict_from_backingstore(
         raise UnknownCloneTypeException(prototype_bs_platform_name)
 
     return clone_bs_dict
+
+
+def get_backingstore():
+    """
+    Fetches the backingstore
+
+    Returns:
+        dict: backingstore details
+
+    """
+    backingstore = OCP(
+        kind=constants.BACKINGSTORE, namespace=config.ENV_DATA["cluster_namespace"]
+    )
+    return backingstore.get(resource_name=constants.DEFAULT_NOOBAA_BACKINGSTORE)

ocs_ci/utility/ibmcloud.py

@@ -10,8 +10,11 @@
 import re
 import requests
 import time
+import ibm_boto3
 import ipaddress
+
 from copy import copy
+from ibm_botocore.client import Config as IBMBotocoreConfig, ClientError
 from json import JSONDecodeError
 from ocs_ci.framework import config
 from ocs_ci.ocs import constants
@@ -956,3 +959,112 @@ def find_free_network_subnets(subnet_cidr, network_prefix=27):
         if is_free:
             logger.info(f"Free set of subnets found: {possible_subnets}")
             return possible_subnets
+
+
+class IBMCloudObjectStorage:
+    """
+    IBM Cloud Object Storage (COS) class
+    """
+
+    def __init__(self, api_key, service_instance_id, endpoint_url):
+        """
+        Initialize all necessary parameters
+
+        Args:
+            api_key (str): API key for IBM Cloud Object Storage (COS)
+            service_instance_id (str): Service instance ID for COS
+            endpoint_url (str): COS endpoint URL
+
+        """
+        self.cos_api_key_id = api_key
+        self.cos_instance_crn = service_instance_id
+        self.cos_endpoint = endpoint_url
+        # create client
+        self.cos_client = ibm_boto3.client(
+            "s3",
+            ibm_api_key_id=self.cos_api_key_id,
+            ibm_service_instance_id=self.cos_instance_crn,
+            config=IBMBotocoreConfig(signature_version="oauth"),
+            endpoint_url=self.cos_endpoint,
+        )
+
+    def get_bucket_objects(self, bucket_name):
+        """
+        Fetches the objects in a bucket
+
+        Args:
+            bucket_name (str): Name of the bucket
+
+        Returns:
+            list: List of objects in a bucket
+
+        """
+        bucket_objects = []
+        logger.info(f"Retrieving bucket contents from {bucket_name}")
+        try:
+            objects = self.cos_client.list_objects(Bucket=bucket_name)
+            for obj in objects.get("Contents", []):
+                bucket_objects.append(obj["Key"])
+        except ClientError as ce:
+            logger.error(f"CLIENT ERROR: {ce}")
+        except Exception as e:
+            logger.error(f"Unable to retrieve bucket contents: {e}")
+        logger.info(f"bucket objects: {bucket_objects}")
+        return bucket_objects
+
+    def delete_objects(self, bucket_name):
+        """
+        Delete objects in a bucket
+
+        Args:
+            bucket_name (str): Name of the bucket
+
+        """
+        objects = self.get_bucket_objects(bucket_name)
+        if objects:
+            try:
+                # Form the delete request
+                delete_request = {"Objects": [{"Key": obj} for obj in objects]}
+                response = self.cos_client.delete_objects(
+                    Bucket=bucket_name, Delete=delete_request
+                )
+                logger.info(f"Deleted items for {bucket_name}")
+                logger.debug(json.dumps(response.get("Deleted"), indent=4))
+            except ClientError as ce:
+                logger.error(f"CLIENT ERROR: {ce}")
+            except Exception as e:
+                logger.error(f"Unable to delete objects: {e}")
+
+    def delete_bucket(self, bucket_name):
+        """
+        Delete the bucket
+
+        Args:
+            bucket_name (str): Name of the bucket
+
+        """
+        logger.info(f"Deleting bucket: {bucket_name}")
+        try:
+            self.delete_objects(bucket_name=bucket_name)
+            self.cos_client.delete_bucket(Bucket=bucket_name)
+            logger.info(f"Bucket: {bucket_name} deleted!")
+        except ClientError as ce:
+            logger.error(f"CLIENT ERROR: {ce}")
+        except Exception as e:
+            logger.error(f"Unable to delete bucket: {e}")
+
+    def get_buckets(self):
+        """
+        Fetches the buckets
+        """
+        buckets = []
+        logger.info("Retrieving list of buckets")
+        try:
+            buckets = self.cos_client.list_buckets()
+            for bucket in buckets["Buckets"]:
+                buckets.append(bucket["Name"])
+        except ClientError as ce:
+            logger.error(f"CLIENT ERROR: {ce}")
+        except Exception as e:
+            logger.error(f"Unable to retrieve list buckets: {e}")
+        return buckets

setup.py

@@ -20,6 +20,7 @@
         "cryptography==42.0.4",
         "docopt==0.6.2",
         "gevent==23.9.1",
+        "ibm-cos-sdk==2.13.5",
         "reportportal-client==3.2.3",
         "requests==2.32.3",
         "paramiko==3.4.0",
@@ -40,7 +41,7 @@
         "pytest_marker_bugzilla>=0.9.3",
         "pyvmomi==7.0",
         "python-hcl2==3.0.1",
-        "python-dateutil==2.8.2",
+        "python-dateutil==2.9.0",
         "pytest-order==1.2.0",
         "funcy==1.14",
         "semantic-version==2.8.5",