From 827ae89fbc769fd840b9a76725d1cadfb0128391 Mon Sep 17 00:00:00 2001 From: akaranjkar-qu <41927856+akaranjkar-qu@users.noreply.github.com> Date: Mon, 1 Apr 2019 04:11:21 -0700 Subject: [PATCH] SDK-345: Add custom resource group and start-stop related params for Azure clusters (#268) --- qds_sdk/cloud/azure_cloud.py | 11 ++++-- qds_sdk/clusterv2.py | 68 ++++++++++++++++++++++++++++++++++-- tests/test_clusterv2.py | 53 ++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py index 682c58d4..435e4993 100755 --- a/qds_sdk/cloud/azure_cloud.py +++ b/qds_sdk/cloud/azure_cloud.py @@ -29,7 +29,8 @@ def set_cloud_config(self, vnet_resource_group_name=None, master_elastic_ip=None, master_static_nic_name=None, - master_static_public_ip_name=None): + master_static_public_ip_name=None, + resource_group_name=None): ''' Args: @@ -74,6 +75,7 @@ def set_cloud_config(self, master_static_public_ip_name: Name of Static Public Ip address that has to be attached to cluster's master node + resource_group_name: Resource group for cluster ''' self.set_compute_config(use_account_compute_creds, compute_tenant_id, @@ -86,6 +88,7 @@ def set_cloud_config(self, self.set_storage_config(storage_access_key, storage_account_name, disk_storage_account_name, disk_storage_account_resource_group_name) + self.resource_group_name = resource_group_name def set_compute_config(self, use_account_compute_creds=None, @@ -147,7 +150,8 @@ def set_cloud_config_from_arguments(self, arguments): subnet_name=arguments.subnet_name, vnet_resource_group_name=arguments.vnet_resource_group_name, master_static_nic_name=arguments.master_static_nic_name, - master_static_public_ip_name=arguments.master_static_public_ip_name) + master_static_public_ip_name=arguments.master_static_public_ip_name, + resource_group_name=arguments.resource_group_name) def create_parser(self, argparser): # compute settings parser @@ -204,6 +208,9 @@ def create_parser(self, argparser): network_config_group.add_argument("--master-static-public-ip-name", dest="master_static_public_ip_name", help="name of public IP to be attached to master node") + network_config_group.add_argument("--resource-group-name", + dest="resource_group_name", + help="resource group for cluster") # storage config settings parser storage_config = argparser.add_argument_group("storage config settings") storage_config.add_argument("--storage-access-key", diff --git a/qds_sdk/clusterv2.py b/qds_sdk/clusterv2.py index 95e7905a..eb3f07f6 100755 --- a/qds_sdk/clusterv2.py +++ b/qds_sdk/clusterv2.py @@ -105,7 +105,11 @@ def get_cluster_create_clone_update(arguments, action): image_uri_overrides=arguments.image_uri_overrides, env_name=arguments.env_name, python_version=arguments.python_version, - r_version=arguments.r_version) + r_version=arguments.r_version, + disable_cluster_pause=arguments.disable_cluster_pause, + paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins, + disable_autoscale_node_pause=arguments.disable_autoscale_node_pause, + paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins) # This will set cloud config settings cloud_config = Qubole.get_cloud() @@ -193,7 +197,11 @@ def set_cluster_info(self, image_uri_overrides=None, env_name=None, python_version=None, - r_version=None): + r_version=None, + disable_cluster_pause=None, + paused_cluster_timeout_mins=None, + disable_autoscale_node_pause=None, + paused_autoscale_node_timeout_mins=None): """ Args: @@ -290,6 +298,14 @@ def set_cluster_info(self, `r_version`: Version of R for environment. (For Spark clusters) + `disable_cluster_pause`: Disable cluster pause + + `paused_cluster_timeout_mins`: Paused cluster timeout in mins + + `disable_autoscale_node_pause`: Disable autoscale node pause + + `paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins + Doc: For getting details about arguments http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters @@ -327,6 +343,8 @@ def set_cluster_info(self, self.set_monitoring(enable_ganglia_monitoring, datadog_api_token, datadog_app_token) self.set_internal(image_uri_overrides) self.set_env_settings(env_name, python_version, r_version) + self.set_start_stop_settings(disable_cluster_pause, paused_cluster_timeout_mins, + disable_autoscale_node_pause, paused_autoscale_node_timeout_mins) def set_datadog_setting(self, datadog_api_token=None, @@ -392,6 +410,20 @@ def set_env_settings(self, env_name=None, python_version=None, r_version=None): self.cluster_info['env_settings']['python_version'] = python_version self.cluster_info['env_settings']['r_version'] = r_version + def set_start_stop_settings(self, + disable_cluster_pause=None, + paused_cluster_timeout_mins=None, + disable_autoscale_node_pause=None, + paused_autoscale_node_timeout_mins=None): + if disable_cluster_pause is not None: + disable_cluster_pause = int(disable_cluster_pause) + self.cluster_info['disable_cluster_pause'] = disable_cluster_pause + self.cluster_info['paused_cluster_timeout_mins'] = paused_cluster_timeout_mins + if disable_autoscale_node_pause is not None: + disable_autoscale_node_pause = int(disable_autoscale_node_pause) + self.cluster_info['disable_autoscale_node_pause'] = disable_autoscale_node_pause + self.cluster_info['paused_autoscale_node_timeout_mins'] = paused_autoscale_node_timeout_mins + @staticmethod def list_info_parser(argparser, action): argparser.add_argument("--id", dest="cluster_id", @@ -641,6 +673,38 @@ def cluster_info_parser(argparser, action): default=None, help="version of R in environment") + start_stop_group = argparser.add_argument_group("start stop settings") + start_stop_group.add_argument("--disable-cluster-pause", + dest="disable_cluster_pause", + action='store_true', + default=None, + help="disable cluster pause") + start_stop_group.add_argument("--no-disable-cluster-pause", + dest="disable_cluster_pause", + action='store_false', + default=None, + help="disable cluster pause") + start_stop_group.add_argument("--paused-cluster-timeout", + dest="paused_cluster_timeout_mins", + default=None, + type=int, + help="paused cluster timeout in min") + start_stop_group.add_argument("--disable-autoscale-node-pause", + dest="disable_autoscale_node_pause", + action='store_true', + default=None, + help="disable autoscale node pause") + start_stop_group.add_argument("--no-disable-autoscale-node-pause", + dest="disable_autoscale_node_pause", + action='store_false', + default=None, + help="disable autoscale node pause") + start_stop_group.add_argument("--paused-autoscale-node-timeout", + dest="paused_autoscale_node_timeout_mins", + default=None, + type=int, + help="paused autoscale node timeout in min") + class ClusterV2(Resource): rest_entity_path = "clusters" diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index 1b33f4e2..0f6f94b8 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -259,6 +259,19 @@ def test_azure_master_static_pip(self): 'master_static_public_ip_name':'pip1'}}, 'cluster_info': {'label': ['test_label']}}) + def test_azure_resource_group_name(self): + sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'AZURE', 'cluster', 'create', '--label', 'test_label', + '--resource-group-name', 'testrg'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + {'cloud_config': { + 'resource_group_name': 'testrg' + }, + 'cluster_info': {'label': ['test_label']}}) + def test_oracle_opc_compute_config(self): sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'ORACLE_OPC', 'cluster', 'create', '--label', 'test_label', '--username', 'testusername', '--password', 'testpassword', @@ -585,6 +598,46 @@ def test_root_disk_size_invalid_v2(self): qds.main() + def test_disable_start_stop(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', + '--disable-cluster-pause', '--disable-autoscale-node-pause'] + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + { + 'cluster_info': { + 'label': ['test_label'], + 'disable_cluster_pause': 1, + 'disable_autoscale_node_pause': 1 + } + }) + def test_start_stop_timeouts(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', + '--no-disable-cluster-pause', '--paused-cluster-timeout', '30', + '--no-disable-autoscale-node-pause', '--paused-autoscale-node-timeout', '60'] + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + { + 'cluster_info': { + 'label': ['test_label'], + 'disable_cluster_pause': 0, + 'paused_cluster_timeout_mins': 30, + 'disable_autoscale_node_pause': 0, + 'paused_autoscale_node_timeout_mins': 60 + } + }) + + def test_start_stop_timeouts_invalid(self): + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', + '--paused-cluster-timeout', 'invalid_value', '--paused-autoscale-node-timeout', 'invalid_value'] + print_command() + with self.assertRaises(SystemExit): + qds.main() + + class TestClusterUpdate(QdsCliTestCase): def test_minimal(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'update', '123']