Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chg: dev: SDK-345: Add custom resource group and start-stop related params for Azure clusters #268

Merged
merged 2 commits into from
Apr 1, 2019
Merged

chg: dev: SDK-345: Add custom resource group and start-stop related params for Azure clusters #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ce6643a
Add custom resource group and start-stop related params for Azure clu…
akaranjkar-qu Mar 19, 2019
0481802
Add custom resource group and start-stop related params for Azure clu…
akaranjkar-qu Mar 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions qds_sdk/cloud/azure_cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ def set_cloud_config(self,
vnet_resource_group_name=None,
master_elastic_ip=None,
master_static_nic_name=None,
master_static_public_ip_name=None):
master_static_public_ip_name=None,
resource_group_name=None):
'''

Args:
Expand Down Expand Up @@ -74,6 +75,7 @@ def set_cloud_config(self,
master_static_public_ip_name: Name of Static Public Ip address that has to be attached
to cluster's master node

resource_group_name: Resource group for cluster
'''

self.set_compute_config(use_account_compute_creds, compute_tenant_id,
Expand All @@ -86,6 +88,7 @@ def set_cloud_config(self,
self.set_storage_config(storage_access_key, storage_account_name,
disk_storage_account_name,
disk_storage_account_resource_group_name)
self.resource_group_name = resource_group_name

def set_compute_config(self,
use_account_compute_creds=None,
Expand Down Expand Up @@ -147,7 +150,8 @@ def set_cloud_config_from_arguments(self, arguments):
subnet_name=arguments.subnet_name,
vnet_resource_group_name=arguments.vnet_resource_group_name,
master_static_nic_name=arguments.master_static_nic_name,
master_static_public_ip_name=arguments.master_static_public_ip_name)
master_static_public_ip_name=arguments.master_static_public_ip_name,
resource_group_name=arguments.resource_group_name)

def create_parser(self, argparser):
# compute settings parser
Expand Down Expand Up @@ -204,6 +208,9 @@ def create_parser(self, argparser):
network_config_group.add_argument("--master-static-public-ip-name",
dest="master_static_public_ip_name",
help="name of public IP to be attached to master node")
network_config_group.add_argument("--resource-group-name",
dest="resource_group_name",
help="resource group for cluster")
# storage config settings parser
storage_config = argparser.add_argument_group("storage config settings")
storage_config.add_argument("--storage-access-key",
Expand Down
68 changes: 66 additions & 2 deletions qds_sdk/clusterv2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,11 @@ def get_cluster_create_clone_update(arguments, action):
image_uri_overrides=arguments.image_uri_overrides,
env_name=arguments.env_name,
python_version=arguments.python_version,
r_version=arguments.r_version)
r_version=arguments.r_version,
disable_cluster_pause=arguments.disable_cluster_pause,
paused_cluster_timeout_mins=arguments.paused_cluster_timeout_mins,
disable_autoscale_node_pause=arguments.disable_autoscale_node_pause,
paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins)

# This will set cloud config settings
cloud_config = Qubole.get_cloud()
Expand Down Expand Up @@ -193,7 +197,11 @@ def set_cluster_info(self,
image_uri_overrides=None,
env_name=None,
python_version=None,
r_version=None):
r_version=None,
disable_cluster_pause=None,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None):
"""
Args:

Expand Down Expand Up @@ -290,6 +298,14 @@ def set_cluster_info(self,

`r_version`: Version of R for environment. (For Spark clusters)

`disable_cluster_pause`: Disable cluster pause

`paused_cluster_timeout_mins`: Paused cluster timeout in mins

`disable_autoscale_node_pause`: Disable autoscale node pause

`paused_autoscale_node_timeout_mins`: Paused autoscale node timeout in mins

Doc: For getting details about arguments
http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#parameters

Expand Down Expand Up @@ -327,6 +343,8 @@ def set_cluster_info(self,
self.set_monitoring(enable_ganglia_monitoring, datadog_api_token, datadog_app_token)
self.set_internal(image_uri_overrides)
self.set_env_settings(env_name, python_version, r_version)
self.set_start_stop_settings(disable_cluster_pause, paused_cluster_timeout_mins,
disable_autoscale_node_pause, paused_autoscale_node_timeout_mins)

def set_datadog_setting(self,
datadog_api_token=None,
Expand Down Expand Up @@ -392,6 +410,20 @@ def set_env_settings(self, env_name=None, python_version=None, r_version=None):
self.cluster_info['env_settings']['python_version'] = python_version
self.cluster_info['env_settings']['r_version'] = r_version

def set_start_stop_settings(self,
disable_cluster_pause=None,
paused_cluster_timeout_mins=None,
disable_autoscale_node_pause=None,
paused_autoscale_node_timeout_mins=None):
if disable_cluster_pause is not None:
disable_cluster_pause = int(disable_cluster_pause)
self.cluster_info['disable_cluster_pause'] = disable_cluster_pause
self.cluster_info['paused_cluster_timeout_mins'] = paused_cluster_timeout_mins
if disable_autoscale_node_pause is not None:
disable_autoscale_node_pause = int(disable_autoscale_node_pause)
self.cluster_info['disable_autoscale_node_pause'] = disable_autoscale_node_pause
self.cluster_info['paused_autoscale_node_timeout_mins'] = paused_autoscale_node_timeout_mins

@staticmethod
def list_info_parser(argparser, action):
argparser.add_argument("--id", dest="cluster_id",
Expand Down Expand Up @@ -641,6 +673,38 @@ def cluster_info_parser(argparser, action):
default=None,
help="version of R in environment")

start_stop_group = argparser.add_argument_group("start stop settings")
start_stop_group.add_argument("--disable-cluster-pause",
dest="disable_cluster_pause",
action='store_true',
default=None,
help="disable cluster pause")
start_stop_group.add_argument("--no-disable-cluster-pause",
dest="disable_cluster_pause",
action='store_false',
default=None,
help="disable cluster pause")
start_stop_group.add_argument("--paused-cluster-timeout",
dest="paused_cluster_timeout_mins",
default=None,
type=int,
help="paused cluster timeout in min")
start_stop_group.add_argument("--disable-autoscale-node-pause",
dest="disable_autoscale_node_pause",
action='store_true',
default=None,
help="disable autoscale node pause")
start_stop_group.add_argument("--no-disable-autoscale-node-pause",
dest="disable_autoscale_node_pause",
action='store_false',
default=None,
help="disable autoscale node pause")
start_stop_group.add_argument("--paused-autoscale-node-timeout",
dest="paused_autoscale_node_timeout_mins",
default=None,
type=int,
help="paused autoscale node timeout in min")

class ClusterV2(Resource):

rest_entity_path = "clusters"
Expand Down
53 changes: 53 additions & 0 deletions tests/test_clusterv2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,19 @@ def test_azure_master_static_pip(self):
'master_static_public_ip_name':'pip1'}},
'cluster_info': {'label': ['test_label']}})

def test_azure_resource_group_name(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'AZURE', 'cluster', 'create', '--label', 'test_label',
'--resource-group-name', 'testrg']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{'cloud_config': {
'resource_group_name': 'testrg'
},
'cluster_info': {'label': ['test_label']}})

def test_oracle_opc_compute_config(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'ORACLE_OPC', 'cluster', 'create', '--label', 'test_label',
'--username', 'testusername', '--password', 'testpassword',
Expand Down Expand Up @@ -585,6 +598,46 @@ def test_root_disk_size_invalid_v2(self):
qds.main()


def test_disable_start_stop(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--disable-cluster-pause', '--disable-autoscale-node-pause']
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{
'cluster_info': {
'label': ['test_label'],
'disable_cluster_pause': 1,
'disable_autoscale_node_pause': 1
}
})
def test_start_stop_timeouts(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--no-disable-cluster-pause', '--paused-cluster-timeout', '30',
'--no-disable-autoscale-node-pause', '--paused-autoscale-node-timeout', '60']
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{
'cluster_info': {
'label': ['test_label'],
'disable_cluster_pause': 0,
'paused_cluster_timeout_mins': 30,
'disable_autoscale_node_pause': 0,
'paused_autoscale_node_timeout_mins': 60
}
})

def test_start_stop_timeouts_invalid(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label',
'--paused-cluster-timeout', 'invalid_value', '--paused-autoscale-node-timeout', 'invalid_value']
print_command()
with self.assertRaises(SystemExit):
qds.main()


class TestClusterUpdate(QdsCliTestCase):
def test_minimal(self):
sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'update', '123']
Expand Down