update extension to newest version (#139)

* update extension to newest version
Azure · Sep 9, 2022 · 49dcd2f · 49dcd2f
1 parent 7320514
commit 49dcd2f
Show file tree

Hide file tree

Showing 37 changed files with 6,317 additions and 8,390 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,11 @@
 SDK Release History
 ===================
 
+9.0.0 (2022-08-30)
+------------------
+
+* Update to match latest Python SDK taking associated breaking changes.
+
 8.0.0 (2020-06-15)
 ------------------
 

diff --git a/azext/batch/batch_extensions_client.py b/azext/batch/batch_extensions_client.py
@@ -52,7 +52,7 @@ class BatchExtensionsClient(BatchServiceClient):
 
     def __init__(self, credentials=None, batch_url=None, subscription_id=None,
                  resource_group=None, batch_account=None, storage_client=None,
-                 storage_endpoint=None, mgmt_credentials=None, mgmt_base_url=None):
+                 storage_endpoint=None, mgmt_credentials=None, mgmt_base_url="https://management.azure.com"):
         credentials, mgmt_credentials, subscription_id = self._configure_credentials(
             credentials, mgmt_credentials, subscription_id)
         super(BatchExtensionsClient, self).__init__(credentials, batch_url=batch_url)

diff --git a/azext/batch/models/_models.py b/azext/batch/models/_models.py
diff --git a/azext/batch/models/_models_py3.py b/azext/batch/models/_models_py3.py
diff --git a/azext/batch/version.py b/azext/batch/version.py
@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-VERSION = "8.0.0"
+VERSION = "9.0.0"
diff --git a/batch-cli-extensions/azext_batch/_params.py b/batch-cli-extensions/azext_batch/_params.py
@@ -10,7 +10,7 @@
 from azure.cli.command_modules.batch._validators import (
     metadata_item_format, certificate_reference_format, validate_json_file,
     environment_setting_format, resource_file_format)
-from azext_batch._validators import validate_pool_settings, validate_client_parameters
+from azext_batch._validators import validate_client_parameters
 
 
 # pylint: disable=line-too-long, too-many-statements
@@ -20,9 +20,8 @@ def load_arguments(self, _):
         c.argument('json_file', type=file_type, help='The file containing the pool to create in JSON format, if this parameter is specified, all other parameters are ignored.', validator=validate_json_file, completer=FilesCompleter())
         c.argument('template', type=file_type, arg_group='Batch Extensions', help='A Batch pool JSON template file. If this parameter is specified, all other parameters are ignored.', completer=FilesCompleter())
         c.argument('parameters', type=file_type, arg_group='Batch Extensions', help='Parameter values for a Batch pool JSON template file. Can only be used with --template.', completer=FilesCompleter())
-        c.argument('application_package_references', nargs='+', validator=validate_pool_settings)
+        c.argument('os_version', arg_group='Pool: Cloud Service Configuration', help='The default value is * which specifies the latest operating system version for the specified OS family.')
         c.argument('certificate_references', nargs='+', type=certificate_reference_format)
-        c.argument('application_licenses', nargs='+')
         c.argument('metadata', nargs='+', type=metadata_item_format)
         c.argument('start_task_command_line', arg_group='Pool: Start Task', help='The command line of the start task. The command line does not run under a shell, and therefore cannot take advantage of shell features such as environment variable expansion. If you want to take advantage of such features, you should invoke the shell in the command line, for example using "cmd /c MyCommand" in Windows or "/bin/sh -c MyCommand" in Linux.')
         c.argument('start_task_resource_files', arg_group='Pool: Start Task', nargs='+', type=resource_file_format, help='A list of files that the Batch service will download to the compute node before running the command line. Space separated resource references in filename=httpurl format.')
@@ -32,23 +31,38 @@ def load_arguments(self, _):
         c.extra('disk_encryption_targets',
                 arg_group="Pool: Virtual Machine Configuration",
                 help='A space seperated list of DiskEncryptionTargets. current possible values include OsDisk and TemporaryDisk.')
+        c.extra('disk_encryption_configuration_targets', options_list=('--targets',),
+                arg_group="Pool: Virtual Machine Configuration: Disk Encryption Configuration Arguments",
+                help='If omitted, no disks on the compute nodes in the pool will be encrypted. On Linux pool, only "TemporaryDisk" is supported; on Windows pool, "OsDisk" and "TemporaryDisk" must be specified. Space seperated target disks to be encrypted. Values can either be OsDisk or TemporaryDisk.')
+        c.extra('node_placement_policy', options_list=('--policy',),
+                arg_group="Pool: Virtual Machine Configuration: Node Placement Configuration Arguments",
+                help='Node placement Policy type on Batch Pools. Allocation policy used by Batch Service to provision the nodes. If not specified, Batch will use the regional policy.  Allowed values: regional, zonal.')
         c.argument('node_agent_sku_id', arg_group="Pool: Virtual Machine Configuration", help='The SKU of the Batch node agent to be provisioned on compute nodes in the pool. The Batch node agent is a program that runs on each node in the pool, and provides the command-and-control interface between the node and the Batch service. There are different implementations of the node agent, known as SKUs, for different operating systems. You must specify a node agent SKU which matches the selected image reference. To get the list of supported node agent SKUs along with their list of verified image references, see the \'List supported node agent SKUs\' operation.')
         c.argument('image', completer=load_supported_images, arg_group="Pool: Virtual Machine Configuration",
                    help="OS image reference. This can be either 'publisher:offer:sku[:version]' format, or a fully qualified ARM image id of the form '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroup}/providers/Microsoft.Compute/images/{imageName}'. If 'publisher:offer:sku[:version]' format, version is optional and if omitted latest will be used. Valid values can be retrieved via 'az batch pool node-agent-skus list'. For example: 'MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:latest'")
 
     with self.argument_context('batch job create') as c:
-        c.argument('json_file', type=file_type, help='The file containing the job to create in JSON format, if this parameter is specified, all other parameters are ignored.', validator=validate_json_file, completer=FilesCompleter())
+        c.argument('json_file', type=file_type, help='A file containing the job specification in JSON (formatted to match the respective REST API body). If this parameter is specified, all \'Job Arguments\' are ignored.', validator=validate_json_file, completer=FilesCompleter())
         c.argument('template', type=file_type, arg_group='Batch Extensions', help='A Batch job JSON template file. If this parameter is specified, all other parameters are ignored.', completer=FilesCompleter())
         c.argument('parameters', type=file_type, arg_group='Batch Extensions', help='Parameter values for a Batch job JSON template file. Can only be used with --template.', completer=FilesCompleter())
-        c.argument('metadata', nargs='+', type=metadata_item_format, help='A list of name-value pairs associated with the job as metadata. The Batch service does not assign any meaning to metadata; it is solely for the use of user code. Space separated values in \'key=value\' format.')
-        c.argument('uses_task_dependencies', action='store_true', help='The flag that determines if this job will use tasks with dependencies. True if flag present.')
+        c.argument('metadata', arg_group='Job', nargs='+', type=metadata_item_format)
+        c.argument('uses_task_dependencies', arg_group='Job', action='store_true', help='The flag that determines if this job will use tasks with dependencies. True if flag present.')
         c.argument('pool_id', arg_group='Job: Pool Info', help='The id of an existing pool. All the tasks of the job will run on the specified pool.')
         c.argument('job_max_task_retry_count', arg_group='Job: Constraints', help='The maximum number of times each task may be retried. The Batch service retries a task if its exit code is nonzero. Note that this value specifically controls the number of retries. The Batch service will try each task once, and may then retry up to this limit. For example, if the maximum retry count is 3, Batch tries a task up to 4 times (one initial try and 3 retries). If the maximum retry count is 0, the Batch service does not retry tasks. If the maximum retry count is -1, the Batch service retries tasks without limit. The default value is 0 (no retries).')
         c.argument('job_max_wall_clock_time', arg_group='Job: Constraints', help='The maximum elapsed time that the job may run, measured from the time the job is created. If the job does not complete within the time limit, the Batch service terminates it and any tasks that are still running. In this case, the termination reason will be MaxWallClockTimeExpiry. If this property is not specified, there is no time limit on how long the job may run. Expected format is an ISO-8601 duration.')
         c.argument('job_manager_task_command_line', arg_group='Job: Job Manager Task', help='The command line of the Job Manager task. The command line does not run under a shell, and therefore cannot take advantage of shell features such as environment variable expansion. If you want to take advantage of such features, you should invoke the shell in the command line, for example using "cmd /c MyCommand" in Windows or "/bin/sh -c MyCommand" in Linux.')
         c.argument('job_manager_task_environment_settings', arg_group='Job: Job Manager Task', type=environment_setting_format, help='A list of environment variable settings for the Job Manager task. Space separated values in \'key=value\' format.')
         c.argument('job_manager_task_resource_files', arg_group='Job: Job Manager Task', type=resource_file_format, help='A list of files that the Batch service will download to the compute node before running the command line. Files listed under this element are located in the task\'s working directory. Space separated resource references in filename=httpurl format.')
         c.argument('job_manager_task_id', arg_group='Job: Job Manager Task', help='A string that uniquely identifies the Job Manager task within the job. The id can contain any combination of alphanumeric characters including hyphens and underscores and cannot contain more than 64 characters.')
+        c.argument('required_slots', arg_group='Job: Job Manager Task', help='The number of scheduling slots that the Task requires to run. The default is 1. A Task can only be scheduled to run on a compute node if the node has enough free scheduling slots available. For multi-instance Tasks, this property is not supported and must not be specified.')
+        c.argument('allow_task_preemption', arg_group='Job')
+        c.argument('max_parallel_tasks', arg_group='Job')
+        c.argument('id', arg_group='Job')
+        c.argument('priority', arg_group='Job')
+
+
+
+
 
     with self.argument_context('batch file upload') as c:
         c.argument('local_path', type=file_type, help='Path to a local file or directory to be uploaded - can include wildcard patterns.')

diff --git a/batch-cli-extensions/azext_batch/commands.py b/batch-cli-extensions/azext_batch/commands.py
@@ -13,9 +13,9 @@
 
 
 logger = get_logger(__name__)
-MINIMUM_UNSUPPORTED_BATCH_VERSION = "7.1"
-MINIMUM_UNSUPPORTED_BMGMT_VERSION = "6.1"
-MINIMUM_UNSUPPORTED_BATCH_EXT_VERSION = "6.1"
+MINIMUM_UNSUPPORTED_BATCH_VERSION = "12.2"
+MINIMUM_UNSUPPORTED_BMGMT_VERSION = "16.3"
+MINIMUM_UNSUPPORTED_BATCH_EXT_VERSION = "9.1"
 
 
 def confirm_version(current, supported, package):

diff --git a/batch-cli-extensions/azext_batch/custom.py b/batch-cli-extensions/azext_batch/custom.py
@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 import multiprocessing
+import os
 from azure.cli.core.util import get_file_json
 from knack.log import get_logger
 from knack.prompting import prompt
@@ -28,19 +29,29 @@ def disk_encryption_target_format(value):
     message = 'Argument {} is not a valid disk_encryption_target'
     raise ValueError(message.format(value))
 
+def node_placement_policy_format(value):
+    """If not specified, Batch will use the regional policy. Possible values include: regional,zonal"""
+    from azext.batch.models import NodePlacementPolicyType
+    if value == 'regional':
+        return NodePlacementPolicyType.regional
+    if value == 'zonal':
+        return NodePlacementPolicyType.zonal
+    message = 'Argument {} is not a valid node_placement_policy'
+    raise ValueError(message.format(value))
 
 def create_pool(client, template=None, parameters=None, json_file=None, id=None, vm_size=None,  # pylint:disable=too-many-arguments, too-many-locals
                 target_dedicated_nodes=None, target_low_priority_nodes=None, auto_scale_formula=None,  # pylint: disable=redefined-builtin
                 enable_inter_node_communication=False, os_family=None, image=None, disk_encryption_targets=None,
                 node_agent_sku_id=None, resize_timeout=None, start_task_command_line=None,
                 start_task_resource_files=None, start_task_wait_for_success=False, application_licenses=None,
-                certificate_references=None, application_package_references=None, metadata=None):
+                certificate_references=None, application_package_references=None, metadata=None, disk_encryption_configuration_targets=None, 
+                node_placement_policy=None, os_version=None, task_slots_per_node=None):
     # pylint: disable=too-many-branches, too-many-statements
     from azext.batch.errors import MissingParameterValue
     from azext.batch.models import (
         PoolAddOptions, StartTask, ImageReference,
         CloudServiceConfiguration, VirtualMachineConfiguration,
-        DiskEncryptionConfiguration)
+        DiskEncryptionConfiguration, NodePlacementConfiguration)
     if template or json_file:
         if template:
             json_obj = None
@@ -78,9 +89,9 @@ def create_pool(client, template=None, parameters=None, json_file=None, id=None,
             pool.enable_auto_scale = False
 
         pool.enable_inter_node_communication = enable_inter_node_communication
-
+        pool.task_slots_per_node = task_slots_per_node
         if os_family:
-            pool.cloud_service_configuration = CloudServiceConfiguration(os_family=os_family)
+            pool.cloud_service_configuration = CloudServiceConfiguration(os_family=os_family,os_version=os_version)
         else:
             if image:
                 version = 'latest'
@@ -94,12 +105,21 @@ def create_pool(client, template=None, parameters=None, json_file=None, id=None,
                         image_reference=ImageReference(publisher=publisher, offer=offer, sku=sku, version=version),
                         node_agent_sku_id=node_agent_sku_id)
                     if disk_encryption_targets:
-                        targets = disk_encryption_targets.split(' ')
+                        targets_list = disk_encryption_targets.split(' ')
+                        parsed_targets = []
+                        for target in targets_list:
+                            parsed_targets.append(
+                                disk_encryption_target_format(target))
+                        pool.virtual_machine_configuration.disk_configuration = DiskEncryptionConfiguration(targets=parsed_targets)
+                    if disk_encryption_configuration_targets:
+                        targets_list = disk_encryption_configuration_targets.split(' ')
                         parsed_targets = []
-                        for target in targets:
+                        for target in targets_list:
                             parsed_targets.append(
                                 disk_encryption_target_format(target))
                         pool.virtual_machine_configuration.disk_configuration = DiskEncryptionConfiguration(targets=parsed_targets)
+                    if node_placement_policy:
+                        pool.virtual_machine_configuration.node_placement_configuration = NodePlacementConfiguration(node_placement_policy_format(node_placement_policy))
                 except ValueError:
                     if '/' not in image:
                         message = ("Incorrect format for VM image. Should be in the format: \n"
@@ -136,7 +156,8 @@ def create_job(client, template=None, parameters=None, json_file=None, id=None,
                pool_id=None, priority=None, uses_task_dependencies=False, metadata=None,
                job_max_wall_clock_time=None, job_max_task_retry_count=None,
                job_manager_task_command_line=None, job_manager_task_environment_settings=None,
-               job_manager_task_id=None, job_manager_task_resource_files=None):
+               job_manager_task_id=None, job_manager_task_resource_files=None, allow_task_preemption=None,
+               max_parallel_tasks=None, required_slots=None):
     # pylint: disable=too-many-branches, too-many-statements
     from azext.batch.errors import MissingParameterValue
     from azext.batch.models import JobManagerTask, JobAddOptions, PoolInformation
@@ -179,14 +200,18 @@ def create_job(client, template=None, parameters=None, json_file=None, id=None,
                                          max_task_retry_count=job_max_task_retry_count)
             job.constraints = constraints
 
-        if metadata:
-            job.metadata = metadata
-
+        job.metadata = metadata
+        job.allow_task_preemption = allow_task_preemption
+        job.max_parallel_tasks = max_parallel_tasks
+
         if job_manager_task_command_line and job_manager_task_id:
             job_manager_task = JobManagerTask(id=job_manager_task_id,
                                               command_line=job_manager_task_command_line,
                                               resource_files=job_manager_task_resource_files,
                                               environment_settings=job_manager_task_environment_settings)  # pylint: disable=line-too-long
+            if required_slots:
+                job_manager_task.required_slots = required_slots
+
             job.job_manager_task = job_manager_task
 
     add_option = JobAddOptions()

diff --git a/batch-cli-extensions/azext_batch/version.py b/batch-cli-extensions/azext_batch/version.py
@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-VERSION = "6.0.0"
+VERSION = "7.0.0"
diff --git a/batch-cli-extensions/setup.py b/batch-cli-extensions/setup.py
@@ -27,7 +27,7 @@
 ]
 
 DEPENDENCIES = [
-    'azure-batch-extensions>=8.0.0,<8.1',
+    'azure-batch-extensions>=8.0.0,<9.1',
     'pycparser==2.18'
 ]
 

diff --git a/doc/packages.md b/doc/packages.md
@@ -40,10 +40,10 @@ An example of an APT package reference in an IaaS pool:
         "imageReference": {
             "publisher": "Canonical",
             "offer": "UbuntuServer",
-            "sku": "16.04.0-LTS",
+            "sku": "18.04-LTS",
             "version": "latest"
         },
-        "nodeAgentSKUId": "batch.node.ubuntu 16.04"
+        "nodeAgentSKUId": "batch.node.ubuntu 18.04"
     },
     "vmSize": "STANDARD_D1",
     "targetDedicated": 5,

diff --git a/requirements.txt b/requirements.txt
@@ -1,8 +1,8 @@
 docutils==0.14
-adal==0.4.7
-applicationinsights==0.10.0
+adal==0.6.0
+applicationinsights==0.11.1
 argcomplete==1.8.0
-colorama==0.3.7
+colorama==0.4.4
 jmespath
 mock==1.3.0
 nose==1.3.7
@@ -13,8 +13,7 @@ pylint==1.9.3
 pyOpenSSL==19.0.0
 pyyaml==5.4
 requests==2.22.0
-setuptools==30.4.0
-six==1.10.0
-tabulate==0.7.5
+setuptools==52.0.0
+six==1.12.0
+tabulate==0.7.7
 vcrpy==1.10.3
-azure-storage-blob==1.1.0
diff --git a/samples/blender-appTemplate/pool.json b/samples/blender-appTemplate/pool.json
@@ -36,18 +36,18 @@
         "osType": {
             "publisher": "Canonical",
             "offer": "UbuntuServer",
-            "sku": "16.04.0-LTS",
+            "sku": "18.04-LTS",
             "version": "latest"
         }
     },    
     "pool": {
         "type": "Microsoft.Batch/batchAccounts/pools",
-        "apiVersion": "2018-12-01",
+        "apiVersion": "2022-06-01",
         "properties": {
             "id": "[parameters('poolId')]",
             "virtualMachineConfiguration": {
                 "imageReference": "[variables('osType')]",
-                "nodeAgentSKUId": "batch.node.ubuntu 16.04"
+                "nodeAgentSKUId": "batch.node.ubuntu 18.04"
             },
             "packageReferences": [
                 {