BatchAI. Initial drop of azure-cli-batchai. (Azure#4600)

* BatchAI. Initial drop of azure-cli-batchai.

* BatchAI. Getting rid of legacy "learning" term.

The old service name "Batch AI Learning" has been changed to "Batch AI",
so we need to update all environment variables names.

* BatchAI. Added scenario for configless cluster and nfs creation

* BatchAI. Update cluster resize and cluster auto-scale.

After adding payload-flattening-threshold=2, cluster.update signature
changed. Updating affected commands.

.github/CODEOWNERS

@@ -11,6 +11,7 @@
 /src/command_modules/azure-cli-appservice/ @yugangw-msft
 /src/command_modules/azure-cli-backup/ @dragonfly91
 /src/command_modules/azure-cli-batch/ @annatisch 
+/src/command_modules/azure-cli-batchai/ @AlexanderYukhanov
 /src/command_modules/azure-cli-cdn/ @tjprescott
 /src/command_modules/azure-cli-cosmosdb/ @dmakwana
 /src/command_modules/azure-cli-cloud/ @derekbekoe

doc/sphinx/azhelpgen/doc_source_map.json

@@ -10,6 +10,7 @@
     "acs": "src/command_modules/azure-cli-acs/azure/cli/command_modules/acs/_help.py",
     "appservice": "src/command_modules/azure-cli-appservice/azure/cli/command_modules/appservice/_help.py",
     "batch": "src/command_modules/azure-cli-batch/azure/cli/command_modules/batch/_help.py",
+    "batchai": "src/command_modules/azure-cli-batchai/azure/cli/command_modules/batchai/_help.py",
     "backup": "src/command_modules/azure-cli-backup/azure/cli/command_modules/backup/_help.py",
     "billing": "src/command_modules/azure-cli-billing/azure/cli/command_modules/billing/_help.py",
     "cdn": "src/command_modules/azure-cli-cdn/azure/cli/command_modules/cdn/_help.py",

src/azure-cli/setup.py

@@ -53,6 +53,7 @@
     'azure-cli-acs',
     'azure-cli-appservice',
     'azure-cli-batch',
+    'azure-cli-batchai',
     'azure-cli-backup',
     'azure-cli-billing',
     'azure-cli-cdn',

src/command_modules/azure-cli-batchai/HISTORY.rst

@@ -0,0 +1,9 @@
+.. :changelog:
+
+Release History
+===============
+
+(unreleased)
+++++++++++++++++++
+
+* Initial release of Batch AI module.

src/command_modules/azure-cli-batchai/MANIFEST.in

@@ -0,0 +1 @@
+include *.rst

src/command_modules/azure-cli-batchai/README.rst

@@ -0,0 +1,4 @@
+Microsoft Azure CLI Batch AI Module
+=========================================
+
+This package is for the `batchai` module.

src/command_modules/azure-cli-batchai/azure/__init__.py

@@ -0,0 +1,7 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import pkg_resources
+
+pkg_resources.declare_namespace(__name__)

src/command_modules/azure-cli-batchai/azure/cli/__init__.py

@@ -0,0 +1,7 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import pkg_resources
+
+pkg_resources.declare_namespace(__name__)

src/command_modules/azure-cli-batchai/azure/cli/command_modules/__init__.py

@@ -0,0 +1,7 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+import pkg_resources
+
+pkg_resources.declare_namespace(__name__)

src/command_modules/azure-cli-batchai/azure/cli/command_modules/batchai/__init__.py

@@ -0,0 +1,14 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import azure.cli.command_modules.batchai._help  # pylint: disable=unused-import
+
+
+def load_params(_):
+    import azure.cli.command_modules.batchai._params  # pylint: disable=redefined-outer-name, unused-variable
+
+
+def load_commands():
+    import azure.cli.command_modules.batchai.commands  # pylint: disable=redefined-outer-name, unused-variable

src/command_modules/azure-cli-batchai/azure/cli/command_modules/batchai/_client_factory.py

@@ -0,0 +1,26 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+
+def batchai_client_factory(_=None):
+    from azure.mgmt.batchai import BatchAIManagementClient
+    from azure.cli.core.commands.client_factory import get_mgmt_service_client
+    return get_mgmt_service_client(BatchAIManagementClient)
+
+
+def cluster_client_factory(_):
+    return batchai_client_factory().clusters
+
+
+def job_client_factory(_):
+    return batchai_client_factory().jobs
+
+
+def file_client_factory(_):
+    return batchai_client_factory().jobs
+
+
+def file_server_client_factory(_):
+    return batchai_client_factory().file_servers

src/command_modules/azure-cli-batchai/azure/cli/command_modules/batchai/_format.py

@@ -0,0 +1,91 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from collections import OrderedDict
+
+
+def cluster_list_table_format(result):
+    """Format cluster list as a table."""
+    table = []
+    for item in result:
+        row = OrderedDict()
+        row['Name'] = item['name']
+        row['Resource Group'] = item['resourceGroup']
+        row['VM Size'] = item['vmSize']
+        row['State'] = item['allocationState']
+        row['Idle'] = str(item['nodeStateCounts']['idleNodeCount'])
+        row['Running'] = str(item['nodeStateCounts']['runningNodeCount'])
+        row['Preparing'] = str(item['nodeStateCounts']['preparingNodeCount'])
+        row['Leaving'] = str(item['nodeStateCounts']['leavingNodeCount'])
+        row['Unusable'] = str(item['nodeStateCounts']['unusableNodeCount'])
+        table.append(row)
+    return table
+
+
+def job_list_table_format(result):
+    """Format job list as a table."""
+    table = []
+    for item in result:
+        row = OrderedDict()
+        row['Name'] = item['name']
+        row['Resource Group'] = item['resourceGroup']
+        cluster = item['cluster']['id'].split('/')[8]
+        row['Cluster'] = cluster
+        row['Cluster RG'] = item['cluster']['resourceGroup']
+        row['Tool'] = item['toolType']
+        row['Nodes'] = item['nodeCount']
+        row['State'] = item['executionState']
+        if item['executionInfo'] and \
+           item['executionInfo']['exitCode'] is not None:
+            row['Exit code'] = str(item['executionInfo']['exitCode'])
+        else:
+            row['Exit code'] = ''
+        table.append(row)
+    return table
+
+
+def file_list_table_format(result):
+    """Format file list as a table."""
+    table = []
+    for item in result:
+        row = OrderedDict()
+        row['Name'] = item['name']
+        row['Size'] = str(item['contentLength'])
+        row['URL'] = item['downloadUrl']
+        table.append(row)
+    return table
+
+
+def file_server_table_format(result):
+    """Format file server list as a table."""
+    table = []
+    for item in result:
+        row = OrderedDict()
+        row['Name'] = item['name']
+        row['Resource Group'] = item['resourceGroup']
+        row['Size'] = item['vmSize']
+        disks = item['dataDisks']
+        if disks:
+            row['Disks'] = '{0} x {1} Gb'.format(disks['diskCount'], disks['diskSizeInGb'])
+        mount_settings = item['mountSettings']
+        if mount_settings:
+            row['Public IP'] = mount_settings['fileServerPublicIp']
+            row['Internal IP'] = mount_settings['fileServerInternalIp']
+            row['Type'] = mount_settings['fileServerType']
+            row['Mount Point'] = mount_settings['mountPoint']
+        table.append(row)
+    return table
+
+
+def remote_login_table_format(result):
+    """Format remote login info list as a table."""
+    table = []
+    for item in result:
+        row = OrderedDict()
+        row['ID'] = item['nodeId']
+        row['IP'] = item['ipAddress']
+        row['Port'] = int(item['port'])
+        table.append(row)
+    return table

src/command_modules/azure-cli-batchai/azure/cli/command_modules/batchai/_help.py

@@ -0,0 +1,122 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure.cli.core.help_files import helps
+
+helps['batchai'] = """
+    type: group
+    short-summary: Batch AI.
+"""
+
+helps['batchai cluster'] = """
+    type: group
+    short-summary: Commands to manage clusters.
+"""
+
+helps['batchai cluster create'] = """
+    type: command
+    short-summary: Create a cluster.
+"""
+
+helps['batchai cluster resize'] = """
+    type: command
+    short-summary: Resize a cluster.
+"""
+
+helps['batchai cluster set-auto-scale-parameters'] = """
+    type: command
+    short-summary: Set auto-scale parameters for a cluster.
+"""
+
+helps['batchai cluster delete'] = """
+    type: command
+    short-summary: Delete a cluster.
+"""
+
+helps['batchai cluster list'] = """
+    type: command
+    short-summary: List clusters.
+"""
+
+helps['batchai cluster show'] = """
+    type: command
+    short-summary: Show information about a cluster.
+"""
+
+helps['batchai cluster list-nodes'] = """
+    type: command
+    short-summary: List remote login information for cluster's nodes.
+"""
+
+helps['batchai job'] = """
+    type: group
+    short-summary: Commands to manage jobs.
+"""
+
+helps['batchai job create'] = """
+    type: command
+    short-summary: Create a job.
+"""
+
+helps['batchai job terminate'] = """
+    type: command
+    short-summary: Terminate a job.
+"""
+
+helps['batchai job delete'] = """
+    type: command
+    short-summary: Delete a job.
+"""
+
+helps['batchai job list'] = """
+    type: command
+    short-summary: List jobs.
+"""
+
+helps['batchai job show'] = """
+    type: command
+    short-summary: Show information about a job.
+"""
+
+helps['batchai job list-nodes'] = """
+    type: command
+    short-summary: List remote login information for nodes on which the job was run.
+"""
+
+helps['batchai file list-files'] = """
+    type: command
+    short-summary: List job's output files in a directory with given id.
+"""
+
+helps['batchai stream-file'] = """
+    type: command
+    short-summary: Output the current content of the file and outputs appended data as the file grows
+                   (similar to 'tail -f').
+"""
+
+helps['batchai file-server'] = """
+    type: group
+    short-summary: Commands to manage file servers.
+"""
+
+helps['batchai file-server create'] = """
+    type: command
+    short-summary: Create a file server.
+"""
+
+helps['batchai file-server delete'] = """
+    type: command
+    short-summary: Delete a file server.
+"""
+
+helps['batchai file-server list'] = """
+    type: command
+    short-summary: List file servers.
+"""
+
+helps['batchai file-server show'] = """
+    type: command
+    short-summary: Show information about a file server.
+"""