[storage-preview] Azcopy- release blob delete/upload/download commands (

Azure#587)

* added azcopy and call

* abstract azcopy from operating system

* implemented basic upload batch for non oauth flow

* initial oauth wiring

new executables with app id refresh token fix

add upload command

* add run-command, and cleaned up upload and download code

* finished logic for up/downloads and run-command

* add 10.0.8 files

* simplified upload/download and added delete

* small fixes and testing

* pylint

* fixed test

* flake8

* fix bug with python script

* add examples for new commands

scripts/ci/test_static.py

@@ -39,7 +39,7 @@ def _get_sdk_module_list(root_dir):
 # check if the current file is a python sdk file
 def _is_sdk_file(file_path):
     # don't bother opening non-python files. e.g pyc files.
-    if not file_path.endswith("py") or file_path.endswith("py3"):
+    if not file_path.endswith(".py"):
         return False
     with open(file_path) as f:
         for line in f:

src/storage-preview/azext_storage_preview/_client_factory.py

@@ -48,8 +48,7 @@ def get_data_service_client(cli_ctx, service_type, account_name, account_key, co
                                               'common._error#_ERROR_STORAGE_MISSING_INFO')
         if _ERROR_STORAGE_MISSING_INFO in str(exc):
             raise ValueError(exc)
-        else:
-            raise CLIError('Unable to obtain data client. Check your connection parameters.')
+        raise CLIError('Unable to obtain data client. Check your connection parameters.')
     # TODO: enable Fiddler
     client.request_callback = _get_add_headers_callback(cli_ctx)
     return client
@@ -98,6 +97,15 @@ def blob_data_service_factory(cli_ctx, kwargs):
                                         token_credential=kwargs.pop('token_credential', None))
 
 
+def cloud_storage_account_service_factory(cli_ctx, kwargs):
+    t_cloud_storage_account = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'common#CloudStorageAccount')
+    account_name = kwargs.pop('account_name', None)
+    account_key = kwargs.pop('account_key', None)
+    sas_token = kwargs.pop('sas_token', None)
+    kwargs.pop('connection_string', None)
+    return t_cloud_storage_account(account_name, account_key, sas_token)
+
+
 def cf_sa(cli_ctx, _):
     return storage_client_factory(cli_ctx).storage_accounts
 

src/storage-preview/azext_storage_preview/_help.py

@@ -51,3 +51,57 @@
     type: command
     short-summary: Updates the data policy rules associated with the specified storage account.
 """
+
+helps['storage azcopy'] = """
+    type: group
+    short-summary: |
+        [EXPERIMENTAL] Manage storage operations utilizing AzCopy.
+    long-summary: |
+        Open issues here: https://github.com/Azure/azure-storage-azcopy
+"""
+
+helps['storage azcopy blob'] = """
+    type: group
+    short-summary: Manage object storage for unstructured data (blobs) using AzCopy.
+"""
+
+helps['storage azcopy blob upload'] = """
+    type: command
+    short-summary: Upload blobs to a storage blob container using AzCopy.
+    examples:
+        - name: Upload a single blob to a container.
+          text: storage azcopy blob upload -c MyContainer --account-name MyStorageAccount -s "path/to/file" -d NewBlob
+        - name: Upload a directory to a container.
+          text: storage azcopy blob upload -c MyContainer --account-name MyStorageAccount -s "path/to/directory" --recursive
+        - name: Upload the contents of a directory to a container.
+          text: storage azcopy blob upload -c MyContainer --account-name MyStorageAccount -s "path/to/directory/*" --recursive
+"""
+
+helps['storage azcopy blob download'] = """
+    type: command
+    short-summary: Download blobs from a storage blob container using AzCopy.
+    examples:
+        - name: Download a single blob from a container.
+          text: storage azcopy blob download -c MyContainer --account-name MyStorageAccount -s "path/to/blob" -d "path/to/file"
+        - name: Download a virtual directory from a container.
+          text: storage azcopy blob download -c MyContainer --account-name MyStorageAccount -s "path/to/virtual_directory" -d "download/path" --recursive
+        - name: Download the contents of a container onto a local file system.
+          text: storage azcopy blob download -c MyContainer --account-name MyStorageAccount -s * -d "download/path" --recursive
+"""
+
+helps['storage azcopy blob delete'] = """
+    type: command
+    short-summary: Delete blobs from a storage blob container using AzCopy.
+    examples:
+        - name: Delete a single blob from a container.
+          text: storage azcopy blob delete -c MyContainer --account-name MyStorageAccount -t TargetBlob
+        - name: Delete all blobs from a container.
+          text: storage azcopy blob delete -c MyContainer --account-name MyStorageAccount --recursive
+        - name: Delete all blobs in a virtual directory.
+          text: storage azcopy blob delete -c MyContainer --account-name MyStorageAccount -t "path/to/virtual_directory" --recursive
+"""
+
+helps['storage azcopy run-command'] = """
+    type: command
+    short-summary: Run a command directly using the AzCopy CLI. Please use SAS tokens for authentication.
+"""

src/storage-preview/azext_storage_preview/_params.py

@@ -8,7 +8,9 @@
                                                 get_three_state_flag)
 
 from ._validators import (get_datetime_type, validate_metadata, validate_custom_domain, process_resource_group,
-                          validate_bypass, validate_encryption_source, storage_account_key_options, validate_key)
+                          validate_bypass, validate_encryption_source, storage_account_key_options, validate_key,
+                          validate_azcopy_upload_destination_url, validate_azcopy_download_source_url,
+                          validate_azcopy_target_url)
 from .profiles import CUSTOM_MGMT_STORAGE
 
 
@@ -161,3 +163,34 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('error_document_404_path', options_list=['--404-document'], arg_group='Static Website',
                    help='Represents the path to the error document that should be shown when an error 404 is issued,'
                         ' in other words, when a browser requests a page that does not exist.')
+
+    with self.argument_context('storage azcopy blob upload') as c:
+        c.extra('destination_container', options_list=['--container', '-c'], required=True)
+        c.extra('destination_path', options_list=['--destination', '-d'],
+                validator=validate_azcopy_upload_destination_url)
+        c.argument('source', options_list=['--source', '-s'])
+        c.argument('recursive', options_list=['--recursive', '-r'], action='store_true')
+        c.ignore('destination')
+
+    with self.argument_context('storage azcopy blob download') as c:
+        c.extra('source_container', options_list=['--container', '-c'], required=True)
+        c.extra('source_path', options_list=['--source', '-s'],
+                validator=validate_azcopy_download_source_url)
+        c.argument('destination', options_list=['--destination', '-d'])
+        c.argument('recursive', options_list=['--recursive', '-r'], action='store_true')
+        c.ignore('source')
+
+    with self.argument_context('storage azcopy blob delete') as c:
+        c.extra('target_container', options_list=['--container', '-c'], required=True)
+        c.extra('target_path', options_list=['--target', '-t'],
+                validator=validate_azcopy_target_url)
+        c.argument('recursive', options_list=['--recursive', '-r'], action='store_true')
+        c.ignore('target')
+
+    # with self.argument_context('storage azcopy blob sync') as c:
+    #     c.argument('destination', options_list=['--destination', '-d'],
+    #                validator=validate_azcopy_container_destination_url)
+    #     c.argument('source', options_list=['--source', '-s'])
+
+    with self.argument_context('storage azcopy run-command') as c:
+        c.positional('command_args', help='Command to run using azcopy. Please start commands with "azcopy ".')

src/storage-preview/azext_storage_preview/_validators.py

@@ -4,11 +4,13 @@
 # --------------------------------------------------------------------------------------------
 
 # pylint: disable=protected-access
+import os
+
 from azure.cli.core.commands.client_factory import get_mgmt_service_client
 from azure.cli.core.commands.validators import validate_key_value_pairs
 from azure.cli.core.profiles import get_sdk
 
-from ._client_factory import get_storage_data_service_client
+from ._client_factory import get_storage_data_service_client, blob_data_service_factory
 from .util import guess_content_type
 from .oauth_token_util import TokenUpdater
 from .profiles import CUSTOM_MGMT_STORAGE
@@ -130,6 +132,62 @@ def get_config_value(section, key, default):
         n.account_key = _query_account_key(cmd.cli_ctx, n.account_name)
 
 
+def validate_azcopy_blob_source_url(cmd, namespace):
+    client = blob_data_service_factory(cmd.cli_ctx, {
+        'account_name': namespace.account_name})
+    blob_name = namespace.source_blob
+    if not blob_name:
+        blob_name = os.path.basename(namespace.destination)
+    url = client.make_blob_url(namespace.source_container, blob_name)
+    namespace.source = url
+    del namespace.source_container
+    del namespace.source_blob
+
+
+def validate_azcopy_container_source_url(cmd, namespace):
+    client = blob_data_service_factory(cmd.cli_ctx, {
+        'account_name': namespace.account_name})
+    url = client.make_blob_url(namespace.source, '')
+    namespace.source = url[:-1]
+
+
+def validate_azcopy_upload_destination_url(cmd, namespace):
+    client = blob_data_service_factory(cmd.cli_ctx, {
+        'account_name': namespace.account_name})
+    destination_path = namespace.destination_path
+    if not destination_path:
+        destination_path = ''
+    url = client.make_blob_url(namespace.destination_container, destination_path)
+    namespace.destination = url
+    del namespace.destination_container
+    del namespace.destination_path
+
+
+def validate_azcopy_download_source_url(cmd, namespace):
+    client = blob_data_service_factory(cmd.cli_ctx, {
+        'account_name': namespace.account_name})
+    source_path = namespace.source_path
+    if not source_path:
+        source_path = ''
+    url = client.make_blob_url(namespace.source_container, source_path)
+    namespace.source = url
+    del namespace.source_container
+    del namespace.source_path
+    print(namespace)
+
+
+def validate_azcopy_target_url(cmd, namespace):
+    client = blob_data_service_factory(cmd.cli_ctx, {
+        'account_name': namespace.account_name})
+    target_path = namespace.target_path
+    if not target_path:
+        target_path = ''
+    url = client.make_blob_url(namespace.target_container, target_path)
+    namespace.target = url
+    del namespace.target_container
+    del namespace.target_path
+
+
 def get_content_setting_validator(settings_class, update, guess_from_file=None):
     def _class_name(class_type):
         return class_type.__module__ + "." + class_type.__class__.__name__
@@ -245,7 +303,6 @@ def get_file_path_validator(default_file_param=None):
     Allows another path-type parameter to be named which can supply a default filename. """
 
     def validator(namespace):
-        import os
         if not hasattr(namespace, 'path'):
             return
 
@@ -281,7 +338,7 @@ def validate_subnet(cmd, namespace):
 
     if (subnet_is_id and not vnet) or (not subnet and not vnet):
         return
-    elif subnet and not subnet_is_id and vnet:
+    if subnet and not subnet_is_id and vnet:
         namespace.subnet = resource_id(
             subscription=get_subscription_id(cmd.cli_ctx),
             resource_group=namespace.resource_group_name,
@@ -323,6 +380,32 @@ def ipv4_range_type(string):
     import re
     ip_format = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}'
     if not re.match("^{}$".format(ip_format), string):
-        if not re.match("^{}-{}$".format(ip_format, ip_format), string):
+        if not re.match("^{ip_format}-{ip_format}$".format(ip_format=ip_format), string):
             raise ValueError
     return string
+
+
+def resource_type_type(loader):
+    """ Returns a function which validates that resource types string contains only a combination of service,
+    container, and object. Their shorthand representations are s, c, and o. """
+
+    def impl(string):
+        t_resources = loader.get_models('common.models#ResourceTypes')
+        if set(string) - set("sco"):
+            raise ValueError
+        return t_resources(_str=''.join(set(string)))
+
+    return impl
+
+
+def services_type(loader):
+    """ Returns a function which validates that services string contains only a combination of blob, queue, table,
+    and file. Their shorthand representations are b, q, t, and f. """
+
+    def impl(string):
+        t_services = loader.get_models('common.models#Services')
+        if set(string) - set("bqtf"):
+            raise ValueError
+        return t_services(_str=''.join(set(string)))
+
+    return impl

src/storage-preview/azext_storage_preview/azcopy/__init__.py

@@ -0,0 +1,4 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------