Knack - Storage Command Module (#5076)

* Update azdev to list commands information

* Convert storage command modules to knack

.flake8

@@ -14,7 +14,6 @@ exclude =
     scripts
     doc
     build_scripts
-    src/command_modules/azure-cli-storage
     src/command_modules/azure-cli-rdbms
     src/command_modules/azure-cli-sql
     src/command_modules/azure-cli-vm

.gitignore

@@ -91,3 +91,6 @@ artifacts/
 
 # Python version
 .python-version
+
+# Pytest
+.cache/

scripts/ci/build.sh

@@ -114,14 +114,13 @@ for name in $(ls src/command_modules | grep azure-cli-); do
     if [ "$name" == "azure-cli-eventgrid" ]; then continue; fi
     if [ "$name" == "azure-cli-find" ]; then continue; fi
     if [ "$name" == "azure-cli-interactive" ]; then continue; fi
-	if [ "$name" == "azure-cli-keyvault" ]; then continue; fi
+    if [ "$name" == "azure-cli-keyvault" ]; then continue; fi
     if [ "$name" == "azure-cli-lab" ]; then continue; fi
     if [ "$name" == "azure-cli-monitor" ]; then continue; fi
     if [ "$name" == "azure-cli-profile" ]; then continue; fi
     if [ "$name" == "azure-cli-rdbms" ]; then continue; fi
     if [ "$name" == "azure-cli-role" ]; then continue; fi
     if [ "$name" == "azure-cli-sql" ]; then continue; fi
-    if [ "$name" == "azure-cli-storage" ]; then continue; fi
     if [ "$name" == "azure-cli-vm" ] ; then continue; fi
     module_name=${name##azure-cli-}
     if [ -d src/command_modules/$name/azure/cli/command_modules/$module_name/tests ]; then

scripts/ci/test_static.sh

@@ -70,7 +70,7 @@ run_style azure.cli.command_modules.resource
 run_style azure.cli.command_modules.role
 run_style azure.cli.command_modules.servicefabric
 #run_style azure.cli.command_modules.sql
-#run_style azure.cli.command_modules.storage
+run_style azure.cli.command_modules.storage
 #run_style azure.cli.command_modules.testsdk
 #run_style azure.cli.command_modules.vm
 

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -605,6 +605,9 @@ def __init__(self, overrides=None, **kwargs):
         self.settings = {}
         self.update(overrides, **kwargs)
 
+    def __repr__(self):
+        return str(vars(self))
+
     def update(self, other=None, **kwargs):
         if other:
             self.settings.update(**other.settings)
@@ -679,6 +682,8 @@ def command(self, name, method_name=None, **kwargs):
         operation = operations_tmpl.format(method_name) if operations_tmpl else None
         self.command_loader._cli_command(command_name, operation, **merged_kwargs)  # pylint: disable=protected-access
 
+        return command_name
+
     def custom_command(self, name, method_name, **kwargs):
         """
         Register a custome CLI command.
@@ -709,6 +714,8 @@ def custom_command(self, name, method_name, **kwargs):
                                          operation=operations_tmpl.format(method_name),
                                          **merged_kwargs)
 
+        return command_name
+
     # pylint: disable=no-self-use
     def _resolve_operation(self, kwargs, name, command_type=None, source_kwarg='command_type'):
 

src/azure-cli-core/azure/cli/core/commands/client_factory.py

@@ -93,7 +93,7 @@ def _get_mgmt_service_client(cli_ctx,
 
     configure_common_settings(cli_ctx, client)
 
-    return (client, subscription_id)
+    return client, subscription_id
 
 
 def get_data_service_client(cli_ctx, service_type, account_name, account_key, connection_string=None,

src/azure-cli-core/azure/cli/core/commands/validators.py

@@ -14,27 +14,27 @@
 
 
 class IterateAction(argparse.Action):  # pylint: disable=too-few-public-methods
-    '''Action used to collect argument values in an IterateValue list
+    """Action used to collect argument values in an IterateValue list
     The application will loop through each value in the IterateValue
     and execeute the associated handler for each
-    '''
+    """
 
     def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace, self.dest, IterateValue(values))
 
 
 class IterateValue(list):
-    '''Marker class to indicate that, when found as a value in the parsed namespace
+    """Marker class to indicate that, when found as a value in the parsed namespace
     from argparse, the handler should be invoked once per value in the list with all
     other values in the parsed namespace frozen.
 
     Typical use is to allow multiple ID parameter to a show command etc.
-    '''
+    """
     pass
 
 
 def validate_tags(ns):
-    ''' Extracts multiple space-separated tags in key[=value] format '''
+    """ Extracts multiple space-separated tags in key[=value] format """
     if isinstance(ns.tags, list):
         tags_dict = {}
         for item in ns.tags:
@@ -43,7 +43,7 @@ def validate_tags(ns):
 
 
 def validate_tag(string):
-    ''' Extracts a single tag in key[=value] format '''
+    """ Extracts a single tag in key[=value] format """
     result = {}
     if string:
         comps = string.split('=', 1)
@@ -52,7 +52,7 @@ def validate_tag(string):
 
 
 def validate_key_value_pairs(string):
-    ''' Validates key-value pairs in the following format: a=b;c=d '''
+    """ Validates key-value pairs in the following format: a=b;c=d """
     result = None
     if string:
         kv_list = [x for x in string.split(';') if '=' in x]     # key-value pairs

src/azure-cli-testsdk/azure/cli/testsdk/patches.py

@@ -13,14 +13,17 @@ def patch_progress_controller(unit_test):
     def _mock_pass(*args, **kwargs):  # pylint: disable=unused-argument
         pass
 
+    def _mock_get_hook(_):
+        return _mock_pass
+
     mock_in_unit_test(
         unit_test, 'azure.cli.core.commands.progress.ProgressHook.update', _mock_pass)
     mock_in_unit_test(
         unit_test, 'azure.cli.core.commands.progress.ProgressHook.add', _mock_pass)
     mock_in_unit_test(
         unit_test, 'azure.cli.core.commands.progress.ProgressHook.end', _mock_pass)
     mock_in_unit_test(
-        unit_test, 'azure.cli.command_modules.storage.blob._update_progress', _mock_pass)
+        unit_test, 'azure.cli.command_modules.storage.operations.blob.get_update_progress_fn', _mock_get_hook)
 
 
 def patch_main_exception_handler(unit_test):

src/command_modules/azure-cli-storage/HISTORY.rst

@@ -3,6 +3,10 @@
 Release History
 ===============
 
+2.0.21
+++++++
+* Knack conversion
+
 2.0.20
 ++++++
 * Update multiapi storage package dependency to 0.1.7

src/command_modules/azure-cli-storage/azure/cli/command_modules/storage/__init__.py

@@ -5,21 +5,21 @@
 
 from azure.cli.core import AzCommandsLoader
 from azure.cli.core.profiles import ResourceType
+from azure.cli.core.commands import AzCommandGroup, AzArgumentContext
 
 import azure.cli.command_modules.storage._help  # pylint: disable=unused-import
 
 
 class StorageCommandsLoader(AzCommandsLoader):
-
     def __init__(self, cli_ctx=None):
         from azure.cli.core.commands import CliCommandType
-        from azure.cli.command_modules.storage._command_type import _StorageCommandGroup
 
         storage_custom = CliCommandType(operations_tmpl='azure.cli.command_modules.storage.custom#{}')
         super(StorageCommandsLoader, self).__init__(cli_ctx=cli_ctx,
                                                     resource_type=ResourceType.DATA_STORAGE,
                                                     custom_command_type=storage_custom,
-                                                    command_group_cls=_StorageCommandGroup)
+                                                    command_group_cls=StorageCommandGroup,
+                                                    argument_context_cls=StorageArgumentContext)
 
     def load_command_table(self, args):
         super(StorageCommandsLoader, self).load_command_table(args)
@@ -32,4 +32,151 @@ def load_arguments(self, command):
         from azure.cli.command_modules.storage._params import load_arguments
         load_arguments(self, command)
 
+
+class StorageArgumentContext(AzArgumentContext):
+    def register_sas_arguments(self):
+        from azure.cli.command_modules.storage._validators import ipv4_range_type, get_datetime_type
+        self.argument('ip', type=ipv4_range_type,
+                      help='Specifies the IP address or range of IP addresses from which to accept requests. Supports '
+                           'only IPv4 style addresses.')
+        self.argument('expiry', type=get_datetime_type(True),
+                      help='Specifies the UTC datetime (Y-m-d\'T\'H:M\'Z\') at which the SAS becomes invalid. Do not '
+                           'use if a stored access policy is referenced with --id that specifies this value.')
+        self.argument('start', type=get_datetime_type(True),
+                      help='Specifies the UTC datetime (Y-m-d\'T\'H:M\'Z\') at which the SAS becomes valid. Do not use '
+                           'if a stored access policy is referenced with --id that specifies this value. Defaults to '
+                           'the time of the request.')
+        self.argument('protocol', options_list=('--https-only',), action='store_const', const='https',
+                      help='Only permit requests made with the HTTPS protocol. If omitted, requests from both the HTTP '
+                           'and HTTPS protocol are permitted.')
+
+    def register_content_settings_argument(self, settings_class, update, arg_group=None, guess_from_file=None):
+        from azure.cli.command_modules.storage._validators import get_content_setting_validator
+
+        self.ignore('content_settings')
+        self.extra('content_type', default=None, help='The content MIME type.', arg_group=arg_group,
+                   validator=get_content_setting_validator(settings_class, update, guess_from_file=guess_from_file))
+        self.extra('content_encoding', default=None, help='The content encoding type.', arg_group=arg_group)
+        self.extra('content_language', default=None, help='The content language.', arg_group=arg_group)
+        self.extra('content_disposition', default=None, arg_group=arg_group,
+                   help='Conveys additional information about how to process the response payload, and can also be '
+                        'used to attach additional metadata.')
+        self.extra('content_cache_control', default=None, help='The cache control string.', arg_group=arg_group)
+        self.extra('content_md5', default=None, help='The content\'s MD5 hash.', arg_group=arg_group)
+
+    def register_path_argument(self, default_file_param=None, options_list=None):
+        from ._validators import get_file_path_validator
+        from .completers import file_path_completer
+
+        path_help = 'The path to the file within the file share.'
+        if default_file_param:
+            path_help = '{} If the file name is omitted, the source file name will be used.'.format(path_help)
+        self.extra('path', options_list=options_list or ('--path', '-p'),
+                   required=default_file_param is None, help=path_help,
+                   validator=get_file_path_validator(default_file_param=default_file_param),
+                   completer=file_path_completer)
+        self.ignore('file_name')
+        self.ignore('directory_name')
+
+    def register_source_uri_arguments(self, validator, blob_only=False):
+        self.argument('copy_source', options_list=('--source-uri', '-u'), validator=validator, required=False,
+                      arg_group='Copy Source')
+        self.extra('source_sas', default=None, arg_group='Copy Source',
+                   help='The shared access signature for the source storage account.')
+        self.extra('source_container', default=None, arg_group='Copy Source',
+                   help='The container name for the source storage account.')
+        self.extra('source_blob', default=None, arg_group='Copy Source',
+                   help='The blob name for the source storage account.')
+        self.extra('source_snapshot', default=None, arg_group='Copy Source',
+                   help='The blob snapshot for the source storage account.')
+        self.extra('source_account_name', default=None, arg_group='Copy Source',
+                   help='The storage account name of the source blob.')
+        self.extra('source_account_key', default=None, arg_group='Copy Source',
+                   help='The storage account key of the source blob.')
+        if not blob_only:
+            self.extra('source_path', default=None, arg_group='Copy Source',
+                       help='The file path for the source storage account.')
+            self.extra('source_share', default=None, arg_group='Copy Source',
+                       help='The share name for the source storage account.')
+
+    def register_common_storage_account_options(self):
+        from azure.cli.core.commands.parameters import get_three_state_flag, get_enum_type
+        from ._validators import validate_encryption_services
+
+        t_access_tier, t_sku_name, t_encryption_services = self.command_loader.get_models(
+            'AccessTier', 'SkuName', 'EncryptionServices', resource_type=ResourceType.MGMT_STORAGE)
+
+        self.argument('https_only', help='Allows https traffic only to storage service.',
+                      arg_type=get_three_state_flag())
+        self.argument('sku', help='The storage account SKU.', arg_type=get_enum_type(t_sku_name))
+        self.argument('assign_identity', action='store_true', resource_type=ResourceType.MGMT_STORAGE,
+                      min_api='2017-06-01',
+                      help='Generate and assign a new Storage Account Identity for this storage account for use '
+                           'with key management services like Azure KeyVault.')
+        self.argument('access_tier', arg_type=get_enum_type(t_access_tier),
+                      help='The access tier used for billing StandardBlob accounts. Cannot be set for StandardLRS, '
+                           'StandardGRS, StandardRAGRS, or PremiumLRS account types. It is required for '
+                           'StandardBlob accounts during creation')
+
+        if t_encryption_services:
+            encryption_choices = list(
+                t_encryption_services._attribute_map.keys())  # pylint: disable=protected-access
+            self.argument('encryption_services', arg_type=get_enum_type(encryption_choices),
+                          resource_type=ResourceType.MGMT_STORAGE, min_api='2016-12-01', nargs='+',
+                          validator=validate_encryption_services, help='Specifies which service(s) to encrypt.')
+
+
+class StorageCommandGroup(AzCommandGroup):
+    def storage_command(self, name, method_name=None, command_type=None, **kwargs):
+        """ Registers an Azure CLI Storage Data Plane command. These commands always include the four parameters which
+        can be used to obtain a storage client: account-name, account-key, connection-string, and sas-token. """
+        if command_type:
+            command_name = self.command(name, method_name, command_type=command_type, **kwargs)
+        else:
+            command_name = self.command(name, method_name, **kwargs)
+        self._register_data_plane_account_arguments(command_name)
+
+    def storage_custom_command(self, name, method_name, **kwargs):
+        command_name = self.custom_command(name, method_name, **kwargs)
+        self._register_data_plane_account_arguments(command_name)
+
+    def get_handler_suppress_404(self):
+        def handler(ex):
+            from azure.cli.core.profiles import get_sdk
+
+            t_error = get_sdk(self.command_loader.cli_ctx,
+                              ResourceType.DATA_STORAGE,
+                              'common._error#AzureMissingResourceHttpError')
+            if isinstance(ex, t_error):
+                return None
+            raise ex
+
+        return handler
+
+    def _register_data_plane_account_arguments(self, command_name):
+        """ Add parameters required to create a storage client """
+        from azure.cli.command_modules.storage._validators import validate_client_parameters
+        command = self.command_loader.command_table[command_name]
+        group_name = 'Storage Account'
+        command.add_argument('account_name', '--account-name', required=False, default=None,
+                             arg_group=group_name,
+                             help='Storage account name. Related environment variable: AZURE_STORAGE_ACCOUNT. Must be '
+                                  'used in conjunction with either storage account key or a SAS token. If neither are '
+                                  'present, the command will try to query the storage account key using the '
+                                  'authenticated Azure account. If a large number of storage commands are executed the '
+                                  'API quota may be hit')
+        command.add_argument('account_key', '--account-key', required=False, default=None,
+                             arg_group=group_name,
+                             help='Storage account key. Must be used in conjunction with storage account name. '
+                                  'Environment variable: AZURE_STORAGE_KEY')
+        command.add_argument('connection_string', '--connection-string', required=False, default=None,
+                             validator=validate_client_parameters, arg_group=group_name,
+                             help='Storage account connection string. Environment variable: '
+                                  'AZURE_STORAGE_CONNECTION_STRING')
+        command.add_argument('sas_token', '--sas-token', required=False, default=None,
+                             arg_group=group_name,
+                             help='A Shared Access Signature (SAS). Must be used in conjunction with storage account '
+                                  'name. Environment variable: AZURE_STORAGE_SAS_TOKEN')
+
+
 COMMAND_LOADER_CLS = StorageCommandsLoader