[KeyVault] KeyVault Round 3 Commands (Azure#1215)

* Key backup/restore/import. Certificate import. Pending certificate
operations.

* Fix build errors and help text.

* Code review fix.

azure-cli.pyproj

@@ -74,33 +74,23 @@
     <Compile Include="azure-cli\setup.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\credential.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\custom.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\container_registry.py" />
+    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\container_registry_management_client.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\credentials.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\exceptions.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_create_parameters.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_credentials.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_move_request.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_name_check_request.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_name_status.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_properties.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_properties_create_parameters.py" />
+    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_paged.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_update_parameters.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\resource_list_registry.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\storage_account_base_properties.py" />
+    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\resource.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\storage_account_properties.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\subscription_notification.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\subscription_properties.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__init__.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\operation.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\registries.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\subscriptions.py" />
+    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\registries_operations.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__init__.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\version.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\__init__.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\repository.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\storage.py" />
-    <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\_arm_utils.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\_constants.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\_factory.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\_format.py" />
@@ -729,9 +719,7 @@
     <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\" />
     <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\" />
     <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\" />
-    <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\" />
     <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\" />
-    <Folder Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__pycache__\" />
     <Folder Include="command_modules\azure-cli-acs\" />
     <Folder Include="command_modules\azure-cli-acs\azure\" />
     <Folder Include="command_modules\azure-cli-acs\azure\cli\" />
@@ -911,46 +899,6 @@
     <Content Include="azure-cli-core\setup.cfg" />
     <Content Include="azure-cli\az.completion.sh" />
     <Content Include="azure-cli\setup.cfg" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_create_parameters.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_credentials.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_move_request.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_name_check_request.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_name_status.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_properties.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_properties_create_parameters.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\registry_update_parameters.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\resource_list_registry.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\storage_account_base_properties.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\storage_account_properties.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\subscription_notification.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\subscription_properties.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__init__.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_create_parameters.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_credentials.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_move_request.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_name_check_request.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_name_status.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_properties.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_properties_create_parameters.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\registry_update_parameters.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\resource_list_registry.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\storage_account_base_properties.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\storage_account_properties.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\subscription_notification.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\subscription_properties.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\models\__pycache__\__init__.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\operation.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\registries.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\subscriptions.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__init__.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__pycache__\operation.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__pycache__\registries.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__pycache__\subscriptions.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\mgmt_acr\operations\__pycache__\__init__.cpython-35.pyc" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\template.existing.json" />
-    <Content Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\template.new.json" />
     <Content Include="command_modules\azure-cli-acr\requirements.txt" />
     <Content Include="command_modules\azure-cli-acs\requirements.txt" />
     <Content Include="command_modules\azure-cli-component\requirements.txt" />

azure-cli.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.24720.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "azure-cli", "azure-cli.pyproj", "{938454F7-93BD-41A7-84B2-3C89D64B969D}"
 EndProject

requirements.txt

@@ -8,6 +8,7 @@ paramiko==2.0.2
 pip
 pygments==2.1.3
 pylint==1.5.4
+pyOpenSSL==16.1.0
 pyyaml==3.11
 requests==2.9.1
 six==1.10.0

src/azure-cli-core/azure/cli/core/commands/_introspection.py

@@ -17,6 +17,7 @@ def extract_full_summary_from_signature(operation):
             summary = lines[:match.regs[0][0]]
         else:
             summary = lines
+    summary = summary.replace('\n', ' ').replace('\r', '')
     return summary
 
 def _option_descriptions(operation):

src/command_modules/azure-cli-keyvault/azure/cli/command_modules/keyvault/_command_type.py

@@ -6,7 +6,7 @@
 import base64
 
 from msrest.paging import Paged
-from msrest.exceptions import ValidationError
+from msrest.exceptions import ValidationError, ClientRequestError
 from msrestazure.azure_operation import AzureOperationPoller
 
 from azure.cli.core.commands import command_table, CliCommand, LongRunningOperation
@@ -78,6 +78,11 @@ def get_token(server, resource, scope): # pylint: disable=unused-argument
                 raise CLIError(ex.inner_exception.error.message)
             except AttributeError:
                 raise CLIError(ex)
+        except ClientRequestError as ex:
+            if 'Failed to establish a new connection' in str(ex.inner_exception):
+                raise CLIError('Max retries exceeded attempting to connect to vault. '
+                               'Try flushing your DNS cache or try again later.')
+            raise CLIError(ex)
 
     name = ' '.join(name.split())
     cmd = CliCommand(name, _execute_command, table_transformer=table_transformer)

src/command_modules/azure-cli-keyvault/azure/cli/command_modules/keyvault/_help.py

@@ -54,6 +54,11 @@
     short-summary: Manage certificates.
 """
 
+helps['keyvault certificate pending'] = """
+    type: group
+    short-summary: Manage pending certificate creation operations.
+"""
+
 helps['keyvault certificate contact'] = """
     type: group
     short-summary: Manage contacts for certificate management.

src/command_modules/azure-cli-keyvault/azure/cli/command_modules/keyvault/_params.py

@@ -20,18 +20,23 @@
 from azure.cli.command_modules.keyvault.keyvaultclient.generated.models import \
     (KeyAttributes, SecretAttributes, CertificateAttributes)
 from azure.cli.command_modules.keyvault._validators import \
-    (datetime_type,
+    (datetime_type, base64_encoded_certificate_type,
      get_attribute_validator,
      vault_base_url_type, validate_key_import_source,
      validate_key_type, validate_key_ops, validate_policy_permissions,
-     validate_principal, validate_resource_group_name)
+     validate_principal, validate_resource_group_name,
+     validate_x509_certificate_chain,
+     process_certificate_cancel_namespace)
 
 # CUSTOM CHOICE LISTS
 
 key_permission_values = ', '.join([x.value for x in KeyPermissions])
 secret_permission_values = ', '.join([x.value for x in SecretPermissions])
 certificate_permission_values = ', '.join([x.value for x in CertificatePermissions])
 json_web_key_op_values = ', '.join([x.value for x in JsonWebKeyOperation])
+secret_file_encoding_values = ['utf8', 'utf16le', 'ucs2', 'ascii']
+secret_file_decoding_values = ['base64', 'hex']
+certificate_file_encoding_values = ['base64']
 
 # KEY ATTRIBUTE PARAMETER REGISTRATION
 
@@ -87,6 +92,10 @@ def register_attributes_argument(scope, name, attr_class, create=False):
 register_cli_argument('keyvault key import', 'pem_password', help='Password of PEM file.', arg_group='Key Source')
 register_cli_argument('keyvault key import', 'byok_file', help='BYOK file containing the key to be imported. Must not be password protected.', arg_group='Key Source')
 
+register_cli_argument('keyvault key backup', 'file_path', options_list=('--file', '-f'), help='Local file path in which to store key backup.')
+
+register_cli_argument('keyvault key restore', 'file_path', options_list=('--file', '-f'), help='Local key backup from which to restore key.')
+
 register_attributes_argument('keyvault key set-attributes', 'key', KeyAttributes)
 
 register_cli_argument('keyvault secret', 'secret_version', options_list=('--version', '-v'), help='The secret version. If omitted, uses the latest version.', default='', required=False)
@@ -96,11 +105,19 @@ def register_attributes_argument(scope, name, attr_class, create=False):
 
 register_cli_argument('keyvault certificate', 'certificate_version', options_list=('--version', '-v'), help='The certificate version. If omitted, uses the latest version.', default='', required=False)
 register_attributes_argument('keyvault certificate create', 'certificate', CertificateAttributes, True)
+register_attributes_argument('keyvault certificate import', 'certificate', CertificateAttributes, True)
 register_attributes_argument('keyvault certificate set-attributes', 'certificate', CertificateAttributes)
-for item in ['create', 'set-attributes']:
+for item in ['create', 'set-attributes', 'import']:
     register_cli_argument('keyvault certificate {}'.format(item), 'certificate_policy', options_list=('--policy', '-p'), help='JSON encoded policy defintion. Use @{file} to load from a file.', type=get_json_object)
 
-register_cli_argument('keyvault certificate contact', 'contact_email', options_list=('--email',), help='Contact e-mail address. Must be unique within the vault.')
+register_cli_argument('keyvault certificate import', 'base64_encoded_certificate', options_list=('--file', '-f'), help='PKCS12 file or PEM file containing the certificate and private key.', type=base64_encoded_certificate_type)
+
+register_cli_argument('keyvault certificate pending merge', 'x509_certificates', options_list=('--file', '-f'), help='File containing the certificate or certificate chain to merge.', validator=validate_x509_certificate_chain)
+register_attributes_argument('keyvault certificate pending merge', 'certificate', CertificateAttributes, True)
+
+register_cli_argument('keyvault certificate pending cancel', 'cancellation_requested', ignore_type, validator=process_certificate_cancel_namespace)
+
+register_cli_argument('keyvault certificate contact', 'contact_email', options_list=('--email',), help='Contact e-mail address. Must be unique.')
 register_cli_argument('keyvault certificate contact', 'contact_name', options_list=('--name',), help='Full contact name.')
 register_cli_argument('keyvault certificate contact', 'contact_phone', options_list=('--phone',), help='Contact phone number.')
 

src/command_modules/azure-cli-keyvault/azure/cli/command_modules/keyvault/_validators.py

@@ -4,7 +4,10 @@
 #---------------------------------------------------------------------------------------------
 
 import argparse
+import base64
+import binascii
 from datetime import datetime
+import re
 
 from azure.mgmt.keyvault import KeyVaultManagementClient
 from azure.mgmt.keyvault.models.key_vault_management_client_enums import \
@@ -22,6 +25,9 @@ def _extract_version(item_id):
 
 # COMMAND NAMESPACE VALIDATORS
 
+def process_certificate_cancel_namespace(namespace):
+    namespace.cancellation_requested = True
+
 # PARAMETER NAMESPACE VALIDATORS
 
 def get_attribute_validator(name, attribute_class, create=False):
@@ -61,6 +67,8 @@ def validate_key_type(ns):
             'hsm': 'RSA-HSM'
         }
         ns.destination = dest_to_type_map[ns.destination]
+        if ns.destination == 'RSA' and hasattr(ns, 'byok_file') and ns.byok_file:
+            raise CLIError('BYOK keys are hardware protected. Omit --protection')
 
 def validate_policy_permissions(ns):
     key_perms = ns.key_permissions
@@ -108,10 +116,33 @@ def validate_resource_group_name(ns):
             "The Resource 'Microsoft.KeyVault/vaults/{}'".format(vault_name) + \
             " not found within subscription")
 
+def validate_x509_certificate_chain(ns):
+    def _load_certificate_as_bytes(file_name):
+        cert_list = []
+        regex = r'-----BEGIN CERTIFICATE-----([^-]+)-----END CERTIFICATE-----'
+        with open(file_name, 'r') as f:
+            cert_data = f.read()
+            for entry in re.findall(regex, cert_data):
+                cert_list.append(base64.b64decode(entry.replace('\n', '')))
+        return cert_list
+
+    ns.x509_certificates = _load_certificate_as_bytes(ns.x509_certificates)
+
 # ARGUMENT TYPES
 
+def base64_encoded_certificate_type(string):
+    """ Loads file and outputs contents as base64 encoded string. """
+    with open(string, 'rb') as f:
+        cert_data = f.read()
+    try:
+        # for PEM files (including automatic endline conversion for Windows)
+        cert_data = cert_data.decode('utf-8').replace('\r\n', '\n')
+    except UnicodeDecodeError:
+        cert_data = binascii.b2a_base64(cert_data).decode('utf-8')
+    return cert_data
+
 def datetime_type(string):
-    ''' Validates UTC datettime in format '%Y-%m-%d\'T\'%H:%M\'Z\''. '''
+    """ Validates UTC datettime in format '%Y-%m-%d\'T\'%H:%M\'Z\''. """
     date_format = '%Y-%m-%dT%H:%MZ'
     return datetime.strptime(string, date_format)