[SQL VM] az sql vm update: Add configuration options for SQL Assess…

…ment pre-requisites (#22672)

* Manage SQLVM Assessment

* Update sqlvm tests

* pylint fixes

* temp

* Add workspace arguments for sql vm update

* Remove unnecessary change in az vm code

* Suppress cred in response body during one-off live test

* fix lingering stylecop issue

* Fix style issue

* code review comments

* remove blank line

Co-authored-by: Vaibhav Devekar <vaibhav.devekar@gmail.com>

scripts/ci/credscan/CredScanSuppressions.json

@@ -327,6 +327,12 @@
       ],
       "_justification": "[SQL] one-off password used for test"
     },
+    {
+      "file": [
+        "src\\azure-cli\\azure\\cli\\command_modules\\sqlvm\\tests\\latest\\recordings\\test_sqlvm_mgmt_assessment.yaml"
+      ],
+      "_justification": "[SQL] response body contains key generated during one-off live test"
+    },
     {
       "placeholder": "SqlPassword",
       "_justification": "[SQL] False alarm about ClientAuthenticationType"

src/azure-cli-testsdk/azure/cli/testsdk/__init__.py

@@ -7,7 +7,8 @@
 
 from .base import ScenarioTest, LiveScenarioTest, LocalContextScenarioTest
 from .preparers import (StorageAccountPreparer, ResourceGroupPreparer, RoleBasedServicePrincipalPreparer,
-                        KeyVaultPreparer, ManagedHSMPreparer, ManagedApplicationPreparer, VirtualNetworkPreparer, VnetNicPreparer)
+                        KeyVaultPreparer, ManagedHSMPreparer, ManagedApplicationPreparer, VirtualNetworkPreparer,
+                        VnetNicPreparer, LogAnalyticsWorkspacePreparer)
 from .exceptions import CliTestError
 from .checkers import (JMESPathCheck, JMESPathCheckExists, JMESPathCheckGreaterThan, NoneCheck, StringCheck,
                        StringContainCheck)
@@ -21,7 +22,7 @@
            'JMESPathCheckExists', 'NoneCheck', 'live_only', 'record_only', 'StringCheck', 'StringContainCheck',
            'get_sha1_hash', 'KeyVaultPreparer', 'JMESPathCheckGreaterThan', 'api_version_constraint',
            'create_random_name', 'MOCKED_USER_NAME', 'MSGraphUserReplacer', 'LocalContextScenarioTest',
-           'VirtualNetworkPreparer', 'VnetNicPreparer']
+           'VirtualNetworkPreparer', 'VnetNicPreparer', 'LogAnalyticsWorkspacePreparer']
 
 
 __version__ = '0.1.0'

src/azure-cli-testsdk/azure/cli/testsdk/preparers.py

@@ -7,7 +7,6 @@
 from datetime import datetime
 
 from .scenario_tests import AbstractPreparer, SingleValueReplacer
-
 from .base import LiveScenarioTest
 from .exceptions import CliTestError
 from .reverse_dependency import get_dummy_cli
@@ -465,6 +464,38 @@ def _get_virtual_network(self, **kwargs):
                        'decorator @{} in front of this VirtualNetworkNic preparer.'
             raise CliTestError(template.format(VnetNicPreparer.__name__))
 
+
+class LogAnalyticsWorkspacePreparer(NoTrafficRecordingPreparer, SingleValueReplacer):
+    def __init__(self, name_prefix='laworkspace', location='eastus2euap', parameter_name='laworkspace',
+                 resource_group_parameter_name='resource_group', skip_delete=False):
+        super(LogAnalyticsWorkspacePreparer, self).__init__(name_prefix, 15)
+        self.cli_ctx = get_dummy_cli()
+        self.location = location
+        self.parameter_name = parameter_name
+        self.resource_group_parameter_name = resource_group_parameter_name
+        self.skip_delete = skip_delete
+
+    def create_resource(self, name, **kwargs):
+        group = self._get_resource_group(**kwargs)
+        template = ('az monitor log-analytics workspace create -l {} -g {} -n {}')
+        self.live_only_execute(self.cli_ctx, template.format(self.location, group, name))
+        return {self.parameter_name: name}
+
+    def remove_resource(self, name, **kwargs):
+        if not self.skip_delete:
+            group = self._get_resource_group(**kwargs)
+            template = ('az monitor log-analytics workspace delete -g {} -n {} --yes')
+            self.live_only_execute(self.cli_ctx, template.format(group, name))
+
+    def _get_resource_group(self, **kwargs):
+        try:
+            return kwargs.get(self.resource_group_parameter_name)
+        except KeyError:
+            template = 'To create a log analytics workspace a resource group is required. Please add ' \
+                       'decorator @{} in front of this preparer.'
+            raise CliTestError(template.format(ResourceGroupPreparer.__name__,
+                                               self.resource_group_parameter_name))
+
 # Utility
 
 

src/azure-cli/azure/cli/command_modules/security/tests/latest/preparers.py

@@ -16,9 +16,6 @@
 sql_server_image = 'MicrosoftSQLServer:SQL2017-WS2016:Enterprise:latest'
 sql_server_vm_size = 'Standard_DS2_v2'
 
-la_workspace_name_prefix = 'laworkspace'
-la_workspace_max_length = 15
-
 
 class SqlVirtualMachinePreparer(AbstractPreparer, SingleValueReplacer):
     def __init__(self, name_prefix=sqlvm_name_prefix, location='eastus2euap',
@@ -52,34 +49,3 @@ def _get_resource_group(self, **kwargs):
                        'decorator @{} in front of this preparer.'
             raise CliTestError(template.format(ResourceGroupPreparer.__name__,
                                                self.resource_group_parameter_name))
-
-
-class LogAnalyticsWorkspacePreparer(AbstractPreparer, SingleValueReplacer):
-    def __init__(self, name_prefix=la_workspace_name_prefix, location='eastus2euap', parameter_name='laworkspace',
-                 resource_group_parameter_name='resource_group', skip_delete=False):
-        super(LogAnalyticsWorkspacePreparer, self).__init__(name_prefix, la_workspace_max_length)
-        self.location = location
-        self.parameter_name = parameter_name
-        self.resource_group_parameter_name = resource_group_parameter_name
-        self.skip_delete = skip_delete
-
-    def create_resource(self, name, **kwargs):
-        group = self._get_resource_group(**kwargs)
-        template = ('az monitor log-analytics workspace create -l {} -g {} -n {}')
-        execute(DummyCli(), template.format(self.location, group, name))
-        return {self.parameter_name: name}
-
-    def remove_resource(self, name, **kwargs):
-        if not self.skip_delete:
-            group = self._get_resource_group(**kwargs)
-            template = ('az monitor log-analytics workspace delete -g {} -n {} --yes')
-            execute(DummyCli(), template.format(group, name))
-
-    def _get_resource_group(self, **kwargs):
-        try:
-            return kwargs.get(self.resource_group_parameter_name)
-        except KeyError:
-            template = 'To create a log analytics workspace a resource group is required. Please add ' \
-                       'decorator @{} in front of this preparer.'
-            raise CliTestError(template.format(ResourceGroupPreparer.__name__,
-                                               self.resource_group_parameter_name))

src/azure-cli/azure/cli/command_modules/security/tests/latest/test_va_sql_scenario.py

@@ -6,11 +6,11 @@
 import time
 import json
 import tempfile
-from azure.cli.testsdk import (ScenarioTest, LiveScenarioTest, ResourceGroupPreparer)
+from azure.cli.testsdk import (ScenarioTest, LiveScenarioTest, ResourceGroupPreparer, LogAnalyticsWorkspacePreparer)
 from azure.cli.testsdk.base import execute
 from azure.cli.core.mock import DummyCli
 from azure.cli.testsdk.exceptions import CliTestError
-from .preparers import (SqlVirtualMachinePreparer, LogAnalyticsWorkspacePreparer)
+from .preparers import (SqlVirtualMachinePreparer)
 
 
 class VulnerabilityAssessmentForSqlTests(LiveScenarioTest):

src/azure-cli/azure/cli/command_modules/sqlvm/_assessment_data_source.py

@@ -0,0 +1,48 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+# pylint: skip-file
+
+data_source_name = "DataSource_CustomLog_SQLAssessment_CL"
+data_source_kind = "CustomLog"
+
+data_source_properties = {
+    "customLogName": "SqlAssessment",
+    "description": "The Sql Assessment results",
+    "extractions": [
+        {
+            "extractionName": "TimeGenerated",
+            "extractionProperties": {
+                "dateTimeExtraction": {
+                    "regex": [
+                        {
+                            "matchIndex": 0,
+                            "pattern": "((\\d{2})|(\\d{4}))-([0-1]\\d)-(([0-3]\\d)|(\\d))T((\\d)|([0-1]\\d)|(2[0-4])):[0-5][0-9]:[0-5][0-9]"
+                        }
+                    ],
+                    "formatString": "yyyy-MM-ddTHH:mm:ssK"
+                }
+            },
+            "extractionType": "DateTime"
+        }
+    ],
+    "inputs": [
+        {
+            "location": {
+                "fileSystemLocations": {
+                    "windowsFileTypeLogPaths": [
+                        "C:\\Windows\\System32\\config\\systemprofile\\AppData\\Local\\Microsoft SQL Server IaaS Agent\\Assessment\\*.csv"
+                    ]
+                }
+            },
+            "recordDelimiter": {
+                "regexDelimiter": {
+                    "matchIndex": 0,
+                    "pattern": "(^.*((\\d{2})|(\\d{4}))-([0-1]\\d)-(([0-3]\\d)|(\\d))T((\\d)|([0-1]\\d)|(2[0-4])):[0-5][0-9]:[0-5][0-9].*$)"
+                }
+            }
+        }
+    ]
+}

src/azure-cli/azure/cli/command_modules/sqlvm/_help.py

@@ -117,9 +117,9 @@
 
 helps['sql vm start-assessment'] = """
 type: command
-short-summary: Starts SQL Best Practice Assessment on SQL virtual machine
+short-summary: Starts SQL best practice assessment on SQL virtual machine
 examples:
-  - name: Starts SQL Best Practice Assessment.
+  - name: Starts SQL best practice assessment.
     text: >
         az sql vm start-assessment -n sqlvm -g myresourcegroup
 """
@@ -158,16 +158,19 @@
   - name: Update a SQL virtual machine billing tag to DR.
     text: >
         az sql vm update -n sqlvm -g myresourcegroup --license-type DR
-  - name: Update a SQL virtual machine to disable SQL Best Practice Assessment.
+  - name: Update a SQL virtual machine to disable SQL best practice assessment.
     text: >
         az sql vm update -n sqlvm -g myresourcegroup --enable-assessment false
-  - name: Update a SQL virtual machine to disable schedule for SQL Best Practice Assessment.
+  - name: Update a SQL virtual machine to disable schedule for SQL best practice assessment.
     text: >
         az sql vm update -n sqlvm -g myresourcegroup --enable-assessment-schedule false
-  - name: Update a SQL virtual machine to enable schedule with weekly interval for SQL Best Practice Assessment.
+  - name: Update a SQL virtual machine to enable schedule with weekly interval for SQL best practice assessment when VM is already associated with a Log Analytics workspace.
     text: >
-        az sql vm update -n sqlvm -g myresourcegroup --enable-assessment true --enable-assessment-schedule true --assessment-weekly-interval 1 --assessment-day-of-week monday --assessment-start-time-local '19:30'
-  - name: Update a SQL virtual machine to enable schedule with monthly occurrence for SQL Best Practice Assessment.
+        az sql vm update -n sqlvm -g myresourcegroup --assessment-weekly-interval 1 --assessment-day-of-week monday --assessment-start-time-local '19:30'
+  - name: Update a SQL virtual machine to enable schedule with monthly occurrence for SQL best practice assessment while associating with a Log Analytics workspace.
     text: >
-        az sql vm update -n sqlvm -g myresourcegroup --enable-assessment true --enable-assessment-schedule true --assessment-monthly-occurrence 1 --assessment-day-of-week monday --assessment-start-time-local '19:30'
+        az sql vm update -n sqlvm -g myresourcegroup --workspace-name myLogAnalyticsWorkspace --workspace-rg myRg --assessment-monthly-occurrence 1 --assessment-day-of-week monday --assessment-start-time-local '19:30'
+  - name: Update a SQL virtual machine to enable SQL best practices assessment without setting a schedule for running assessment on-demand
+    text: >
+        az sql vm update -n sqlvm -g myresourcegroup --enable-assessment true
 """

src/azure-cli/azure/cli/command_modules/sqlvm/_params.py

@@ -346,3 +346,7 @@ def load_arguments(self, _):
                    options_list=['--assessment-start-time-local', '--am-time'],
                    help='Time of the day in HH:mm format. Examples include 17:30, 05:13.',
                    validator=validate_assessment_start_time_local)
+        c.argument('workspace_name',
+                   help='Name of the Log Analytics workspace to associate with VM.')
+        c.argument('workspace_rg',
+                   help='Resource group containing the Log Analytics workspace.')

src/azure-cli/azure/cli/command_modules/sqlvm/_util.py

@@ -3,6 +3,21 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
+from azure.core.exceptions import HttpResponseError
+from azure.core.paging import ItemPaged
+from azure.cli.command_modules.vm._client_factory import cf_log_analytics_data_sources, cf_log_analytics
+from azure.cli.core.commands.client_factory import get_subscription_id
+from azure.cli.command_modules.vm.custom import (
+    list_extensions,
+    set_extension
+)
+from azure.mgmt.loganalytics.models import DataSource
+from ._assessment_data_source import data_source_name, data_source_kind, data_source_properties
+
+WINDOWS_LA_EXT_NAME = 'MicrosoftMonitoringAgent'
+WINDOWS_LA_EXT_PUBLISHER = 'Microsoft.EnterpriseCloud.Monitoring'
+WINDOWS_LA_EXT_VERSION = '1.0'
+
 
 def get_sqlvirtualmachine_management_client(cli_ctx):
     from azure.cli.core.commands.client_factory import get_mgmt_service_client
@@ -26,3 +41,79 @@ def get_sqlvirtualmachine_sql_virtual_machines_operations(cli_ctx, _):
 
 def get_sqlvirtualmachine_operations(cli_ctx, _):
     return get_sqlvirtualmachine_management_client(cli_ctx).operations
+
+
+def _get_log_analytics_client(cmd):
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+    return cf_log_analytics(cmd.cli_ctx, subscription_id)
+
+
+def get_workspace_id_from_log_analytics_extension(cmd, resource_group_name, sql_virtual_machine_name):
+    '''
+    Get workspace id from Log Analytics extension on VM
+    '''
+    extensions = list_extensions(cmd, resource_group_name, sql_virtual_machine_name)
+    for ext in extensions:
+        if (ext.publisher == WINDOWS_LA_EXT_PUBLISHER and
+                ext.type_properties_type == WINDOWS_LA_EXT_NAME):
+            return ext.settings.get('workspaceId', None)
+
+    return None
+
+
+def set_log_analytics_extension(cmd, resource_group_name, vm_name, workspace_rg, workspace_name):
+    '''
+    Deploy Log Analytics extension to the Windows VM
+    '''
+    log_client = _get_log_analytics_client(cmd)
+
+    # Get workspace id and key
+    customer_id = log_client.workspaces.get(workspace_rg, workspace_name).customer_id
+    settings = {
+        'workspaceId': customer_id,
+        'stopOnMultipleConnections': 'true'
+    }
+    primary_shared_key = log_client.shared_keys.get_shared_keys(workspace_rg, workspace_name).primary_shared_key
+    protected_settings = {
+        'workspaceKey': primary_shared_key
+    }
+
+    return set_extension(cmd, resource_group_name, vm_name,
+                         WINDOWS_LA_EXT_NAME,
+                         WINDOWS_LA_EXT_PUBLISHER,
+                         WINDOWS_LA_EXT_VERSION,
+                         settings,
+                         protected_settings), customer_id
+
+
+def get_workspace_in_sub(cmd, workspace_id):
+    '''
+    Get workspace details for given workspace id
+    '''
+    log_client = _get_log_analytics_client(cmd)
+    obj_list = log_client.workspaces.list()
+    workspaces = list(obj_list) if isinstance(obj_list, ItemPaged) else obj_list  # Convert iterable to list
+    return next((w for w in workspaces if w.customer_id == workspace_id), None)
+
+
+def validate_and_set_assessment_custom_log(cmd, workspace_name, workspace_rg):
+    '''
+    Validate and deploy custom log definition for assessment feature
+    '''
+    subscription_id = get_subscription_id(cmd.cli_ctx)
+    data_sources_client = cf_log_analytics_data_sources(cmd.cli_ctx, subscription_id)
+
+    try:
+        # Verify if required custom log definition already exists
+        data_sources_client.get(workspace_rg, workspace_name, data_source_name)
+    except HttpResponseError as err:
+        # Required custom log definition does not exist so deploy it
+        if err.status_code == 404:
+            data_source = DataSource(kind=data_source_kind,
+                                     properties=data_source_properties)
+            data_sources_client.create_or_update(workspace_rg,
+                                                 workspace_name,
+                                                 data_source_name,
+                                                 data_source)
+        else:
+            raise err