Skip to content

Commit

Permalink
[Monitor Control Service] Update Data Collection Rules (#4716)
Browse files Browse the repository at this point in the history
* Bump azure-mgmt-amcs

* refine changelog

Co-authored-by: Zeng Taoxu <v-taoxuzeng@microsoft.com>
  • Loading branch information
necusjz and ZengTaoxu authored Apr 22, 2022
1 parent dcc774a commit cff8558
Show file tree
Hide file tree
Showing 34 changed files with 6,060 additions and 5,760 deletions.
7 changes: 7 additions & 0 deletions src/monitor-control-service/HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
Release History
===============

0.3.0
++++++
* `az monitor data-collection rule create`: Add parameter --rule-file.
* `az monitor data-collection rule create`: Remove parameters --data-flows, --log-analytics, --monitor-metrics, --performance-counters, --windows-event-logs, --syslog and --extensions.
* `az monitor data-collection rule association list`: Add parameter --data-collection-endpoint-name.
* Bump api version from 2021-04-01 to 2021_09_01_preview.

0.2.0
++++++
* GA release.
Expand Down
6 changes: 5 additions & 1 deletion src/monitor-control-service/azext_amcs/generated/_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,16 @@
helps['monitor data-collection rule association list'] = """
type: command
short-summary: "Lists associations for the specified data collection rule. And Lists associations for the \
specified resource."
specified data collection endpoint. And Lists associations for the specified resource."
examples:
- name: List associations for specified data collection rule
text: |-
az monitor data-collection rule association list --rule-name "myCollectionRule" --resource-group \
"myResourceGroup"
- name: List associations for specified data collection endpoint
text: |-
az monitor data-collection rule association list --data-collection-endpoint-name \
"myDataCollectionEndpointName" --resource-group "myResourceGroup"
- name: List associations for specified resource
text: |-
az monitor data-collection rule association list --resource "subscriptions/703362b3-f278-4e4b-9179-c76ea\
Expand Down
2 changes: 2 additions & 0 deletions src/monitor-control-service/azext_amcs/generated/_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ def load_arguments(self, _):
c.argument('resource_group_name', resource_group_name_type)
c.argument('data_collection_rule_name', options_list=['--rule-name'], type=str, help='The name of the data '
'collection rule. The name is case insensitive.')
c.argument('data_collection_endpoint_name', options_list=['--endpoint-name'], type=str, help='The name of the data collection endpoint. The name '
'is case insensitive.')
c.argument('resource_uri', options_list=['--resource'], type=str, help='The identifier of the resource.')

with self.argument_context('monitor data-collection rule association show') as c:
Expand Down
4 changes: 4 additions & 0 deletions src/monitor-control-service/azext_amcs/generated/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,14 @@ def monitor_data_collection_endpoint_delete(client,
def monitor_data_collection_rule_association_list(client,
resource_group_name=None,
data_collection_rule_name=None,
data_collection_endpoint_name=None,
resource_uri=None):
if resource_group_name and data_collection_rule_name is not None:
return client.list_by_rule(resource_group_name=resource_group_name,
data_collection_rule_name=data_collection_rule_name)
elif resource_group_name and data_collection_endpoint_name is not None:
return client.list_by_data_collection_endpoint(resource_group_name=resource_group_name,
data_collection_endpoint_name=data_collection_endpoint_name)
return client.list_by_resource(resource_uri=resource_uri)


Expand Down
96 changes: 3 additions & 93 deletions src/monitor-control-service/azext_amcs/manual/_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,103 +165,13 @@
type: command
short-summary: "Create a data collection rule."
parameters:
- name: --data-flows
short-summary: "The specification of data flows."
long-summary: |
Usage: --data-flows streams=XX1 streams=XX2 destinations=XX1 destinations=XX2
streams: Required. List of streams for this data flow.
destinations: Required. List of destinations for this data flow.
Multiple actions can be specified by using more than one --data-flows argument.
- name: --log-analytics
short-summary: "List of Log Analytics destinations."
long-summary: |
Usage: --log-analytics resource-id=XX name=XX
resource-id: Required. The resource ID of the Log Analytics workspace.
name: Required. A friendly name for the destination. This name should be unique across all destinations \
(regardless of type) within the data collection rule.
Multiple actions can be specified by using more than one --log-analytics argument.
- name: --monitor-metrics
short-summary: "Azure Monitor Metrics destination."
long-summary: |
Usage: --monitor-metrics name=XX
name: Required. A friendly name for the destination. This name should be unique across all destinations \
(regardless of type) within the data collection rule.
- name: --performance-counters
short-summary: "The list of performance counter data source configurations."
long-summary: |
Usage: --performance-counters streams=XX1 streams=XX2 \
sampling-frequency=XX counter-specifiers=XX1 counter-specifiers=XX2 name=XX
streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
will be used for this data and usually what table in Log Analytics the data will be sent to.
sampling-frequency: Required. The number of seconds between consecutive counter measurements \
(samples).
counter-specifiers: Required. A list of specifier names of the performance counters you want to collect. \
Use a wildcard (*) to collect a counter for all instances. To get a list of performance counters on Windows, run the \
command 'typeperf'.
name: Required. A friendly name for the data source. This name should be unique across all data sources \
(regardless of type) within the data collection rule.
Multiple actions can be specified by using more than one --performance-counters argument.
- name: --windows-event-logs
short-summary: "The list of Windows Event Log data source configurations."
long-summary: |
Usage: --windows-event-logs streams=XX1 streams=XX2 x-path-queries=XX1 \
x-path-queries=XX2 name=XX
streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
will be used for this data and usually what table in Log Analytics the data will be sent to.
x-path-queries: Required. A list of Windows Event Log queries in XPATH format.
name: Required. A friendly name for the data source. This name should be unique across all data sources \
(regardless of type) within the data collection rule.
Multiple actions can be specified by using more than one --windows-event-logs argument.
- name: --syslog
short-summary: "The list of Syslog data source configurations."
long-summary: |
Usage: --syslog streams=XX1 streams=XX2 facility-names=XX1 facility-names=XX2 log-levels=XX1 log-levels=XX2 \
name=XX
streams: Required. List of streams that this data source will be sent to. A stream indicates what schema \
will be used for this data and usually what table in Log Analytics the data will be sent to.
facility-names: Required. The list of facility names.
log-levels: The log levels to collect.
name: Required. A friendly name for the data source. This name should be unique across all data sources \
(regardless of type) within the data collection rule.
Multiple actions can be specified by using more than one --syslog argument.
- name: --rule-file
short-summary: "The json file for rule parameters."
examples:
- name: Create data collection rule
text: |-
az monitor data-collection rule create --resource-group "myResourceGroup" --location "eastus" \
--name "myCollectionRule" \
--data-flows destinations="centralWorkspace" streams="Microsoft-Perf" streams="Microsoft-Syslog" \
streams="Microsoft-WindowsEvent" \
--log-analytics name="centralWorkspace" \
resource-id="/subscriptions/703362b3-f278-4e4b-9179-c76eaf41ffc2/resourceGroups/myResourceGroup/providers/Mic\
rosoft.OperationalInsights/workspaces/centralTeamWorkspace" \
--performance-counters name="cloudTeamCoreCounters" counter-specifiers="\\\\Processor(_Total)\\\\% Processor Time" \
counter-specifiers="\\\\Memory\\\\Committed Bytes" counter-specifiers="\\\\LogicalDisk(_Total)\\\\Free Megabytes" \
counter-specifiers="\\\\PhysicalDisk(_Total)\\\\Avg. Disk Queue Length" sampling-frequency=15 \
streams="Microsoft-Perf" \
--performance-counters name="appTeamExtraCounters" \
counter-specifiers="\\\\Process(_Total)\\\\Thread Count" sampling-frequency=30 \
streams="Microsoft-Perf" \
--syslog name="cronSyslog" facility-names="cron" log-levels="Debug" log-levels="Critical" log-levels="Emergency" \
streams="Microsoft-Syslog" \
--syslog name="syslogBase" facility-names="syslog" log-levels="Alert" log-levels="Critical" log-levels="Emergency" \
streams="Microsoft-Syslog" \
--windows-event-logs name="cloudSecurityTeamEvents" streams="Microsoft-WindowsEvent" \
x-path-queries="Security!" \
--windows-event-logs name="appTeam1AppEvents" streams="Microsoft-WindowsEvent" \
x-path-queries="System![System[(Level = 1 or Level = 2 or Level = 3)]]" \
x-path-queries="Application!*[System[(Level = 1 or Level = 2 or Level = 3)]]"
--name "myCollectionRule" --rule-file "C:\samples\dcrEx1.json"
"""

helps['monitor data-collection rule update'] = """
Expand Down
1 change: 1 addition & 0 deletions src/monitor-control-service/azext_amcs/manual/_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def load_arguments(self, _):

with self.argument_context('monitor data-collection rule create') as c:
c.argument('data_collection_rule_name', id_part=None)
c.argument('rule_file', type=str, help='The json file for rule parameters.', required=True)
c.argument('location', arg_type=get_location_type(self.cli_ctx), required=False,
validator=get_default_location_from_resource_group)

Expand Down
57 changes: 40 additions & 17 deletions src/monitor-control-service/azext_amcs/manual/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,29 +94,52 @@ def _data_collection_rules_create(client,
def data_collection_rules_create(client,
resource_group_name,
data_collection_rule_name,
rule_file,
location=None,
tags=None,
description=None,
data_flows=None,
destinations__log_analytics=None,
destinations__azure_monitor_metrics=None,
data_sources__performance_counters=None,
data_sources__windows_event_logs=None,
data_sources__syslog=None,
data_sources__extensions=None):
description=None):
from azure.cli.core.util import get_file_json
from azure.cli.core.azclierror import FileOperationError, UnclassifiedUserFault
body = {}
body['location'] = location
body['tags'] = tags
body['description'] = description
body['data_flows'] = data_flows
body['destinations'] = {}
body['destinations']['log_analytics'] = destinations__log_analytics
body['destinations']['azure_monitor_metrics'] = destinations__azure_monitor_metrics
body['data_sources'] = {}
body['data_sources']['performance_counters'] = data_sources__performance_counters
body['data_sources']['windows_event_logs'] = data_sources__windows_event_logs
body['data_sources']['syslog'] = data_sources__syslog
body['data_sources']['extensions'] = data_sources__extensions
try:
json_data = get_file_json(rule_file)
except FileNotFoundError:
raise FileOperationError("No such file: " + str(rule_file))
except IsADirectoryError:
raise FileOperationError("Is a directory: " + str(rule_file))
except PermissionError:
raise FileOperationError("Permission denied: " + str(rule_file))
except OSError as e:
raise UnclassifiedUserFault(e)
for key_prop in json_data:
if key_prop == 'properties':
data = json_data['properties']
else:
data = json_data
for key in data:
if key == 'dataSources':
body['data_sources'] = {}
for key_ds in data['dataSources']:
if key_ds == 'performanceCounters':
body['data_sources']['performance_counters'] = data['dataSources']['performanceCounters']
if key_ds == 'windowsEventLogs':
body['data_sources']['windows_event_logs'] = data['dataSources']['windowsEventLogs']
if key_ds == 'syslog':
body['data_sources']['syslog'] = data['dataSources']['syslog']
if key_ds == 'extensions':
body['data_sources']['extensions'] = data['dataSources']['extensions']
if key == 'destinations':
body['destinations'] = {}
for key_de in data['destinations']:
if key_de == 'logAnalytics':
body['destinations']['log_analytics'] = data['destinations']['logAnalytics']
if key_de == 'azureMonitorMetrics':
body['destinations']['azure_monitor_metrics'] = data['destinations']['azureMonitorMetrics']
if key == 'dataFlows':
body['data_flows'] = data['dataFlows']
return _data_collection_rules_create(client,
resource_group_name=resource_group_name,
data_collection_rule_name=data_collection_rule_name,
Expand Down
116 changes: 116 additions & 0 deletions src/monitor-control-service/azext_amcs/tests/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# coding=utf-8
# --------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
#
# Code generated by Microsoft (R) AutoRest Code Generator.
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------
import inspect
import logging
import os
import sys
import traceback
import datetime as dt

from azure.core.exceptions import AzureError
from azure.cli.testsdk.exceptions import CliTestError, CliExecutionError, JMESPathCheckAssertionError


logger = logging.getLogger('azure.cli.testsdk')
logger.addHandler(logging.StreamHandler())
__path__ = __import__('pkgutil').extend_path(__path__, __name__)
exceptions = []
test_map = dict()
SUCCESSED = "successed"
FAILED = "failed"


def try_manual(func):
def import_manual_function(origin_func):
from importlib import import_module
decorated_path = inspect.getfile(origin_func).lower()
module_path = __path__[0].lower()
if not decorated_path.startswith(module_path):
raise Exception("Decorator can only be used in submodules!")
manual_path = os.path.join(
decorated_path[module_path.rfind(os.path.sep) + 1:])
manual_file_path, manual_file_name = os.path.split(manual_path)
module_name, _ = os.path.splitext(manual_file_name)
manual_module = "..manual." + \
".".join(manual_file_path.split(os.path.sep) + [module_name, ])
return getattr(import_module(manual_module, package=__name__), origin_func.__name__)

def get_func_to_call():
func_to_call = func
try:
func_to_call = import_manual_function(func)
logger.info("Found manual override for %s(...)", func.__name__)
except (ImportError, AttributeError):
pass
return func_to_call

def wrapper(*args, **kwargs):
func_to_call = get_func_to_call()
logger.info("running %s()...", func.__name__)
try:
test_map[func.__name__] = dict()
test_map[func.__name__]["result"] = SUCCESSED
test_map[func.__name__]["error_message"] = ""
test_map[func.__name__]["error_stack"] = ""
test_map[func.__name__]["error_normalized"] = ""
test_map[func.__name__]["start_dt"] = dt.datetime.utcnow()
ret = func_to_call(*args, **kwargs)
except (AssertionError, AzureError, CliTestError, CliExecutionError, SystemExit,
JMESPathCheckAssertionError) as e:
use_exception_cache = os.getenv("TEST_EXCEPTION_CACHE")
if use_exception_cache is None or use_exception_cache.lower() != "true":
raise
test_map[func.__name__]["end_dt"] = dt.datetime.utcnow()
test_map[func.__name__]["result"] = FAILED
test_map[func.__name__]["error_message"] = str(e).replace("\r\n", " ").replace("\n", " ")[:500]
test_map[func.__name__]["error_stack"] = traceback.format_exc().replace(
"\r\n", " ").replace("\n", " ")[:500]
logger.info("--------------------------------------")
logger.info("step exception: %s", e)
logger.error("--------------------------------------")
logger.error("step exception in %s: %s", func.__name__, e)
logger.info(traceback.format_exc())
exceptions.append((func.__name__, sys.exc_info()))
else:
test_map[func.__name__]["end_dt"] = dt.datetime.utcnow()
return ret

if inspect.isclass(func):
return get_func_to_call()
return wrapper


def calc_coverage(filename):
filename = filename.split(".")[0]
coverage_name = filename + "_coverage.md"
with open(coverage_name, "w") as f:
f.write("|Scenario|Result|ErrorMessage|ErrorStack|ErrorNormalized|StartDt|EndDt|\n")
total = len(test_map)
covered = 0
for k, v in test_map.items():
if not k.startswith("step_"):
total -= 1
continue
if v["result"] == SUCCESSED:
covered += 1
f.write("|{step_name}|{result}|{error_message}|{error_stack}|{error_normalized}|{start_dt}|"
"{end_dt}|\n".format(step_name=k, **v))
f.write("Coverage: {}/{}\n".format(covered, total))
print("Create coverage\n", file=sys.stderr)


def raise_if():
if exceptions:
if len(exceptions) <= 1:
raise exceptions[0][1][1]
message = "{}\nFollowed with exceptions in other steps:\n".format(str(exceptions[0][1][1]))
message += "\n".join(["{}: {}".format(h[0], h[1][1]) for h in exceptions[1:]])
raise exceptions[0][1][0](message).with_traceback(exceptions[0][1][2])
Loading

0 comments on commit cff8558

Please sign in to comment.