Add Azure Monitor output plugin (#4089)

Gopkg.toml

@@ -221,3 +221,7 @@
 [[override]]
   source = "https://github.com/fsnotify/fsnotify/archive/v1.4.7.tar.gz"
   name = "gopkg.in/fsnotify.v1"
+
+[[constraint]]
+  name = "github.com/Azure/go-autorest"
+  version = "10.12.0"

docs/LICENSE_OF_DEPENDENCIES.md

@@ -8,6 +8,7 @@ following works:
 - github.com/aerospike/aerospike-client-go [APACHE](https://github.com/aerospike/aerospike-client-go/blob/master/LICENSE)
 - github.com/amir/raidman [PUBLIC DOMAIN](https://github.com/amir/raidman/blob/master/UNLICENSE)
 - github.com/armon/go-metrics [MIT](https://github.com/armon/go-metrics/blob/master/LICENSE)
+- github.com/Azure/go-autorest [APACHE](https://github.com/Azure/go-autorest/blob/master/LICENSE)
 - github.com/aws/aws-sdk-go [APACHE](https://github.com/aws/aws-sdk-go/blob/master/LICENSE.txt)
 - github.com/beorn7/perks [MIT](https://github.com/beorn7/perks/blob/master/LICENSE)
 - github.com/boltdb/bolt [MIT](https://github.com/boltdb/bolt/blob/master/LICENSE)
@@ -19,6 +20,7 @@ following works:
 - github.com/couchbase/goutils [MIT](https://github.com/couchbase/go-couchbase/blob/master/LICENSE)
 - github.com/dancannon/gorethink [APACHE](https://github.com/dancannon/gorethink/blob/master/LICENSE)
 - github.com/davecgh/go-spew [ISC](https://github.com/davecgh/go-spew/blob/master/LICENSE)
+- github.com/dimchansky/utfbom [APACHE](https://github.com/dimchansky/utfbom/blob/master/LICENSE)
 - github.com/docker/docker [APACHE](https://github.com/docker/docker/blob/master/LICENSE)
 - github.com/docker/cli [APACHE](https://github.com/docker/cli/blob/master/LICENSE)
 - github.com/eapache/go-resiliency [MIT](https://github.com/eapache/go-resiliency/blob/master/LICENSE)

internal/models/running_output.go

@@ -114,6 +114,13 @@ func (ro *RunningOutput) AddMetric(m telegraf.Metric) {
 		m, _ = metric.New(name, tags, fields, t, tp)
 	}
 
+	if output, ok := ro.Output.(telegraf.AggregatingOutput); ok {
+		ro.Lock()
+		defer ro.Unlock()
+		output.Add(m)
+		return
+	}
+
 	ro.metrics.Add(m)
 	if ro.metrics.Len() == ro.MetricBatchSize {
 		batch := ro.metrics.Batch(ro.MetricBatchSize)
@@ -127,6 +134,12 @@ func (ro *RunningOutput) AddMetric(m telegraf.Metric) {
 
 // Write writes all cached points to this output.
 func (ro *RunningOutput) Write() error {
+	if output, ok := ro.Output.(telegraf.AggregatingOutput); ok {
+		metrics := output.Push()
+		ro.metrics.Add(metrics...)
+		output.Reset()
+	}
+
 	nFails, nMetrics := ro.failMetrics.Len(), ro.metrics.Len()
 	ro.BufferSize.Set(int64(nFails + nMetrics))
 	log.Printf("D! Output [%s] buffer fullness: %d / %d metrics. ",

output.go

@@ -13,6 +13,12 @@ type Output interface {
 	Write(metrics []Metric) error
 }
 
+type AggregatingOutput interface {
+	Add(in Metric)
+	Push() []Metric
+	Reset()
+}
+
 type ServiceOutput interface {
 	// Connect to the Output
 	Connect() error

plugins/outputs/all/all.go

@@ -4,6 +4,7 @@ import (
 	_ "github.com/influxdata/telegraf/plugins/outputs/amon"
 	_ "github.com/influxdata/telegraf/plugins/outputs/amqp"
 	_ "github.com/influxdata/telegraf/plugins/outputs/application_insights"
+	_ "github.com/influxdata/telegraf/plugins/outputs/azure_monitor"
 	_ "github.com/influxdata/telegraf/plugins/outputs/cloudwatch"
 	_ "github.com/influxdata/telegraf/plugins/outputs/cratedb"
 	_ "github.com/influxdata/telegraf/plugins/outputs/datadog"

plugins/outputs/azure_monitor/README.md

@@ -0,0 +1,139 @@
+# Azure Monitor
+
+This plugin will send custom metrics to Azure Monitor. Azure Monitor has a
+metric resolution of one minute. To handle this in Telegraf, the Azure Monitor
+output plugin will automatically aggregates metrics into one minute buckets,
+which are then sent to Azure Monitor on every flush interval.
+
+The metrics from each input plugin will be written to a separate Azure Monitor
+namespace, prefixed with `Telegraf/` by default. The field name for each
+metric is written as the Azure Monitor metric name. All field values are
+written as a summarized set that includes: min, max, sum, count. Tags are
+written as a dimension on each Azure Monitor metric.
+
+Since Azure Monitor only accepts numeric values, string-typed fields are
+dropped by default. There is a configuration option (`strings_as_dimensions`)
+to retain fields that contain strings as extra dimensions. Azure Monitor
+allows a maximum of 10 dimensions per metric so any dimensions over that
+amount will be deterministically dropped.
+
+### Configuration:
+
+```toml
+[[outputs.azure_monitor]]
+  ## Timeout for HTTP writes.
+  # timeout = "20s"
+
+  ## Set the namespace prefix, defaults to "Telegraf/<input-name>".
+  # namespace_prefix = "Telegraf/"
+
+  ## Azure Monitor doesn't have a string value type, so convert string
+  ## fields to dimensions (a.k.a. tags) if enabled. Azure Monitor allows
+  ## a maximum of 10 dimensions so Telegraf will only send the first 10
+  ## alphanumeric dimensions.
+  # strings_as_dimensions = false
+
+  ## Both region and resource_id must be set or be available via the
+  ## Instance Metadata service on Azure Virtual Machines.
+  #
+  ## Azure Region to publish metrics against.
+  ##   ex: region = "southcentralus"
+  # region = ""
+  #
+  ## The Azure Resource ID against which metric will be logged, e.g.
+  ##   ex: resource_id = "/subscriptions/<subscription_id>/resourceGroups/<resource_group>/providers/Microsoft.Compute/virtualMachines/<vm_name>"
+  # resource_id = ""
+```
+
+### Setup
+
+1. [Register the `microsoft.insights` resource provider in your Azure subscription][resource provider].
+2. If using Managed Service Identities to authenticate an Azure VM,
+   [enable system-assigned managed identity][enable msi].
+2. Use a region that supports Azure Monitor Custom Metrics,
+   For regions with Custom Metrics support, an endpoint will be available with
+   the format `https://<region>.monitoring.azure.com`. The following regions
+   are currently known to be supported:
+    - East US (eastus)
+    - West US 2 (westus2)
+    - South Central US (southcentralus)
+    - West Central US (westcentralus)
+    - North Europe (northeurope)
+    - West Europe (westeurope)
+    - Southeast Asia (southeastasia)
+
+[resource provider]: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-manager-supported-services
+[enable msi]: https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/qs-configure-portal-windows-vm
+
+### Region and Resource ID
+
+The plugin will attempt to discover the region and resource ID using the Azure
+VM Instance Metadata service. If Telegraf is not running on a virtual machine
+or the VM Instance Metadata service is not available, the following variables
+are required for the output to function.
+
+* region
+* resource_id
+
+### Authentication
+
+This plugin uses one of several different types of authenticate methods. The
+preferred authentication methods are different from the *order* in which each
+authentication is checked. Here are the preferred authentication methods:
+
+1. Managed Service Identity (MSI) token
+    - This is the prefered authentication method. Telegraf will automatically
+      authenticate using this method when running on Azure VMs.
+2. AAD Application Tokens (Service Principals)
+    - Primarily useful if Telegraf is writing metrics for other resources.
+      [More information][principal].
+    - A Service Principal or User Principal needs to be assigned the `Monitoring
+      Contributor` roles.
+3. AAD User Tokens (User Principals)
+    - Allows Telegraf to authenticate like a user. It is best to use this method
+      for development.
+
+[principal]: https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-application-objects
+
+The plugin will authenticate using the first available of the
+following configurations:
+
+1. **Client Credentials**: Azure AD Application ID and Secret.
+
+    Set the following Telegraf configuration variables:
+
+    - `azure_tenant_id`: Specifies the Tenant to which to authenticate.
+    - `azure_client_id`: Specifies the app client ID to use.
+    - `azure_client_secret`: Specifies the app secret to use.
+
+    Or set the following environment variables:
+
+    - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
+    - `AZURE_CLIENT_ID`: Specifies the app client ID to use.
+    - `AZURE_CLIENT_SECRET`: Specifies the app secret to use.
+
+2. **Client Certificate**: Azure AD Application ID and X.509 Certificate.
+
+    - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
+    - `AZURE_CLIENT_ID`: Specifies the app client ID to use.
+    - `AZURE_CERTIFICATE_PATH`: Specifies the certificate Path to use.
+    - `AZURE_CERTIFICATE_PASSWORD`: Specifies the certificate password to use.
+
+3. **Resource Owner Password**: Azure AD User and Password. This grant type is
+   *not recommended*, use device login instead if you need interactive login.
+
+    - `AZURE_TENANT_ID`: Specifies the Tenant to which to authenticate.
+    - `AZURE_CLIENT_ID`: Specifies the app client ID to use.
+    - `AZURE_USERNAME`: Specifies the username to use.
+    - `AZURE_PASSWORD`: Specifies the password to use.
+
+4. **Azure Managed Service Identity**: Delegate credential management to the
+   platform. Requires that code is running in Azure, e.g. on a VM. All
+   configuration is handled by Azure. See [Azure Managed Service Identity][msi]
+   for more details. Only available when using the [Azure Resource Manager][arm].
+
+[msi]: https://docs.microsoft.com/en-us/azure/active-directory/msi-overview
+[arm]: https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-overview
+
+**Note: As shown above, the last option (#4) is the preferred way to
+authenticate when running Telegraf on Azure VMs.