fix: ensure aliyuncms metrics accept array, fix discovery (#10850)

(cherry picked from commit 03847d8)

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/Shopify/sarama v1.32.0
 	github.com/aerospike/aerospike-client-go/v5 v5.7.0
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
-	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529
 	github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
 	github.com/antchfx/jsonquery v1.1.5
 	github.com/antchfx/xmlquery v1.3.9

go.sum

@@ -266,8 +266,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C
 github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
-github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I=
-github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
+github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529 h1:qAt5MZ3Ukwx/JMAiaagQhNQMBZLcmJbnweBoK3EeHxI=
+github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU=
 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ=
 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=

plugins/inputs/aliyuncms/aliyuncms.go

@@ -258,10 +258,16 @@ func (s *AliyunCMS) Init() error {
 		if metric.Dimensions != "" {
 			metric.dimensionsUdObj = map[string]string{}
 			metric.dimensionsUdArr = []map[string]string{}
+
+			// first try to unmarshal as an object
 			err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdObj)
 			if err != nil {
+				// then try to unmarshal as an array
 				err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdArr)
-				return errors.Errorf("Can't parse dimensions (it is neither obj, nor array) %q :%v", metric.Dimensions, err)
+
+				if err != nil {
+					return errors.Errorf("cannot parse dimensions (neither obj, nor array) %q :%v", metric.Dimensions, err)
+				}
 			}
 		}
 	}
@@ -544,14 +550,15 @@ L:
 				metric.discoveryTags[instanceID][tagKey] = tagValue
 			}
 
-			//Preparing dimensions (first adding dimensions that comes from discovery data)
-			metric.requestDimensions = append(
-				metric.requestDimensions,
-				map[string]string{s.dimensionKey: instanceID})
+			//if no dimension configured in config file, use discovery data
+			if len(metric.dimensionsUdArr) == 0 && len(metric.dimensionsUdObj) == 0 {
+				metric.requestDimensions = append(
+					metric.requestDimensions,
+					map[string]string{s.dimensionKey: instanceID})
+			}
 		}
 
-		//Get final dimension (need to get full lis of
-		//what was provided in config + what comes from discovery
+		//add dimensions filter from config file
 		if len(metric.dimensionsUdArr) != 0 {
 			metric.requestDimensions = append(metric.requestDimensions, metric.dimensionsUdArr...)
 		}

plugins/inputs/aliyuncms/aliyuncms_test.go

@@ -199,6 +199,107 @@ func TestPluginInitialize(t *testing.T) {
 	}
 }
 
+func TestPluginMetricsInitialize(t *testing.T) {
+	var err error
+
+	plugin := new(AliyunCMS)
+	plugin.Log = testutil.Logger{Name: inputTitle}
+	plugin.Regions = []string{"cn-shanghai"}
+	plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.Regions)
+	if err != nil {
+		t.Fatalf("Can't create discovery tool object: %v", err)
+	}
+
+	httpResp := &http.Response{
+		StatusCode: 200,
+		Body: io.NopCloser(bytes.NewBufferString(
+			`{
+				"LoadBalancers":
+					{
+						"LoadBalancer": [
+ 							{"LoadBalancerId":"bla"}
+                        ]
+                    },
+				"TotalCount": 1,
+				"PageSize": 1,
+				"PageNumber": 1
+			}`)),
+	}
+	mockCli, err := getMockSdkCli(httpResp)
+	if err != nil {
+		t.Fatalf("Can't create mock sdk cli: %v", err)
+	}
+	plugin.dt.cli = map[string]aliyunSdkClient{plugin.Regions[0]: &mockCli}
+
+	tests := []struct {
+		name                string
+		project             string
+		accessKeyID         string
+		accessKeySecret     string
+		expectedErrorString string
+		regions             []string
+		discoveryRegions    []string
+		metrics             []*Metric
+	}{
+		{
+			name:            "Valid project",
+			project:         "acs_slb_dashboard",
+			regions:         []string{"cn-shanghai"},
+			accessKeyID:     "dummy",
+			accessKeySecret: "dummy",
+			metrics: []*Metric{
+				{
+					MetricNames: []string{},
+					Dimensions:  `{"instanceId": "i-abcdefgh123456"}`,
+				},
+			},
+		},
+		{
+			name:            "Valid project",
+			project:         "acs_slb_dashboard",
+			regions:         []string{"cn-shanghai"},
+			accessKeyID:     "dummy",
+			accessKeySecret: "dummy",
+			metrics: []*Metric{
+				{
+					MetricNames: []string{},
+					Dimensions:  `[{"instanceId": "p-example"},{"instanceId": "q-example"}]`,
+				},
+			},
+		},
+		{
+			name:                "Valid project",
+			project:             "acs_slb_dashboard",
+			regions:             []string{"cn-shanghai"},
+			accessKeyID:         "dummy",
+			accessKeySecret:     "dummy",
+			expectedErrorString: `cannot parse dimensions (neither obj, nor array) "[" :unexpected end of JSON input`,
+			metrics: []*Metric{
+				{
+					MetricNames: []string{},
+					Dimensions:  `[`,
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			plugin.Project = tt.project
+			plugin.AccessKeyID = tt.accessKeyID
+			plugin.AccessKeySecret = tt.accessKeySecret
+			plugin.Regions = tt.regions
+			plugin.Metrics = tt.metrics
+
+			if tt.expectedErrorString != "" {
+				require.EqualError(t, plugin.Init(), tt.expectedErrorString)
+			} else {
+				require.Equal(t, nil, plugin.Init())
+			}
+		})
+	}
+}
+
 func TestUpdateWindow(t *testing.T) {
 	duration, _ := time.ParseDuration("1m")
 	internalDuration := config.Duration(duration)

plugins/inputs/aliyuncms/discovery.go

@@ -3,7 +3,6 @@ package aliyuncms
 import (
 	"encoding/json"
 	"reflect"
-	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -91,7 +90,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
 	var (
 		dscReq                = map[string]discoveryRequest{}
 		cli                   = map[string]aliyunSdkClient{}
-		parseRootKey          = regexp.MustCompile(`Describe(.*)`)
 		responseRootKey       string
 		responseObjectIDKey   string
 		err                   error
@@ -112,12 +110,15 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
 		switch project {
 		case "acs_ecs_dashboard":
 			dscReq[region] = ecs.CreateDescribeInstancesRequest()
+			responseRootKey = "Instances"
 			responseObjectIDKey = "InstanceId"
 		case "acs_rds_dashboard":
 			dscReq[region] = rds.CreateDescribeDBInstancesRequest()
+			responseRootKey = "Items"
 			responseObjectIDKey = "DBInstanceId"
 		case "acs_slb_dashboard":
 			dscReq[region] = slb.CreateDescribeLoadBalancersRequest()
+			responseRootKey = "LoadBalancers"
 			responseObjectIDKey = "LoadBalancerId"
 		case "acs_memcache":
 			return nil, noDiscoverySupportErr
@@ -137,6 +138,7 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
 			//req.InitWithApiInfo("oss", "2014-08-15", "DescribeDBInstances", "oss", "openAPI")
 		case "acs_vpc_eip":
 			dscReq[region] = vpc.CreateDescribeEipAddressesRequest()
+			responseRootKey = "EipAddresses"
 			responseObjectIDKey = "AllocationId"
 		case "acs_kvstore":
 			return nil, noDiscoverySupportErr
@@ -235,35 +237,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred
 		return nil, errors.Errorf("Can't build discovery request for project: %q,\nregions: %v", project, regions)
 	}
 
-	//Getting response root key (if not set already). This is to be able to parse discovery responses
-	//As they differ per object type
-	//Discovery requests are of the same type per every region, so pick the first one
-	rpcReq, err := getRPCReqFromDiscoveryRequest(dscReq[regions[0]])
-	//This means that the discovery request is not of proper type/kind
-	if err != nil {
-		return nil, errors.Errorf("Can't parse rpc request object from  discovery request %v", dscReq[regions[0]])
-	}
-
-	/*
-		The action name is of the following format Describe<Project related title for managed instances>,
-		For example: DescribeLoadBalancers -> for SLB project, or DescribeInstances for ECS project
-		We will use it to construct root key name in the discovery API response.
-		It follows the following logic: for 'DescribeLoadBalancers' action in discovery request we get the response
-		in json of the following structure:
-		{
-			 ...
-			 "LoadBalancers": {
-				 "LoadBalancer": [ here comes objects, one per every instance]
-			}
-		}
-		As we can see, the root key is a part of action name, except first word (part) 'Describe'
-	*/
-	result := parseRootKey.FindStringSubmatch(rpcReq.GetActionName())
-	if result == nil || len(result) != 2 {
-		return nil, errors.Errorf("Can't parse the discovery response root key from request action name %q", rpcReq.GetActionName())
-	}
-	responseRootKey = result[1]
-
 	return &discoveryTool{
 		req:                dscReq,
 		cli:                cli,
@@ -313,9 +286,9 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse)
 			if !foundDataItem {
 				return nil, errors.Errorf("Didn't find array item in root key %q", key)
 			}
-		case "TotalCount":
+		case "TotalCount", "TotalRecordCount":
 			pdResp.totalCount = int(val.(float64))
-		case "PageSize":
+		case "PageSize", "PageRecordCount":
 			pdResp.pageSize = int(val.(float64))
 		case "PageNumber":
 			pdResp.pageNumber = int(val.(float64))