From bf3b566cda6c8f8503b527d1759ab01c245447df Mon Sep 17 00:00:00 2001 From: Joshua Powers Date: Tue, 12 Apr 2022 15:52:03 -0600 Subject: [PATCH] fix: ensure aliyuncms metrics accept array, fix discovery (#10850) (cherry picked from commit 03847d8cac00bed07944a6cd24f20055806ac420) --- go.mod | 2 +- go.sum | 4 +- plugins/inputs/aliyuncms/aliyuncms.go | 21 +++-- plugins/inputs/aliyuncms/aliyuncms_test.go | 101 +++++++++++++++++++++ plugins/inputs/aliyuncms/discovery.go | 39 ++------ 5 files changed, 124 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index 5de5bef3ef781..90766c6953b82 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/Shopify/sarama v1.32.0 github.com/aerospike/aerospike-client-go/v5 v5.7.0 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 - github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 + github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 github.com/antchfx/jsonquery v1.1.5 github.com/antchfx/xmlquery v1.3.9 diff --git a/go.sum b/go.sum index 5781935345482..d13c267af62ea 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:C github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529 h1:qAt5MZ3Ukwx/JMAiaagQhNQMBZLcmJbnweBoK3EeHxI= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= diff --git a/plugins/inputs/aliyuncms/aliyuncms.go b/plugins/inputs/aliyuncms/aliyuncms.go index 1dc20d7187853..380cbad01a023 100644 --- a/plugins/inputs/aliyuncms/aliyuncms.go +++ b/plugins/inputs/aliyuncms/aliyuncms.go @@ -258,10 +258,16 @@ func (s *AliyunCMS) Init() error { if metric.Dimensions != "" { metric.dimensionsUdObj = map[string]string{} metric.dimensionsUdArr = []map[string]string{} + + // first try to unmarshal as an object err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdObj) if err != nil { + // then try to unmarshal as an array err := json.Unmarshal([]byte(metric.Dimensions), &metric.dimensionsUdArr) - return errors.Errorf("Can't parse dimensions (it is neither obj, nor array) %q :%v", metric.Dimensions, err) + + if err != nil { + return errors.Errorf("cannot parse dimensions (neither obj, nor array) %q :%v", metric.Dimensions, err) + } } } } @@ -544,14 +550,15 @@ L: metric.discoveryTags[instanceID][tagKey] = tagValue } - //Preparing dimensions (first adding dimensions that comes from discovery data) - metric.requestDimensions = append( - metric.requestDimensions, - map[string]string{s.dimensionKey: instanceID}) + //if no dimension configured in config file, use discovery data + if len(metric.dimensionsUdArr) == 0 && len(metric.dimensionsUdObj) == 0 { + metric.requestDimensions = append( + metric.requestDimensions, + map[string]string{s.dimensionKey: instanceID}) + } } - //Get final dimension (need to get full lis of - //what was provided in config + what comes from discovery + //add dimensions filter from config file if len(metric.dimensionsUdArr) != 0 { metric.requestDimensions = append(metric.requestDimensions, metric.dimensionsUdArr...) } diff --git a/plugins/inputs/aliyuncms/aliyuncms_test.go b/plugins/inputs/aliyuncms/aliyuncms_test.go index 7e346a6ae9b8e..ed1c8d7e645cb 100644 --- a/plugins/inputs/aliyuncms/aliyuncms_test.go +++ b/plugins/inputs/aliyuncms/aliyuncms_test.go @@ -199,6 +199,107 @@ func TestPluginInitialize(t *testing.T) { } } +func TestPluginMetricsInitialize(t *testing.T) { + var err error + + plugin := new(AliyunCMS) + plugin.Log = testutil.Logger{Name: inputTitle} + plugin.Regions = []string{"cn-shanghai"} + plugin.dt, err = getDiscoveryTool("acs_slb_dashboard", plugin.Regions) + if err != nil { + t.Fatalf("Can't create discovery tool object: %v", err) + } + + httpResp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString( + `{ + "LoadBalancers": + { + "LoadBalancer": [ + {"LoadBalancerId":"bla"} + ] + }, + "TotalCount": 1, + "PageSize": 1, + "PageNumber": 1 + }`)), + } + mockCli, err := getMockSdkCli(httpResp) + if err != nil { + t.Fatalf("Can't create mock sdk cli: %v", err) + } + plugin.dt.cli = map[string]aliyunSdkClient{plugin.Regions[0]: &mockCli} + + tests := []struct { + name string + project string + accessKeyID string + accessKeySecret string + expectedErrorString string + regions []string + discoveryRegions []string + metrics []*Metric + }{ + { + name: "Valid project", + project: "acs_slb_dashboard", + regions: []string{"cn-shanghai"}, + accessKeyID: "dummy", + accessKeySecret: "dummy", + metrics: []*Metric{ + { + MetricNames: []string{}, + Dimensions: `{"instanceId": "i-abcdefgh123456"}`, + }, + }, + }, + { + name: "Valid project", + project: "acs_slb_dashboard", + regions: []string{"cn-shanghai"}, + accessKeyID: "dummy", + accessKeySecret: "dummy", + metrics: []*Metric{ + { + MetricNames: []string{}, + Dimensions: `[{"instanceId": "p-example"},{"instanceId": "q-example"}]`, + }, + }, + }, + { + name: "Valid project", + project: "acs_slb_dashboard", + regions: []string{"cn-shanghai"}, + accessKeyID: "dummy", + accessKeySecret: "dummy", + expectedErrorString: `cannot parse dimensions (neither obj, nor array) "[" :unexpected end of JSON input`, + metrics: []*Metric{ + { + MetricNames: []string{}, + Dimensions: `[`, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + plugin.Project = tt.project + plugin.AccessKeyID = tt.accessKeyID + plugin.AccessKeySecret = tt.accessKeySecret + plugin.Regions = tt.regions + plugin.Metrics = tt.metrics + + if tt.expectedErrorString != "" { + require.EqualError(t, plugin.Init(), tt.expectedErrorString) + } else { + require.Equal(t, nil, plugin.Init()) + } + }) + } +} + func TestUpdateWindow(t *testing.T) { duration, _ := time.ParseDuration("1m") internalDuration := config.Duration(duration) diff --git a/plugins/inputs/aliyuncms/discovery.go b/plugins/inputs/aliyuncms/discovery.go index a6fe5471beecf..c287d07a388c2 100644 --- a/plugins/inputs/aliyuncms/discovery.go +++ b/plugins/inputs/aliyuncms/discovery.go @@ -3,7 +3,6 @@ package aliyuncms import ( "encoding/json" "reflect" - "regexp" "strconv" "strings" "sync" @@ -91,7 +90,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred var ( dscReq = map[string]discoveryRequest{} cli = map[string]aliyunSdkClient{} - parseRootKey = regexp.MustCompile(`Describe(.*)`) responseRootKey string responseObjectIDKey string err error @@ -112,12 +110,15 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred switch project { case "acs_ecs_dashboard": dscReq[region] = ecs.CreateDescribeInstancesRequest() + responseRootKey = "Instances" responseObjectIDKey = "InstanceId" case "acs_rds_dashboard": dscReq[region] = rds.CreateDescribeDBInstancesRequest() + responseRootKey = "Items" responseObjectIDKey = "DBInstanceId" case "acs_slb_dashboard": dscReq[region] = slb.CreateDescribeLoadBalancersRequest() + responseRootKey = "LoadBalancers" responseObjectIDKey = "LoadBalancerId" case "acs_memcache": return nil, noDiscoverySupportErr @@ -137,6 +138,7 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred //req.InitWithApiInfo("oss", "2014-08-15", "DescribeDBInstances", "oss", "openAPI") case "acs_vpc_eip": dscReq[region] = vpc.CreateDescribeEipAddressesRequest() + responseRootKey = "EipAddresses" responseObjectIDKey = "AllocationId" case "acs_kvstore": return nil, noDiscoverySupportErr @@ -235,35 +237,6 @@ func newDiscoveryTool(regions []string, project string, lg telegraf.Logger, cred return nil, errors.Errorf("Can't build discovery request for project: %q,\nregions: %v", project, regions) } - //Getting response root key (if not set already). This is to be able to parse discovery responses - //As they differ per object type - //Discovery requests are of the same type per every region, so pick the first one - rpcReq, err := getRPCReqFromDiscoveryRequest(dscReq[regions[0]]) - //This means that the discovery request is not of proper type/kind - if err != nil { - return nil, errors.Errorf("Can't parse rpc request object from discovery request %v", dscReq[regions[0]]) - } - - /* - The action name is of the following format Describe, - For example: DescribeLoadBalancers -> for SLB project, or DescribeInstances for ECS project - We will use it to construct root key name in the discovery API response. - It follows the following logic: for 'DescribeLoadBalancers' action in discovery request we get the response - in json of the following structure: - { - ... - "LoadBalancers": { - "LoadBalancer": [ here comes objects, one per every instance] - } - } - As we can see, the root key is a part of action name, except first word (part) 'Describe' - */ - result := parseRootKey.FindStringSubmatch(rpcReq.GetActionName()) - if result == nil || len(result) != 2 { - return nil, errors.Errorf("Can't parse the discovery response root key from request action name %q", rpcReq.GetActionName()) - } - responseRootKey = result[1] - return &discoveryTool{ req: dscReq, cli: cli, @@ -313,9 +286,9 @@ func (dt *discoveryTool) parseDiscoveryResponse(resp *responses.CommonResponse) if !foundDataItem { return nil, errors.Errorf("Didn't find array item in root key %q", key) } - case "TotalCount": + case "TotalCount", "TotalRecordCount": pdResp.totalCount = int(val.(float64)) - case "PageSize": + case "PageSize", "PageRecordCount": pdResp.pageSize = int(val.(float64)) case "PageNumber": pdResp.pageNumber = int(val.(float64))