Skip to content

Create on metric description per cloudwatch stat #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aywrite wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Create on metric description per cloudwatch stat #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ee32ed5
Fix race condition when fetching resources (#4)
aywrite Apr 15, 2020
9b78ebe
One stat per metric
aywrite Apr 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions base/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,18 @@ import (
"github.com/aws/aws-sdk-go/service/cloudwatch"
)

type configMetric struct {
type ConfigMetric struct {
Copy link

@hound hound bot Apr 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported type ConfigMetric should have comment or be unexported

Copy link

@hound hound bot Apr 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported type ConfigMetric should have comment or be unexported

Copy link

@hound hound bot Apr 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported type ConfigMetric should have comment or be unexported

AWSMetric string `yaml:"metric"` // The Cloudwatch metric to use
Help string `yaml:"help"` // Custom help text for the generated metric
Dimensions []*cloudwatch.Dimension `yaml:"dimensions"` // The resource dimensions to generate individual series for (via labels)
Statistics []*string `yaml:"statistics"` // List of AWS statistics to use.
OutputName string `yaml:"output_name"` // Allows override of the generate metric name
RangeSeconds int64 `yaml:"range_seconds"` // How far back to request data for in seconds.
PeriodSeconds int64 `yaml:"period_seconds"` // Granularity of results from cloudwatch API.
RangeSeconds int64 `yaml:"range_seconds"` // How far back to request data for in seconds.
}

type metric struct {
Data map[string][]*configMetric `yaml:",omitempty,inline"` // Map from namespace to list of metrics to scrape.
Data map[string][]*ConfigMetric `yaml:",omitempty,inline"` // Map from namespace to list of metrics to scrape.
}

// Config represents the exporter configuration passed which is read at runtime from a YAML file.
Expand All @@ -40,17 +40,16 @@ type Config struct {
}

// ConstructMetrics generates a map of MetricDescriptions keyed by CloudWatch namespace using the defaults provided in Config.
func (c *Config) ConstructMetrics(defaults map[string]map[string]*MetricDescription) map[string][]*MetricDescription {
func (c *Config) ConstructMetrics(defaults map[string]map[string]*ConfigMetric) map[string][]*MetricDescription {
mds := make(map[string][]*MetricDescription)
for namespace, metrics := range c.Metrics.Data {

if len(metrics) <= 0 {
if len(metrics) == 0 {
if namespaceDefaults, ok := defaults[namespace]; ok {
for key, defaultMetric := range namespaceDefaults {
metrics = append(metrics, &configMetric{
metrics = append(metrics, &ConfigMetric{
AWSMetric: key,
OutputName: *defaultMetric.OutputName,
Help: *defaultMetric.Help,
OutputName: defaultMetric.OutputName,
Help: defaultMetric.Help,
PeriodSeconds: defaultMetric.PeriodSeconds,
RangeSeconds: defaultMetric.RangeSeconds,
Dimensions: defaultMetric.Dimensions,
Expand Down Expand Up @@ -83,24 +82,25 @@ func (c *Config) ConstructMetrics(defaults map[string]map[string]*MetricDescript
help := metric.Help
if help == "" {
if d, ok := defaults[namespace][metric.AWSMetric]; ok {
help = *d.Help
help = d.Help
}
}

// TODO handle dimensions
// TODO move metricName function here / apply to output name
// TODO one stat per metric
mds[namespace] = append(mds[namespace], &MetricDescription{
Help: &help,
OutputName: &name,
Dimensions: metric.Dimensions,
PeriodSeconds: period,
RangeSeconds: rangeSeconds,
Statistic: metric.Statistics,

Namespace: namespace,
AWSMetric: metric.AWSMetric,
})
for _, stat := range metric.Statistics {
mds[namespace] = append(mds[namespace], &MetricDescription{
Help: &help,
OutputName: &name,
Dimensions: metric.Dimensions,
PeriodSeconds: period,
RangeSeconds: rangeSeconds,
Statistic: *stat,

Namespace: namespace,
AWSMetric: metric.AWSMetric,
})
}
}
}
return mds
Expand Down
157 changes: 78 additions & 79 deletions base/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
log "github.com/sirupsen/logrus"
)

var alphaRegex, _ = regexp.Compile("[^a-zA-Z0-9]+")
var alphaRegex = regexp.MustCompile("[^a-zA-Z0-9]+")

// TagDescription represents an AWS tag key value pair
type TagDescription struct {
Expand All @@ -49,7 +49,7 @@ type MetricDescription struct {
Dimensions []*cloudwatch.Dimension
PeriodSeconds int64
RangeSeconds int64
Statistic []*string
Statistic string

timestamps map[awsLabels]*time.Time
mutex sync.RWMutex
Expand Down Expand Up @@ -89,9 +89,9 @@ type ResourceDescription struct {
Query []*cloudwatch.MetricDataQuery
}

func (md *MetricDescription) metricName(stat string) *string {
func (md *MetricDescription) metricName() *string {
suffix := ""
switch stat {
switch md.Statistic {
case "Average":
// For backwards compatibility we have to omit the _avg
suffix = ""
Expand Down Expand Up @@ -120,7 +120,7 @@ func (rd *RegionDescription) BuildARN(s *string, r *string) (string, error) {
return a.String(), nil
}

func (rd *RegionDescription) buildFilters() error {
func (rd *RegionDescription) saveFilters() {
filters := []*ec2.Filter{}
for _, tag := range rd.Tags {
f := &ec2.Filter{
Expand All @@ -130,16 +130,20 @@ func (rd *RegionDescription) buildFilters() error {
filters = append(filters, f)
}
rd.Filters = filters
return nil
}

func (rd *RegionDescription) saveAccountID() error {
session := iam.New(rd.Session)
input := iam.GetUserInput{}
user, err := session.GetUser(&input)
h.LogError(err)
if err != nil {
return err
}

a, err := arn.Parse(*user.User.Arn)
h.LogError(err)
if err != nil {
return err
}
rd.AccountID = &a.AccountID

return nil
Expand All @@ -153,13 +157,16 @@ func (rd *RegionDescription) Init(s *session.Session, td []*TagDescription, metr
rd.Tags = td

err := rd.saveAccountID()
h.LogErrorExit(err)
if err != nil {
return fmt.Errorf("error saving account id: %s", err)
}

err = rd.buildFilters()
h.LogErrorExit(err)
rd.saveFilters()

err = rd.CreateNamespaceDescriptions(metrics)
h.LogErrorExit(err)
if err != nil {
return fmt.Errorf("error creating namespaces: %s", err)
}

return nil
}
Expand Down Expand Up @@ -196,9 +203,9 @@ func (nd *NamespaceDescription) GatherMetrics(cw *cloudwatch.CloudWatch) {
for _, md := range nd.Metrics {
go func(md *MetricDescription) {
nd.Mutex.RLock()
result, err := md.getData(cw, nd.Resources, nd)
result, err := md.getData(cw, nd.Resources)
nd.Mutex.RUnlock()
h.LogError(err)
h.LogIfError(err)
md.saveData(result, *nd.Parent.Region)
}(md)
}
Expand Down Expand Up @@ -232,70 +239,63 @@ func (md *MetricDescription) BuildQuery(rds []*ResourceDescription) ([]*cloudwat
for _, rd := range rds {
dimensions := rd.Dimensions
dimensions = append(dimensions, md.Dimensions...)
for _, stat := range md.Statistic {
cm := &cloudwatch.MetricDataQuery{
Id: rd.queryID(*stat),
MetricStat: &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
MetricName: &md.AWSMetric,
Namespace: rd.Parent.Namespace,
Dimensions: dimensions,
},
Stat: stat,
Period: aws.Int64(md.PeriodSeconds),
cm := &cloudwatch.MetricDataQuery{
Id: rd.queryID(*&md.Statistic),
MetricStat: &cloudwatch.MetricStat{
Metric: &cloudwatch.Metric{
MetricName: &md.AWSMetric,
Namespace: rd.Parent.Namespace,
Dimensions: dimensions,
},
// We hardcode the label so that we can rely on the ordering in
// saveData.
Label: aws.String((&awsLabels{*stat, *rd.Name, *rd.ID, *rd.Type, *rd.Parent.Parent.Region}).String()),
ReturnData: aws.Bool(true),
}
query = append(query, cm)
Stat: &md.Statistic,
Period: aws.Int64(md.PeriodSeconds),
},
// We hardcode the label so that we can rely on the ordering in
// saveData.
Label: aws.String((&awsLabels{*rd.Name, *rd.ID, *rd.Type, *rd.Parent.Parent.Region}).String()),
ReturnData: aws.Bool(true),
}
query = append(query, cm)
}
return query, nil
}

type awsLabels struct {
statistic string
name string
id string
rType string
region string
name string
id string
rType string
region string
}

func (l *awsLabels) String() string {
return fmt.Sprintf("%s %s %s %s %s", l.statistic, l.name, l.id, l.rType, l.region)
return fmt.Sprintf("%s %s %s %s", l.name, l.id, l.rType, l.region)
}

func awsLabelsFromString(s string) (*awsLabels, error) {
stringLabels := strings.Split(s, " ")
if len(stringLabels) < 5 {
return nil, fmt.Errorf("expected at least five labels, got %s", s)
if len(stringLabels) < 4 {
return nil, fmt.Errorf("expected at least four labels, got %s", s)
}
labels := awsLabels{
statistic: stringLabels[len(stringLabels)-5],
name: stringLabels[len(stringLabels)-4],
id: stringLabels[len(stringLabels)-3],
rType: stringLabels[len(stringLabels)-2],
region: stringLabels[len(stringLabels)-1],
name: stringLabels[len(stringLabels)-4],
id: stringLabels[len(stringLabels)-3],
rType: stringLabels[len(stringLabels)-2],
region: stringLabels[len(stringLabels)-1],
}
return &labels, nil
}

func (md *MetricDescription) saveData(c *cloudwatch.GetMetricDataOutput, region string) {
newData := map[string][]*promMetric{}
for _, stat := range md.Statistic {
// pre-allocate in case the last resource for a stat goes away
newData[*stat] = []*promMetric{}
}
newData := []*promMetric{}

for _, data := range c.MetricDataResults {
if len(data.Values) <= 0 {
continue
}

labels, err := awsLabelsFromString(*data.Label)
if err != nil {
h.LogError(err)
h.LogIfError(err)
continue
}

Expand All @@ -305,7 +305,7 @@ func (md *MetricDescription) saveData(c *cloudwatch.GetMetricDataOutput, region
}

value := 0.0
switch labels.statistic {
switch md.Statistic {
case "Average":
value, err = h.Average(values)
case "Sum":
Expand All @@ -317,45 +317,44 @@ func (md *MetricDescription) saveData(c *cloudwatch.GetMetricDataOutput, region
case "SampleCount":
value, err = h.Sum(values)
default:
err = fmt.Errorf("unknown statistic type: %s", labels.statistic)
err = fmt.Errorf("unknown statistic type: %s", md.Statistic)
}
if err != nil {
h.LogError(err)
h.LogIfError(err)
continue
}

newData[labels.statistic] = append(newData[labels.statistic], &promMetric{value, []string{labels.name, labels.id, labels.rType, labels.region}})
newData = append(newData, &promMetric{value, []string{labels.name, labels.id, labels.rType, labels.region}})
}
for stat, data := range newData {
name := *md.metricName(stat)
opts := prometheus.Opts{
Name: name,
Help: *md.Help,
}
labels := []string{"name", "id", "type", "region"}

exporter.mutex.Lock()
if _, ok := exporter.data[name+region]; !ok {
if stat == "Sum" {
exporter.data[name+region] = NewBatchCounterVec(opts, labels)
} else {
exporter.data[name+region] = NewBatchGaugeVec(opts, labels)
}
}
exporter.mutex.Unlock()

exporter.mutex.RLock()
exporter.data[name+region].BatchUpdate(data)
exporter.mutex.RUnlock()
name := *md.metricName()
opts := prometheus.Opts{
Name: name,
Help: *md.Help,
}
labels := []string{"name", "id", "type", "region"}

exporter.mutex.Lock()
if _, ok := exporter.data[name+region]; !ok {
if md.Statistic == "Sum" {
exporter.data[name+region] = NewBatchCounterVec(opts, labels)
} else {
exporter.data[name+region] = NewBatchGaugeVec(opts, labels)
}
}
exporter.mutex.Unlock()

exporter.mutex.RLock()
exporter.data[name+region].BatchUpdate(newData)
exporter.mutex.RUnlock()
}

func (md *MetricDescription) filterValues(data *cloudwatch.MetricDataResult, labels *awsLabels) []*float64 {
// In the case of a counter we need to remove any datapoints which have
// already been added to the counter, otherwise if the poll intervals
// overlap we will double count some data.
values := data.Values
if labels.statistic == "Sum" {
if md.Statistic == "Sum" {
md.mutex.Lock()
defer md.mutex.Unlock()
if md.timestamps == nil {
Expand Down Expand Up @@ -448,12 +447,12 @@ func (rd *RegionDescription) TagsFound(tl interface{}) bool {
return false
}

func (md *MetricDescription) getData(cw *cloudwatch.CloudWatch, rds []*ResourceDescription, nd *NamespaceDescription) (*cloudwatch.GetMetricDataOutput, error) {
func (md *MetricDescription) getData(cw *cloudwatch.CloudWatch, rds []*ResourceDescription) (*cloudwatch.GetMetricDataOutput, error) {
query, err := md.BuildQuery(rds)
if len(query) < 1 {
if len(query) == 0 {
return &cloudwatch.GetMetricDataOutput{}, nil
}
h.LogError(err)
h.LogIfError(err)

end := time.Now().Round(5 * time.Minute)
start := end.Add(-time.Duration(md.RangeSeconds) * time.Second)
Expand All @@ -464,7 +463,7 @@ func (md *MetricDescription) getData(cw *cloudwatch.CloudWatch, rds []*ResourceD
MetricDataQueries: query,
}
result, err := cw.GetMetricData(&input)
h.LogError(err)
h.LogIfError(err)

return result, err
}
Expand Down
1 change: 0 additions & 1 deletion base/prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,5 +132,4 @@ func NewBatchCounterVec(opts prometheus.Opts, labels []string) *BatchCounterVec
labels,
),
}

}
Loading