Skip to content

Commit

Permalink
feat: implement a Keptn metrics provider
Browse files Browse the repository at this point in the history
Add a Keptn metrics provider for two resources:
* KeptnMetric: Verify the value of a single metric.
* Analysis (via AnalysisDefinition): Run a Keptn analysis over an
  interval validating SLOs.

Signed-off-by: Florian Bacher <florian.bacher@dynatrace.com>
  • Loading branch information
bacherfl authored and aryan9600 committed Jun 10, 2024
1 parent adc6059 commit 29c8bf5
Show file tree
Hide file tree
Showing 12 changed files with 741 additions and 9 deletions.
1 change: 1 addition & 0 deletions artifacts/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1298,6 +1298,7 @@ spec:
- newrelic
- graphite
- dynatrace
- keptn
address:
description: API address of this provider
type: string
Expand Down
13 changes: 13 additions & 0 deletions charts/flagger/templates/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,19 @@ rules:
- update
- patch
- delete
- apiGroups:
- metrics.keptn.sh
resources:
- keptnmetrics
- analyses
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- nonResourceURLs:
- /version
verbs:
Expand Down
1 change: 1 addition & 0 deletions cmd/flagger/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ func main() {
fromEnv("EVENT_WEBHOOK_URL", eventWebhook),
clusterName,
noCrossNamespaceRefs,
cfg,
)

// leader election context
Expand Down
62 changes: 62 additions & 0 deletions docs/gitbook/usage/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -668,3 +668,65 @@ Reference the template in the canary analysis:
max: 1000
interval: 1m
```
## Keptn
You can create custom metric checks using the Keptn provider.
This Provider allows to verify either the value of a single [KeptnMetric](https://keptn.sh/stable/docs/reference/crd-reference/metric/),
representing the value of a single metric,
or of a [Keptn Analysis](https://keptn.sh/stable/docs/reference/crd-reference/analysis/),
which provides a flexible grading logic for analysing and prioritising a number of different
metric values coming from different data sources.
This provider requires [Keptn](https://keptn.sh/stable/docs/installation/) to be installed in the cluster.
Example for a Keptn metric template:
```yaml
apiVersion: flagger.app/v1beta1
kind: MetricTemplate
metadata:
name: response-time
namespace: istio-system
spec:
provider:
type: keptn
query: keptnmetric/my-namespace/response-time/2m/reporter=destination
```
This will reference the `KeptnMetric` with the name `response-time` in
the namespace `my-namespace`, which could look like the following:

```yaml
apiVersion: metrics.keptn.sh/v1beta1
kind: KeptnMetric
metadata:
name: response-time
namespace: my-namespace
spec:
fetchIntervalSeconds: 10
provider:
name: my-prometheus-keptn-provider
query: histogram_quantile(0.8, sum by(le) (rate(http_server_request_latency_seconds_bucket{status_code='200',
job='simple-go-backend'}[5m[])))
```

The `query` contains the following components, which are divided by `/` characters:

```
<type>/<namespace>/<resource-name>/<timeframe>/<arguments>
```

* **type (required)**: Must be either `keptnmetric` or `analysis`.
* **namespace (required)**: The namespace of the referenced `KeptnMetric`/`AnalysisDefinition`.
* **resource-name (required):** The name of the referenced `KeptnMetric`/`AnalysisDefinition`.
* **timeframe (optional)**: The timeframe used for the Analysis.
This will usually be set to the same value as the analysis interval of a `Canary`.
Only relevant if the `type` is set to `analysis`.
* **arguments (optional)**: Arguments to be passed to an `Analysis`.
Arguments are passed as a list of key value pairs, separated by `;` characters,
e.g. `foo=bar;bar=foo`.
Only relevant if the `type` is set to `analysis`.

For the type `analysis`, the value returned by the provider is either `0`
(if the analysis failed), or `1` (analysis passed).
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/go-logr/zapr v1.3.0
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.12.4
github.com/hashicorp/go-retryablehttp v0.7.7
github.com/influxdata/influxdb-client-go/v2 v2.13.0
github.com/prometheus/client_golang v1.19.1
github.com/stretchr/testify v1.9.0
go.uber.org/zap v1.27.0
golang.org/x/sync v0.7.0
google.golang.org/api v0.182.0
google.golang.org/genproto v0.0.0-20240528184218-531527333157
google.golang.org/grpc v1.64.0
Expand Down Expand Up @@ -47,7 +49,6 @@ require (
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
Expand Down Expand Up @@ -77,7 +78,6 @@ require (
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/oauth2 v0.20.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
Expand Down
1 change: 1 addition & 0 deletions kustomize/base/flagger/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1298,6 +1298,7 @@ spec:
- newrelic
- graphite
- dynatrace
- keptn
address:
description: API address of this provider
type: string
Expand Down
13 changes: 13 additions & 0 deletions kustomize/base/flagger/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,19 @@ rules:
- update
- patch
- delete
- apiGroups:
- metrics.keptn.sh
resources:
- keptnmetrics
- analyses
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- nonResourceURLs:
- /version
verbs:
Expand Down
5 changes: 5 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"sync"
"time"

"k8s.io/client-go/rest"

"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -49,6 +51,7 @@ const controllerAgentName = "flagger"

// Controller is managing the canary objects and schedules canary deployments
type Controller struct {
kubeConfig *rest.Config
kubeClient kubernetes.Interface
flaggerClient clientset.Interface
flaggerInformers Informers
Expand Down Expand Up @@ -91,6 +94,7 @@ func NewController(
eventWebhook string,
clusterName string,
noCrossNamespaceRefs bool,
kubeConfig *rest.Config,
) *Controller {
logger.Debug("Creating event broadcaster")
flaggerscheme.AddToScheme(scheme.Scheme)
Expand All @@ -105,6 +109,7 @@ func NewController(
recorder.SetInfo(version, meshProvider)

ctrl := &Controller{
kubeConfig: kubeConfig,
kubeClient: kubeClient,
flaggerClient: flaggerClient,
flaggerInformers: flaggerInformers,
Expand Down
4 changes: 2 additions & 2 deletions pkg/controller/scheduler_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (c *Controller) checkMetricProviderAvailability(canary *flaggerv1.Canary) e
}

factory := providers.Factory{}
provider, err := factory.Provider(metric.Interval, template.Spec.Provider, credentials)
provider, err := factory.Provider(metric.Interval, template.Spec.Provider, credentials, c.kubeConfig)
if err != nil {
return fmt.Errorf("metric template %s.%s provider %s error: %v",
metric.TemplateRef.Name, namespace, template.Spec.Provider.Type, err)
Expand Down Expand Up @@ -260,7 +260,7 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool {
}

factory := providers.Factory{}
provider, err := factory.Provider(metric.Interval, template.Spec.Provider, credentials)
provider, err := factory.Provider(metric.Interval, template.Spec.Provider, credentials, c.kubeConfig)
if err != nil {
c.recordEventErrorf(canary, "Metric template %s.%s provider %s error: %v",
metric.TemplateRef.Name, namespace, template.Spec.Provider.Type, err)
Expand Down
9 changes: 4 additions & 5 deletions pkg/metrics/providers/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,12 @@ package providers

import (
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
rest "k8s.io/client-go/rest"
)

type Factory struct{}

func (factory Factory) Provider(
metricInterval string,
provider flaggerv1.MetricTemplateProvider,
credentials map[string][]byte,
) (Interface, error) {
func (factory Factory) Provider(metricInterval string, provider flaggerv1.MetricTemplateProvider, credentials map[string][]byte, config *rest.Config) (Interface, error) {
switch provider.Type {
case "prometheus":
return NewPrometheusProvider(provider, credentials)
Expand All @@ -44,6 +41,8 @@ func (factory Factory) Provider(
return NewInfluxdbProvider(provider, credentials)
case "dynatrace":
return NewDynatraceProvider(metricInterval, provider, credentials)
case "keptn":
return NewKeptnProvider(config)
default:
return NewPrometheusProvider(provider, credentials)
}
Expand Down
Loading

0 comments on commit 29c8bf5

Please sign in to comment.