Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support notifications on rollout events using notifications-engine #1175

Merged
merged 5 commits into from
Jun 18, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,9 @@ plugin-darwin: ui/dist
cp -r ui/dist/app/* server/static
CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-darwin-amd64 ./cmd/kubectl-argo-rollouts

.PHONY: plugin-docs
plugin-docs:
go run ./hack/gen-plugin-docs/main.go
.PHONY: docs
docs:
go run ./hack/gen-docs/main.go

.PHONY: builder-image
builder-image:
Expand Down Expand Up @@ -259,7 +259,7 @@ clean:
precheckin: test lint

.PHONY: release-docs
release-docs: plugin-docs
release-docs: docs
docker run --rm -it \
-v ~/.ssh:/root/.ssh \
-v ${CURRENT_DIR}:/docs \
Expand Down
10 changes: 10 additions & 0 deletions cmd/rollouts-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,13 @@ func newCommand() *cobra.Command {
clusterDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, metav1.NamespaceAll, instanceIDTweakListFunc)
// 3. We finally need an istio dynamic informer factory which does not use a tweakListFunc.
istioDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, namespace, nil)

controllerNamespaceInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
kubeClient,
resyncDuration,
kubeinformers.WithNamespace(defaults.Namespace()))
configMapInformer := controllerNamespaceInformerFactory.Core().V1().ConfigMaps()
secretInformer := controllerNamespaceInformerFactory.Core().V1().Secrets()
cm := controller.NewManager(
namespace,
kubeClient,
Expand All @@ -153,6 +160,8 @@ func newCommand() *cobra.Command {
tolerantinformer.NewTolerantClusterAnalysisTemplateInformer(clusterDynamicInformerFactory),
istioDynamicInformerFactory.ForResource(istioutil.GetIstioVirtualServiceGVR()).Informer(),
istioDynamicInformerFactory.ForResource(istioutil.GetIstioDestinationRuleGVR()).Informer(),
configMapInformer,
secretInformer,
resyncDuration,
instanceID,
metricsPort,
Expand All @@ -166,6 +175,7 @@ func newCommand() *cobra.Command {
clusterDynamicInformerFactory.Start(stopCh)
}
kubeInformerFactory.Start(stopCh)
controllerNamespaceInformerFactory.Start(stopCh)
jobInformerFactory.Start(stopCh)

// Check if Istio installed on cluster before starting dynamicInformerFactory
Expand Down
51 changes: 40 additions & 11 deletions controller/controller.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
package controller

import (
"encoding/json"
"fmt"
"time"

"github.com/argoproj/argo-rollouts/utils/queue"

"github.com/argoproj/notifications-engine/pkg/api"
"github.com/argoproj/notifications-engine/pkg/controller"
"github.com/pkg/errors"
smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
log "github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -28,11 +30,14 @@ import (
"github.com/argoproj/argo-rollouts/controller/metrics"
"github.com/argoproj/argo-rollouts/experiments"
"github.com/argoproj/argo-rollouts/ingress"
"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned"
rolloutscheme "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/scheme"
informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1"
"github.com/argoproj/argo-rollouts/rollout"
"github.com/argoproj/argo-rollouts/service"
"github.com/argoproj/argo-rollouts/utils/defaults"
"github.com/argoproj/argo-rollouts/utils/queue"
"github.com/argoproj/argo-rollouts/utils/record"
)

Expand Down Expand Up @@ -61,12 +66,13 @@ const (

// Manager is the controller implementation for Argo-Rollout resources
type Manager struct {
metricsServer *metrics.MetricsServer
rolloutController *rollout.Controller
experimentController *experiments.Controller
analysisController *analysis.Controller
serviceController *service.Controller
ingressController *ingress.Controller
metricsServer *metrics.MetricsServer
rolloutController *rollout.Controller
experimentController *experiments.Controller
analysisController *analysis.Controller
serviceController *service.Controller
ingressController *ingress.Controller
notificationsController controller.NotificationController

rolloutSynced cache.InformerSynced
experimentSynced cache.InformerSynced
Expand All @@ -77,6 +83,8 @@ type Manager struct {
ingressSynced cache.InformerSynced
jobSynced cache.InformerSynced
replicasSetSynced cache.InformerSynced
configMapSynced cache.InformerSynced
secretSynced cache.InformerSynced

rolloutWorkqueue workqueue.RateLimitingInterface
serviceWorkqueue workqueue.RateLimitingInterface
Expand Down Expand Up @@ -110,6 +118,8 @@ func NewManager(
clusterAnalysisTemplateInformer informers.ClusterAnalysisTemplateInformer,
istioVirtualServiceInformer cache.SharedIndexInformer,
istioDestinationRuleInformer cache.SharedIndexInformer,
configMapInformer coreinformers.ConfigMapInformer,
secretInformer coreinformers.SecretInformer,
resyncPeriod time.Duration,
instanceID string,
metricsPort int,
Expand Down Expand Up @@ -139,8 +149,22 @@ func NewManager(
ingressWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Ingresses")

refResolver := rollout.NewInformerBasedWorkloadRefResolver(namespace, dynamicclientset, discoveryClient, rolloutWorkqueue, rolloutsInformer.Informer())

recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal)
apiFactory := api.NewFactory(record.NewAPIFactorySettings(), defaults.Namespace(), secretInformer.Informer(), configMapInformer.Informer())
recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal, apiFactory)
notificationsController := controller.NewController(dynamicclientset.Resource(v1alpha1.RolloutGVR), rolloutsInformer.Informer(), apiFactory,
controller.WithToUnstructured(func(obj metav1.Object) (*unstructured.Unstructured, error) {
data, err := json.Marshal(obj)
if err != nil {
return nil, err
}
res := &unstructured.Unstructured{}
err = json.Unmarshal(data, res)
if err != nil {
return nil, err
}
return res, nil
}),
)

rolloutController := rollout.NewController(rollout.ControllerConfig{
Namespace: namespace,
Expand Down Expand Up @@ -229,6 +253,8 @@ func NewManager(
analysisTemplateSynced: analysisTemplateInformer.Informer().HasSynced,
clusterAnalysisTemplateSynced: clusterAnalysisTemplateInformer.Informer().HasSynced,
replicasSetSynced: replicaSetInformer.Informer().HasSynced,
configMapSynced: configMapInformer.Informer().HasSynced,
secretSynced: secretInformer.Informer().HasSynced,
rolloutWorkqueue: rolloutWorkqueue,
experimentWorkqueue: experimentWorkqueue,
analysisRunWorkqueue: analysisRunWorkqueue,
Expand All @@ -239,6 +265,7 @@ func NewManager(
ingressController: ingressController,
experimentController: experimentController,
analysisController: analysisController,
notificationsController: notificationsController,
dynamicClientSet: dynamicclientset,
refResolver: refResolver,
namespace: namespace,
Expand All @@ -260,7 +287,7 @@ func (c *Manager) Run(rolloutThreadiness, serviceThreadiness, ingressThreadiness

// Wait for the caches to be synced before starting workers
log.Info("Waiting for controller's informer caches to sync")
if ok := cache.WaitForCacheSync(stopCh, c.serviceSynced, c.ingressSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced); !ok {
if ok := cache.WaitForCacheSync(stopCh, c.serviceSynced, c.ingressSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced, c.configMapSynced, c.secretSynced); !ok {
return fmt.Errorf("failed to wait for caches to sync")
}
// only wait for cluster scoped informers to sync if we are running in cluster-wide mode
Expand All @@ -277,6 +304,8 @@ func (c *Manager) Run(rolloutThreadiness, serviceThreadiness, ingressThreadiness
go wait.Until(func() { c.ingressController.Run(ingressThreadiness, stopCh) }, time.Second, stopCh)
go wait.Until(func() { c.experimentController.Run(experimentThreadiness, stopCh) }, time.Second, stopCh)
go wait.Until(func() { c.analysisController.Run(analysisThreadiness, stopCh) }, time.Second, stopCh)
go wait.Until(func() { c.notificationsController.Run(rolloutThreadiness, stopCh) }, time.Second, stopCh)

log.Info("Started controller")

go func() {
Expand Down
134 changes: 134 additions & 0 deletions docs/features/notifications.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
# Notifications
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a note:

!!! important

    Available since v1.1

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

has v1.1 been released yet? i see that the release tags only go up to v1.0.2


!!! important
Available since v1.1

Argo Rollouts provides notifications powered by the [Notifications Engine](https://github.com/argoproj/notifications-engine).
Controller administrators can leverage flexible systems of triggers and templates to configure notifications requested
by the end users. The end-users can subscribe to the configured triggers by adding an annotation to the Rollout objects.

## Configuration

The trigger defines the condition when the notification should be sent as well as the notification content template.
Default Argo Rollouts comes with a list of built-in triggers that cover the most important events of Argo Rollout live-cycle.
Both triggers and templates are configured in the `argo-rollouts-notification-configmap` ConfigMap. In order to get
started quickly, you can use pre-configured notification templates defined in [notifications-install.yaml](../../manifests/notifications-install.yaml).

If you are leveraging Kustomize it is recommended to include [notifications-install.yaml](../../manifests/notifications-install.yaml) as a remote
resource into your `kustomization.yaml` file:

```yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
- https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml
- https://github.com/argoproj/argo-rollouts/releases/latest/download/notifications-install.yaml
```

After including the `argo-rollouts-notification-configmap` ConfigMap the administrator needs to configure integration
with the required notifications service such as Slack or MS Teams. An example below demonstrates Slack integration:

```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: argo-rollouts-notification-configmap
data:
service.slack: |
token: $slack-token
---
apiVersion: v1
kind: Secret
metadata:
name: argo-rollouts-notification-secret
stringData:
slack-token: <my-slack-token>
```

Learn more about supported services and configuration settings in services [documentation](../generated/notification-services/overview.md).
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This link is broken

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

link is referencing file is generated directory. So these files are not committed to git.


## Subscriptions

The end-users can start leveraging notifications using `notifications.argoproj.io/subscribe.<trigger>.<service>: <recipient>` annotation.
For example, the following annotation subscribes two Slack channels to notifications about canary rollout step completion:

```yaml
---
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
name: rollout-canary
annotations:
notifications.argoproj.io/subscribe.on-rollout-step-completed.slack: my-channel1;my-channel2

```

Annotation key consists of following parts:

* `on-rollout-step-completed` - trigger name
* `slack` - notification service name
* `my-channel1;my-channel2` - a semicolon separated list of recipients

## Customization

The Rollout administrator can customize the notifications by configuring notification templates and custom triggers
in `argo-rollouts-notification-configmap` ConfigMap.

### Templates

The notification template is a stateless function that generates the notification content. The template is leveraging
[html/template](https://golang.org/pkg/html/template/) golang package. It is meant to be reusable and can be referenced by multiple triggers.

An example below demonstrates a sample template:

```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: argo-rollouts-notification-configmap
data:
template.my-purple-template: |
message: |
Rollout {{.rollout.metadata.name}} has purple image
slack:
attachments: |
[{
"title": "{{ .rollout.metadata.name}}",
"color": "#800080"
}]
```

Each template has access to the following fields:

- `rollout` holds the rollout object.
- `recipient` holds the recipient name.

The `message` field of the template definition allows creating a basic notification for any notification service. You can
leverage notification service-specific fields to create complex notifications. For example using service-specific you can
add blocks and attachments for Slack, subject for Email or URL path, and body for Webhook. See corresponding service
[documentation](../generated/notification-services/overview.md) for more information.

### Custom Triggers

In addition to custom notification template administrator and configure custom triggers. Custom trigger defines the
condition when the notification should be sent. The definition includes name, condition and notification templates reference.
The condition is a predicate expression that returns true if the notification should be sent. The trigger condition
evaluation is powered by [antonmedv/expr](https://github.com/antonmedv/expr).
The condition language syntax is described at [Language-Definition.md](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md).

The trigger is configured in `argo-rollouts-notification-configmap` ConfigMap. For example the following trigger sends a notification
when rollout pod spec uses `argoproj/rollouts-demo:purple` image:

```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: argocd-notifications-cm
data:
trigger.on-purple: |
- send: [my-purple-template]
when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple'
```

Each condition might use several templates. Typically each template is responsible for generating a service-specific notification part.
20 changes: 20 additions & 0 deletions examples/notifications/configmap.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: argo-rollouts-notification-configmap
data:
service.slack: |
token: $slack-token
## Custom Trigger
trigger.on-purple: |
- send: [my-purple-template]
when: rollout.spec.template.spec.containers[0].image == 'argoproj/rollouts-demo:purple'
template.my-purple-template: |
message: |
Rollout {{.rollout.metadata.name}} has purple image
slack:
attachments: |
[{
"title": "{{ .rollout.metadata.name}}",
"color": "#800080"
}]
8 changes: 8 additions & 0 deletions examples/notifications/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
- ../../manifests/notifications

patchesStrategicMerge:
- configmap.yaml
Loading