From 3e00ae1b09e7af4c6d538957883e49e45b6371b3 Mon Sep 17 00:00:00 2001 From: Matthew Magaldi Date: Mon, 23 Jul 2018 08:53:58 -0400 Subject: [PATCH] implementing signals as microservice deployments (#49) * implementing signals as microservice deployments, making signals resilient to pod failures and moving escalation logic after requeue failures * updating Makefile and CONTRIBUTING guide * fixing failing tests --- .travis.yml | 10 - CHANGELOG.md | 7 +- CONTRIBUTING.md | 5 +- DEPENDENCIES.md | 1 + Gopkg.lock | 213 ++++- Gopkg.toml | 5 +- Makefile | 65 +- README.md | 5 +- ROADMAP.md | 1 - cmd/sensor-controller/main.go | 6 +- common/common.go | 5 +- common/util.go | 56 +- common/util_test.go | 19 + controller/Dockerfile | 8 - controller/config.go | 10 +- controller/config_test.go | 4 + controller/controller.go | 53 +- controller/controller_test.go | 4 +- controller/operator.go | 42 +- controller/operator_test.go | 63 -- controller/plugin-manager.go | 98 --- controller/signal-manager.go | 63 ++ controller/signal.go | 256 +++--- docs/index.md | 5 +- docs/plugin-guide.md | 27 - docs/quickstart.md | 2 +- docs/signal-guide.md | 17 +- examples/webhook-sensor.yaml | 8 +- hack/generate-proto.sh | 13 +- .../manifests/argo-events-cluster-roles.yaml | 4 +- hack/k8s/manifests/argo-events-sa.yaml | 2 +- .../sensor-controller-configmap.yaml | 1 - hack/k8s/manifests/services/artifact.yaml | 37 + hack/k8s/manifests/services/calendar.yaml | 37 + hack/k8s/manifests/services/resource.yaml | 37 + hack/k8s/manifests/services/stream.yaml | 37 + hack/k8s/manifests/services/webhook.yaml | 42 + pkg/apis/sensor/v1alpha1/generated.pb.go | 403 +++++----- pkg/apis/sensor/v1alpha1/generated.proto | 3 - pkg/apis/sensor/v1alpha1/openapi_generated.go | 7 - pkg/apis/sensor/v1alpha1/types.go | 3 - sdk/doc.go | 2 + {shared => sdk}/errors.go | 2 +- sdk/interface.go | 59 ++ sdk/micro_client.go | 52 ++ sdk/micro_server.go | 88 +++ sdk/signal.micro.go | 192 +++++ sdk/signal.pb.go | 737 ++++++++++++++++++ sdk/signal.proto | 31 + shared/doc.go | 4 + shared/grpc.go | 76 -- shared/interface.go | 86 -- shared/micro.go | 40 + shared/rpc.go | 46 -- shared/signal.pb.go | 184 ----- shared/signal.proto | 16 - signals/README.md | 56 ++ signals/artifact/micro/Dockerfile | 3 + signals/artifact/micro/artifact_service.go | 58 ++ signals/artifact/s3.go | 116 ++- signals/artifact/s3_test.go | 16 + signals/calendar/calendar.go | 130 +++ .../{signal_test.go => calendar_test.go} | 47 +- signals/calendar/micro/Dockerfile | 3 + signals/calendar/micro/calendar_service.go | 35 + signals/calendar/signal.go | 136 ---- signals/resource/micro/Dockerfile | 3 + signals/resource/micro/resource_signal.go | 45 ++ signals/resource/{signal.go => resource.go} | 108 +-- signals/stream/README.md | 25 +- signals/stream/builtin/amqp/amqp.go | 155 ++-- signals/stream/builtin/amqp/micro/Dockerfile | 3 + .../stream/builtin/amqp/micro/amqp_service.go | 35 + signals/stream/builtin/doc.go | 2 +- signals/stream/builtin/kafka/kafka.go | 154 ++-- signals/stream/builtin/kafka/kafka_test.go | 66 +- signals/stream/builtin/kafka/micro/Dockerfile | 3 + .../builtin/kafka/micro/kafka_service.go | 35 + signals/stream/builtin/mqtt/micro/Dockerfile | 3 + .../stream/builtin/mqtt/micro/mqtt_service.go | 35 + signals/stream/builtin/mqtt/mqtt.go | 118 +-- signals/stream/builtin/mqtt/mqtt_test.go | 18 +- signals/stream/builtin/nats/micro/Dockerfile | 3 + .../stream/builtin/nats/micro/nats_service.go | 35 + signals/stream/builtin/nats/nats.go | 114 ++- signals/stream/builtin/nats/nats_test.go | 35 +- signals/stream/custom/doc.go | 2 +- signals/webhook/micro/Dockerfile | 3 + signals/webhook/micro/client/Dockerfile | 3 + signals/webhook/micro/client/main.go | 82 ++ signals/webhook/micro/webhook_service.go | 35 + signals/webhook/signal.go | 110 --- signals/webhook/webhook.go | 117 +++ .../{signal_test.go => webhook_test.go} | 23 +- 94 files changed, 3419 insertions(+), 1850 deletions(-) delete mode 100644 .travis.yml delete mode 100644 controller/plugin-manager.go create mode 100644 controller/signal-manager.go delete mode 100644 docs/plugin-guide.md create mode 100644 hack/k8s/manifests/services/artifact.yaml create mode 100644 hack/k8s/manifests/services/calendar.yaml create mode 100644 hack/k8s/manifests/services/resource.yaml create mode 100644 hack/k8s/manifests/services/stream.yaml create mode 100644 hack/k8s/manifests/services/webhook.yaml create mode 100644 sdk/doc.go rename {shared => sdk}/errors.go (95%) create mode 100644 sdk/interface.go create mode 100644 sdk/micro_client.go create mode 100644 sdk/micro_server.go create mode 100644 sdk/signal.micro.go create mode 100644 sdk/signal.pb.go create mode 100644 sdk/signal.proto create mode 100644 shared/doc.go delete mode 100644 shared/grpc.go delete mode 100644 shared/interface.go create mode 100644 shared/micro.go delete mode 100644 shared/rpc.go delete mode 100644 shared/signal.pb.go delete mode 100644 shared/signal.proto create mode 100644 signals/README.md create mode 100644 signals/artifact/micro/Dockerfile create mode 100644 signals/artifact/micro/artifact_service.go create mode 100644 signals/calendar/calendar.go rename signals/calendar/{signal_test.go => calendar_test.go} (69%) create mode 100644 signals/calendar/micro/Dockerfile create mode 100644 signals/calendar/micro/calendar_service.go delete mode 100644 signals/calendar/signal.go create mode 100644 signals/resource/micro/Dockerfile create mode 100644 signals/resource/micro/resource_signal.go rename signals/resource/{signal.go => resource.go} (76%) create mode 100644 signals/stream/builtin/amqp/micro/Dockerfile create mode 100644 signals/stream/builtin/amqp/micro/amqp_service.go create mode 100644 signals/stream/builtin/kafka/micro/Dockerfile create mode 100644 signals/stream/builtin/kafka/micro/kafka_service.go create mode 100644 signals/stream/builtin/mqtt/micro/Dockerfile create mode 100644 signals/stream/builtin/mqtt/micro/mqtt_service.go create mode 100644 signals/stream/builtin/nats/micro/Dockerfile create mode 100644 signals/stream/builtin/nats/micro/nats_service.go create mode 100644 signals/webhook/micro/Dockerfile create mode 100644 signals/webhook/micro/client/Dockerfile create mode 100644 signals/webhook/micro/client/main.go create mode 100644 signals/webhook/micro/webhook_service.go delete mode 100644 signals/webhook/signal.go create mode 100644 signals/webhook/webhook.go rename signals/webhook/{signal_test.go => webhook_test.go} (78%) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e229f16082ba..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,10 +0,0 @@ -language: go -go: - - "1.10" - -script: - - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - - dep version - - dep ensure -vendor-only - - make test coverage - - make controller executor-job diff --git a/CHANGELOG.md b/CHANGELOG.md index 20609a2e077c..f3d090648dfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog -## 1.0.0 (tbd) +## v0.5-beta1 (tbd) ++ Signals as separate deployments [#49](https://github.com/argoproj/argo-events/pull/49) ++ Fixed code-gen bug [#46](https://github.com/argoproj/argo-events/issues/46) ++ Filters for signals [#26](https://github.com/argoproj/argo-events/issues/26) + +## v0.5-alpha1 + Initial release \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61705e177812..bec15e1201a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,8 +26,5 @@ See the [quickstart guide](./docs/quickstart.md) for help in getting started. ## Changing Types If you're making a change to the `pkg/apis/sensor/v1alpha1` package, please ensure you re-run the K8 code-generator scripts found in the `/hack` folder. First, ensure you have the `generate-groups.sh` script at the path: `vendor/k8s.io/code-generator/`. Next run the following commands in order: ``` -$ hack/update-codegen.sh -$ hack/verify-codegen.sh -$ hack/update-codeapigen.sh -$ hack/generate-proto.sh +$ make codegen ``` diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index 23591b698930..a8277691f64b 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -7,6 +7,7 @@ | k8s.io/apimachinery | Apache-2.0 | | k8s.io/code-generator | Apache-2.0 | | k8s.io/client-go | Apache-2.0 | +| micro/go-micro | Apache-2.0 | | eclipse/paho.mqtt.golang | EPL-1.0 | | ghodss/yaml | Apache-2.0 | | minio/minio-go | Apache-2.0 | diff --git a/Gopkg.lock b/Gopkg.lock index fee0d64a8fd9..8fbb52a2406f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -88,6 +88,15 @@ revision = "358ee7663966325963d4e8b2e1fbd570c5195153" version = "v1.38.1" +[[projects]] + name = "github.com/go-log/log" + packages = [ + ".", + "log" + ] + revision = "37e2e1f19306361e1fc152a1839f1236149cb4e4" + version = "v0.1.0" + [[projects]] name = "github.com/go-openapi/jsonpointer" packages = ["."] @@ -166,7 +175,6 @@ "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/empty", "ptypes/timestamp" ] revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" @@ -209,17 +217,26 @@ ] revision = "9cad4c3443a7200dd6400aef47183728de563a38" +[[projects]] + name = "github.com/hashicorp/consul" + packages = [ + "api", + "watch" + ] + revision = "39f93f011e591c842acc8053a7f5972aa6e592fd" + version = "v1.2.1" + [[projects]] branch = "master" - name = "github.com/hashicorp/go-hclog" + name = "github.com/hashicorp/go-cleanhttp" packages = ["."] - revision = "ff2cf002a8dd750586d91dddd4470c341f981fe1" + revision = "d5fe4b57a186c716b0e00b8c301cbd9b4182694d" [[projects]] branch = "master" - name = "github.com/hashicorp/go-plugin" + name = "github.com/hashicorp/go-rootcerts" packages = ["."] - revision = "e8d22c780116115ae5624720c9af0c97afe4f551" + revision = "6bb64b370b90e7ef1fa532be9e591a81c3493e00" [[projects]] branch = "master" @@ -231,10 +248,10 @@ revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] - branch = "master" - name = "github.com/hashicorp/yamux" - packages = ["."] - revision = "3520598351bb3500a49ae9563f5539666ae0a27c" + name = "github.com/hashicorp/serf" + packages = ["coordinate"] + revision = "d6574a5bb1226678d7010325fb6c985db20ee458" + version = "v0.8.1" [[projects]] branch = "master" @@ -270,6 +287,147 @@ ] revision = "efc7eb8984d6655c26b5c9d2e65c024e5767c37c" +[[projects]] + branch = "master" + name = "github.com/micro/cli" + packages = ["."] + revision = "3d2263bb092286dde294a31049c92665f8373820" + +[[projects]] + name = "github.com/micro/go-grpc" + packages = ["."] + revision = "52024085c07e420ed8795c95006255a75feb500e" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "github.com/micro/go-log" + packages = ["."] + revision = "cbfa9447f9b655f8e66583237e51f40ca9bff860" + +[[projects]] + name = "github.com/micro/go-micro" + packages = [ + ".", + "broker", + "broker/codec", + "broker/codec/json", + "broker/http", + "client", + "cmd", + "codec", + "codec/jsonrpc", + "codec/protorpc", + "errors", + "metadata", + "registry", + "registry/consul", + "registry/mdns", + "selector", + "selector/cache", + "server", + "server/debug", + "server/debug/proto", + "transport", + "transport/codec", + "transport/http" + ] + revision = "5372707d0ec1ebd736bec36818ddb2e700606589" + version = "v0.9.0" + +[[projects]] + branch = "master" + name = "github.com/micro/go-plugins" + packages = [ + "client/grpc", + "registry/kubernetes", + "registry/kubernetes/client", + "registry/kubernetes/client/api", + "registry/kubernetes/client/watch", + "selector/cache", + "selector/static", + "server/grpc" + ] + revision = "8be6afe2f1228157fc5f0ba608a296a4fd00441d" + +[[projects]] + branch = "master" + name = "github.com/micro/go-rcache" + packages = ["."] + revision = "a581a57b513351792b9f21fc8b95d87b1c5a95da" + +[[projects]] + branch = "master" + name = "github.com/micro/grpc-go" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclog", + "internal", + "internal/backoff", + "internal/channelz", + "internal/envconfig", + "internal/grpcrand", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "ce3486ae08f0271df80a4c86fd7ca3b006308762" + +[[projects]] + name = "github.com/micro/kubernetes" + packages = ["go/micro"] + revision = "1b3cb85ccc7a367049117e263edc1a26a98b20b5" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/micro/mdns" + packages = ["."] + revision = "cdf30746f9f70dc7a366d3d885792a092fd83102" + +[[projects]] + name = "github.com/micro/protoc-gen-micro" + packages = [ + ".", + "generator", + "plugin/micro" + ] + revision = "656f330b069fe6ea1ce5013282e5914d6a1c3719" + version = "v0.4.0" + +[[projects]] + branch = "master" + name = "github.com/micro/util" + packages = [ + "go/lib/addr", + "go/lib/grpc", + "go/lib/net", + "go/lib/tls" + ] + revision = "4b7ed83e8520ab8fe6e5ef61fa2ede94b32f8602" + +[[projects]] + name = "github.com/miekg/dns" + packages = ["."] + revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" + version = "v1.0.8" + [[projects]] name = "github.com/minio/minio-go" packages = [ @@ -291,9 +449,15 @@ [[projects]] branch = "master" - name = "github.com/mitchellh/go-testing-interface" + name = "github.com/mitchellh/hashstructure" + packages = ["."] + revision = "2bca23e0e452137f789efbc8610126fd8b94f73b" + +[[projects]] + branch = "master" + name = "github.com/mitchellh/mapstructure" packages = ["."] - revision = "a61a99592b77c9ba629d254a693acffaeb4b7e28" + revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac" [[projects]] name = "github.com/modern-go/concurrent" @@ -337,10 +501,10 @@ version = "v1.0.0" [[projects]] - name = "github.com/oklog/run" + name = "github.com/pborman/uuid" packages = ["."] - revision = "4dadeb3030eda0273a12382bb2348ffc7c9d1a39" - version = "v1.0.0" + revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" + version = "v1.1" [[projects]] branch = "master" @@ -363,6 +527,12 @@ revision = "1958fd8fff7f115e79725b1288e0b878b3e06b00" version = "v2.0.3" +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] name = "github.com/pmezard/go-difflib" packages = ["difflib"] @@ -438,6 +608,8 @@ "bcrypt", "blake2b", "blowfish", + "ed25519", + "ed25519/internal/edwards25519", "ssh/terminal" ] revision = "a2144134853fc9a27a7b1e3eb4f19f1a76df13c9" @@ -446,18 +618,23 @@ branch = "master" name = "golang.org/x/net" packages = [ + "bpf", "context", "http/httpguts", "http2", "http2/hpack", "idna", + "internal/iana", + "internal/socket", "internal/socks", "internal/timeseries", + "ipv4", + "ipv6", "proxy", "trace", "websocket" ] - revision = "81d44fd177a98d09fe3bc40a5a78d495d3f042ea" + revision = "a680a1efc54dd51c040b3b5ce4939ea3cf2ea0d1" [[projects]] branch = "master" @@ -504,7 +681,7 @@ "imports", "internal/fastwalk" ] - revision = "32950ab3be12acf6d472893021373669979907ab" + revision = "4735f53fb37db012e63d1a62914417980fb3c023" [[projects]] branch = "master" @@ -525,8 +702,6 @@ "encoding", "encoding/proto", "grpclog", - "health", - "health/grpc_health_v1", "internal", "internal/backoff", "internal/channelz", @@ -789,6 +964,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5825a13b72c61f5672319f333388e1823d98f7e8f76a009b50cedcc77fe40436" + inputs-digest = "444489fe54d843be31d012830ae9c72c25e0c116de569adddd5e7287368d6170" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 9d59d1be1a39..a9e4ac30aa55 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -10,6 +10,7 @@ required = [ "github.com/golang/protobuf/protoc-gen-go", "github.com/gogo/protobuf/protoc-gen-gofast", "github.com/gogo/protobuf/protoc-gen-gogofast", + "github.com/micro/protoc-gen-micro", ] [[constraint]] @@ -76,9 +77,9 @@ required = [ name = "github.com/Shopify/sarama" version = "1.16.0" -[[constraint]] +[[override]] + name = "github.com/micro/go-plugins" branch = "master" - name = "github.com/hashicorp/go-plugin" [prune] go-tests = true diff --git a/Makefile b/Makefile index 239c79616497..c94b14979094 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,12 @@ - +PACKAGE=github.com/argoproj/argo-events CURRENT_DIR=$(shell pwd) DIST_DIR=${CURRENT_DIR}/dist -PLUGIN_DIR=${DIST_DIR}/plugins VERSION=$(shell cat ${CURRENT_DIR}/VERSION) BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') GIT_COMMIT=$(shell git rev-parse HEAD) GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi) GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi) -PLUGINS=$(shell find . \( -type d -and -path '*/signals/stream/builtin/*' \)) override LDFLAGS += \ -X ${PACKAGE}.version=${VERSION} \ @@ -43,13 +41,21 @@ protogen: clientgen: ./hack/update-codegen.sh +.PHONY: openapigen +openapi-gen: + ./hack/update-openapigen.sh + .PHONY: codegen -codegen: protogen clientgen +codegen: clientgen openapigen protogen -# Build the project -.PHONY: all controller controller-image clean test +# this is the default stream service +STREAM=nats -all: controller-image +# Build the project images +.DELETE_ON_ERROR: +all: controller-image artifact-image calendar-image resource-image webhook-image stream-image + +.PHONY: all controller controller-image clean test # Sensor controller controller: @@ -58,19 +64,46 @@ controller: controller-linux: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make controller -controller-image: controller-linux stream-plugins-linux +controller-image: controller-linux docker build -t $(IMAGE_PREFIX)sensor-controller:$(IMAGE_TAG) -f ./controller/Dockerfile . @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)sensor-controller:$(IMAGE_TAG) ; fi -# Plugins -stream-plugins: - go build -v -ldflags '${LDFLAGS}' -o ${PLUGIN_DIR}/nats ./signals/stream/builtin/nats - go build -v -ldflags '${LDFLAGS}' -o ${PLUGIN_DIR}/mqtt ./signals/stream/builtin/mqtt - go build -v -ldflags '${LDFLAGS}' -o ${PLUGIN_DIR}/kafka ./signals/stream/builtin/kafka - go build -v -ldflags '${LDFLAGS}' -o ${PLUGIN_DIR}/amqp ./signals/stream/builtin/amqp +# signal microservice binaries +artifact: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/artifact-signal ./signals/artifact/micro + +calendar: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/calendar-signal ./signals/calendar/micro + +resource: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/resource-signal ./signals/resource/micro + +webhook: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/webhook-signal ./signals/webhook/micro + +stream: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${STREAM}-signal ./signals/stream/builtin/${STREAM}/micro + +# signal microservice docker images +artifact-image: artifact + docker build -t $(IMAGE_PREFIX)artifact-signal:$(IMAGE_TAG) -f ./signals/artifact/micro/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)artifact-signal:$(IMAGE_TAG) ; fi + +calendar-image: calendar + docker build -t $(IMAGE_PREFIX)calendar-signal:$(IMAGE_TAG) -f ./signals/calendar/micro/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)calendar-signal:$(IMAGE_TAG) ; fi + +resource-image: resource + docker build -t $(IMAGE_PREFIX)resource-signal:$(IMAGE_TAG) -f ./signals/resource/micro/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)resource-signal:$(IMAGE_TAG) ; fi + +webhook-image: webhook + docker build -t $(IMAGE_PREFIX)webhook-signal:$(IMAGE_TAG) -f ./signals/webhook/micro/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)webhook-signal:$(IMAGE_TAG) ; fi -stream-plugins-linux: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make stream-plugins +stream-image: stream + docker build -t $(IMAGE_PREFIX)stream-$(STREAM)-signal:$(IMAGE_TAG) -f ./signals/stream/builtin/$(STREAM)/micro/Dockerfile . + @if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)stream-$(STREAM)-signal:$(IMAGE_TAG) ; fi test: go test $(shell go list ./... | grep -v /vendor/) -race -short -v diff --git a/README.md b/README.md index 1ce38cadd19d..e2bffc5f4809 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## What is Argo Events? Argo Events is an open source event-based dependency manager for Kubernetes. The core concept of the project are `sensors` which are implemented as Kubernetes-native Custom Resource Definition that define a set of dependencies (signals) and actions (triggers). The sensor's triggers will only be fired after it's signals have been resolved. `Sensors` can trigger once or repeatedly. - Define multiple dependencies from a variety of signal sources -- Define dependency constraints and build plugins to support business-level constraint logic +- Plugin custom signal microservices to support business-level constraint logic - Trigger messages and Kubernetes object creation after successful dependency resolution - Trigger escalation after errors, or dependency constraint failures - Build and manage a distributed, cross-team, event-driven architecture @@ -12,7 +12,8 @@ Argo Events is an open source event-based dependency manager for Kubernetes. The ## Why Argo Events? - Containers. Designed from the ground-up as Kubernetes-native. - Extremely lightweight. All signals, with the exception of calendar-based signals, are event-driven, meaning that there is no polling involved. -- High performance. Each `sensor` runs in its own Kubernetes job, enabling high bandwidth for processing near real-time events. +- Configurable. Choose which signals to support and only deploy those you want to Kubernetes. +- Scalability & Resilient. Signal sources run as stateless microservices enabling you to scale simply by increasing the deployment replicas. - Simple or Complex dependencies. Manage everything from simple, linear, real-time dependencies to complex, multi-source, batch job dependencies. ## Getting Started diff --git a/ROADMAP.md b/ROADMAP.md index ca3a1e9922f1..6aed351bb74f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,7 +11,6 @@ ## Design Phase - Pluggable Calendar definitions that extend a common Calendar interface - Pass in credentials for connection to various signal sources -- S3 Bucket Notification Handling - Implement GC for old sensors diff --git a/cmd/sensor-controller/main.go b/cmd/sensor-controller/main.go index c425f5516621..6516450f8a7c 100644 --- a/cmd/sensor-controller/main.go +++ b/cmd/sensor-controller/main.go @@ -46,13 +46,13 @@ func main() { panic(err) } - // stream signal plugins - pluginMgr, err := controller.NewPluginManager() + // stream signal micro services + signalMgr, err := controller.NewSignalManager(logger.Sugar()) if err != nil { panic(err) } - controller := controller.NewSensorController(restConfig, configMap, pluginMgr, logger.Sugar()) + controller := controller.NewSensorController(restConfig, configMap, signalMgr, logger.Sugar()) err = controller.ResyncConfig() if err != nil { panic(err) diff --git a/common/common.go b/common/common.go index 4c0d444993ec..9d87464c02db 100644 --- a/common/common.go +++ b/common/common.go @@ -22,9 +22,6 @@ import ( // SENSOR CONTROLLER CONSTANTS const ( - // DefaultSensorControllerNamespace is the default namespace where the sensor controller is installed - DefaultSensorControllerNamespace = "default" - // DefaultSensorControllerDeploymentName is the default deployment name of the sensor controller DefaultSensorControllerDeploymentName = "sensor-controller" @@ -41,7 +38,7 @@ const ( // LabelKeyComplete is the label to mark sensors as complete LabelKeyComplete = sensor.FullName + "/complete" - // EnvVarNamespace contains the namespace of the controller & jobs + // EnvVarNamespace contains the namespace of the controller & services EnvVarNamespace = "SENSOR_NAMESPACE" // EnvVarConfigMap is the name of the configmap to use for the controller diff --git a/common/util.go b/common/util.go index c5310b8f4f47..10227775526a 100644 --- a/common/util.go +++ b/common/util.go @@ -17,10 +17,13 @@ limitations under the License. package common import ( + "errors" "fmt" + "io/ioutil" + "os" "time" - "k8s.io/apimachinery/pkg/api/errors" + apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" @@ -29,6 +32,20 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const namespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + +var ( + // DefaultSensorControllerNamespace is the default namespace where the sensor controller is installed + DefaultSensorControllerNamespace = "default" + + // ErrReadNamespace occurs when the namespace cannot be read from a Kubernetes pod's service account token + ErrReadNamespace = errors.New("Could not read namespace from service account secret") +) + +func init() { + RefreshNamespace() +} + // DefaultRetry is a default retry backoff settings when retrying API calls var DefaultRetry = wait.Backoff{ Steps: 5, @@ -40,7 +57,7 @@ var DefaultRetry = wait.Backoff{ // IsRetryableKubeAPIError returns if the error is a retryable kubernetes error func IsRetryableKubeAPIError(err error) bool { // get original error if it was wrapped - if errors.IsNotFound(err) || errors.IsForbidden(err) || errors.IsInvalid(err) || errors.IsMethodNotSupported(err) { + if apierr.IsNotFound(err) || apierr.IsForbidden(err) || apierr.IsInvalid(err) || apierr.IsMethodNotSupported(err) { return false } return true @@ -77,3 +94,38 @@ func ServerResourceForGroupVersionKind(disco discovery.DiscoveryInterface, gvk s } return nil, fmt.Errorf("Server is unable to handle %s", gvk) } + +// detectNamespace attemps to read the namespace from the mounted service account token +// Note that this will return an error if running outside a Kubernetes pod +func detectNamespace() (string, error) { + // Make sure it's a file and we can read it + if s, e := os.Stat(namespacePath); e != nil { + return "", e + } else if s.IsDir() { + return "", ErrReadNamespace + } + + // Read the file, and cast to a string + ns, e := ioutil.ReadFile(namespacePath) + return string(ns), e +} + +// RefreshNamespace performs waterfall logic for choosing a "default" namespace +// this function is run as part of an init() function +func RefreshNamespace() { + // 1 - env variable + nm, ok := os.LookupEnv(EnvVarNamespace) + if ok { + DefaultSensorControllerNamespace = nm + return + } + + // 2 - pod service account token + nm, err := detectNamespace() + if err == nil { + DefaultSensorControllerNamespace = nm + } + + // 3 - use the DefaultSensorControllerNamespace + return +} diff --git a/common/util_test.go b/common/util_test.go index ccb20e97a9d2..6c1b119e380c 100644 --- a/common/util_test.go +++ b/common/util_test.go @@ -18,6 +18,7 @@ package common import ( "fmt" + "os" "testing" "github.com/stretchr/testify/assert" @@ -60,3 +61,21 @@ func TestServerResourceForGroupVersionKind(t *testing.T) { assert.NotNil(t, err) assert.Nil(t, apiResource) } + +func TestResolveNamespace(t *testing.T) { + defer os.Unsetenv(EnvVarNamespace) + + RefreshNamespace() + assert.Equal(t, "default", DefaultSensorControllerNamespace) + + // TODO: now write the namespace file + + // now set the env variable + err := os.Setenv(EnvVarNamespace, "test") + if err != nil { + t.Error(err) + } + + RefreshNamespace() + assert.Equal(t, "test", DefaultSensorControllerNamespace) +} diff --git a/controller/Dockerfile b/controller/Dockerfile index 5287ed603b1e..f3e930cabb8e 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -1,11 +1,3 @@ FROM alpine:3.7 - COPY dist/sensor-controller / - -# add the stream plugin binaries -COPY dist/plugins/ /plugins/ - -ENV STREAM_PLUGIN_DIR=plugins -RUN chmod -R 777 /plugins - CMD [ "/sensor-controller" ] diff --git a/controller/config.go b/controller/config.go index bd33288e0aec..04cdc86f0342 100644 --- a/controller/config.go +++ b/controller/config.go @@ -19,7 +19,6 @@ package controller import ( "context" "fmt" - "os" "github.com/argoproj/argo-events/common" "github.com/ghodss/yaml" @@ -92,11 +91,7 @@ func (c *SensorController) newControllerConfigMapWatch() *cache.ListWatch { // ResyncConfig reloads the controller config from the configmap func (c *SensorController) ResyncConfig() error { - namespace, _ := os.LookupEnv(common.EnvVarNamespace) - if namespace == "" { - namespace = common.DefaultSensorControllerNamespace - } - cmClient := c.kubeClientset.CoreV1().ConfigMaps(namespace) + cmClient := c.kubeClientset.CoreV1().ConfigMaps(common.DefaultSensorControllerNamespace) cm, err := cmClient.Get(c.ConfigMap, metav1.GetOptions{}) if err != nil { return err @@ -115,6 +110,9 @@ func (c *SensorController) updateConfig(cm *apiv1.ConfigMap) error { if err != nil { return err } + if config.Namespace == "" { + config.Namespace = common.DefaultSensorControllerNamespace + } c.Config = config return nil } diff --git a/controller/config_test.go b/controller/config_test.go index df4c94de8141..a9a451d2217e 100644 --- a/controller/config_test.go +++ b/controller/config_test.go @@ -56,6 +56,7 @@ func TestNewControllerConfigMapWatch(t *testing.T) { } func TestResyncConfig(t *testing.T) { + defer os.Unsetenv(common.EnvVarNamespace) controller := SensorController{ ConfigMap: "sensor-controller-configmap", ConfigMapNS: "testing", @@ -67,6 +68,9 @@ func TestResyncConfig(t *testing.T) { os.Setenv(common.EnvVarNamespace, "testing") + // Note: need to refresh the namespace + common.RefreshNamespace() + // fail when the configmap does not have key 'config' configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controller/controller.go b/controller/controller.go index f924f2319dc5..04daa65051e8 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "errors" "sync" "time" @@ -31,12 +32,11 @@ import ( "github.com/argoproj/argo-events" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" sensorclientset "github.com/argoproj/argo-events/pkg/client/clientset/versioned" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" ) const ( - sensorResyncPeriod = 20 * time.Minute - pluginHealthCheckPeriod = 20 * time.Second + sensorResyncPeriod = 20 * time.Minute ) // SensorControllerConfig contain the configuration settings for the sensor controller @@ -67,26 +67,26 @@ type SensorController struct { informer cache.SharedIndexInformer queue workqueue.RateLimitingInterface - // enables access to plugin signals - pluginMgr *PluginManager + // enables access to signals micro services + signalMgr *SignalManager // inventory for all types of signal implementations - signalMu sync.Mutex - signals map[string]shared.Signaler + signalMu sync.Mutex + signalStreams map[string]sdk.SignalService_ListenService log *zap.SugaredLogger } // NewSensorController creates a new Controller -func NewSensorController(rest *rest.Config, configMap string, pluginMgr *PluginManager, log *zap.SugaredLogger) *SensorController { +func NewSensorController(rest *rest.Config, configMap string, signalMgr *SignalManager, log *zap.SugaredLogger) *SensorController { return &SensorController{ ConfigMap: configMap, kubeConfig: rest, kubeClientset: kubernetes.NewForConfigOrDie(rest), sensorClientset: sensorclientset.NewForConfigOrDie(rest), queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), - pluginMgr: pluginMgr, - signals: make(map[string]shared.Signaler), + signalMgr: signalMgr, + signalStreams: make(map[string]sdk.SignalService_ListenService), log: log, } } @@ -117,35 +117,47 @@ func (c *SensorController) processNextItem() bool { } ctx := newSensorOperationCtx(sensor, c) - err = ctx.operate() - c.handleErr(err, key) + err = c.handleErr(ctx.operate(), key) + if err != nil { + // now let's escalate the sensor + // the context should have the most up-to-date version + c.log.Infof("escalating sensor to level %s via %s message", ctx.s.Spec.Escalation.Level, ctx.s.Spec.Escalation.Message.Stream.Type) + err := sendMessage(&ctx.s.Spec.Escalation.Message) + if err != nil { + c.log.Panicf("failed escalating sensor '%s'", key) + } + } return true } -// handleErr checks if an error happened and make sure we will retry later - do we want to retry later? probably -func (c *SensorController) handleErr(err error, key interface{}) { +// handleErr checks if an error happened and make sure we will retry later +// returns an error if unable to handle the error +func (c *SensorController) handleErr(err error, key interface{}) error { if err == nil { - // Forget about the #AddRateLimited history of key on every successful synch + // Forget about the #AddRateLimited history of key on every successful sync // Ensure future updates for this key are not delayed because of outdated error history c.queue.Forget(key) - return + return nil } - if c.queue.NumRequeues(key) < 3 { - c.log.Errorf("Error syncing sensor %v: %v", key, err) + // due to the base delay of 5ms of the DefaultControllerRateLimiter + // requeues will happen very quickly even after a signal pod goes down + // we want to give the signal pod a chance to come back up so we give a genorous number of retries + if c.queue.NumRequeues(key) < 20 { + c.log.Errorf("Error syncing sensor '%v': %v", key, err) // Re-enqueue the key rate limited. This key will be processed later again. c.queue.AddRateLimited(key) - return + return nil } + return errors.New("exceeded max requeues") } // Run executes the controller func (c *SensorController) Run(ctx context.Context, ssThreads, signalThreads int) { defer c.queue.ShutDown() - defer c.pluginMgr.Close() c.log.Infof("sensor controller (version: %s) (instance: %s) starting", axis.GetVersion(), c.Config.InstanceID) _, err := c.watchControllerConfigMap(ctx) @@ -156,7 +168,6 @@ func (c *SensorController) Run(ctx context.Context, ssThreads, signalThreads int c.informer = c.newSensorInformer() go c.informer.Run(ctx.Done()) - go c.pluginMgr.Monitor(ctx.Done()) if !cache.WaitForCacheSync(ctx.Done(), c.informer.HasSynced) { c.log.Panicf("timed out waiting for the caches to sync") diff --git a/controller/controller_test.go b/controller/controller_test.go index b298f5f2fc4f..ddabac9baf67 100644 --- a/controller/controller_test.go +++ b/controller/controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/client-go/util/workqueue" fakesensor "github.com/argoproj/argo-events/pkg/client/clientset/versioned/fake" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" ) // fakeController is a wrapper around the sensorController to allow efficient test setup/cleanup @@ -46,7 +46,7 @@ func (f *fakeController) setup(namespace string) { sensorClientset: fakesensor.NewSimpleClientset(), log: zap.NewNop().Sugar(), queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), - signals: make(map[string]shared.Signaler), + signalStreams: make(map[string]sdk.SignalService_ListenService), } } diff --git a/controller/operator.go b/controller/operator.go index 5dac274b0483..85b29ad3c0f8 100644 --- a/controller/operator.go +++ b/controller/operator.go @@ -24,6 +24,7 @@ import ( "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" @@ -117,21 +118,8 @@ func (soc *sOperationCtx) operate() error { } } - if soc.s.IsComplete() { - // escalate - if !soc.s.Status.Escalated { - soc.log.Warnf("escalating sensor to level %s via %s message", soc.s.Spec.Escalation.Level, soc.s.Spec.Escalation.Message.Stream.Type) - err := sendMessage(&soc.s.Spec.Escalation.Message) - if err != nil { - return err - } - soc.s.Status.Escalated = true - soc.updated = true - } else { - soc.log.Debug("sensor already escalated") - } - } - + // if we get here - we know the signals are running + soc.markSensorPhase(v1alpha1.NodePhaseActive, false, "listening for signal events") return nil } @@ -178,16 +166,24 @@ func (soc *sOperationCtx) persistUpdates() { time.Sleep(1 * time.Second) } -// todo: implement me +// reapplyUpdate by fetching a new version of the sensor and updating the status +// TODO: use patch here? func (soc *sOperationCtx) reapplyUpdate(sensorClient client.SensorInterface) error { - attempt := 1 - for { - _, err := sensorClient.Get(soc.s.ObjectMeta.Name, metav1.GetOptions{}) - - if attempt > 5 { - return err + return wait.ExponentialBackoff(common.DefaultRetry, func() (bool, error) { + s, err := sensorClient.Get(soc.s.Name, metav1.GetOptions{}) + if err != nil { + return false, err } - } + s.Status = soc.s.Status + soc.s, err = sensorClient.Update(s) + if err != nil { + if !common.IsRetryableKubeAPIError(err) { + return false, err + } + return false, nil + } + return true, nil + }) } // create a new node diff --git a/controller/operator_test.go b/controller/operator_test.go index 71e144544e85..116b842cd614 100644 --- a/controller/operator_test.go +++ b/controller/operator_test.go @@ -17,11 +17,8 @@ limitations under the License. package controller import ( - "strconv" "testing" - "github.com/nats-io/gnatsd/server" - "github.com/nats-io/gnatsd/test" "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -192,63 +189,3 @@ func TestReRunSensor(t *testing.T) { resourceSignalNode := soc.getNodeByName(sampleSensor.Spec.Signals[1].Name) assert.Equal(t, v1alpha1.NodePhaseNew, resourceSignalNode.Phase) } - -func TestEscalationSent(t *testing.T) { - fake := newFakeController() - defer fake.teardown() - - natsEmbeddedServerOpts := server.Options{ - Host: "localhost", - Port: 4225, - NoLog: true, - NoSigs: true, - MaxControlLine: 256, - } - testServer := test.RunServer(&natsEmbeddedServerOpts) - defer testServer.Shutdown() - - sampleSensor.Spec.Escalation = v1alpha1.EscalationPolicy{ - Level: "High", - Message: v1alpha1.Message{ - Body: "esclating this sensor on failure", - Stream: v1alpha1.Stream{ - Type: "NATS", - URL: "nats://" + natsEmbeddedServerOpts.Host + ":" + strconv.Itoa(natsEmbeddedServerOpts.Port), - Attributes: map[string]string{"subject": "escalation"}, - }, - }, - } - - sampleSensor.Status = v1alpha1.SensorStatus{ - Escalated: false, - Phase: v1alpha1.NodePhaseError, - Nodes: make(map[string]v1alpha1.NodeStatus), - } - for _, signal := range sampleSensor.Spec.Signals { - nodeID := sampleSensor.NodeID(signal.Name) - sampleSensor.Status.Nodes[nodeID] = v1alpha1.NodeStatus{ - ID: nodeID, - Name: signal.Name, - DisplayName: signal.Name, - Type: v1alpha1.NodeTypeSignal, - Phase: v1alpha1.NodePhaseError, - StartedAt: metav1.Time{}, - Message: "failed node reason", - } - } - - sensor, err := fake.sensorClientset.ArgoprojV1alpha1().Sensors(fake.Config.Namespace).Create(&sampleSensor) - assert.Nil(t, err) - soc := newSensorOperationCtx(sensor, fake.SensorController) - - // if sensor not yet escalated, make sure we escalate and update it - soc.operate() - assert.True(t, soc.s.Status.Escalated) - assert.True(t, soc.updated) - - // second pass through, verify we don't escalate and update - soc.updated = false // reset this field - soc.operate() - assert.True(t, soc.s.Status.Escalated) - assert.False(t, soc.updated) -} diff --git a/controller/plugin-manager.go b/controller/plugin-manager.go deleted file mode 100644 index c0736883e31f..000000000000 --- a/controller/plugin-manager.go +++ /dev/null @@ -1,98 +0,0 @@ -package controller - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - "github.com/argoproj/argo-events/shared" - "github.com/hashicorp/go-plugin" -) - -// PluginManager helps manage the various plugins organized under one directory -type PluginManager struct { - dir string - clients map[string]*plugin.Client -} - -// NewPluginManager creates a new PluginManager -func NewPluginManager() (*PluginManager, error) { - dir := os.Getenv("STREAM_PLUGIN_DIR") - mgr := PluginManager{ - dir: dir, - clients: make(map[string]*plugin.Client), - } - plugins, err := plugin.Discover("*", dir) - if err != nil { - return nil, err - } - for _, pluginFile := range plugins { - c := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: shared.Handshake, - Plugins: shared.PluginMap, - Cmd: exec.Command(pluginFile), - AllowedProtocols: []plugin.Protocol{ - plugin.ProtocolNetRPC, plugin.ProtocolGRPC, - }, - }) - _, file := filepath.Split(pluginFile) - lowerFile := strings.ToLower(file) - fmt.Printf("adding plugin '%s'\n", lowerFile) - mgr.clients[lowerFile] = c - } - return &mgr, nil -} - -// Dispense the interface with the given name -// NOTE: assumes the name matches the file name and the plugin name -func (pm *PluginManager) Dispense(name string) (interface{}, error) { - lowercase := strings.ToLower(name) - client, ok := pm.clients[lowercase] - if !ok { - return nil, fmt.Errorf("unknown plugin '%s'", name) - } - protocol, err := client.Client() - if err != nil { - return nil, err - } - iface, err := protocol.Dispense(shared.SignalPluginName) - if err != nil { - return nil, err - } - return iface, nil -} - -// Monitor the plugins -func (pm *PluginManager) Monitor(done <-chan struct{}) { - timer := time.NewTimer(pluginHealthCheckPeriod) - for { - select { - case <-timer.C: - for name, client := range pm.clients { - proto, err := client.Client() - if err != nil { - panic(fmt.Errorf("failed to retrieve the plugin client protocol. cause: %s", err)) - } - fmt.Printf("pinging plugin '%s'", name) - err = proto.Ping() - if err != nil { - panic(fmt.Errorf("signal plugin client connection failed. cause: %s", err)) - } - } - case <-done: - timer.Stop() - return - } - } - -} - -// Close kills all the manager's clients -func (pm *PluginManager) Close() { - for _, client := range pm.clients { - client.Kill() - } -} diff --git a/controller/signal-manager.go b/controller/signal-manager.go new file mode 100644 index 000000000000..693176d39c31 --- /dev/null +++ b/controller/signal-manager.go @@ -0,0 +1,63 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + "sync" + + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/shared" + "go.uber.org/zap" +) + +// SignalManager helps manage the various signals microservices +// TODO: clients may become disconnected during operations so we need a way to remove bad clients +// this can be accomplished through a health-checking routine +type SignalManager struct { + microClient *shared.MicroSignalClient + sync.Mutex + clients map[string]sdk.SignalClient + log *zap.SugaredLogger +} + +// NewSignalManager creates a new SignalManager +func NewSignalManager(log *zap.SugaredLogger) (*SignalManager, error) { + mgr := SignalManager{ + microClient: shared.NewMicroSignalClient(), + clients: make(map[string]sdk.SignalClient), + log: log, + } + // TODO: add to cache of the builtin signal services + return &mgr, nil +} + +// Dispense the signal client with the given name +// NOTE: assumes the name matches the service name +func (pm *SignalManager) Dispense(name string) (sdk.SignalClient, error) { + pm.Lock() + defer pm.Unlock() + lowercase := strings.ToLower(name) + //client, ok := pm.clients[lowercase] + //if !ok { + c := pm.microClient.NewSignalService(lowercase) + pm.log.Debugf("Dispensed '%s' signal service", lowercase) + pm.clients[name] = c + return c, nil + //} + //return client, nil +} diff --git a/controller/signal.go b/controller/signal.go index 254f4ac1390e..b5dc3b8c4c86 100644 --- a/controller/signal.go +++ b/controller/signal.go @@ -1,60 +1,68 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controller import ( - "errors" + "context" + "fmt" + "io" + "time" + "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" - "github.com/argoproj/argo-events/signals/artifact" - "github.com/argoproj/argo-events/signals/calendar" - "github.com/argoproj/argo-events/signals/resource" - "github.com/argoproj/argo-events/signals/webhook" + "github.com/argoproj/argo-events/sdk" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" ) -var ( - ErrInvalidSignalType = errors.New("signal type unknown") -) - -// signalCtx is the context for handling signal -type signalCtx struct { - nodename string - obj *v1alpha1.Signal - plugin shared.Signaler -} - func (soc *sOperationCtx) processSignal(signal v1alpha1.Signal) (*v1alpha1.NodeStatus, error) { soc.log.Debugf("evaluating signal '%s'", signal.Name) node := soc.getNodeByName(signal.Name) - if node != nil && node.IsComplete() { - soc.stopSignal(&signal) - return node, nil + if node != nil && node.Phase == v1alpha1.NodePhaseComplete { + return node, soc.controller.stopSignal(node.ID) } if node == nil { node = soc.initializeNode(signal.Name, v1alpha1.NodeTypeSignal, v1alpha1.NodePhaseNew) } - // if signal is new (phase=new) == signal not present (signaler=nil) if node.Phase == v1alpha1.NodePhaseNew { - // can the node phase and the status of the signal watch ever introduce discrepency? - if !soc.signalIsPresent(signal.Name) { + if !soc.signalIsPresent(node.ID) { + // under normal operations, the signal stream is not present when the node is new + // we now attempt to establish a bi-directional stream with our signal microservice + // and watch the stream for events. err := soc.watchSignal(&signal) if err != nil { return nil, err } } else { - // we get here if the phase == New and the signaler already exists -- should never happen... - // in any case, let's log a warning as this is unintended but let's keep the signaler running to receive events - // TODO: add check to see if plugin is running? - soc.log.Warn("signal is new however signal plugin is already present") + // we get here if the stream already exists even though the node phase is new. + // this can happen if the sensor was deleted and re-created. + // let's log a warning but let's keep the stream running to receive events. + // TODO: add check to see if service is running and we're reading from the stream? + soc.log.Info("WARNING: signal '%s' is new however signal stream is already present", signal.Name) } } else { - if !soc.signalIsPresent(signal.Name) { - // signal is active but signaler plugin is missing... meaning we could have been missing events for this signal - // we could get here if the controller crashes... - // plugin may have crashed? - soc.log.Warn("signal is active however signal plugin is missing, potentially could have missed events! attempting to restart plugin.") + if !soc.signalIsPresent(node.ID) { + // we get here if the signal is in active or error phase and the signal stream is missing. + // this means that we had down-time for this signal - we could have missed events. + // this can happen if the controller or a signal pod serving the stream goes down. + // let's log a warning and attempt to re-establish a stream and watch for events. + soc.log.Infof("WARNING: event stream for signal '%s' is missing - could have missed events! reconnecting stream...", signal.Name) err := soc.watchSignal(&signal) if err != nil { return nil, err @@ -72,125 +80,163 @@ func (soc *sOperationCtx) processSignal(signal v1alpha1.Signal) (*v1alpha1.NodeS return soc.markNodePhase(signal.Name, v1alpha1.NodePhaseComplete), nil } - return soc.markNodePhase(signal.Name, v1alpha1.NodePhaseActive), nil + return soc.markNodePhase(signal.Name, v1alpha1.NodePhaseActive, "stream established"), nil } // checks to see if the signal is present -// TODO: include a check on the signaler interface to check if signal is active? -func (soc *sOperationCtx) signalIsPresent(name string) bool { - nodeID := soc.s.NodeID(name) +// TODO: include a check on the stream interface to check if stream is still open? +func (soc *sOperationCtx) signalIsPresent(nodeID string) bool { soc.controller.signalMu.Lock() - _, ok := soc.controller.signals[nodeID] + _, ok := soc.controller.signalStreams[nodeID] soc.controller.signalMu.Unlock() return ok } -// resolveSignaler is a helper method to find the correct Signaler for this signal definition -func (soc *sOperationCtx) resolveSignaler(signal *v1alpha1.Signal) (shared.Signaler, error) { - var signaler shared.Signaler +// resolveClient is a helper method to find the correct SignalClient for this signal +func (soc *sOperationCtx) resolveClient(signal *v1alpha1.Signal) (sdk.SignalClient, error) { + var typ string switch signal.GetType() { case v1alpha1.SignalTypeStream: - // these come from plugins - plugin, err := soc.controller.pluginMgr.Dispense(signal.Stream.Type) - if err != nil { - return nil, err - } - signaler = plugin.(shared.Signaler) - case v1alpha1.SignalTypeArtifact: - streamPlugin, err := soc.controller.pluginMgr.Dispense(signal.Artifact.Target.Type) - if err != nil { - return nil, err - } - streamSignaler := streamPlugin.(shared.Signaler) - signaler = artifact.New(streamSignaler, soc.controller.kubeClientset, soc.controller.Config.Namespace) - case v1alpha1.SignalTypeCalendar: - signaler = calendar.New() - case v1alpha1.SignalTypeResource: - signaler = resource.New(soc.controller.kubeConfig) - case v1alpha1.SignalTypeWebhook: - signaler = webhook.New() + typ = signal.Stream.Type default: - return nil, ErrInvalidSignalType + typ = string(signal.GetType()) } - return signaler, nil + client, err := soc.controller.signalMgr.Dispense(typ) + if err != nil { + return nil, err + } + return client, nil } // adds a new signal to the controller's signals func (soc *sOperationCtx) watchSignal(signal *v1alpha1.Signal) error { - signaler, err := soc.resolveSignaler(signal) + client, err := soc.resolveClient(signal) if err != nil { return err } - events, err := signaler.Start(signal) + // create the context for this stream + ctx := context.Background() + var cancel context.CancelFunc + if signal.Deadline > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(signal.Deadline)) + } else { + ctx, cancel = context.WithCancel(ctx) + } + + stream, err := client.Listen(ctx, signal) if err != nil { + cancel() return err } nodeID := soc.s.NodeID(signal.Name) soc.controller.signalMu.Lock() - soc.controller.signals[nodeID] = signaler + soc.controller.signalStreams[nodeID] = stream soc.controller.signalMu.Unlock() - go soc.controller.listenForEvents(soc.s.Name, signal.Name, nodeID, events) + go soc.controller.listenOnStream(soc.s.Name, signal.Name, nodeID, stream, cancel) return nil } -func (soc *sOperationCtx) stopSignal(signal *v1alpha1.Signal) { - nodeID := soc.s.NodeID(signal.Name) - soc.controller.signalMu.Lock() - signaler, ok := soc.controller.signals[nodeID] - soc.controller.signals[nodeID] = nil - delete(soc.controller.signals, nodeID) - soc.controller.signalMu.Unlock() - if ok && signaler != nil { - if err := signaler.Stop(); err != nil { - soc.log.Warnf("failed to stop the signaler '%s'. cause: %s", nodeID, err) +// stop the signal by: +// 1. deleting the stream from the controller's signalStreams map +// 2. sending the terminate signal on the stream and close it +// NOTE: this is a method on the controller +func (c *SensorController) stopSignal(nodeID string) error { + c.signalMu.Lock() + stream, ok := c.signalStreams[nodeID] + c.signalStreams[nodeID] = nil + delete(c.signalStreams, nodeID) + c.signalMu.Unlock() + if ok && stream != nil { + if err := stream.Send(sdk.Terminate); err != nil { + return err + } + if err := stream.Close(); err != nil { + return err } } + return nil } -// listens for events on the events channel -// meant to be run as a separate goroutine -// this will terminate once the events chan is closed -// the events chan should be closed by the future operations on this sensor -func (c *SensorController) listenForEvents(sensor, signal, nodeID string, events <-chan *v1alpha1.Event) { +// listens for events on the event stream. meant to be run as a separate goroutine +// this will terminate once the stream receives an EOF indicating it has completed or it encounters a stream error. +// NOTE: this is a method on the controller +func (c *SensorController) listenOnStream(sensor, signal, nodeID string, stream sdk.SignalService_ListenService, cancel context.CancelFunc) { + // TODO: possible context leak if we don't utilize cancel + //defer cancel() sensors := c.sensorClientset.ArgoprojV1alpha1().Sensors(c.Config.Namespace) - for event := range events { - c.log.Infof("sensor '%s' received event for signal '%s'", sensor, signal) - // todo: store the event in an external store + for { + in, streamErr := stream.Recv() + if streamErr == io.EOF { + return + } + // retrieve the sensor & node as this will be needed to update accordingly s, err := sensors.Get(sensor, metav1.GetOptions{}) if err != nil { - // we can get here if the sensor was removed after completion or deleted and the signaler was not stopped - // we should attempt to log a warning and stop the signaler - c.log.Warnf("sensor '%s' cannot be found due to: %s. Stopping signaler...", sensor, err) - c.signalMu.Lock() - defer c.signalMu.Unlock() - signaler, ok := c.signals[nodeID] - if !ok { - c.log.Panicf("signaler '%s' not found, unable to stop listenForEvents() gracefully", nodeID) - } - err := signaler.Stop() + // we can get here if the sensor was deleted and the stream was not closed + // we should attempt to log a warning and stop & close the signal stream + // TODO: why call stopSignal() when we have access to the stream within this func? + c.log.Warnf("sensor '%s' cannot be found due to: %s. Stopping signal '%s' event stream...", sensor, err, signal) + err := c.stopSignal(nodeID) if err != nil { - c.log.Panicf("failed to stop signaler '%s'", nodeID) + c.log.Panicf("failed to stop signal stream '%s': %s", nodeID, err) } } - // requeue this sensor with the event context - /* - key, err := cache.MetaNamespaceKeyFunc(s) - if err == nil { - c.queue.Add(key) - } - */ + phase := s.Status.Phase + msg := s.Status.Message node, ok := s.Status.Nodes[nodeID] if !ok { + // TODO: should we re-initialize this node? c.log.Panicf("'%s' node is missing from sensor's nodes", nodeID) } - node.LatestEvent = &v1alpha1.EventWrapper{Event: *event} - s.Status.Nodes[nodeID] = node - _, err = sensors.Update(s) + + if streamErr != nil { + c.log.Infof("ERROR: sensor '%s' signal '%s' stream failed: %s", sensor, signal, streamErr) + // error received from the stream + // remove the stream from the signalStreams map + c.signalMu.Lock() + c.signalStreams[nodeID] = nil + delete(c.signalStreams, nodeID) + c.signalMu.Unlock() + + // mark the sensor & node as error phase + phase = v1alpha1.NodePhaseError + msg = fmt.Sprintf("signal '%s' encountered stream err: %s", signal, streamErr) + node.Phase = v1alpha1.NodePhaseError + node.Message = streamErr.Error() + } else { + c.log.Debugf("sensor '%s' received event for signal '%s'", sensor, signal) + node.LatestEvent = &v1alpha1.EventWrapper{Event: *in.Event} + } + + // TODO: perform a Patch here instead as the sensor could become stale + // and this exponential backoff is slow + err = wait.ExponentialBackoff(common.DefaultRetry, func() (bool, error) { + s, err := sensors.Get(sensor, metav1.GetOptions{}) + if err != nil { + return false, err + } + s.Status.Nodes[nodeID] = node + s.Status.Phase = phase + s.Status.Message = msg + _, err = sensors.Update(s) + if err != nil { + if !common.IsRetryableKubeAPIError(err) { + return false, err + } + return false, nil + } + return true, nil + }) if err != nil { - c.log.Panicf("failed to update sensor with event") + c.log.Panicf("failed to update sensor: %s", err) + } + + // finally check if there was a streamErr, we must return + if streamErr != nil { + return } } } diff --git a/docs/index.md b/docs/index.md index 0be197b97a05..078f406cf7b4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3,13 +3,14 @@ ## Why Argo Events? - Containers. Designed from the ground-up as Kubernetes-native. - Extremely lightweight. All signals, with exception of calendar based signals, are event-driven, meaning there is no polling involved. -- High performance. Each `sensor` runs in its own Kubernetes job enabling high bandwidth for processing near-real time events. +- Configurable. Choose which signals to support and only deploy those you want to Kubernetes. +- Scalability & Resilient. Signal sources run as stateless microservices enabling you to scale simply by increasing the deployment replicas. - Simple or Complex dependencies. Manage everything from simple, linear, real-time dependencies to complex, multi-source batch job dependencies. ## Basics Argo Events is an open source event-based dependency manager for Kubernetes. The core concept of the project are `sensors` which are implemented as a Kubernetes-native Custom Resource Definition that define a set of dependencies (inputs) and actions (outputs). The sensor's actions will only be triggered after it's dependencies have been resolved. - Define multiple dependencies from a variety of sources -- Define dependency constraints and build plugins to support business-level constraint logic +- Plugin custom signal microservices to support business-level constraint logic - Trigger messages and Kubernetes object creation after successful dependency resolution - Trigger escalation after errors, or dependency constraint failures - Build and manage a distributed, cross-team, event-driven architecture diff --git a/docs/plugin-guide.md b/docs/plugin-guide.md deleted file mode 100644 index c8d9a9bd0fe9..000000000000 --- a/docs/plugin-guide.md +++ /dev/null @@ -1,27 +0,0 @@ -# Plugging in Signals -This is a walkthrough for how to plugin different signals. I am leveraging [hashicorp's go-plugin](https://github.com/hashicorp/go-plugin) in implementing the signal interface and RPC. Currently, `Stream` signals are only supported. See the [Stream Signals](../signals/stream/README.md) for an overview of use cases and design. - -## 1. Follow the [quickstart](quickstart.md) to get Minikube up & running - -## 2. Build a Go binary plugin. See the `signals/stream/builtin/nats` implementation for an example for how to do this. - -## 3. Modify the `controller/Dockerfile` to copy the plugin binary to the `STREAM_PLUGIN_DIR` directory. -Note that the binary name should be equal to the Signal.Stream.Type field value. - -## 4. Build the controller Dockerfile -``` -$ make controller-image -``` - -## 5. Create a sensor -``` -$ k create -f examples/nats-sensor.yaml -``` - -## 6. Trigger the signal execution -This depends on your signal implementation. For `NATS`, you can use `github.com/shogsbro/natscat` you just have to expose the NATS service externally as a `LoadBalancer`. -``` -$ go get github.com/shogsbro/natscat -$ cd $GOPATH/src/github.shogsbro/natscat -$ ./natscat -S http://192.168.99.100:32472 -s bucketevents "test" -``` diff --git a/docs/quickstart.md b/docs/quickstart.md index dc42b2a27660..09eab740e8bd 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -45,7 +45,7 @@ kubectl create -f hack/k8s/manifests/* ``` $ helm init ... -$ helm install stable/minio --name artifacts +$ helm install stable/minio --name artifacts --set service.type=LoadBalancer ... $ #Verify that the minio pod, the minio service and minio secret are present diff --git a/docs/signal-guide.md b/docs/signal-guide.md index 34100d138475..583a1ee15d31 100644 --- a/docs/signal-guide.md +++ b/docs/signal-guide.md @@ -6,15 +6,15 @@ A `signal` is a dependency, namely: - A message on a queue - An object created in S3 - A repeated calendar schedule -- A kubernetes resource -- A HTTP notification +- A Kubernetes resource +- An HTTP notification ### Prerequisites You need a working Kubernetes cluster at version >= 1.9. You will also need to install the `sensor-controller` into the cluster. This controller is is responsible for managing the `sensor` resources. -In order to take advantage of the various signal types, you may need to install compatible message platforms (e.g. amqp, mmqp, NATS, etc..) and s3 api compatible object storage servers (e.g. Minio, Rook, CEPH, NetApp). See the [signal guide](signal-guide.md) for more information about installing messaging platforms and [artifact guide](artifact-guide.md) for installing object stores. +In order to take advantage of the various signal types, you may need to install compatible message platforms (e.g. amqp, mmqp, NATS, etc..) and s3 api compatible object storage servers (e.g. Minio, Rook, CEPH, NetApp). See the [artifact guide](artifact-guide.md) for installing object stores. ## Sensor Controller -The `sensor-controller` is responsible for managing the `Sensor` resources and creating `sensor-executor` jobs. +The `sensor-controller` is responsible for managing the `Sensor` resources, listening on sensor signals, and executing sensor triggers. The following types of signals are supported: - Artifact signals which can include things like S3 Bucket Notifications etc.. - Stream signals which subscribe to messages on a queue or a topic @@ -26,11 +26,10 @@ The following types of signals are supported: - Resource triggers produce Kubernetes objects - Message triggers produce messages on a streaming platform -## Sensor Executor -The `sensor-executor` is responsible for listening for various signals and updating the sensor resource with updates. There is a one-to-one mapping from sensor resource to executor job. Jobs are named in the following format: `{sensor-name}-sensor` and the associated job's pods are named in the following format: `{sensor-name}-sensor-{id}`. On successful resolution of a sensor, the job terminates and the sensor is marked as `Successful`. If an error occurs during signal processing, the sensor is marked as `Error`. +## Signal Deployments +Signals are configured as separate deployments to the main sensor controller. Signals are registered as stateless [micro](https://github.com/micro/go-micro) "microservices". Users can specify in the deployment spec for each signal how many replica pods to run for the particular sensor in order to increase event bandwidth available and also make signals more resilient. For help in building and running these see the [Signals](../signals/README.md) README. - -## Types of Signal Dependencies +## Types of Signals ### Calendar Time-based signals can include signals based on a [cron]() schedule or an [interval duration](https://golang.org/pkg/time/#ParseDuration). In addition, calendar signals currently support a `recurrence` field in which to specify special exclusion dates for which this signal will not produce an event. Eventually, we hope to support a calendar-plugin or interface where users can configure special handling calendar/business logic. @@ -46,7 +45,7 @@ Supports watching Kubernetes resources. Users can specify `group`, `version`, `k Supports S3 artifact signals in the form of `bucket-notifications` via [Minio](https://docs.minio.io/docs/minio-bucket-notification-guide). Note that a supported notification target must be running, exposed, and configured in the Minio server. For more information, please refer to the [artifact guide](artifact-guide.md). ### Message Streams / Brokers -Supports a generic specification for message stream signals. Currently, there is a push signals toward being extensible and one solution we are investigating is toward classifying signals with common definitions and building a plugin-based architecture for supporting the specific signal implementations. The following defines the currently supported types of stream signals and examples of how to define them. +Supports a generic specification for messages received on a queue and/or though messaging server. The following are the `builtin` supported stream signals. Users can build their own signals by adding implementations to the `custom` package. #### NATS [Nats](https://nats.io/) is an open-sourced, lightweight, secure, and scalable messaging system for cloud native applications and microservices architecture. It is currently a hosted CNCF Project. We are currently experimenting with using NATS as a solution for signals (inputs) and triggers (outputs), however `NATS Streaming`, the data streaming system powered by NATS, offers many additional [features](https://nats.io/documentation/streaming/nats-streaming-intro/) on top of the core NATS platform that we believe are very desirable and definite future enhancements. diff --git a/examples/webhook-sensor.yaml b/examples/webhook-sensor.yaml index c087d44fa847..81951d88efa7 100644 --- a/examples/webhook-sensor.yaml +++ b/examples/webhook-sensor.yaml @@ -1,18 +1,16 @@ -apiVersion: core.events/v1alpha1 +apiVersion: argoproj.io/v1alpha1 kind: Sensor metadata: name: webhook-example - namespace: cloud-native-scheduler labels: sensors.argoproj.io/controller-instanceid: axis spec: - repeat: true signals: - name: webhook webhook: port: 9000 - endpoint: "/app" - method: "POST" + endpoint: /app + method: POST triggers: - name: done-workflow resource: diff --git a/hack/generate-proto.sh b/hack/generate-proto.sh index 9ce926b7fcf0..016aaf2e43d2 100755 --- a/hack/generate-proto.sh +++ b/hack/generate-proto.sh @@ -44,24 +44,27 @@ go-to-protobuf \ # server/*/.pb.go from .proto files. golang/protobuf and gogo/protobuf can be used # interchangeably. The difference in the options are: # 1. protoc-gen-go - official golang/protobuf -go build -i -o dist/protoc-gen-go ./vendor/github.com/golang/protobuf/protoc-gen-go -GOPROTOBINARY=go +#go build -i -o dist/protoc-gen-go ./vendor/github.com/golang/protobuf/protoc-gen-go +#GOPROTOBINARY=go # 2. protoc-gen-gofast - fork of golang golang/protobuf. Faster code generation -#go build -i -o dist/protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast -#GOPROTOBINARY=gofast +go build -i -o dist/protoc-gen-gofast ./vendor/github.com/gogo/protobuf/protoc-gen-gofast +GOPROTOBINARY=gofast # 3. protoc-gen-gogofast - faster code generation and gogo extensions and flexibility in controlling # the generated go code (e.g. customizing field names, nullable fields) #go build -i -o dist/protoc-gen-gogofast ./vendor/github.com/gogo/protobuf/protoc-gen-gogofast #GOPROTOBINARY=gogofast +go build -i -o dist/protoc-gen-micro ./vendor/github.com/micro/protoc-gen-micro + # Generate job//(.pb.go) -PROTO_FILES=$(find $PROJECT_ROOT \( -name "*.proto" -and -path '*/shared/*' \)) +PROTO_FILES=$(find $PROJECT_ROOT \( -name "*.proto" -and -path '*/sdk/*' \)) for i in ${PROTO_FILES}; do protoc \ -I${PROJECT_ROOT} \ -I/usr/local/include \ -I./vendor \ -I$GOPATH/src \ + --micro_out=$GOPATH/src \ --${GOPROTOBINARY}_out=plugins=grpc:$GOPATH/src \ $i done \ No newline at end of file diff --git a/hack/k8s/manifests/argo-events-cluster-roles.yaml b/hack/k8s/manifests/argo-events-cluster-roles.yaml index 803c79ea7fcc..e9844ee26b35 100644 --- a/hack/k8s/manifests/argo-events-cluster-roles.yaml +++ b/hack/k8s/manifests/argo-events-cluster-roles.yaml @@ -24,5 +24,5 @@ rules: resources: ["workflows"] verbs: ["create", "delete"] - apiGroups: [""] - resources: ["configmaps", "secrets"] - verbs: ["get", "watch", "list"] + resources: ["configmaps", "secrets", "pods"] + verbs: ["get", "watch", "list", "patch"] diff --git a/hack/k8s/manifests/argo-events-sa.yaml b/hack/k8s/manifests/argo-events-sa.yaml index 852c82e0b1ac..10e482a8c791 100644 --- a/hack/k8s/manifests/argo-events-sa.yaml +++ b/hack/k8s/manifests/argo-events-sa.yaml @@ -1,4 +1,4 @@ -# All argoe-events services are bound to the "argo-events" service account. +# All argo-events services are bound to the "argo-events" service account. # In RBAC enabled setups, this SA is bound to specific roles. apiVersion: v1 kind: ServiceAccount diff --git a/hack/k8s/manifests/sensor-controller-configmap.yaml b/hack/k8s/manifests/sensor-controller-configmap.yaml index de8a326703e6..01a9b642d2a7 100644 --- a/hack/k8s/manifests/sensor-controller-configmap.yaml +++ b/hack/k8s/manifests/sensor-controller-configmap.yaml @@ -6,4 +6,3 @@ metadata: data: config: | instanceID: axis - namespace: default diff --git a/hack/k8s/manifests/services/artifact.yaml b/hack/k8s/manifests/services/artifact.yaml new file mode 100644 index 000000000000..34f814c77a70 --- /dev/null +++ b/hack/k8s/manifests/services/artifact.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: signal-artifact +spec: + replicas: 1 + template: + metadata: + labels: + app: artifact + spec: + serviceAccountName: argo-events-sa + containers: + - name: artifact + image: argoproj/artifact-signal:latest + imagePullPolicy: IfNotPresent + env: + - name: MICRO_SERVER_ADDRESS + value: 0.0.0.0:8080 + - name: MICRO_BROKER_ADDRESS + value: 0.0.0.0:10001 + ports: + - containerPort: 8080 + name: micro-port +--- +apiVersion: v1 +kind: Service +metadata: + name: artifact + labels: + app: artifact +spec: + ports: + - name: micro-port + port: 8080 + selector: + app: artifact diff --git a/hack/k8s/manifests/services/calendar.yaml b/hack/k8s/manifests/services/calendar.yaml new file mode 100644 index 000000000000..1e7dd474c7da --- /dev/null +++ b/hack/k8s/manifests/services/calendar.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: signal-calendar +spec: + replicas: 1 + template: + metadata: + labels: + app: calendar + spec: + serviceAccountName: argo-events-sa + containers: + - name: calendar + image: argoproj/calendar-signal:latest + imagePullPolicy: IfNotPresent + env: + - name: MICRO_SERVER_ADDRESS + value: 0.0.0.0:8080 + - name: MICRO_BROKER_ADDRESS + value: 0.0.0.0:10001 + ports: + - containerPort: 8080 + name: micro-port +--- +apiVersion: v1 +kind: Service +metadata: + name: calendar + labels: + app: calendar +spec: + ports: + - name: micro-port + port: 8080 + selector: + app: calendar diff --git a/hack/k8s/manifests/services/resource.yaml b/hack/k8s/manifests/services/resource.yaml new file mode 100644 index 000000000000..e8e31eecf567 --- /dev/null +++ b/hack/k8s/manifests/services/resource.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: signal-resource +spec: + replicas: 1 + template: + metadata: + labels: + app: resource + spec: + serviceAccountName: argo-events-sa + containers: + - name: resource + image: argoproj/resource-signal:latest + imagePullPolicy: IfNotPresent + env: + - name: MICRO_SERVER_ADDRESS + value: 0.0.0.0:8080 + - name: MICRO_BROKER_ADDRESS + value: 0.0.0.0:10001 + ports: + - containerPort: 8080 + name: micro-port +--- +apiVersion: v1 +kind: Service +metadata: + name: resource + labels: + app: resource +spec: + ports: + - name: micro-port + port: 8080 + selector: + app: resource diff --git a/hack/k8s/manifests/services/stream.yaml b/hack/k8s/manifests/services/stream.yaml new file mode 100644 index 000000000000..ca00171fe79e --- /dev/null +++ b/hack/k8s/manifests/services/stream.yaml @@ -0,0 +1,37 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: signal-stream-nats +spec: + replicas: 1 + template: + metadata: + labels: + app: stream-nats + spec: + serviceAccountName: argo-events-sa + containers: + - name: stream-nats + image: argoproj/stream-nats-signal:latest + imagePullPolicy: IfNotPresent + env: + - name: MICRO_SERVER_ADDRESS + value: 0.0.0.0:8080 + - name: MICRO_BROKER_ADDRESS + value: 0.0.0.0:10001 + ports: + - containerPort: 8080 + name: micro-port +--- +apiVersion: v1 +kind: Service +metadata: + name: stream-nats + labels: + app: stream-nats +spec: + ports: + - name: micro-port + port: 8080 + selector: + app: stream-nats diff --git a/hack/k8s/manifests/services/webhook.yaml b/hack/k8s/manifests/services/webhook.yaml new file mode 100644 index 000000000000..cab7aa698aa6 --- /dev/null +++ b/hack/k8s/manifests/services/webhook.yaml @@ -0,0 +1,42 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: signal-webhook +spec: + replicas: 1 + template: + metadata: + labels: + app: webhook + spec: + serviceAccountName: argo-events-sa + containers: + - name: webhook + image: argoproj/webhook-signal:latest + imagePullPolicy: IfNotPresent + env: + - name: MICRO_SERVER_ADDRESS + value: 0.0.0.0:8080 + - name: MICRO_BROKER_ADDRESS + value: 0.0.0.0:10001 + ports: + - containerPort: 8080 + name: micro-port + - containerPort: 7070 + name: webhook-port +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook + labels: + app: webhook +spec: + type: LoadBalancer + ports: + - name: micro-port + port: 8080 + - name: webhook-port + port: 7070 + selector: + app: webhook diff --git a/pkg/apis/sensor/v1alpha1/generated.pb.go b/pkg/apis/sensor/v1alpha1/generated.pb.go index b518a201591b..9eee90f20539 100644 --- a/pkg/apis/sensor/v1alpha1/generated.pb.go +++ b/pkg/apis/sensor/v1alpha1/generated.pb.go @@ -44,7 +44,7 @@ const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package func (m *ArtifactLocation) Reset() { *m = ArtifactLocation{} } func (*ArtifactLocation) ProtoMessage() {} func (*ArtifactLocation) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{0} + return fileDescriptor_generated_5470019f7b19666c, []int{0} } func (m *ArtifactLocation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -72,7 +72,7 @@ var xxx_messageInfo_ArtifactLocation proto.InternalMessageInfo func (m *ArtifactSignal) Reset() { *m = ArtifactSignal{} } func (*ArtifactSignal) ProtoMessage() {} func (*ArtifactSignal) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{1} + return fileDescriptor_generated_5470019f7b19666c, []int{1} } func (m *ArtifactSignal) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -100,7 +100,7 @@ var xxx_messageInfo_ArtifactSignal proto.InternalMessageInfo func (m *CalendarSignal) Reset() { *m = CalendarSignal{} } func (*CalendarSignal) ProtoMessage() {} func (*CalendarSignal) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{2} + return fileDescriptor_generated_5470019f7b19666c, []int{2} } func (m *CalendarSignal) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -128,7 +128,7 @@ var xxx_messageInfo_CalendarSignal proto.InternalMessageInfo func (m *EscalationPolicy) Reset() { *m = EscalationPolicy{} } func (*EscalationPolicy) ProtoMessage() {} func (*EscalationPolicy) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{3} + return fileDescriptor_generated_5470019f7b19666c, []int{3} } func (m *EscalationPolicy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -156,7 +156,7 @@ var xxx_messageInfo_EscalationPolicy proto.InternalMessageInfo func (m *Event) Reset() { *m = Event{} } func (*Event) ProtoMessage() {} func (*Event) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{4} + return fileDescriptor_generated_5470019f7b19666c, []int{4} } func (m *Event) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -184,7 +184,7 @@ var xxx_messageInfo_Event proto.InternalMessageInfo func (m *EventContext) Reset() { *m = EventContext{} } func (*EventContext) ProtoMessage() {} func (*EventContext) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{5} + return fileDescriptor_generated_5470019f7b19666c, []int{5} } func (m *EventContext) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -212,7 +212,7 @@ var xxx_messageInfo_EventContext proto.InternalMessageInfo func (m *EventWrapper) Reset() { *m = EventWrapper{} } func (*EventWrapper) ProtoMessage() {} func (*EventWrapper) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{6} + return fileDescriptor_generated_5470019f7b19666c, []int{6} } func (m *EventWrapper) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -240,7 +240,7 @@ var xxx_messageInfo_EventWrapper proto.InternalMessageInfo func (m *GroupVersionKind) Reset() { *m = GroupVersionKind{} } func (*GroupVersionKind) ProtoMessage() {} func (*GroupVersionKind) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{7} + return fileDescriptor_generated_5470019f7b19666c, []int{7} } func (m *GroupVersionKind) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -268,7 +268,7 @@ var xxx_messageInfo_GroupVersionKind proto.InternalMessageInfo func (m *Message) Reset() { *m = Message{} } func (*Message) ProtoMessage() {} func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{8} + return fileDescriptor_generated_5470019f7b19666c, []int{8} } func (m *Message) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -296,7 +296,7 @@ var xxx_messageInfo_Message proto.InternalMessageInfo func (m *NodeStatus) Reset() { *m = NodeStatus{} } func (*NodeStatus) ProtoMessage() {} func (*NodeStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{9} + return fileDescriptor_generated_5470019f7b19666c, []int{9} } func (m *NodeStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -324,7 +324,7 @@ var xxx_messageInfo_NodeStatus proto.InternalMessageInfo func (m *ResourceFilter) Reset() { *m = ResourceFilter{} } func (*ResourceFilter) ProtoMessage() {} func (*ResourceFilter) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{10} + return fileDescriptor_generated_5470019f7b19666c, []int{10} } func (m *ResourceFilter) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -352,7 +352,7 @@ var xxx_messageInfo_ResourceFilter proto.InternalMessageInfo func (m *ResourceObject) Reset() { *m = ResourceObject{} } func (*ResourceObject) ProtoMessage() {} func (*ResourceObject) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{11} + return fileDescriptor_generated_5470019f7b19666c, []int{11} } func (m *ResourceObject) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -380,7 +380,7 @@ var xxx_messageInfo_ResourceObject proto.InternalMessageInfo func (m *ResourceSignal) Reset() { *m = ResourceSignal{} } func (*ResourceSignal) ProtoMessage() {} func (*ResourceSignal) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{12} + return fileDescriptor_generated_5470019f7b19666c, []int{12} } func (m *ResourceSignal) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -408,7 +408,7 @@ var xxx_messageInfo_ResourceSignal proto.InternalMessageInfo func (m *RetryStrategy) Reset() { *m = RetryStrategy{} } func (*RetryStrategy) ProtoMessage() {} func (*RetryStrategy) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{13} + return fileDescriptor_generated_5470019f7b19666c, []int{13} } func (m *RetryStrategy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -436,7 +436,7 @@ var xxx_messageInfo_RetryStrategy proto.InternalMessageInfo func (m *S3Artifact) Reset() { *m = S3Artifact{} } func (*S3Artifact) ProtoMessage() {} func (*S3Artifact) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{14} + return fileDescriptor_generated_5470019f7b19666c, []int{14} } func (m *S3Artifact) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -464,7 +464,7 @@ var xxx_messageInfo_S3Artifact proto.InternalMessageInfo func (m *S3Bucket) Reset() { *m = S3Bucket{} } func (*S3Bucket) ProtoMessage() {} func (*S3Bucket) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{15} + return fileDescriptor_generated_5470019f7b19666c, []int{15} } func (m *S3Bucket) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -492,7 +492,7 @@ var xxx_messageInfo_S3Bucket proto.InternalMessageInfo func (m *S3Filter) Reset() { *m = S3Filter{} } func (*S3Filter) ProtoMessage() {} func (*S3Filter) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{16} + return fileDescriptor_generated_5470019f7b19666c, []int{16} } func (m *S3Filter) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -520,7 +520,7 @@ var xxx_messageInfo_S3Filter proto.InternalMessageInfo func (m *Sensor) Reset() { *m = Sensor{} } func (*Sensor) ProtoMessage() {} func (*Sensor) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{17} + return fileDescriptor_generated_5470019f7b19666c, []int{17} } func (m *Sensor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -548,7 +548,7 @@ var xxx_messageInfo_Sensor proto.InternalMessageInfo func (m *SensorList) Reset() { *m = SensorList{} } func (*SensorList) ProtoMessage() {} func (*SensorList) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{18} + return fileDescriptor_generated_5470019f7b19666c, []int{18} } func (m *SensorList) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -576,7 +576,7 @@ var xxx_messageInfo_SensorList proto.InternalMessageInfo func (m *SensorSpec) Reset() { *m = SensorSpec{} } func (*SensorSpec) ProtoMessage() {} func (*SensorSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{19} + return fileDescriptor_generated_5470019f7b19666c, []int{19} } func (m *SensorSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -604,7 +604,7 @@ var xxx_messageInfo_SensorSpec proto.InternalMessageInfo func (m *SensorStatus) Reset() { *m = SensorStatus{} } func (*SensorStatus) ProtoMessage() {} func (*SensorStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{20} + return fileDescriptor_generated_5470019f7b19666c, []int{20} } func (m *SensorStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -632,7 +632,7 @@ var xxx_messageInfo_SensorStatus proto.InternalMessageInfo func (m *Signal) Reset() { *m = Signal{} } func (*Signal) ProtoMessage() {} func (*Signal) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{21} + return fileDescriptor_generated_5470019f7b19666c, []int{21} } func (m *Signal) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -660,7 +660,7 @@ var xxx_messageInfo_Signal proto.InternalMessageInfo func (m *SignalConstraints) Reset() { *m = SignalConstraints{} } func (*SignalConstraints) ProtoMessage() {} func (*SignalConstraints) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{22} + return fileDescriptor_generated_5470019f7b19666c, []int{22} } func (m *SignalConstraints) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -688,7 +688,7 @@ var xxx_messageInfo_SignalConstraints proto.InternalMessageInfo func (m *Stream) Reset() { *m = Stream{} } func (*Stream) ProtoMessage() {} func (*Stream) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{23} + return fileDescriptor_generated_5470019f7b19666c, []int{23} } func (m *Stream) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -716,7 +716,7 @@ var xxx_messageInfo_Stream proto.InternalMessageInfo func (m *TimeConstraints) Reset() { *m = TimeConstraints{} } func (*TimeConstraints) ProtoMessage() {} func (*TimeConstraints) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{24} + return fileDescriptor_generated_5470019f7b19666c, []int{24} } func (m *TimeConstraints) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -744,7 +744,7 @@ var xxx_messageInfo_TimeConstraints proto.InternalMessageInfo func (m *Trigger) Reset() { *m = Trigger{} } func (*Trigger) ProtoMessage() {} func (*Trigger) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{25} + return fileDescriptor_generated_5470019f7b19666c, []int{25} } func (m *Trigger) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -772,7 +772,7 @@ var xxx_messageInfo_Trigger proto.InternalMessageInfo func (m *URI) Reset() { *m = URI{} } func (*URI) ProtoMessage() {} func (*URI) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{26} + return fileDescriptor_generated_5470019f7b19666c, []int{26} } func (m *URI) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -800,7 +800,7 @@ var xxx_messageInfo_URI proto.InternalMessageInfo func (m *WebhookSignal) Reset() { *m = WebhookSignal{} } func (*WebhookSignal) ProtoMessage() {} func (*WebhookSignal) Descriptor() ([]byte, []int) { - return fileDescriptor_generated_4c80987033103634, []int{27} + return fileDescriptor_generated_5470019f7b19666c, []int{27} } func (m *WebhookSignal) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1802,14 +1802,6 @@ func (m *SensorStatus) MarshalTo(dAtA []byte) (int, error) { i += n30 } } - dAtA[i] = 0x30 - i++ - if m.Escalated { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ return i, nil } @@ -2470,7 +2462,6 @@ func (m *SensorStatus) Size() (n int) { n += mapEntrySize + 1 + sovGenerated(uint64(mapEntrySize)) } } - n += 2 return n } @@ -2913,7 +2904,6 @@ func (this *SensorStatus) String() string { `CompletedAt:` + strings.Replace(strings.Replace(this.CompletedAt.String(), "Time", "v1.Time", 1), `&`, ``, 1) + `,`, `Message:` + fmt.Sprintf("%v", this.Message) + `,`, `Nodes:` + mapStringForNodes + `,`, - `Escalated:` + fmt.Sprintf("%v", this.Escalated) + `,`, `}`, }, "") return s @@ -6626,26 +6616,6 @@ func (m *SensorStatus) Unmarshal(dAtA []byte) error { } m.Nodes[mapkey] = *mapvalue iNdEx = postIndex - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Escalated", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Escalated = bool(v != 0) default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -8059,163 +8029,162 @@ var ( ) func init() { - proto.RegisterFile("github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto", fileDescriptor_generated_4c80987033103634) -} - -var fileDescriptor_generated_4c80987033103634 = []byte{ - // 2461 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x59, 0xcd, 0x6f, 0x1c, 0x49, - 0x15, 0x4f, 0xcf, 0x97, 0xc7, 0x6f, 0x92, 0xd8, 0x5b, 0xec, 0x61, 0x64, 0xb1, 0x76, 0xd4, 0x2b, - 0x56, 0x01, 0x25, 0x3d, 0x1b, 0x1b, 0x50, 0x40, 0x0a, 0xc4, 0x63, 0x3b, 0x89, 0x37, 0x4e, 0xd6, - 0x5b, 0x93, 0x64, 0x45, 0x58, 0x09, 0xca, 0xdd, 0xe5, 0x99, 0x8e, 0x7b, 0xba, 0x7b, 0xab, 0x6a, - 0xbc, 0x19, 0x09, 0xad, 0x22, 0xb4, 0x12, 0x12, 0x5a, 0xb4, 0x2b, 0x24, 0x10, 0x87, 0xbd, 0x70, - 0xe0, 0xc4, 0x85, 0x13, 0x7f, 0x00, 0x17, 0x72, 0x5c, 0x24, 0x0e, 0x7b, 0xc1, 0x22, 0x46, 0xe2, - 0x8f, 0xc8, 0x09, 0xd5, 0x47, 0x57, 0xb7, 0x67, 0x62, 0x36, 0xf6, 0xcc, 0xc2, 0xc5, 0x72, 0xbf, - 0xf7, 0xea, 0xf7, 0xaa, 0x5e, 0xd5, 0xfb, 0x1c, 0xb8, 0xd5, 0x0d, 0x45, 0x6f, 0xb0, 0xe3, 0xf9, - 0x49, 0xbf, 0x45, 0x58, 0x37, 0x49, 0x59, 0xf2, 0x48, 0xfd, 0x73, 0x99, 0xee, 0xd3, 0x58, 0xf0, - 0x56, 0xba, 0xd7, 0x6d, 0x91, 0x34, 0xe4, 0x2d, 0x4e, 0x63, 0x9e, 0xb0, 0xd6, 0xfe, 0x15, 0x12, - 0xa5, 0x3d, 0x72, 0xa5, 0xd5, 0xa5, 0x31, 0x65, 0x44, 0xd0, 0xc0, 0x4b, 0x59, 0x22, 0x12, 0x74, - 0x35, 0x47, 0xf2, 0x32, 0x24, 0xf5, 0xcf, 0x4f, 0x34, 0x92, 0x97, 0xee, 0x75, 0x3d, 0x89, 0xe4, - 0x69, 0x24, 0x2f, 0x43, 0x5a, 0xb8, 0x5c, 0xd8, 0x43, 0x37, 0xe9, 0x26, 0x2d, 0x05, 0xb8, 0x33, - 0xd8, 0x55, 0x5f, 0xea, 0x43, 0xfd, 0xa7, 0x15, 0x2d, 0xb8, 0x7b, 0x57, 0xb9, 0x17, 0x26, 0x72, - 0x57, 0x2d, 0x3f, 0x61, 0xb4, 0xb5, 0x3f, 0xb6, 0x99, 0x85, 0x6f, 0xe7, 0x32, 0x7d, 0xe2, 0xf7, - 0xc2, 0x98, 0xb2, 0x61, 0x7e, 0x94, 0x3e, 0x15, 0xe4, 0x45, 0xab, 0x5a, 0xc7, 0xad, 0x62, 0x83, - 0x58, 0x84, 0x7d, 0x3a, 0xb6, 0xe0, 0xbb, 0x5f, 0xb6, 0x80, 0xfb, 0x3d, 0xda, 0x27, 0x63, 0xeb, - 0x56, 0x8e, 0x5b, 0x37, 0x10, 0x61, 0xd4, 0x0a, 0x63, 0xc1, 0x05, 0x1b, 0x5d, 0xe4, 0xa6, 0x30, - 0xbf, 0xca, 0x44, 0xb8, 0x4b, 0x7c, 0xb1, 0x95, 0xf8, 0x44, 0x84, 0x49, 0x8c, 0xde, 0x83, 0x12, - 0x5f, 0x69, 0x3a, 0x17, 0x9c, 0x8b, 0x8d, 0xe5, 0x75, 0xef, 0xb4, 0x37, 0xe0, 0x75, 0x56, 0x32, - 0xe4, 0x76, 0xed, 0xf0, 0x60, 0xa9, 0xd4, 0x59, 0xc1, 0x25, 0xbe, 0xe2, 0xfe, 0xaa, 0x04, 0xe7, - 0x33, 0x46, 0x27, 0xec, 0xc6, 0x24, 0x42, 0x3d, 0xa8, 0x09, 0xc2, 0xba, 0x54, 0x18, 0xa5, 0xd7, - 0x27, 0x50, 0x2a, 0x18, 0x25, 0xfd, 0xf6, 0xf9, 0xa7, 0x07, 0x4b, 0x67, 0x0e, 0x0f, 0x96, 0x6a, - 0xf7, 0x14, 0x2e, 0x36, 0xf8, 0xe8, 0x53, 0x07, 0xe6, 0xc9, 0xc8, 0x79, 0x9b, 0x25, 0xa5, 0xf4, - 0xad, 0xd3, 0x2b, 0x1d, 0xb5, 0x60, 0xbb, 0x69, 0xd4, 0x8f, 0xd9, 0x16, 0x8f, 0x69, 0x77, 0x3f, - 0x73, 0xe0, 0xfc, 0x1a, 0x89, 0x68, 0x1c, 0x10, 0x66, 0xec, 0x71, 0x09, 0xea, 0xf2, 0x8e, 0x83, - 0x41, 0x44, 0x95, 0x45, 0x66, 0xdb, 0xf3, 0x06, 0xb0, 0xde, 0x31, 0x74, 0x6c, 0x25, 0xa4, 0x74, - 0x18, 0x0b, 0xca, 0xf6, 0x49, 0xa4, 0x8e, 0x52, 0x90, 0xde, 0x34, 0x74, 0x6c, 0x25, 0x90, 0x07, - 0xc0, 0xa8, 0x3f, 0x60, 0x8c, 0xc6, 0x3e, 0x6d, 0x96, 0x2f, 0x94, 0x2f, 0xce, 0xb6, 0xcf, 0x1f, - 0x1e, 0x2c, 0x01, 0xb6, 0x54, 0x5c, 0x90, 0x70, 0xff, 0xe0, 0xc0, 0xfc, 0x06, 0xf7, 0x49, 0xa4, - 0x76, 0xbb, 0x9d, 0x44, 0xa1, 0x3f, 0x44, 0xaf, 0x43, 0x35, 0xa2, 0xfb, 0x34, 0x32, 0xbb, 0x3b, - 0x67, 0xf4, 0x55, 0xb7, 0x24, 0x11, 0x6b, 0x1e, 0x8a, 0x60, 0xa6, 0x4f, 0x39, 0x27, 0x5d, 0x6a, - 0x2c, 0xbc, 0x7a, 0x7a, 0x0b, 0xdf, 0xd1, 0x40, 0xed, 0x39, 0xa3, 0x69, 0xc6, 0x10, 0x70, 0xa6, - 0xc2, 0xfd, 0x9d, 0x03, 0xd5, 0x0d, 0x89, 0x82, 0xde, 0x87, 0x19, 0x3f, 0x89, 0x05, 0x7d, 0x9c, - 0x3d, 0xa7, 0x1b, 0xa7, 0xd7, 0xab, 0x10, 0xd7, 0x34, 0x5a, 0xae, 0xdc, 0x10, 0x70, 0xa6, 0x07, - 0x7d, 0x1d, 0x2a, 0x01, 0x11, 0x44, 0x9d, 0xf3, 0x6c, 0xbb, 0x7e, 0x78, 0xb0, 0x54, 0x59, 0x27, - 0x82, 0x60, 0x45, 0x75, 0xff, 0x58, 0x83, 0xb3, 0x45, 0x20, 0xd4, 0x82, 0x59, 0xa5, 0xf8, 0xde, - 0x30, 0xcd, 0x2e, 0xf8, 0x15, 0x83, 0x3d, 0xbb, 0x91, 0x31, 0x70, 0x2e, 0x83, 0xd6, 0x61, 0xde, - 0x7e, 0x3c, 0xa0, 0x8c, 0x67, 0xaf, 0x76, 0x36, 0x7f, 0x69, 0x1b, 0x23, 0x7c, 0x3c, 0xb6, 0x02, - 0xbd, 0x05, 0xc8, 0x8f, 0x92, 0x41, 0xa0, 0x44, 0x79, 0x86, 0x53, 0x56, 0x38, 0x0b, 0x06, 0x07, - 0xad, 0x8d, 0x49, 0xe0, 0x17, 0xac, 0x42, 0x04, 0x6a, 0x3c, 0x19, 0x30, 0x9f, 0x36, 0x2b, 0xca, - 0xc6, 0xd7, 0x4e, 0x6f, 0xe3, 0xfb, 0x78, 0xb3, 0x0d, 0xd2, 0x57, 0x3b, 0x0a, 0x10, 0x1b, 0x60, - 0xf4, 0x4d, 0x98, 0x51, 0x4b, 0x37, 0xd7, 0x9b, 0x55, 0xb5, 0x47, 0x6b, 0xff, 0x0d, 0x4d, 0xc6, - 0x19, 0x1f, 0xfd, 0x38, 0x33, 0x68, 0xd8, 0xa7, 0xcd, 0x9a, 0xda, 0xd0, 0xb7, 0x3c, 0x1d, 0x0e, - 0xbd, 0x62, 0x38, 0xcc, 0x37, 0x21, 0xa3, 0xb5, 0xb7, 0x7f, 0xc5, 0x93, 0x2b, 0x46, 0x8d, 0x1f, - 0xf6, 0xad, 0xf1, 0xc3, 0x3e, 0x45, 0x8f, 0x60, 0x56, 0x47, 0xdc, 0xfb, 0x78, 0xab, 0x39, 0x33, - 0x8d, 0xd3, 0x9e, 0x93, 0xba, 0x3a, 0x19, 0x26, 0xce, 0xe1, 0xd1, 0x77, 0xa0, 0xa1, 0xde, 0x94, - 0x79, 0x1b, 0x75, 0x75, 0xee, 0xaf, 0x99, 0xed, 0x35, 0xd6, 0x72, 0x16, 0x2e, 0xca, 0xa1, 0x5f, - 0x3a, 0x00, 0xf4, 0xb1, 0xa0, 0xb1, 0xbc, 0x1b, 0xde, 0x9c, 0xbd, 0x50, 0xbe, 0xd8, 0x58, 0x7e, - 0x30, 0x9d, 0x67, 0xef, 0x6d, 0x58, 0xe0, 0x8d, 0x58, 0xb0, 0x61, 0x1b, 0x99, 0xed, 0x40, 0xce, - 0xc0, 0x05, 0xed, 0x0b, 0xd7, 0x60, 0x6e, 0x64, 0x09, 0x9a, 0x87, 0xf2, 0x1e, 0x1d, 0xea, 0xa7, - 0x8e, 0xe5, 0xbf, 0xe8, 0x55, 0xa8, 0xee, 0x93, 0x68, 0xa0, 0x43, 0xc3, 0x2c, 0xd6, 0x1f, 0xdf, - 0x2f, 0x5d, 0x75, 0xdc, 0xdf, 0x3a, 0xc6, 0x5b, 0xde, 0x65, 0x24, 0x4d, 0x29, 0x43, 0x01, 0x54, - 0xd5, 0x7e, 0x8d, 0x37, 0xff, 0x70, 0xc2, 0x63, 0xe5, 0xd1, 0x4a, 0x7d, 0x62, 0x0d, 0x8e, 0x2e, - 0x40, 0x85, 0x53, 0xaa, 0xdd, 0xaa, 0xde, 0x3e, 0x6b, 0x64, 0x2a, 0x1d, 0x4a, 0x63, 0xac, 0x38, - 0xee, 0x47, 0x0e, 0xcc, 0xdf, 0x64, 0xc9, 0x20, 0x35, 0x3e, 0x70, 0x3b, 0x8c, 0x03, 0x19, 0x09, - 0xbb, 0x92, 0x36, 0x1a, 0x09, 0x95, 0x20, 0xd6, 0x3c, 0xf9, 0x92, 0xf7, 0x8f, 0x78, 0xad, 0x7d, - 0xc9, 0x99, 0x8b, 0x65, 0x7c, 0xb9, 0x8d, 0xbd, 0x30, 0x0e, 0x8c, 0x57, 0xda, 0x6d, 0x48, 0x5d, - 0x58, 0x71, 0xdc, 0xdf, 0x38, 0x90, 0x45, 0x3f, 0x29, 0xbd, 0x93, 0x04, 0xc6, 0xb0, 0xb9, 0x74, - 0x3b, 0x09, 0x86, 0x58, 0x71, 0x64, 0x6a, 0xe5, 0x2a, 0x25, 0x9a, 0x18, 0x3c, 0xc5, 0xd4, 0xaa, - 0xbf, 0xb1, 0xc1, 0x77, 0xff, 0x5a, 0x01, 0xb8, 0x9b, 0x04, 0xb4, 0x23, 0x88, 0x18, 0x70, 0xb4, - 0x00, 0xa5, 0x30, 0x30, 0x1b, 0x03, 0xb3, 0xa4, 0xb4, 0xb9, 0x8e, 0x4b, 0x61, 0x20, 0xb7, 0x1d, - 0x93, 0xbe, 0xb9, 0xfb, 0x7c, 0xdb, 0x77, 0x49, 0x9f, 0x62, 0xc5, 0x91, 0x7e, 0x10, 0x84, 0x3c, - 0x8d, 0xc8, 0x50, 0x12, 0x8d, 0x35, 0xac, 0x1f, 0xac, 0xe7, 0x2c, 0x5c, 0x94, 0x43, 0x97, 0xa0, - 0x22, 0xa4, 0xdf, 0x54, 0x8e, 0xc4, 0xc6, 0x8a, 0xf4, 0x91, 0xe7, 0x07, 0x4b, 0x75, 0xb9, 0x3d, - 0xe5, 0x3c, 0x4a, 0x0a, 0xbd, 0x09, 0xd5, 0xb4, 0x47, 0x38, 0x35, 0xe1, 0x25, 0x0b, 0x81, 0xd5, - 0x6d, 0x49, 0x7c, 0x7e, 0xb0, 0x34, 0x2b, 0xe5, 0xd5, 0x07, 0xd6, 0x82, 0x32, 0xce, 0x70, 0x41, - 0x98, 0xa0, 0xc1, 0xaa, 0x98, 0x24, 0xce, 0x74, 0x32, 0x10, 0x9c, 0xe3, 0x21, 0x22, 0x7d, 0xbf, - 0x9f, 0x46, 0x54, 0xc3, 0xcf, 0x9c, 0x18, 0xbe, 0x10, 0x27, 0x2c, 0x0c, 0x2e, 0x62, 0xca, 0x87, - 0x98, 0xa5, 0xe4, 0xfa, 0xd1, 0x87, 0x38, 0x9a, 0x4f, 0xd1, 0x10, 0x1a, 0x11, 0x11, 0x94, 0x0b, - 0xe5, 0x25, 0xcd, 0xd9, 0xa9, 0x64, 0x52, 0xe3, 0xd2, 0xed, 0x39, 0xb9, 0xcb, 0xad, 0x1c, 0x1e, - 0x17, 0x75, 0xb9, 0xbf, 0xaf, 0xc0, 0x79, 0x4c, 0x75, 0x16, 0xb8, 0x11, 0x46, 0x82, 0x32, 0xf4, - 0x06, 0xd4, 0x52, 0x46, 0x77, 0xc3, 0xc7, 0xe6, 0x45, 0xd9, 0x47, 0xb8, 0xad, 0xa8, 0xd8, 0x70, - 0xd1, 0xcf, 0xa0, 0x16, 0x91, 0x1d, 0x1a, 0xf1, 0x66, 0x49, 0xc5, 0xc0, 0x7b, 0xa7, 0xdf, 0xf0, - 0xd1, 0x1d, 0x78, 0x5b, 0x0a, 0x56, 0x47, 0x40, 0xab, 0x5d, 0x13, 0xb1, 0xd1, 0x29, 0xab, 0xcb, - 0x06, 0x89, 0xe3, 0x44, 0xa8, 0x5a, 0x89, 0xab, 0xea, 0xaa, 0xb1, 0xfc, 0xa3, 0xa9, 0xed, 0x61, - 0x35, 0xc7, 0xd6, 0x1b, 0xb1, 0x37, 0x5e, 0xe0, 0xe0, 0xe2, 0x16, 0xe4, 0x8b, 0xf5, 0x19, 0x95, - 0x05, 0x7f, 0x7b, 0x68, 0x52, 0xf5, 0xa9, 0x5e, 0xec, 0x5a, 0x06, 0x82, 0x73, 0xbc, 0x85, 0xef, - 0x41, 0xa3, 0x60, 0x96, 0x93, 0x44, 0xf9, 0x85, 0x1f, 0xc0, 0xfc, 0xe8, 0x69, 0x4e, 0x94, 0x25, - 0x3e, 0x2b, 0xbc, 0x91, 0xb7, 0x77, 0x1e, 0x51, 0x5f, 0x55, 0x55, 0x32, 0x76, 0xf0, 0x94, 0xf8, - 0x63, 0x55, 0xd5, 0xdd, 0x8c, 0x81, 0x73, 0x19, 0xf4, 0xf1, 0xff, 0xa6, 0x19, 0x78, 0xf5, 0xe5, - 0x1a, 0x81, 0xc2, 0xdb, 0x2d, 0x4f, 0xeb, 0xed, 0x6a, 0xcb, 0xbc, 0xec, 0xdb, 0x9d, 0xef, 0x8e, - 0x64, 0x37, 0xf3, 0x60, 0x26, 0x30, 0xc6, 0x68, 0xbe, 0xcc, 0xeb, 0xd5, 0x51, 0x0e, 0x1e, 0xd3, - 0x3e, 0xc1, 0xf3, 0x72, 0xff, 0x52, 0xca, 0x9f, 0x87, 0x69, 0xaa, 0x4e, 0xfc, 0x3c, 0x22, 0xa8, - 0xed, 0x2a, 0xbf, 0x33, 0x6f, 0xe2, 0xd6, 0xb4, 0xfc, 0x58, 0x57, 0xbb, 0xfa, 0x7f, 0x6c, 0x74, - 0xbc, 0xd8, 0xfe, 0xe5, 0xff, 0xa7, 0xfd, 0xdd, 0x39, 0x38, 0x87, 0xa9, 0x60, 0xc3, 0x8e, 0x60, - 0x44, 0xd0, 0xee, 0xd0, 0xfd, 0x47, 0x09, 0x20, 0xef, 0xea, 0xd1, 0x6b, 0x85, 0x0b, 0x69, 0x37, - 0x0c, 0x70, 0xf9, 0x36, 0x1d, 0xea, 0xdb, 0x79, 0x90, 0xd5, 0x6d, 0x3a, 0xcd, 0x5f, 0x3f, 0x52, - 0x76, 0x3d, 0x3f, 0x58, 0x6a, 0x15, 0x26, 0x34, 0xfd, 0x30, 0x0e, 0x13, 0xfd, 0xf7, 0x72, 0x37, - 0xf1, 0xee, 0x26, 0x22, 0xdc, 0x0d, 0xb5, 0x5f, 0xe4, 0x0d, 0x91, 0xa9, 0xd4, 0x76, 0xed, 0xbd, - 0x68, 0xf3, 0xb4, 0x27, 0x19, 0x51, 0xfc, 0x97, 0x1b, 0x49, 0xa1, 0xce, 0x57, 0xda, 0x03, 0x7f, - 0x8f, 0x0a, 0xe3, 0x08, 0x13, 0x69, 0xd2, 0x48, 0x85, 0x4e, 0xde, 0x50, 0xb0, 0xd5, 0xe2, 0xfe, - 0xbb, 0x04, 0x96, 0x2c, 0xdb, 0x7a, 0x1a, 0x07, 0x69, 0x12, 0x9a, 0xca, 0xb7, 0xd0, 0xd6, 0x6f, - 0x18, 0x3a, 0xb6, 0x12, 0x32, 0x41, 0xee, 0xe8, 0xad, 0x96, 0x8e, 0x26, 0x48, 0xa3, 0xc4, 0x70, - 0xa5, 0x1c, 0xa3, 0xdd, 0xbc, 0xef, 0xb3, 0x72, 0x58, 0x51, 0xb1, 0xe1, 0xea, 0xa1, 0x02, 0xa7, - 0xfe, 0x80, 0xe9, 0x6a, 0xaa, 0x5e, 0x1c, 0x2a, 0x68, 0x3a, 0xb6, 0x12, 0xe8, 0x01, 0xcc, 0x12, - 0xdf, 0xa7, 0x9c, 0xdf, 0xa6, 0x43, 0x55, 0x4d, 0x35, 0x96, 0xbf, 0x51, 0xc8, 0x32, 0x9e, 0x9f, - 0x30, 0x2a, 0x73, 0x4a, 0x87, 0xfa, 0x8c, 0x8a, 0xdb, 0x74, 0xd8, 0xa1, 0x11, 0xf5, 0x45, 0xc2, - 0x72, 0x17, 0x5c, 0xcd, 0xd6, 0xe3, 0x1c, 0x4a, 0xe2, 0xf2, 0x6c, 0x89, 0xa9, 0xb7, 0x4e, 0x8a, - 0x6b, 0x59, 0x38, 0x87, 0x72, 0x1f, 0x4a, 0x3b, 0x9f, 0xb0, 0xb4, 0x78, 0x03, 0x6a, 0x7c, 0xb0, - 0x2b, 0xe5, 0x46, 0x2c, 0xdc, 0x51, 0x54, 0x6c, 0xb8, 0x32, 0xf4, 0xd4, 0x3a, 0xea, 0xf6, 0xd1, - 0x4f, 0xa1, 0x2e, 0xb3, 0xa9, 0x1a, 0x0d, 0xe8, 0xe6, 0xe5, 0xcd, 0x97, 0xcb, 0xbd, 0x3a, 0x6e, - 0xdf, 0xa1, 0x82, 0xe4, 0xdd, 0x56, 0x4e, 0xc3, 0x16, 0x15, 0xed, 0x42, 0x85, 0xa7, 0xd4, 0x37, - 0x11, 0x6a, 0x92, 0x61, 0x9d, 0xfa, 0xee, 0xa4, 0xd4, 0x2f, 0xf4, 0x3e, 0x29, 0xf5, 0xb1, 0xc2, - 0x47, 0xb1, 0x6c, 0x23, 0x64, 0x5d, 0x6f, 0x7c, 0xee, 0xc6, 0xc4, 0x9a, 0x14, 0x5a, 0xb1, 0x99, - 0x90, 0xdf, 0xd8, 0x68, 0x71, 0xff, 0xe6, 0x00, 0x68, 0xc1, 0xad, 0x90, 0x0b, 0xf4, 0xde, 0x98, - 0x21, 0xbd, 0x97, 0x33, 0xa4, 0x5c, 0xad, 0xcc, 0x68, 0x5f, 0x6f, 0x46, 0x29, 0x18, 0x91, 0x42, - 0x35, 0x14, 0xb4, 0x9f, 0xd5, 0x8c, 0xd7, 0x27, 0x3d, 0x5b, 0xde, 0x05, 0x6e, 0x4a, 0x58, 0xac, - 0xd1, 0xdd, 0x5f, 0x97, 0xb3, 0x33, 0x49, 0xc3, 0xa2, 0x3d, 0x98, 0xe1, 0x2a, 0x33, 0xf1, 0xa6, - 0x33, 0xb1, 0x5e, 0x05, 0x94, 0x57, 0xf3, 0xfa, 0x9b, 0xe3, 0x4c, 0x03, 0x4a, 0xa0, 0x2e, 0x58, - 0xd8, 0xed, 0x52, 0x96, 0x9d, 0x72, 0x82, 0x61, 0xdc, 0x3d, 0x8d, 0x94, 0xdb, 0xd4, 0x10, 0x38, - 0xb6, 0x4a, 0xd0, 0x87, 0x00, 0xd4, 0x4e, 0x0d, 0x27, 0xcf, 0x63, 0xa3, 0x13, 0xc8, 0xc2, 0x10, - 0xc2, 0x72, 0x70, 0x41, 0xa3, 0x8e, 0x73, 0x29, 0x25, 0xc2, 0x44, 0xaf, 0x42, 0x9c, 0x93, 0x54, - 0x6c, 0xb8, 0xee, 0xdf, 0x2b, 0x70, 0xb6, 0xf8, 0x22, 0xf3, 0xa6, 0xd0, 0x39, 0x55, 0x53, 0x58, - 0xfa, 0x6a, 0x9b, 0xc2, 0xf2, 0x57, 0xdb, 0x14, 0x56, 0xbe, 0xa4, 0x29, 0xdc, 0x87, 0x6a, 0x9c, - 0x04, 0x94, 0x37, 0xab, 0xea, 0x0d, 0xbd, 0x33, 0x9d, 0x28, 0xe0, 0x49, 0x93, 0x9a, 0xf2, 0xd4, - 0xba, 0x8e, 0xa2, 0x61, 0xad, 0x4e, 0x0d, 0x4c, 0xf5, 0xdd, 0xd2, 0x40, 0xe5, 0x81, 0x7a, 0x61, - 0x66, 0x97, 0x31, 0x70, 0x2e, 0xb3, 0xf0, 0xa1, 0x9e, 0x45, 0x1c, 0x5b, 0x39, 0x3e, 0x2c, 0x56, - 0x8e, 0x13, 0x05, 0xce, 0x7c, 0xe4, 0x51, 0xac, 0x3f, 0x3f, 0xa9, 0x41, 0xcd, 0xd4, 0x9d, 0xd9, - 0xb0, 0xc3, 0x39, 0x76, 0xd8, 0x71, 0x09, 0xea, 0x01, 0x25, 0x41, 0x14, 0xc6, 0x7a, 0x3f, 0xe5, - 0xdc, 0xb3, 0xd6, 0x0d, 0x1d, 0x5b, 0x09, 0x14, 0xd8, 0x89, 0x4e, 0x79, 0x4a, 0x13, 0x1d, 0x18, - 0x9f, 0xe6, 0x20, 0x06, 0xf5, 0xac, 0x41, 0x31, 0xc5, 0xcf, 0xad, 0xc9, 0x5b, 0x22, 0x13, 0xa6, - 0xce, 0xca, 0x93, 0x65, 0x34, 0x6c, 0xf5, 0x48, 0x9d, 0xbe, 0xf9, 0x21, 0xc4, 0x14, 0x11, 0x13, - 0xe8, 0x3c, 0xfa, 0x93, 0x8a, 0xd6, 0x99, 0xd1, 0xb0, 0xd5, 0x23, 0x75, 0x32, 0x53, 0x9c, 0x9b, - 0x02, 0x63, 0x0a, 0x65, 0x7e, 0x51, 0x67, 0x46, 0xc3, 0x56, 0x0f, 0x8a, 0x61, 0xe6, 0x03, 0xba, - 0xd3, 0x4b, 0x92, 0x3d, 0x33, 0xe4, 0xb9, 0x79, 0x7a, 0x95, 0xef, 0x6a, 0x20, 0xa3, 0xb1, 0x21, - 0xbd, 0xd6, 0x90, 0x70, 0xa6, 0x04, 0xfd, 0xdc, 0x51, 0x53, 0x65, 0x2e, 0x18, 0x09, 0x63, 0xc1, - 0xd5, 0xe8, 0xa7, 0xb1, 0x7c, 0x7b, 0xd2, 0x74, 0xb3, 0x96, 0x43, 0x1e, 0x19, 0x51, 0x67, 0x44, - 0x5c, 0x54, 0xea, 0x3e, 0x71, 0xe0, 0x95, 0xb1, 0x75, 0x68, 0x0f, 0x2a, 0x22, 0x34, 0xce, 0xd1, - 0x58, 0xde, 0x9c, 0x20, 0x27, 0x85, 0x7d, 0x5a, 0xdc, 0x90, 0xf5, 0x33, 0x35, 0xcd, 0x57, 0x4a, - 0xdc, 0x4f, 0x64, 0x65, 0xa6, 0x9f, 0xf7, 0x05, 0x33, 0x28, 0x1c, 0x71, 0xca, 0xc2, 0x70, 0xf0, - 0x35, 0x28, 0x0f, 0x58, 0xf6, 0x83, 0x9a, 0x6d, 0x6e, 0xee, 0xe3, 0x2d, 0x2c, 0xe9, 0xe8, 0x23, - 0x07, 0x80, 0x08, 0xc1, 0xc2, 0x9d, 0x81, 0xa0, 0x59, 0xc7, 0xbe, 0x3d, 0xa9, 0x2b, 0x7a, 0xab, - 0x16, 0x72, 0x64, 0xd6, 0x9e, 0x33, 0x70, 0x41, 0xef, 0xc2, 0x35, 0x98, 0x1b, 0x59, 0x72, 0xa2, - 0x36, 0xf9, 0x4f, 0x0e, 0xcc, 0x8d, 0x58, 0x0e, 0xbd, 0x0d, 0x55, 0x95, 0x7e, 0xcc, 0x9d, 0x9c, - 0x24, 0xd7, 0xd8, 0xe0, 0xad, 0x52, 0x19, 0xd6, 0x38, 0x68, 0x0b, 0x2a, 0x5c, 0x24, 0xe9, 0x29, - 0x52, 0x63, 0x5e, 0x89, 0x8a, 0x24, 0xc5, 0x0a, 0xc5, 0xfd, 0xb8, 0x0c, 0x33, 0xa6, 0xde, 0x78, - 0x89, 0xd0, 0x5a, 0x74, 0xef, 0xa9, 0x75, 0xf1, 0xba, 0x12, 0x3f, 0xd6, 0xbd, 0x7b, 0x79, 0x3e, - 0x2d, 0x4f, 0xeb, 0x77, 0xcf, 0xc6, 0x0b, 0xd3, 0xf1, 0x13, 0x07, 0xce, 0x31, 0x9a, 0x46, 0xb6, - 0x43, 0x37, 0xa1, 0xfa, 0xe6, 0x24, 0x67, 0x2c, 0x34, 0xfc, 0xed, 0x57, 0x0e, 0x0f, 0x96, 0x8e, - 0xce, 0x00, 0xf0, 0x51, 0x85, 0xee, 0x9f, 0x4b, 0x50, 0xbe, 0x8f, 0x37, 0x55, 0x77, 0xe4, 0xf7, - 0xa8, 0xbd, 0x8c, 0xbc, 0xb0, 0x57, 0x54, 0x6c, 0xb8, 0xf2, 0xca, 0x06, 0xdc, 0x8c, 0x54, 0x0a, - 0x57, 0x76, 0x9f, 0x53, 0x86, 0x15, 0x47, 0x66, 0xc3, 0x94, 0x70, 0xfe, 0x41, 0xc2, 0xb2, 0x5f, - 0x41, 0x6c, 0x36, 0xdc, 0x36, 0x74, 0x6c, 0x25, 0x24, 0x5e, 0x2f, 0xe1, 0xc2, 0x54, 0x2e, 0x16, - 0xef, 0x56, 0xc2, 0x05, 0x56, 0x1c, 0x29, 0x91, 0x26, 0x4c, 0xa8, 0x8c, 0x52, 0xcd, 0x25, 0xb6, - 0x13, 0x26, 0xb0, 0xe2, 0x28, 0x09, 0x22, 0x7a, 0x2a, 0xfe, 0x17, 0x30, 0xb6, 0x89, 0xe8, 0x61, - 0xc5, 0x41, 0xaf, 0x43, 0xf5, 0xfd, 0x01, 0x65, 0x43, 0x15, 0xaf, 0x0b, 0xbf, 0xf2, 0xbc, 0x23, - 0x89, 0x58, 0xf3, 0xe4, 0xc6, 0x77, 0x19, 0xe9, 0xf6, 0x69, 0x2c, 0xcc, 0x74, 0xdd, 0x6e, 0xfc, - 0x86, 0xa1, 0x63, 0x2b, 0xe1, 0xfe, 0xc2, 0x81, 0x73, 0x47, 0x82, 0xf7, 0x09, 0x1b, 0xfe, 0xec, - 0x58, 0xa5, 0x63, 0x8f, 0xf5, 0x06, 0xd4, 0xfa, 0x54, 0xf4, 0x92, 0x60, 0xb4, 0xd5, 0xbf, 0xa3, - 0xa8, 0xd8, 0x70, 0xdb, 0xde, 0xd3, 0x67, 0x8b, 0x67, 0x3e, 0x7f, 0xb6, 0x78, 0xe6, 0x8b, 0x67, - 0x8b, 0x67, 0x9e, 0x1c, 0x2e, 0x3a, 0x4f, 0x0f, 0x17, 0x9d, 0xcf, 0x0f, 0x17, 0x9d, 0x2f, 0x0e, - 0x17, 0x9d, 0x7f, 0x1e, 0x2e, 0x3a, 0x9f, 0xfe, 0x6b, 0xf1, 0xcc, 0xc3, 0x7a, 0xf6, 0x44, 0xfe, - 0x13, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x90, 0x97, 0x76, 0xde, 0x23, 0x00, 0x00, + proto.RegisterFile("github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto", fileDescriptor_generated_5470019f7b19666c) +} + +var fileDescriptor_generated_5470019f7b19666c = []byte{ + // 2444 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x59, 0xcd, 0x6f, 0x1b, 0xc7, + 0x15, 0xf7, 0xf2, 0x4b, 0xe4, 0xa3, 0x6d, 0x29, 0xd3, 0x1c, 0x08, 0xa1, 0x91, 0x8c, 0x0d, 0x1a, + 0xb8, 0x85, 0xbd, 0x8c, 0xa5, 0xb6, 0x70, 0x0b, 0xb8, 0xb5, 0x28, 0xc9, 0xb6, 0x62, 0xd9, 0x51, + 0x86, 0xb6, 0x83, 0xba, 0x01, 0xda, 0xd1, 0x72, 0x44, 0xae, 0xb5, 0xdc, 0xdd, 0xcc, 0x0c, 0x15, + 0x13, 0x28, 0x02, 0xa3, 0x08, 0x50, 0xa0, 0x48, 0x91, 0xa0, 0x40, 0x8b, 0x1e, 0x72, 0x29, 0x8a, + 0x9e, 0x7a, 0xe9, 0xa9, 0x7f, 0x40, 0x2f, 0xf5, 0x31, 0xbd, 0xe5, 0x52, 0xa1, 0x56, 0x81, 0xfe, + 0x11, 0x3e, 0x15, 0xf3, 0xb1, 0xb3, 0x4b, 0xd2, 0x6a, 0x2c, 0x91, 0x69, 0x2f, 0x82, 0xf6, 0xbd, + 0x37, 0xbf, 0x37, 0xf3, 0xe6, 0x7d, 0x0e, 0xe1, 0x56, 0x37, 0x10, 0xbd, 0xc1, 0xae, 0xe7, 0xc7, + 0xfd, 0x26, 0x61, 0xdd, 0x38, 0x61, 0xf1, 0x23, 0xf5, 0xcf, 0x65, 0x7a, 0x40, 0x23, 0xc1, 0x9b, + 0xc9, 0x7e, 0xb7, 0x49, 0x92, 0x80, 0x37, 0x39, 0x8d, 0x78, 0xcc, 0x9a, 0x07, 0x57, 0x48, 0x98, + 0xf4, 0xc8, 0x95, 0x66, 0x97, 0x46, 0x94, 0x11, 0x41, 0x3b, 0x5e, 0xc2, 0x62, 0x11, 0xa3, 0xab, + 0x19, 0x92, 0x97, 0x22, 0xa9, 0x7f, 0x7e, 0xa2, 0x91, 0xbc, 0x64, 0xbf, 0xeb, 0x49, 0x24, 0x4f, + 0x23, 0x79, 0x29, 0xd2, 0xe2, 0xe5, 0xdc, 0x1e, 0xba, 0x71, 0x37, 0x6e, 0x2a, 0xc0, 0xdd, 0xc1, + 0x9e, 0xfa, 0x52, 0x1f, 0xea, 0x3f, 0xad, 0x68, 0xd1, 0xdd, 0xbf, 0xca, 0xbd, 0x20, 0x96, 0xbb, + 0x6a, 0xfa, 0x31, 0xa3, 0xcd, 0x83, 0x89, 0xcd, 0x2c, 0x7e, 0x3b, 0x93, 0xe9, 0x13, 0xbf, 0x17, + 0x44, 0x94, 0x0d, 0xb3, 0xa3, 0xf4, 0xa9, 0x20, 0x2f, 0x5a, 0xd5, 0x3c, 0x6e, 0x15, 0x1b, 0x44, + 0x22, 0xe8, 0xd3, 0x89, 0x05, 0xdf, 0xfd, 0xb2, 0x05, 0xdc, 0xef, 0xd1, 0x3e, 0x99, 0x58, 0xb7, + 0x7a, 0xdc, 0xba, 0x81, 0x08, 0xc2, 0x66, 0x10, 0x09, 0x2e, 0xd8, 0xf8, 0x22, 0x37, 0x81, 0x85, + 0x35, 0x26, 0x82, 0x3d, 0xe2, 0x8b, 0xed, 0xd8, 0x27, 0x22, 0x88, 0x23, 0xf4, 0x1e, 0x14, 0xf8, + 0x6a, 0xc3, 0xb9, 0xe0, 0x5c, 0xac, 0xaf, 0x6c, 0x78, 0xa7, 0xbd, 0x01, 0xaf, 0xbd, 0x9a, 0x22, + 0xb7, 0x2a, 0x47, 0x87, 0xcb, 0x85, 0xf6, 0x2a, 0x2e, 0xf0, 0x55, 0xf7, 0x57, 0x05, 0x38, 0x9f, + 0x32, 0xda, 0x41, 0x37, 0x22, 0x21, 0xea, 0x41, 0x45, 0x10, 0xd6, 0xa5, 0xc2, 0x28, 0xbd, 0x3e, + 0x85, 0x52, 0xc1, 0x28, 0xe9, 0xb7, 0xce, 0x3f, 0x3d, 0x5c, 0x3e, 0x73, 0x74, 0xb8, 0x5c, 0xb9, + 0xa7, 0x70, 0xb1, 0xc1, 0x47, 0x9f, 0x3a, 0xb0, 0x40, 0xc6, 0xce, 0xdb, 0x28, 0x28, 0xa5, 0x6f, + 0x9d, 0x5e, 0xe9, 0xb8, 0x05, 0x5b, 0x0d, 0xa3, 0x7e, 0xc2, 0xb6, 0x78, 0x42, 0xbb, 0xfb, 0x99, + 0x03, 0xe7, 0xd7, 0x49, 0x48, 0xa3, 0x0e, 0x61, 0xc6, 0x1e, 0x97, 0xa0, 0x2a, 0xef, 0xb8, 0x33, + 0x08, 0xa9, 0xb2, 0x48, 0xad, 0xb5, 0x60, 0x00, 0xab, 0x6d, 0x43, 0xc7, 0x56, 0x42, 0x4a, 0x07, + 0x91, 0xa0, 0xec, 0x80, 0x84, 0xea, 0x28, 0x39, 0xe9, 0x2d, 0x43, 0xc7, 0x56, 0x02, 0x79, 0x00, + 0x8c, 0xfa, 0x03, 0xc6, 0x68, 0xe4, 0xd3, 0x46, 0xf1, 0x42, 0xf1, 0x62, 0xad, 0x75, 0xfe, 0xe8, + 0x70, 0x19, 0xb0, 0xa5, 0xe2, 0x9c, 0x84, 0xfb, 0x47, 0x07, 0x16, 0x36, 0xb9, 0x4f, 0x42, 0xb5, + 0xdb, 0x9d, 0x38, 0x0c, 0xfc, 0x21, 0x7a, 0x1d, 0xca, 0x21, 0x3d, 0xa0, 0xa1, 0xd9, 0xdd, 0x39, + 0xa3, 0xaf, 0xbc, 0x2d, 0x89, 0x58, 0xf3, 0x50, 0x08, 0x73, 0x7d, 0xca, 0x39, 0xe9, 0x52, 0x63, + 0xe1, 0xb5, 0xd3, 0x5b, 0xf8, 0x8e, 0x06, 0x6a, 0xcd, 0x1b, 0x4d, 0x73, 0x86, 0x80, 0x53, 0x15, + 0xee, 0xef, 0x1c, 0x28, 0x6f, 0x4a, 0x14, 0xf4, 0x3e, 0xcc, 0xf9, 0x71, 0x24, 0xe8, 0xe3, 0xd4, + 0x9d, 0x6e, 0x9c, 0x5e, 0xaf, 0x42, 0x5c, 0xd7, 0x68, 0x99, 0x72, 0x43, 0xc0, 0xa9, 0x1e, 0xf4, + 0x75, 0x28, 0x75, 0x88, 0x20, 0xea, 0x9c, 0x67, 0x5b, 0xd5, 0xa3, 0xc3, 0xe5, 0xd2, 0x06, 0x11, + 0x04, 0x2b, 0xaa, 0xfb, 0xa7, 0x0a, 0x9c, 0xcd, 0x03, 0xa1, 0x26, 0xd4, 0x94, 0xe2, 0x7b, 0xc3, + 0x24, 0xbd, 0xe0, 0x57, 0x0c, 0x76, 0x6d, 0x33, 0x65, 0xe0, 0x4c, 0x06, 0x6d, 0xc0, 0x82, 0xfd, + 0x78, 0x40, 0x19, 0x4f, 0xbd, 0xb6, 0x96, 0x79, 0xda, 0xe6, 0x18, 0x1f, 0x4f, 0xac, 0x40, 0x6f, + 0x01, 0xf2, 0xc3, 0x78, 0xd0, 0x51, 0xa2, 0x3c, 0xc5, 0x29, 0x2a, 0x9c, 0x45, 0x83, 0x83, 0xd6, + 0x27, 0x24, 0xf0, 0x0b, 0x56, 0x21, 0x02, 0x15, 0x1e, 0x0f, 0x98, 0x4f, 0x1b, 0x25, 0x65, 0xe3, + 0x6b, 0xa7, 0xb7, 0xf1, 0x7d, 0xbc, 0xd5, 0x02, 0x19, 0xab, 0x6d, 0x05, 0x88, 0x0d, 0x30, 0xfa, + 0x26, 0xcc, 0xa9, 0xa5, 0x5b, 0x1b, 0x8d, 0xb2, 0xda, 0xa3, 0xb5, 0xff, 0xa6, 0x26, 0xe3, 0x94, + 0x8f, 0x7e, 0x9c, 0x1a, 0x34, 0xe8, 0xd3, 0x46, 0x45, 0x6d, 0xe8, 0x5b, 0x9e, 0x4e, 0x87, 0x5e, + 0x3e, 0x1d, 0x66, 0x9b, 0x90, 0xd9, 0xda, 0x3b, 0xb8, 0xe2, 0xc9, 0x15, 0xe3, 0xc6, 0x0f, 0xfa, + 0xd6, 0xf8, 0x41, 0x9f, 0xa2, 0x47, 0x50, 0xd3, 0x19, 0xf7, 0x3e, 0xde, 0x6e, 0xcc, 0xcd, 0xe2, + 0xb4, 0xe7, 0xa4, 0xae, 0x76, 0x8a, 0x89, 0x33, 0x78, 0xf4, 0x1d, 0xa8, 0x2b, 0x9f, 0x32, 0xbe, + 0x51, 0x55, 0xe7, 0xfe, 0x9a, 0xd9, 0x5e, 0x7d, 0x3d, 0x63, 0xe1, 0xbc, 0x1c, 0xfa, 0xa5, 0x03, + 0x40, 0x1f, 0x0b, 0x1a, 0xc9, 0xbb, 0xe1, 0x8d, 0xda, 0x85, 0xe2, 0xc5, 0xfa, 0xca, 0x83, 0xd9, + 0xb8, 0xbd, 0xb7, 0x69, 0x81, 0x37, 0x23, 0xc1, 0x86, 0x2d, 0x64, 0xb6, 0x03, 0x19, 0x03, 0xe7, + 0xb4, 0x2f, 0x5e, 0x83, 0xf9, 0xb1, 0x25, 0x68, 0x01, 0x8a, 0xfb, 0x74, 0xa8, 0x5d, 0x1d, 0xcb, + 0x7f, 0xd1, 0xab, 0x50, 0x3e, 0x20, 0xe1, 0x40, 0xa7, 0x86, 0x1a, 0xd6, 0x1f, 0xdf, 0x2f, 0x5c, + 0x75, 0xdc, 0xdf, 0x3a, 0x26, 0x5a, 0xde, 0x65, 0x24, 0x49, 0x28, 0x43, 0x1d, 0x28, 0xab, 0xfd, + 0x9a, 0x68, 0xfe, 0xe1, 0x94, 0xc7, 0xca, 0xb2, 0x95, 0xfa, 0xc4, 0x1a, 0x1c, 0x5d, 0x80, 0x12, + 0xa7, 0x54, 0x87, 0x55, 0xb5, 0x75, 0xd6, 0xc8, 0x94, 0xda, 0x94, 0x46, 0x58, 0x71, 0xdc, 0x8f, + 0x1c, 0x58, 0xb8, 0xc9, 0xe2, 0x41, 0x62, 0x62, 0xe0, 0x76, 0x10, 0x75, 0x64, 0x26, 0xec, 0x4a, + 0xda, 0x78, 0x26, 0x54, 0x82, 0x58, 0xf3, 0xa4, 0x27, 0x1f, 0x8c, 0x44, 0xad, 0xf5, 0xe4, 0x34, + 0xc4, 0x52, 0xbe, 0xdc, 0xc6, 0x7e, 0x10, 0x75, 0x4c, 0x54, 0xda, 0x6d, 0x48, 0x5d, 0x58, 0x71, + 0xdc, 0xdf, 0x38, 0x90, 0x66, 0x3f, 0x29, 0xbd, 0x1b, 0x77, 0x8c, 0x61, 0x33, 0xe9, 0x56, 0xdc, + 0x19, 0x62, 0xc5, 0x91, 0xa5, 0x95, 0xab, 0x92, 0x68, 0x72, 0xf0, 0x0c, 0x4b, 0xab, 0xfe, 0xc6, + 0x06, 0xdf, 0xfd, 0x5b, 0x09, 0xe0, 0x6e, 0xdc, 0xa1, 0x6d, 0x41, 0xc4, 0x80, 0xa3, 0x45, 0x28, + 0x04, 0x1d, 0xb3, 0x31, 0x30, 0x4b, 0x0a, 0x5b, 0x1b, 0xb8, 0x10, 0x74, 0xe4, 0xb6, 0x23, 0xd2, + 0x37, 0x77, 0x9f, 0x6d, 0xfb, 0x2e, 0xe9, 0x53, 0xac, 0x38, 0x32, 0x0e, 0x3a, 0x01, 0x4f, 0x42, + 0x32, 0x94, 0x44, 0x63, 0x0d, 0x1b, 0x07, 0x1b, 0x19, 0x0b, 0xe7, 0xe5, 0xd0, 0x25, 0x28, 0x09, + 0x19, 0x37, 0xa5, 0x91, 0xdc, 0x58, 0x92, 0x31, 0xf2, 0xfc, 0x70, 0xb9, 0x2a, 0xb7, 0xa7, 0x82, + 0x47, 0x49, 0xa1, 0x37, 0xa1, 0x9c, 0xf4, 0x08, 0xa7, 0x26, 0xbd, 0xa4, 0x29, 0xb0, 0xbc, 0x23, + 0x89, 0xcf, 0x0f, 0x97, 0x6b, 0x52, 0x5e, 0x7d, 0x60, 0x2d, 0x28, 0xf3, 0x0c, 0x17, 0x84, 0x09, + 0xda, 0x59, 0x13, 0xd3, 0xe4, 0x99, 0x76, 0x0a, 0x82, 0x33, 0x3c, 0x44, 0x64, 0xec, 0xf7, 0x93, + 0x90, 0x6a, 0xf8, 0xb9, 0x13, 0xc3, 0xe7, 0xf2, 0x84, 0x85, 0xc1, 0x79, 0x4c, 0xe9, 0x88, 0x69, + 0x49, 0xae, 0x8e, 0x3a, 0xe2, 0x78, 0x3d, 0x45, 0x43, 0xa8, 0x87, 0x44, 0x50, 0x2e, 0x54, 0x94, + 0x34, 0x6a, 0x33, 0xa9, 0xa4, 0x26, 0xa4, 0x5b, 0xf3, 0x72, 0x97, 0xdb, 0x19, 0x3c, 0xce, 0xeb, + 0x72, 0x7f, 0x5f, 0x82, 0xf3, 0x98, 0xea, 0x2a, 0x70, 0x23, 0x08, 0x05, 0x65, 0xe8, 0x0d, 0xa8, + 0x24, 0x8c, 0xee, 0x05, 0x8f, 0x8d, 0x47, 0x59, 0x27, 0xdc, 0x51, 0x54, 0x6c, 0xb8, 0xe8, 0x67, + 0x50, 0x09, 0xc9, 0x2e, 0x0d, 0x79, 0xa3, 0xa0, 0x72, 0xe0, 0xbd, 0xd3, 0x6f, 0x78, 0x74, 0x07, + 0xde, 0xb6, 0x82, 0xd5, 0x19, 0xd0, 0x6a, 0xd7, 0x44, 0x6c, 0x74, 0xca, 0xee, 0xb2, 0x4e, 0xa2, + 0x28, 0x16, 0xaa, 0x57, 0xe2, 0xaa, 0xbb, 0xaa, 0xaf, 0xfc, 0x68, 0x66, 0x7b, 0x58, 0xcb, 0xb0, + 0xf5, 0x46, 0xec, 0x8d, 0xe7, 0x38, 0x38, 0xbf, 0x05, 0xe9, 0xb1, 0x3e, 0xa3, 0xb2, 0xe1, 0x6f, + 0x0d, 0x4d, 0xa9, 0x3e, 0x95, 0xc7, 0xae, 0xa7, 0x20, 0x38, 0xc3, 0x5b, 0xfc, 0x1e, 0xd4, 0x73, + 0x66, 0x39, 0x49, 0x96, 0x5f, 0xfc, 0x01, 0x2c, 0x8c, 0x9f, 0xe6, 0x44, 0x55, 0xe2, 0xb3, 0x9c, + 0x8f, 0xbc, 0xbd, 0xfb, 0x88, 0xfa, 0xaa, 0xab, 0x92, 0xb9, 0x83, 0x27, 0xc4, 0x9f, 0xe8, 0xaa, + 0xee, 0xa6, 0x0c, 0x9c, 0xc9, 0xa0, 0x8f, 0xff, 0x37, 0xc3, 0xc0, 0xab, 0x2f, 0x37, 0x08, 0xe4, + 0x7c, 0xb7, 0x38, 0x2b, 0xdf, 0xd5, 0x96, 0x79, 0x59, 0xdf, 0x5d, 0xe8, 0x8e, 0x55, 0x37, 0xe3, + 0x30, 0x53, 0x18, 0x63, 0xbc, 0x5e, 0x66, 0xfd, 0xea, 0x38, 0x07, 0x4f, 0x68, 0x9f, 0xc2, 0xbd, + 0xdc, 0xbf, 0x16, 0x32, 0xf7, 0x30, 0x43, 0xd5, 0x89, 0xdd, 0x23, 0x84, 0xca, 0x9e, 0x8a, 0x3b, + 0xe3, 0x13, 0xb7, 0x66, 0x15, 0xc7, 0xba, 0xdb, 0xd5, 0xff, 0x63, 0xa3, 0xe3, 0xc5, 0xf6, 0x2f, + 0xfe, 0x3f, 0xed, 0xef, 0xce, 0xc3, 0x39, 0x4c, 0x05, 0x1b, 0xb6, 0x05, 0x23, 0x82, 0x76, 0x87, + 0xee, 0x3f, 0x0a, 0x00, 0xd9, 0x54, 0x8f, 0x5e, 0xcb, 0x5d, 0x48, 0xab, 0x6e, 0x80, 0x8b, 0xb7, + 0xe9, 0x50, 0xdf, 0xce, 0x83, 0xb4, 0x6f, 0xd3, 0x65, 0xfe, 0xfa, 0x48, 0xdb, 0xf5, 0xfc, 0x70, + 0xb9, 0x99, 0x7b, 0xa1, 0xe9, 0x07, 0x51, 0x10, 0xeb, 0xbf, 0x97, 0xbb, 0xb1, 0x77, 0x37, 0x16, + 0xc1, 0x5e, 0xa0, 0xe3, 0x22, 0x1b, 0x88, 0x4c, 0xa7, 0xb6, 0x67, 0xef, 0x45, 0x9b, 0xa7, 0x35, + 0xcd, 0x13, 0xc5, 0x7f, 0xb9, 0x91, 0x04, 0xaa, 0x7c, 0xb5, 0x35, 0xf0, 0xf7, 0xa9, 0x30, 0x81, + 0x30, 0x95, 0x26, 0x8d, 0x94, 0x9b, 0xe4, 0x0d, 0x05, 0x5b, 0x2d, 0xee, 0xbf, 0x0b, 0x60, 0xc9, + 0x72, 0xac, 0xa7, 0x51, 0x27, 0x89, 0x03, 0xd3, 0xf9, 0xe6, 0xc6, 0xfa, 0x4d, 0x43, 0xc7, 0x56, + 0x42, 0x16, 0xc8, 0x5d, 0xbd, 0xd5, 0xc2, 0x68, 0x81, 0x34, 0x4a, 0x0c, 0x57, 0xca, 0x31, 0xda, + 0xcd, 0xe6, 0x3e, 0x2b, 0x87, 0x15, 0x15, 0x1b, 0xae, 0x7e, 0x54, 0xe0, 0xd4, 0x1f, 0x30, 0xdd, + 0x4d, 0x55, 0xf3, 0x8f, 0x0a, 0x9a, 0x8e, 0xad, 0x04, 0x7a, 0x00, 0x35, 0xe2, 0xfb, 0x94, 0xf3, + 0xdb, 0x74, 0xa8, 0xba, 0xa9, 0xfa, 0xca, 0x37, 0x72, 0x55, 0xc6, 0xf3, 0x63, 0x46, 0x65, 0x4d, + 0x69, 0x53, 0x9f, 0x51, 0x71, 0x9b, 0x0e, 0xdb, 0x34, 0xa4, 0xbe, 0x88, 0x59, 0x16, 0x82, 0x6b, + 0xe9, 0x7a, 0x9c, 0x41, 0x49, 0x5c, 0x9e, 0x2e, 0x31, 0xfd, 0xd6, 0x49, 0x71, 0x2d, 0x0b, 0x67, + 0x50, 0xee, 0x43, 0x69, 0xe7, 0x13, 0xb6, 0x16, 0x6f, 0x40, 0x85, 0x0f, 0xf6, 0xa4, 0xdc, 0x98, + 0x85, 0xdb, 0x8a, 0x8a, 0x0d, 0x57, 0xa6, 0x9e, 0x4a, 0x5b, 0xdd, 0x3e, 0xfa, 0x29, 0x54, 0x65, + 0x35, 0x55, 0x4f, 0x03, 0x7a, 0x78, 0x79, 0xf3, 0xe5, 0x6a, 0xaf, 0xce, 0xdb, 0x77, 0xa8, 0x20, + 0xd9, 0xb4, 0x95, 0xd1, 0xb0, 0x45, 0x45, 0x7b, 0x50, 0xe2, 0x09, 0xf5, 0x4d, 0x86, 0x9a, 0xe6, + 0xb1, 0x4e, 0x7d, 0xb7, 0x13, 0xea, 0xe7, 0x66, 0x9f, 0x84, 0xfa, 0x58, 0xe1, 0xa3, 0x48, 0x8e, + 0x11, 0xb2, 0xaf, 0x37, 0x31, 0x77, 0x63, 0x6a, 0x4d, 0x0a, 0x2d, 0x3f, 0x4c, 0xc8, 0x6f, 0x6c, + 0xb4, 0xb8, 0x7f, 0x77, 0x00, 0xb4, 0xe0, 0x76, 0xc0, 0x05, 0x7a, 0x6f, 0xc2, 0x90, 0xde, 0xcb, + 0x19, 0x52, 0xae, 0x56, 0x66, 0xb4, 0xde, 0x9b, 0x52, 0x72, 0x46, 0xa4, 0x50, 0x0e, 0x04, 0xed, + 0xa7, 0x3d, 0xe3, 0xf5, 0x69, 0xcf, 0x96, 0x4d, 0x81, 0x5b, 0x12, 0x16, 0x6b, 0x74, 0xf7, 0xd7, + 0xc5, 0xf4, 0x4c, 0xd2, 0xb0, 0x68, 0x1f, 0xe6, 0xb8, 0xaa, 0x4c, 0xbc, 0xe1, 0x4c, 0xad, 0x57, + 0x01, 0x65, 0xdd, 0xbc, 0xfe, 0xe6, 0x38, 0xd5, 0x80, 0x62, 0xa8, 0x0a, 0x16, 0x74, 0xbb, 0x94, + 0xa5, 0xa7, 0x9c, 0xe2, 0x31, 0xee, 0x9e, 0x46, 0xca, 0x6c, 0x6a, 0x08, 0x1c, 0x5b, 0x25, 0xe8, + 0x43, 0x00, 0x6a, 0x5f, 0x0d, 0xa7, 0xaf, 0x63, 0xe3, 0x2f, 0x90, 0xb9, 0x47, 0x08, 0xcb, 0xc1, + 0x39, 0x8d, 0x3a, 0xcf, 0x25, 0x94, 0x08, 0x93, 0xbd, 0x72, 0x79, 0x4e, 0x52, 0xb1, 0xe1, 0xba, + 0x7f, 0x28, 0xc1, 0xd9, 0xbc, 0x47, 0x66, 0x43, 0xa1, 0x73, 0xaa, 0xa1, 0xb0, 0xf0, 0xd5, 0x0e, + 0x85, 0xc5, 0xaf, 0x76, 0x28, 0x2c, 0x7d, 0xc9, 0x50, 0x78, 0x00, 0xe5, 0x28, 0xee, 0x50, 0xde, + 0x28, 0x2b, 0x1f, 0x7a, 0x67, 0x36, 0x59, 0xc0, 0x93, 0x26, 0x35, 0xed, 0xa9, 0x0d, 0x1d, 0x45, + 0xc3, 0x5a, 0xdd, 0xe2, 0x87, 0xfa, 0x69, 0xe1, 0xd8, 0x46, 0xf0, 0x61, 0xbe, 0x11, 0x9c, 0x2a, + 0x0f, 0x66, 0x2f, 0x18, 0xf9, 0x76, 0xf2, 0x93, 0x0a, 0x54, 0x4c, 0x1b, 0x99, 0xbe, 0x5d, 0x38, + 0xc7, 0xbe, 0x5d, 0x5c, 0x82, 0x6a, 0x87, 0x92, 0x4e, 0x18, 0x44, 0x7a, 0x3f, 0xc5, 0x2c, 0x50, + 0x36, 0x0c, 0x1d, 0x5b, 0x09, 0xd4, 0xb1, 0x0f, 0x34, 0xc5, 0x19, 0x3d, 0xd0, 0xc0, 0xe4, 0xe3, + 0x0c, 0x62, 0x50, 0x4d, 0xe7, 0x0d, 0xd3, 0xcb, 0xdc, 0x9a, 0x7e, 0xc2, 0x31, 0x59, 0xe7, 0xac, + 0x3c, 0x59, 0x4a, 0xc3, 0x56, 0x8f, 0xd4, 0xe9, 0x9b, 0xdf, 0x35, 0x4c, 0x4f, 0x30, 0x85, 0xce, + 0xd1, 0x5f, 0x48, 0xb4, 0xce, 0x94, 0x86, 0xad, 0x1e, 0xa9, 0x93, 0x99, 0x5e, 0xdb, 0xf4, 0x0b, + 0x33, 0xe8, 0xda, 0xf3, 0x3a, 0x53, 0x1a, 0xb6, 0x7a, 0x50, 0x04, 0x73, 0x1f, 0xd0, 0xdd, 0x5e, + 0x1c, 0xef, 0x9b, 0x37, 0x9b, 0x9b, 0xa7, 0x57, 0xf9, 0xae, 0x06, 0x32, 0x1a, 0xeb, 0x32, 0x08, + 0x0d, 0x09, 0xa7, 0x4a, 0xd0, 0xcf, 0x1d, 0xf5, 0x48, 0xcc, 0x05, 0x23, 0x41, 0x24, 0xb8, 0x7a, + 0xc9, 0xa9, 0xaf, 0xdc, 0x9e, 0xb6, 0x7a, 0xac, 0x67, 0x90, 0x23, 0x2f, 0xce, 0x29, 0x11, 0xe7, + 0x95, 0xba, 0x4f, 0x1c, 0x78, 0x65, 0x62, 0x1d, 0xda, 0x87, 0x92, 0x08, 0x4c, 0x70, 0xd4, 0x57, + 0xb6, 0xa6, 0x28, 0x31, 0x41, 0x9f, 0xe6, 0x37, 0x64, 0xe3, 0x4c, 0x3d, 0xce, 0x2b, 0x25, 0xee, + 0x27, 0xb2, 0xd1, 0xd2, 0xee, 0x7d, 0xc1, 0xbc, 0xfb, 0x8d, 0x05, 0x65, 0xee, 0xad, 0xef, 0x35, + 0x28, 0x0e, 0x58, 0xfa, 0xfb, 0x98, 0x9d, 0x55, 0xee, 0xe3, 0x6d, 0x2c, 0xe9, 0xe8, 0x23, 0x07, + 0x80, 0x08, 0xc1, 0x82, 0xdd, 0x81, 0xa0, 0xe9, 0x00, 0xbe, 0x33, 0x6d, 0x28, 0x7a, 0x6b, 0x16, + 0x72, 0xec, 0xe9, 0x3c, 0x63, 0xe0, 0x9c, 0xde, 0xc5, 0x6b, 0x30, 0x3f, 0xb6, 0xe4, 0x44, 0x53, + 0xef, 0x9f, 0x1d, 0x98, 0x1f, 0xb3, 0x1c, 0x7a, 0x1b, 0xca, 0xaa, 0x9a, 0x98, 0x3b, 0x39, 0x49, + 0xe9, 0xb0, 0xb9, 0x58, 0x55, 0x26, 0xac, 0x71, 0xd0, 0x36, 0x94, 0xb8, 0x88, 0x93, 0x53, 0x54, + 0xba, 0xac, 0xb1, 0x14, 0x71, 0x82, 0x15, 0x8a, 0xfb, 0x71, 0x11, 0xe6, 0x4c, 0xfb, 0xf0, 0x12, + 0xa9, 0x35, 0x1f, 0xde, 0x33, 0x1b, 0xca, 0x75, 0x63, 0x7d, 0x6c, 0x78, 0xf7, 0xb2, 0xf2, 0x58, + 0x9c, 0xd5, 0xcf, 0x98, 0xf5, 0x17, 0x56, 0xd7, 0x27, 0x0e, 0x9c, 0x63, 0x34, 0x09, 0xed, 0xc0, + 0x6d, 0x52, 0xf5, 0xcd, 0x69, 0xce, 0x98, 0x9b, 0xdf, 0x5b, 0xaf, 0x1c, 0x1d, 0x2e, 0x8f, 0x8e, + 0xf4, 0x78, 0x54, 0xa1, 0xfb, 0x97, 0x02, 0x14, 0xef, 0xe3, 0x2d, 0x35, 0xec, 0xf8, 0x3d, 0x6a, + 0x2f, 0x23, 0xeb, 0xd3, 0x15, 0x15, 0x1b, 0xae, 0xbc, 0xb2, 0x01, 0x37, 0x2f, 0x24, 0xb9, 0x2b, + 0xbb, 0xcf, 0x29, 0xc3, 0x8a, 0x23, 0xab, 0x61, 0x42, 0x38, 0xff, 0x20, 0x66, 0xe9, 0x8f, 0x1a, + 0xb6, 0x1a, 0xee, 0x18, 0x3a, 0xb6, 0x12, 0x12, 0xaf, 0x17, 0x73, 0x61, 0x1a, 0x11, 0x8b, 0x77, + 0x2b, 0xe6, 0x02, 0x2b, 0x8e, 0x94, 0x48, 0x62, 0x26, 0x54, 0x45, 0x29, 0x67, 0x12, 0x3b, 0x31, + 0x13, 0x58, 0x71, 0x94, 0x04, 0x11, 0x3d, 0x95, 0xff, 0x73, 0x18, 0x3b, 0x44, 0xf4, 0xb0, 0xe2, + 0xa0, 0xd7, 0xa1, 0xfc, 0xfe, 0x80, 0xb2, 0xa1, 0xca, 0xd7, 0xb9, 0x1f, 0x6d, 0xde, 0x91, 0x44, + 0xac, 0x79, 0x72, 0xe3, 0x7b, 0x8c, 0x74, 0xfb, 0x34, 0x12, 0xe6, 0xb1, 0xdc, 0x6e, 0xfc, 0x86, + 0xa1, 0x63, 0x2b, 0xe1, 0xfe, 0xc2, 0x81, 0x73, 0x23, 0xc9, 0xfb, 0x84, 0xf3, 0x7b, 0x7a, 0xac, + 0xc2, 0xb1, 0xc7, 0x7a, 0x03, 0x2a, 0x7d, 0x2a, 0x7a, 0x71, 0x67, 0x7c, 0x72, 0xbf, 0xa3, 0xa8, + 0xd8, 0x70, 0x5b, 0xde, 0xd3, 0x67, 0x4b, 0x67, 0x3e, 0x7f, 0xb6, 0x74, 0xe6, 0x8b, 0x67, 0x4b, + 0x67, 0x9e, 0x1c, 0x2d, 0x39, 0x4f, 0x8f, 0x96, 0x9c, 0xcf, 0x8f, 0x96, 0x9c, 0x2f, 0x8e, 0x96, + 0x9c, 0x7f, 0x1e, 0x2d, 0x39, 0x9f, 0xfe, 0x6b, 0xe9, 0xcc, 0xc3, 0x6a, 0xea, 0x22, 0xff, 0x09, + 0x00, 0x00, 0xff, 0xff, 0xad, 0x04, 0x89, 0x9e, 0xad, 0x23, 0x00, 0x00, } diff --git a/pkg/apis/sensor/v1alpha1/generated.proto b/pkg/apis/sensor/v1alpha1/generated.proto index b2af95702580..8545ee6ff56e 100644 --- a/pkg/apis/sensor/v1alpha1/generated.proto +++ b/pkg/apis/sensor/v1alpha1/generated.proto @@ -307,9 +307,6 @@ message SensorStatus { // Nodes is a mapping between a node ID and the node's status // it records the states for the FSM of this sensor. map nodes = 5; - - // Escalated is a flag for whether this sensor was escalated - optional bool escalated = 6; } // Signal describes a dependency diff --git a/pkg/apis/sensor/v1alpha1/openapi_generated.go b/pkg/apis/sensor/v1alpha1/openapi_generated.go index d5433309cd06..849d97a75953 100644 --- a/pkg/apis/sensor/v1alpha1/openapi_generated.go +++ b/pkg/apis/sensor/v1alpha1/openapi_generated.go @@ -937,13 +937,6 @@ func schema_pkg_apis_sensor_v1alpha1_SensorStatus(ref common.ReferenceCallback) }, }, }, - "escalated": { - SchemaProps: spec.SchemaProps{ - Description: "Escalated is a flag for whether this sensor was escalated", - Type: []string{"boolean"}, - Format: "", - }, - }, }, Required: []string{"phase"}, }, diff --git a/pkg/apis/sensor/v1alpha1/types.go b/pkg/apis/sensor/v1alpha1/types.go index 30ff1c9d3554..f49d14c86833 100644 --- a/pkg/apis/sensor/v1alpha1/types.go +++ b/pkg/apis/sensor/v1alpha1/types.go @@ -276,9 +276,6 @@ type SensorStatus struct { // Nodes is a mapping between a node ID and the node's status // it records the states for the FSM of this sensor. Nodes map[string]NodeStatus `json:"nodes,omitempty" protobuf:"bytes,5,rep,name=nodes"` - - // Escalated is a flag for whether this sensor was escalated - Escalated bool `json:"escalated,omitempty" protobuf:"bytes,6,opt,name=escalated"` } // NodeStatus describes the status for an individual node in the sensor's FSM. diff --git a/sdk/doc.go b/sdk/doc.go new file mode 100644 index 000000000000..f88c0878841c --- /dev/null +++ b/sdk/doc.go @@ -0,0 +1,2 @@ +// Package sdk contains the necessary files for rpc communication +package sdk diff --git a/shared/errors.go b/sdk/errors.go similarity index 95% rename from shared/errors.go rename to sdk/errors.go index ec5477c99529..672eead11817 100644 --- a/shared/errors.go +++ b/sdk/errors.go @@ -1,4 +1,4 @@ -package shared +package sdk import "errors" diff --git a/sdk/interface.go b/sdk/interface.go new file mode 100644 index 000000000000..f9a9f2636bc0 --- /dev/null +++ b/sdk/interface.go @@ -0,0 +1,59 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sdk + +import ( + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + client "github.com/micro/go-micro/client" + context "golang.org/x/net/context" +) + +type key int + +const ( + CloudEventsKey key = 0 + CloudEventsVersion string = "v1.0" + ContextExtensionErrorKey string = "error" +) + +// Listener handles signal lifecycle +// Must be implemented by all signal service providers +// TODO: this should be the interface implemented by all the signal sources to be a signal provider +type Listener interface { + Listen(*v1alpha1.Signal, <-chan struct{}) (<-chan *v1alpha1.Event, error) +} + +// ArtifactListener is the interface for listening with artifacts +// In addition to including the basic Listener interface, this also +// enables access to read an artifact object to include in the event data payload +type ArtifactListener interface { + Listener + // TODO: change to use io.Reader and io.Closer interfaces? + Read(loc *v1alpha1.ArtifactLocation, key string) ([]byte, error) +} + +// SignalClient is the interface for signal clients +type SignalClient interface { + Listen(context.Context, *v1alpha1.Signal, ...client.CallOption) (SignalService_ListenService, error) + handshake(*v1alpha1.Signal, SignalService_ListenService) error +} + +// SignalServer is the interface for signal servers +type SignalServer interface { + SignalServiceHandler + handshake(SignalService_ListenStream, <-chan struct{}) (<-chan *v1alpha1.Event, error) +} diff --git a/sdk/micro_client.go b/sdk/micro_client.go new file mode 100644 index 000000000000..c15044bfc8a9 --- /dev/null +++ b/sdk/micro_client.go @@ -0,0 +1,52 @@ +package sdk + +import ( + "errors" + + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + client "github.com/micro/go-micro/client" + context "golang.org/x/net/context" +) + +type microSignalClient struct { + impl SignalService +} + +// Terminate is SignalContext to send in order to stop the signal +var Terminate = &SignalContext{Done: true} + +// NewMicroSignalClient creates a Micro compatible SignalClient from a micro.Client +func NewMicroSignalClient(name string, c client.Client) SignalClient { + return µSignalClient{impl: NewSignalService(name, c)} +} + +// Listen to the signal +func (m *microSignalClient) Listen(ctx context.Context, signal *v1alpha1.Signal, opts ...client.CallOption) (SignalService_ListenService, error) { + // #200 - gRPC server defaults to 5s request timeout on stream. setting -1 means indefinite for streams. + opts = append(opts, client.WithRequestTimeout(-1)) + stream, err := m.impl.Listen(ctx, opts...) + if err != nil { + return nil, err + } + err = m.handshake(signal, stream) + if err != nil { + return nil, err + } + return stream, nil +} + +// Handshake performs the initial signal handshaking with the server +func (m *microSignalClient) handshake(signal *v1alpha1.Signal, stream SignalService_ListenService) error { + err := stream.Send(&SignalContext{Signal: signal}) + if err != nil { + return err + } + ack, err := stream.Recv() + if err != nil { + return err + } + if ack.Done { + return nil + } + return errors.New("handshake ack failed") +} diff --git a/sdk/micro_server.go b/sdk/micro_server.go new file mode 100644 index 000000000000..92608f019263 --- /dev/null +++ b/sdk/micro_server.go @@ -0,0 +1,88 @@ +package sdk + +import ( + "context" + "errors" + io "io" + "log" + + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" +) + +type microSignalServer struct { + impl Listener +} + +var ack = &EventContext{Done: true} + +// NewMicroSignalServer creates a Micro compatible SignalServer from the Listener implementation +func NewMicroSignalServer(lis Listener) SignalServer { + return µSignalServer{impl: lis} +} + +// Listen implements the SignalServiceHandler interface +func (m *microSignalServer) Listen(ctx context.Context, stream SignalService_ListenStream) error { + done := make(chan struct{}) + + // perform the initial handshake + events, err := m.handshake(stream, done) + if err != nil { + return err + } + + // start the receive goroutine to monitor context updates + go func() { + for { + sigCtx, err := stream.Recv() + // check if the signal is same and was updated/different? + if err == io.EOF { + return + } + if err != nil { + log.Panicf("encountered error on stream recv: %s", err) + } + if sigCtx.Done { + close(done) + } + } + }() + for { + select { + case event, ok := <-events: + if !ok { + // finished + return stream.Close() + } + err := stream.Send(&EventContext{Event: event}) + if err != nil { + log.Printf("microSignalServer failed to send event: %s", err) + return err + } + case <-ctx.Done(): + // TODO: think about swallowing context errors and just closing the done channel, but keep the program from panicing + close(done) + return ctx.Err() + } + } +} + +// Handshake performs the initial handshake for the signal server. +// This blocks until the initial receive and send have completed or an error is encountered. +func (m *microSignalServer) handshake(stream SignalService_ListenStream, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + sigCtx, err := stream.Recv() + if err != nil { + return nil, err + } + if sigCtx.Done { + return nil, errors.New("signal context done before started") + } + events, err := m.impl.Listen(sigCtx.Signal.DeepCopy(), done) + if err != nil { + return nil, err + } + err = stream.Send(ack) + if err != nil { + return nil, err + } + return events, nil +} diff --git a/sdk/signal.micro.go b/sdk/signal.micro.go new file mode 100644 index 000000000000..9d9b5cf5ad62 --- /dev/null +++ b/sdk/signal.micro.go @@ -0,0 +1,192 @@ +// Code generated by protoc-gen-micro. DO NOT EDIT. +// source: sdk/signal.proto + +/* +Package sdk is a generated protocol buffer package. + +It is generated from these files: + sdk/signal.proto + +It has these top-level messages: + SignalContext + EventContext +*/ +package sdk + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + +import ( + client "github.com/micro/go-micro/client" + server "github.com/micro/go-micro/server" + context "context" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ client.Option +var _ server.Option + +// Client API for SignalService service + +type SignalService interface { + // Listen to the signal. Events are streamed back to the client. + // + // The following describes the mechanics for the stream. + // The first send MUST contain a non-nil signal. + // This first send MUST be followed by a first receive which contains + // a confirmation in the form of done=true to signify that the Signal is set up + // correctly. + // + // Later sends should only signify if the signal is done. Later sends do not + // update the signal process attached to this channel. + Listen(ctx context.Context, opts ...client.CallOption) (SignalService_ListenService, error) +} + +type signalService struct { + c client.Client + name string +} + +func NewSignalService(name string, c client.Client) SignalService { + if c == nil { + c = client.NewClient() + } + if len(name) == 0 { + name = "sdk" + } + return &signalService{ + c: c, + name: name, + } +} + +func (c *signalService) Listen(ctx context.Context, opts ...client.CallOption) (SignalService_ListenService, error) { + req := c.c.NewRequest(c.name, "SignalService.Listen", &SignalContext{}) + stream, err := c.c.Stream(ctx, req, opts...) + if err != nil { + return nil, err + } + return &signalServiceListen{stream}, nil +} + +type SignalService_ListenService interface { + SendMsg(interface{}) error + RecvMsg(interface{}) error + Close() error + Send(*SignalContext) error + Recv() (*EventContext, error) +} + +type signalServiceListen struct { + stream client.Stream +} + +func (x *signalServiceListen) Close() error { + return x.stream.Close() +} + +func (x *signalServiceListen) SendMsg(m interface{}) error { + return x.stream.Send(m) +} + +func (x *signalServiceListen) RecvMsg(m interface{}) error { + return x.stream.Recv(m) +} + +func (x *signalServiceListen) Send(m *SignalContext) error { + return x.stream.Send(m) +} + +func (x *signalServiceListen) Recv() (*EventContext, error) { + m := new(EventContext) + err := x.stream.Recv(m) + if err != nil { + return nil, err + } + return m, nil +} + +// Server API for SignalService service + +type SignalServiceHandler interface { + // Listen to the signal. Events are streamed back to the client. + // + // The following describes the mechanics for the stream. + // The first send MUST contain a non-nil signal. + // This first send MUST be followed by a first receive which contains + // a confirmation in the form of done=true to signify that the Signal is set up + // correctly. + // + // Later sends should only signify if the signal is done. Later sends do not + // update the signal process attached to this channel. + Listen(context.Context, SignalService_ListenStream) error +} + +func RegisterSignalServiceHandler(s server.Server, hdlr SignalServiceHandler, opts ...server.HandlerOption) { + type signalService interface { + Listen(ctx context.Context, stream server.Stream) error + } + type SignalService struct { + signalService + } + h := &signalServiceHandler{hdlr} + s.Handle(s.NewHandler(&SignalService{h}, opts...)) +} + +type signalServiceHandler struct { + SignalServiceHandler +} + +func (h *signalServiceHandler) Listen(ctx context.Context, stream server.Stream) error { + return h.SignalServiceHandler.Listen(ctx, &signalServiceListenStream{stream}) +} + +type SignalService_ListenStream interface { + SendMsg(interface{}) error + RecvMsg(interface{}) error + Close() error + Send(*EventContext) error + Recv() (*SignalContext, error) +} + +type signalServiceListenStream struct { + stream server.Stream +} + +func (x *signalServiceListenStream) Close() error { + return x.stream.Close() +} + +func (x *signalServiceListenStream) SendMsg(m interface{}) error { + return x.stream.Send(m) +} + +func (x *signalServiceListenStream) RecvMsg(m interface{}) error { + return x.stream.Recv(m) +} + +func (x *signalServiceListenStream) Send(m *EventContext) error { + return x.stream.Send(m) +} + +func (x *signalServiceListenStream) Recv() (*SignalContext, error) { + m := new(SignalContext) + if err := x.stream.Recv(m); err != nil { + return nil, err + } + return m, nil +} diff --git a/sdk/signal.pb.go b/sdk/signal.pb.go new file mode 100644 index 000000000000..546803f13eb4 --- /dev/null +++ b/sdk/signal.pb.go @@ -0,0 +1,737 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: sdk/signal.proto + +package sdk // import "github.com/argoproj/argo-events/sdk" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import v1alpha1 "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + +import context "golang.org/x/net/context" +import grpc "google.golang.org/grpc" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type SignalContext struct { + Signal *v1alpha1.Signal `protobuf:"bytes,1,opt,name=signal" json:"signal,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SignalContext) Reset() { *m = SignalContext{} } +func (m *SignalContext) String() string { return proto.CompactTextString(m) } +func (*SignalContext) ProtoMessage() {} +func (*SignalContext) Descriptor() ([]byte, []int) { + return fileDescriptor_signal_4408fb801ca10653, []int{0} +} +func (m *SignalContext) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SignalContext) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SignalContext.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *SignalContext) XXX_Merge(src proto.Message) { + xxx_messageInfo_SignalContext.Merge(dst, src) +} +func (m *SignalContext) XXX_Size() int { + return m.Size() +} +func (m *SignalContext) XXX_DiscardUnknown() { + xxx_messageInfo_SignalContext.DiscardUnknown(m) +} + +var xxx_messageInfo_SignalContext proto.InternalMessageInfo + +func (m *SignalContext) GetSignal() *v1alpha1.Signal { + if m != nil { + return m.Signal + } + return nil +} + +func (m *SignalContext) GetDone() bool { + if m != nil { + return m.Done + } + return false +} + +type EventContext struct { + Event *v1alpha1.Event `protobuf:"bytes,1,opt,name=event" json:"event,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *EventContext) Reset() { *m = EventContext{} } +func (m *EventContext) String() string { return proto.CompactTextString(m) } +func (*EventContext) ProtoMessage() {} +func (*EventContext) Descriptor() ([]byte, []int) { + return fileDescriptor_signal_4408fb801ca10653, []int{1} +} +func (m *EventContext) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *EventContext) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_EventContext.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *EventContext) XXX_Merge(src proto.Message) { + xxx_messageInfo_EventContext.Merge(dst, src) +} +func (m *EventContext) XXX_Size() int { + return m.Size() +} +func (m *EventContext) XXX_DiscardUnknown() { + xxx_messageInfo_EventContext.DiscardUnknown(m) +} + +var xxx_messageInfo_EventContext proto.InternalMessageInfo + +func (m *EventContext) GetEvent() *v1alpha1.Event { + if m != nil { + return m.Event + } + return nil +} + +func (m *EventContext) GetDone() bool { + if m != nil { + return m.Done + } + return false +} + +func init() { + proto.RegisterType((*SignalContext)(nil), "sdk.SignalContext") + proto.RegisterType((*EventContext)(nil), "sdk.EventContext") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for SignalService service + +type SignalServiceClient interface { + // Listen to the signal. Events are streamed back to the client. + // + // The following describes the mechanics for the stream. + // The first send MUST contain a non-nil signal. + // This first send MUST be followed by a first receive which contains + // a confirmation in the form of done=true to signify that the Signal is set up + // correctly. + // + // Later sends should only signify if the signal is done. Later sends do not + // update the signal process attached to this channel. + Listen(ctx context.Context, opts ...grpc.CallOption) (SignalService_ListenClient, error) +} + +type signalServiceClient struct { + cc *grpc.ClientConn +} + +func NewSignalServiceClient(cc *grpc.ClientConn) SignalServiceClient { + return &signalServiceClient{cc} +} + +func (c *signalServiceClient) Listen(ctx context.Context, opts ...grpc.CallOption) (SignalService_ListenClient, error) { + stream, err := c.cc.NewStream(ctx, &_SignalService_serviceDesc.Streams[0], "/sdk.SignalService/Listen", opts...) + if err != nil { + return nil, err + } + x := &signalServiceListenClient{stream} + return x, nil +} + +type SignalService_ListenClient interface { + Send(*SignalContext) error + Recv() (*EventContext, error) + grpc.ClientStream +} + +type signalServiceListenClient struct { + grpc.ClientStream +} + +func (x *signalServiceListenClient) Send(m *SignalContext) error { + return x.ClientStream.SendMsg(m) +} + +func (x *signalServiceListenClient) Recv() (*EventContext, error) { + m := new(EventContext) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// Server API for SignalService service + +type SignalServiceServer interface { + // Listen to the signal. Events are streamed back to the client. + // + // The following describes the mechanics for the stream. + // The first send MUST contain a non-nil signal. + // This first send MUST be followed by a first receive which contains + // a confirmation in the form of done=true to signify that the Signal is set up + // correctly. + // + // Later sends should only signify if the signal is done. Later sends do not + // update the signal process attached to this channel. + Listen(SignalService_ListenServer) error +} + +func RegisterSignalServiceServer(s *grpc.Server, srv SignalServiceServer) { + s.RegisterService(&_SignalService_serviceDesc, srv) +} + +func _SignalService_Listen_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SignalServiceServer).Listen(&signalServiceListenServer{stream}) +} + +type SignalService_ListenServer interface { + Send(*EventContext) error + Recv() (*SignalContext, error) + grpc.ServerStream +} + +type signalServiceListenServer struct { + grpc.ServerStream +} + +func (x *signalServiceListenServer) Send(m *EventContext) error { + return x.ServerStream.SendMsg(m) +} + +func (x *signalServiceListenServer) Recv() (*SignalContext, error) { + m := new(SignalContext) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _SignalService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "sdk.SignalService", + HandlerType: (*SignalServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Listen", + Handler: _SignalService_Listen_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "sdk/signal.proto", +} + +func (m *SignalContext) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SignalContext) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Signal != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintSignal(dAtA, i, uint64(m.Signal.Size())) + n1, err := m.Signal.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 + } + if m.Done { + dAtA[i] = 0x10 + i++ + if m.Done { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *EventContext) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *EventContext) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Event != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintSignal(dAtA, i, uint64(m.Event.Size())) + n2, err := m.Event.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n2 + } + if m.Done { + dAtA[i] = 0x10 + i++ + if m.Done { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeVarintSignal(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *SignalContext) Size() (n int) { + var l int + _ = l + if m.Signal != nil { + l = m.Signal.Size() + n += 1 + l + sovSignal(uint64(l)) + } + if m.Done { + n += 2 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *EventContext) Size() (n int) { + var l int + _ = l + if m.Event != nil { + l = m.Event.Size() + n += 1 + l + sovSignal(uint64(l)) + } + if m.Done { + n += 2 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovSignal(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozSignal(x uint64) (n int) { + return sovSignal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *SignalContext) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SignalContext: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SignalContext: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signal", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSignal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Signal == nil { + m.Signal = &v1alpha1.Signal{} + } + if err := m.Signal.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Done", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Done = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipSignal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthSignal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *EventContext) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: EventContext: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: EventContext: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Event", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSignal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Event == nil { + m.Event = &v1alpha1.Event{} + } + if err := m.Event.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Done", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSignal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Done = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipSignal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthSignal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipSignal(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowSignal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowSignal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowSignal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthSignal + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowSignal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipSignal(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthSignal = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowSignal = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("sdk/signal.proto", fileDescriptor_signal_4408fb801ca10653) } + +var fileDescriptor_signal_4408fb801ca10653 = []byte{ + // 288 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0xd1, 0x3f, 0x4b, 0x3b, 0x31, + 0x18, 0xc0, 0xf1, 0x5f, 0x7e, 0xea, 0x21, 0x51, 0x41, 0x33, 0x95, 0x0e, 0x47, 0xa9, 0xcb, 0x2d, + 0x26, 0xb6, 0xc5, 0x59, 0xf1, 0x0f, 0x38, 0x38, 0x5d, 0x11, 0xc4, 0x45, 0xd2, 0xe6, 0x21, 0x3d, + 0x53, 0x93, 0x90, 0xc4, 0x43, 0x07, 0xdf, 0x87, 0x2f, 0xc9, 0xd1, 0x97, 0x20, 0xe7, 0x1b, 0x91, + 0x26, 0x3d, 0x54, 0x10, 0x04, 0xb7, 0x87, 0x0c, 0xdf, 0x4f, 0xf2, 0x04, 0x6f, 0x7b, 0xa1, 0x98, + 0xaf, 0xa4, 0xe6, 0x73, 0x6a, 0x9d, 0x09, 0x86, 0xac, 0x78, 0xa1, 0xba, 0xe7, 0xb2, 0x0a, 0xb3, + 0xfb, 0x09, 0x9d, 0x9a, 0x3b, 0xc6, 0x9d, 0x34, 0xd6, 0x99, 0xdb, 0x38, 0xec, 0x41, 0x0d, 0x3a, + 0x78, 0x66, 0x95, 0x64, 0xdc, 0x56, 0x9e, 0x79, 0xd0, 0xde, 0x38, 0x56, 0x0f, 0xf8, 0xdc, 0xce, + 0xf8, 0x80, 0x49, 0xd0, 0xe0, 0x78, 0x00, 0x91, 0x72, 0xfd, 0x27, 0xbc, 0x35, 0x8e, 0xf9, 0x13, + 0xa3, 0x03, 0x3c, 0x04, 0x72, 0x85, 0xb3, 0xe4, 0x75, 0x50, 0x0f, 0x15, 0x1b, 0xc3, 0x23, 0xfa, + 0x69, 0xd1, 0xd6, 0x8a, 0xc3, 0x4d, 0xb2, 0xa8, 0x55, 0x92, 0x2e, 0x2c, 0x9a, 0x2c, 0xda, 0x5a, + 0x34, 0x85, 0xcb, 0x65, 0x8f, 0x10, 0xbc, 0x2a, 0x8c, 0x86, 0xce, 0xff, 0x1e, 0x2a, 0xd6, 0xcb, + 0x38, 0xf7, 0x1f, 0xf1, 0xe6, 0xd9, 0xa2, 0xd2, 0xea, 0x97, 0x78, 0x2d, 0x56, 0x97, 0xf8, 0xe1, + 0xdf, 0xf1, 0x98, 0x2d, 0x53, 0xed, 0x27, 0x7a, 0x78, 0xda, 0xbe, 0x7c, 0x0c, 0xae, 0xae, 0xa6, + 0x40, 0x46, 0x38, 0xbb, 0xa8, 0x7c, 0x00, 0x4d, 0x08, 0xf5, 0x42, 0xd1, 0x6f, 0x7b, 0xe9, 0xee, + 0xc4, 0xb3, 0xaf, 0x97, 0x2d, 0xd0, 0x3e, 0x3a, 0x3e, 0x78, 0x69, 0x72, 0xf4, 0xda, 0xe4, 0xe8, + 0xad, 0xc9, 0xd1, 0xf3, 0x7b, 0xfe, 0xef, 0x7a, 0xf7, 0xb7, 0xbf, 0xf1, 0x42, 0x4d, 0xb2, 0xb8, + 0xfd, 0xd1, 0x47, 0x00, 0x00, 0x00, 0xff, 0xff, 0x65, 0x44, 0x0a, 0x7e, 0xe0, 0x01, 0x00, 0x00, +} diff --git a/sdk/signal.proto b/sdk/signal.proto new file mode 100644 index 000000000000..67d85fec1f07 --- /dev/null +++ b/sdk/signal.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; +option go_package = "github.com/argoproj/argo-events/sdk"; + +package sdk; + +import "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto"; + +message SignalContext { + github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Signal signal = 1; + bool done = 2; +} + +message EventContext { + github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Event event = 1; + bool done = 2; +} + +// SignalService enables communication between signal microservices and the sensor controller. +service SignalService { + // Listen to the signal. Events are streamed back to the client. + // + // The following describes the mechanics for the stream. + // The first send MUST contain a non-nil signal. + // This first send MUST be followed by a first receive which contains + // a confirmation in the form of done=true to signify that the Signal is set up + // correctly. + // + // Later sends should only signify if the signal is done. Later sends do not + // update the signal process attached to this channel. + rpc Listen(stream SignalContext) returns (stream EventContext); +} diff --git a/shared/doc.go b/shared/doc.go new file mode 100644 index 000000000000..fc9f25d35639 --- /dev/null +++ b/shared/doc.go @@ -0,0 +1,4 @@ +// Package shared contains the necessary files for registering +// a micro service for the sensor controller +// This package should not be built as part of a signal server! +package shared diff --git a/shared/grpc.go b/shared/grpc.go deleted file mode 100644 index 26f227b37d49..000000000000 --- a/shared/grpc.go +++ /dev/null @@ -1,76 +0,0 @@ -package shared - -import ( - "context" - fmt "fmt" - "io" - "log" - - "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - empty "github.com/golang/protobuf/ptypes/empty" -) - -// GRPCClient is an implementation of Stream that talks over gRPC. -type GRPCClient struct{ client SignalClient } - -func (c *GRPCClient) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - ch := make(chan *v1alpha1.Event, 32) - stream, err := c.client.Start(context.Background(), signal) - if err != nil { - return nil, err - } - go func() { - defer close(ch) - for { - event, err := stream.Recv() - if err == io.EOF { - // we are done - log.Printf("GRPCClient: received io.EOF, finishing sending") - return - } - if err != nil { - // error during processing - should we use callbacks here? - panic(err) - } - ch <- event - } - }() - return ch, nil -} - -func (c *GRPCClient) Stop() error { - _, err := c.client.Stop(context.Background(), &empty.Empty{}) - return err -} - -// GRPCServer is the gRPC server that GRPCClient talks to. -type GRPCServer struct { - // This is the real implementation - Impl Signaler -} - -func (s *GRPCServer) Start(signal *v1alpha1.Signal, stream Signal_StartServer) error { - events, err := s.Impl.Start(signal) - if err != nil { - return err - } - for { - select { - case event, ok := <-events: - if !ok { - // finished - return nil - } - err := stream.Send(event) - if err != nil { - return err - } - case <-stream.Context().Done(): - return fmt.Errorf("GRPCServer: context finished") - } - } -} - -func (s *GRPCServer) Stop(context.Context, *empty.Empty) (*empty.Empty, error) { - return &empty.Empty{}, s.Impl.Stop() -} diff --git a/shared/interface.go b/shared/interface.go deleted file mode 100644 index e7bc8675330b..000000000000 --- a/shared/interface.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2018 BlackRock, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package shared - -import ( - "net/rpc" - - "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - plugin "github.com/hashicorp/go-plugin" - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" -) - -const ( - CloudEventsVersion = "v1.0" - ContextExtensionErrorKey = "error" - SignalPluginName = "signaler" -) - -// Handshake is a common handshake that is shared by plugin and host. -var Handshake = plugin.HandshakeConfig{ - ProtocolVersion: 1, - MagicCookieKey: "SIGNAL_PLUGIN", - MagicCookieValue: "signal", -} - -// PluginMap is the map of plugins we can dispense. -var PluginMap = map[string]plugin.Plugin{ - SignalPluginName: &signalPlugin{}, -} - -// Signaler is the interface for signaling -type Signaler interface { - Start(*v1alpha1.Signal) (<-chan *v1alpha1.Event, error) - Stop() error -} - -// ArtifactSignaler is the interface for signaling with artifacts -// In addition to including the basic Signaler interface, this also -// enables access to read an artifact object to include in the event data payload -type ArtifactSignaler interface { - Signaler - // todo: change to use io.Reader and io.Closer interfaces? - Read(*v1alpha1.ArtifactLocation, string) ([]byte, error) -} - -// NewPlugin creates a base signal plugin -func NewPlugin(impl Signaler) plugin.Plugin { - return &signalPlugin{Impl: impl} -} - -// signalPlugin is the implementation of plugin.Plugin so we can serve/consume this. -type signalPlugin struct { - Impl Signaler -} - -func (p *signalPlugin) Server(*plugin.MuxBroker) (interface{}, error) { - return &RPCServer{Impl: p.Impl}, nil -} - -func (p *signalPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { - return &RPCClient{client: c}, nil -} - -func (p *signalPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { - RegisterSignalServer(s, &GRPCServer{Impl: p.Impl}) - return nil -} - -func (p *signalPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { - return &GRPCClient{client: NewSignalClient(c)}, nil -} diff --git a/shared/micro.go b/shared/micro.go new file mode 100644 index 000000000000..d9e12877bc2e --- /dev/null +++ b/shared/micro.go @@ -0,0 +1,40 @@ +package shared + +import ( + "github.com/argoproj/argo-events/sdk" + micro "github.com/micro/go-micro" + "github.com/micro/go-micro/client" + k8s "github.com/micro/kubernetes/go/micro" +) + +// MicroSignalClient is the micro Client() +// NOTE: we can't use a default "static" client because tests use this package +// and will fail with: /var/run/secrets/kubernetes.io/serviceaccount: no such file or directory +type MicroSignalClient struct { + client client.Client +} + +/* +func init() { + svc := k8s.NewService(micro.Name("signal.client")) + svc.Init() + defaultMicroSignalService = microSignalClient{client: svc.Client()} +} + +var defaultMicroSignalService microSignalClient + +// DefaultMicroSignalClient is the default MicroSignalClient +var DefaultMicroSignalClient = &defaultMicroSignalService +*/ + +// NewMicroSignalClient returns a new Micro Signal Client +func NewMicroSignalClient() *MicroSignalClient { + svc := k8s.NewService(micro.Name("signal.client")) + svc.Init() + return &MicroSignalClient{client: svc.Client()} +} + +// NewSignalService creates a new SignalService for the following service name +func (m *MicroSignalClient) NewSignalService(name string) sdk.SignalClient { + return sdk.NewMicroSignalClient(name, m.client) +} diff --git a/shared/rpc.go b/shared/rpc.go deleted file mode 100644 index eb14f0319cd5..000000000000 --- a/shared/rpc.go +++ /dev/null @@ -1,46 +0,0 @@ -package shared - -import ( - "net/rpc" - - v1alpha1 "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" -) - -type RPCClient struct{ client *rpc.Client } - -func (c *RPCClient) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - var resp <-chan *v1alpha1.Event - err := c.client.Call("Plugin.Start", map[string]interface{}{ - "signal": signal, - }, &resp) - if err != nil { - return nil, err - } - return resp, nil -} - -func (c *RPCClient) Stop() error { - var resp error - err := c.client.Call("Plugin.Stop", map[string]interface{}{}, &resp) - if err != nil { - return err - } - return resp -} - -// RPCServer is the RPC server that RPCClient talks to, conforming to -// the requirements of net/rpc -type RPCServer struct { - // This is the real implementation - Impl Signaler -} - -func (s *RPCServer) Start(args map[string]interface{}, resp *interface{}) error { - events, err := s.Impl.Start(args["signal"].(*v1alpha1.Signal)) - *resp = events - return err -} - -func (s *RPCServer) Stop(args map[string]interface{}, resp *interface{}) error { - return s.Impl.Stop() -} diff --git a/shared/signal.pb.go b/shared/signal.pb.go deleted file mode 100644 index b72dd6902a89..000000000000 --- a/shared/signal.pb.go +++ /dev/null @@ -1,184 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: shared/signal.proto - -package shared // import "github.com/argoproj/argo-events/shared" - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" -import v1alpha1 "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" -import empty "github.com/golang/protobuf/ptypes/empty" - -import ( - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// Client API for Signal service - -type SignalClient interface { - // Start listening. Events are streamed back to the client. - Start(ctx context.Context, in *v1alpha1.Signal, opts ...grpc.CallOption) (Signal_StartClient, error) - // Stop listening. This should terminate the above event stream. - Stop(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) -} - -type signalClient struct { - cc *grpc.ClientConn -} - -func NewSignalClient(cc *grpc.ClientConn) SignalClient { - return &signalClient{cc} -} - -func (c *signalClient) Start(ctx context.Context, in *v1alpha1.Signal, opts ...grpc.CallOption) (Signal_StartClient, error) { - stream, err := grpc.NewClientStream(ctx, &_Signal_serviceDesc.Streams[0], c.cc, "/shared.Signal/Start", opts...) - if err != nil { - return nil, err - } - x := &signalStartClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Signal_StartClient interface { - Recv() (*v1alpha1.Event, error) - grpc.ClientStream -} - -type signalStartClient struct { - grpc.ClientStream -} - -func (x *signalStartClient) Recv() (*v1alpha1.Event, error) { - m := new(v1alpha1.Event) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *signalClient) Stop(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := grpc.Invoke(ctx, "/shared.Signal/Stop", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// Server API for Signal service - -type SignalServer interface { - // Start listening. Events are streamed back to the client. - Start(*v1alpha1.Signal, Signal_StartServer) error - // Stop listening. This should terminate the above event stream. - Stop(context.Context, *empty.Empty) (*empty.Empty, error) -} - -func RegisterSignalServer(s *grpc.Server, srv SignalServer) { - s.RegisterService(&_Signal_serviceDesc, srv) -} - -func _Signal_Start_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(v1alpha1.Signal) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(SignalServer).Start(m, &signalStartServer{stream}) -} - -type Signal_StartServer interface { - Send(*v1alpha1.Event) error - grpc.ServerStream -} - -type signalStartServer struct { - grpc.ServerStream -} - -func (x *signalStartServer) Send(m *v1alpha1.Event) error { - return x.ServerStream.SendMsg(m) -} - -func _Signal_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SignalServer).Stop(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/shared.Signal/Stop", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SignalServer).Stop(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -var _Signal_serviceDesc = grpc.ServiceDesc{ - ServiceName: "shared.Signal", - HandlerType: (*SignalServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Stop", - Handler: _Signal_Stop_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Start", - Handler: _Signal_Start_Handler, - ServerStreams: true, - }, - }, - Metadata: "shared/signal.proto", -} - -func init() { proto.RegisterFile("shared/signal.proto", fileDescriptor_signal_9f8b96b2afc4ddad) } - -var fileDescriptor_signal_9f8b96b2afc4ddad = []byte{ - // 226 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x90, 0x31, 0x4b, 0xc4, 0x40, - 0x10, 0x85, 0x09, 0x68, 0x8a, 0x94, 0x2b, 0x58, 0xc4, 0x5e, 0x6c, 0x9c, 0xf1, 0x14, 0x6c, 0x15, - 0xe1, 0xc0, 0x3e, 0x9d, 0x8d, 0x6c, 0xbc, 0x71, 0x13, 0x2f, 0xb7, 0x33, 0xec, 0xce, 0x1d, 0xf8, - 0x1f, 0xfc, 0x6f, 0xfe, 0x25, 0xc9, 0x0e, 0xc1, 0x4a, 0x84, 0xeb, 0x66, 0xdf, 0x63, 0xdf, 0xf7, - 0x78, 0xcd, 0x59, 0x1e, 0x7c, 0xa2, 0x0d, 0xe6, 0x31, 0x44, 0x3f, 0x81, 0x24, 0x56, 0x76, 0xb5, - 0x89, 0xed, 0x73, 0x18, 0x75, 0xd8, 0xf7, 0xf0, 0xc6, 0x3b, 0xf4, 0x29, 0xb0, 0x24, 0xfe, 0x28, - 0xc7, 0x35, 0x1d, 0x28, 0x6a, 0x46, 0xd9, 0x06, 0xf4, 0x32, 0x66, 0xcc, 0x14, 0x33, 0x27, 0x3c, - 0xac, 0xfc, 0x24, 0x83, 0x5f, 0x61, 0xa0, 0x48, 0xc9, 0x2b, 0x6d, 0x2c, 0xb1, 0xbd, 0x08, 0xcc, - 0x61, 0x22, 0x2c, 0xaf, 0x7e, 0xff, 0x8e, 0xb4, 0x13, 0xfd, 0x34, 0xf3, 0xf6, 0xbb, 0x6a, 0xea, - 0xae, 0xf0, 0xdd, 0x57, 0xd5, 0x9c, 0x76, 0xea, 0x93, 0xba, 0x47, 0xf8, 0x85, 0xc3, 0x02, 0x2f, - 0xc7, 0xab, 0xc1, 0x41, 0xb6, 0x01, 0x66, 0x38, 0x18, 0x1c, 0x16, 0x38, 0x58, 0x56, 0xfb, 0x70, - 0x7c, 0xc2, 0x7a, 0xf6, 0x6f, 0x2a, 0x77, 0xdf, 0x9c, 0x74, 0xca, 0xe2, 0xce, 0xc1, 0xfa, 0xc3, - 0xd2, 0x1f, 0xd6, 0x73, 0xff, 0xf6, 0x0f, 0xfd, 0xe9, 0xea, 0xe5, 0xf2, 0xbf, 0xe9, 0x6c, 0xe2, - 0xbe, 0x2e, 0x3f, 0xef, 0x7e, 0x02, 0x00, 0x00, 0xff, 0xff, 0x63, 0xa2, 0x5c, 0x06, 0x88, 0x01, - 0x00, 0x00, -} diff --git a/shared/signal.proto b/shared/signal.proto deleted file mode 100644 index 13b70b7c5e80..000000000000 --- a/shared/signal.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; -option go_package = "github.com/argoproj/argo-events/shared"; - -package shared; - -import "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1/generated.proto"; -import "google/protobuf/empty.proto"; - -// Signal enables communication between signal plugins and the sensor executor. -service Signal { - // Start listening. Events are streamed back to the client. - rpc Start(github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Signal) returns (stream github.com.argoproj.argo_events.pkg.apis.sensor.v1alpha1.Event); - - // Stop listening. This should terminate the above event stream. - rpc Stop(google.protobuf.Empty) returns (google.protobuf.Empty); -} diff --git a/signals/README.md b/signals/README.md new file mode 100644 index 000000000000..c5d4d3c9d2b2 --- /dev/null +++ b/signals/README.md @@ -0,0 +1,56 @@ +# Micro Signals +This package is meant to prove out the feature of leveraging [Micro](https://github.com/micro/go-micro) on [Kubernetes](https://github.com/micro/kubernetes) for building signals as stateless microservices. + +## Pre-requisites +- Running Kubernetes cluster >v1.9 +- kubectl correctly configured and pointing to your cluster + +## Getting started +1. First, modify the [argo-events-cluster-roles.yaml](../hack/k8s/manifests/argo-events-cluster-roles.yaml) file to use the correct namespace that you wish to deploy the sensor controller + signal microservices. +2. Create the `argo-events-sa` service account, associated cluster roles, and cluster role bindings. +``` +$ k apply -f hack/k8s/manifests/argo-events-sa.yaml +$ k apply -f hack/k8s/manifests/argo-events-cluster-roles.yaml +``` +3. Create Sensor CRD resources +``` +$ k apply -f hack/k8s/manifests/sensor-crd.yaml +``` +4. Build the latest images +``` +$ make all +``` +5. Install the sensor controller +``` +$ k apply -f hack/k8s/manifests/sensor-controller-configmap.yaml +# k apply -f hack/k8s/manifests/sensor-controller-deployment.yaml +``` + +## Running a Webhook Signal +1. Create the Webhook Signal Microservice +``` +$ k apply -f hack/k8s/manifests/services/webhook.yaml +``` +2. Create a webhook sensor +``` +$ k apply -f examples/webhook-sensor.yaml +``` +3. Trigger the webhook +Using [Postman](https://www.getpostman.com/) or curl, send a POST request to the webhook service. + +## Running a NATS Signal +1. Create the NATS Signal Microservice +``` +$ k apply -f hack/k8s/manifests/services/stream.yaml +``` +2. Create a NATS sensor +``` +$ k apply -f examples/nats-sensor.yaml +``` +3. Send a NATS message +For `NATS`, you can use `github.com/shogsbro/natscat` you just make sure to expose the NATS service externally as a `LoadBalancer`. +``` +$ go get github.com/shogsbro/natscat +$ cd $GOPATH/src/github.shogsbro/natscat +$ ./natscat -S {insert NATS client endpoint here} -s bucketevents "test" +``` diff --git a/signals/artifact/micro/Dockerfile b/signals/artifact/micro/Dockerfile new file mode 100644 index 000000000000..f0293f1b6118 --- /dev/null +++ b/signals/artifact/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/artifact-signal / +CMD [ "/artifact-signal" ] diff --git a/signals/artifact/micro/artifact_service.go b/signals/artifact/micro/artifact_service.go new file mode 100644 index 000000000000..b3140505e6da --- /dev/null +++ b/signals/artifact/micro/artifact_service.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/argoproj/argo-events/common" + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/artifact" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" + "k8s.io/client-go/kubernetes" +) + +func main() { + svc := k8s.NewService(micro.Name("artifact")) + svc.Init() + + // kubernetes configuration + kubeConfig, _ := os.LookupEnv(common.EnvVarKubeConfig) + rest, err := common.GetClientConfig(kubeConfig) + if err != nil { + panic(err) + } + kubeclient := kubernetes.NewForConfigOrDie(rest) + + // namespace configuration + nm := common.DefaultSensorControllerNamespace + + // stream configuration + // TODO: make this configurable while running through github.com/micro/go-config + stream, ok := os.LookupEnv("stream-signal") + if !ok { + stream = "nats" + } + streamClient := sdk.NewMicroSignalClient(stream, svc.Client()) + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(artifact.New(streamClient, kubeclient, nm))) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/artifact/s3.go b/signals/artifact/s3.go index 8ad9c4e994c5..9709f9c4fcac 100644 --- a/signals/artifact/s3.go +++ b/signals/artifact/s3.go @@ -1,16 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package artifact import ( + "context" "encoding/json" "errors" "fmt" + "io" "io/ioutil" + "log" "strconv" "strings" "time" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" "github.com/argoproj/argo-events/store" "github.com/golang/protobuf/proto" minio "github.com/minio/minio-go" @@ -23,47 +42,77 @@ const ( ISO8601 = "2006-01-02T15:04:05:07Z" ) -// s3 is a plugin for an S3 artifact signal +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +// Listen() methods CAN retrieve fields from the s3 struct. type s3 struct { - kubeClient kubernetes.Interface - namespace string - signal *v1alpha1.Signal - streamSignaler shared.Signaler + kubeClient kubernetes.Interface + namespace string + streamClient sdk.SignalClient } // New creates a new S3 signal -func New(streamSignaler shared.Signaler, kubeClient kubernetes.Interface, nm string) shared.ArtifactSignaler { +func New(client sdk.SignalClient, kubeClient kubernetes.Interface, nm string) sdk.ArtifactListener { return &s3{ - streamSignaler: streamSignaler, - kubeClient: kubeClient, - namespace: nm, + streamClient: client, + kubeClient: kubeClient, + namespace: nm, } } -// Start the artifact signal -func (s *s3) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { +func (s *s3) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + ctx := context.TODO() streamSignal, err := extractAndCreateStreamSignal(signal) if err != nil { return nil, err } - streamEvents, err := s.streamSignaler.Start(streamSignal) + stream, err := s.streamClient.Listen(ctx, streamSignal) if err != nil { return nil, err } + + // read from stream & write to streamEvents + streamEvents := make(chan *v1alpha1.Event) + go func() { + defer close(streamEvents) + for { + in, err := stream.Recv() + if err == io.EOF { + return + } + if err != nil { + log.Panicf("signal target stream received error: %s", err) + } + streamEvents <- in.Event + } + }() + events := make(chan *v1alpha1.Event) - go s.interceptFilterAndEnhanceEvents(events, streamEvents) - return events, nil -} -// Stop the artifact signal -func (s *s3) Stop() error { - return s.streamSignaler.Stop() + // start stream receiver + go s.interceptFilterAndEnhanceEvents(signal, events, streamEvents) + + // wait for stop signal + go func() { + <-done + close(events) + // TODO: should we cancel or gracefully shutdown and first send Terminate msg + err := stream.Send(sdk.Terminate) + if err != nil { + log.Panicf("failed to terminate stream signal: %s", err) + } + err = stream.Close() + if err != nil { + log.Panicf("failed to close stream: %s", err) + } + }() + return events, nil } // method should be invoked as a separate go routine within the artifact Start method // intercepts the receive-only msgs off the stream, filters them, and writes artifact events // to the sendCh. -func (s *s3) interceptFilterAndEnhanceEvents(sendCh chan *v1alpha1.Event, recvCh <-chan *v1alpha1.Event) { +func (s *s3) interceptFilterAndEnhanceEvents(sig *v1alpha1.Signal, sendCh chan *v1alpha1.Event, recvCh <-chan *v1alpha1.Event) { defer close(sendCh) for streamEvent := range recvCh { // todo: apply general filtering on cloudEvents @@ -76,10 +125,10 @@ func (s *s3) interceptFilterAndEnhanceEvents(sendCh chan *v1alpha1.Event, recvCh continue } if notification.Err != nil { - event.Context.Extensions[shared.ContextExtensionErrorKey] = notification.Err.Error() + event.Context.Extensions[sdk.ContextExtensionErrorKey] = notification.Err.Error() } for _, record := range notification.Records { - if ok := applyFilter(&record, s.signal.Artifact); !ok { + if ok := applyFilter(&record, sig.Artifact.ArtifactLocation); !ok { // this record failed to pass the filter so we ignore it continue } @@ -99,9 +148,9 @@ func (s *s3) interceptFilterAndEnhanceEvents(sendCh chan *v1alpha1.Event, recvCh event.Context.EventID = record.S3.Object.ETag // read the actual s3 artifact to put into the event data - b, err := s.Read(&s.signal.Artifact.ArtifactLocation, record.S3.Object.Key) + b, err := s.Read(&sig.Artifact.ArtifactLocation, record.S3.Object.Key) if err != nil { - event.Context.Extensions[shared.ContextExtensionErrorKey] = err.Error() + event.Context.Extensions[sdk.ContextExtensionErrorKey] = err.Error() } event.Data = b sendCh <- event @@ -117,6 +166,7 @@ func extractAndCreateStreamSignal(artifactSignal *v1alpha1.Signal) (*v1alpha1.Si } return &v1alpha1.Signal{ Name: fmt.Sprintf("%s-artifact-stream", artifactSignal.Name), + Deadline: artifactSignal.Deadline, Stream: &artifactSignal.Artifact.Target, Constraints: artifactSignal.Constraints, }, nil @@ -127,15 +177,15 @@ func extractAndCreateStreamSignal(artifactSignal *v1alpha1.Signal) (*v1alpha1.Si // 1. notification bucket name must equal the S3 bucket // 2. notification event name must equal the signal S3 event // 3. notification object must pass the prefix and suffix string literals -func applyFilter(notification *minio.NotificationEvent, signal *v1alpha1.ArtifactSignal) bool { - if signal.S3.Filter != nil { - return notification.S3.Bucket.Name == signal.S3.Bucket && - notification.EventName == string(signal.S3.Event) && - strings.HasPrefix(notification.S3.Object.Key, signal.S3.Filter.Prefix) && - strings.HasSuffix(notification.S3.Object.Key, signal.S3.Filter.Suffix) +func applyFilter(notification *minio.NotificationEvent, loc v1alpha1.ArtifactLocation) bool { + if loc.S3.Filter != nil { + return notification.S3.Bucket.Name == loc.S3.Bucket && + notification.EventName == string(loc.S3.Event) && + strings.HasPrefix(notification.S3.Object.Key, loc.S3.Filter.Prefix) && + strings.HasSuffix(notification.S3.Object.Key, loc.S3.Filter.Suffix) } - return notification.S3.Bucket.Name == signal.S3.Bucket && - notification.EventName == string(signal.S3.Event) + return notification.S3.Bucket.Name == loc.S3.Bucket && + notification.EventName == string(loc.S3.Event) } func getMetaTimestamp(tStr string) metav1.Time { @@ -151,7 +201,7 @@ func (s *s3) Read(loc *v1alpha1.ArtifactLocation, key string) ([]byte, error) { if err != nil { return nil, err } - client, err := store.NewMinioClient(s.signal.Artifact.S3, *creds) + client, err := store.NewMinioClient(loc.S3, *creds) if err != nil { return nil, err } diff --git a/signals/artifact/s3_test.go b/signals/artifact/s3_test.go index 26d2c42fae48..2ab83fa49042 100644 --- a/signals/artifact/s3_test.go +++ b/signals/artifact/s3_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package artifact import ( diff --git a/signals/calendar/calendar.go b/signals/calendar/calendar.go new file mode 100644 index 000000000000..ebd2493271bd --- /dev/null +++ b/signals/calendar/calendar.go @@ -0,0 +1,130 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package calendar + +import ( + "fmt" + "log" + "time" + + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + "github.com/argoproj/argo-events/sdk" + cronlib "github.com/robfig/cron" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + EventType = "com.github.argoproj.calendar" +) + +// Next is a function to compute the next signal time from a given time +type Next func(time.Time) time.Time + +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type calendar struct{} + +// New creates a new calendar listener +func New() sdk.Listener { + return new(calendar) +} + +func (c *calendar) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + schedule, err := resolveSchedule(signal.Calendar) + if err != nil { + return nil, err + } + exclusionDates := parseExclusionDates(signal.Calendar.Recurrence) + + events := make(chan *v1alpha1.Event) + + var next Next + next = func(last time.Time) time.Time { + nextT := schedule.Next(last) + nextYear := nextT.Year() + nextMonth := nextT.Month() + nextDay := nextT.Day() + for _, exDate := range exclusionDates { + // if exDate == nextEvent, then we need to skip this and get the next + if exDate.Year() == nextYear && exDate.Month() == nextMonth && exDate.Day() == nextDay { + return next(nextT) + } + } + return nextT + } + + // start handling events + go c.handleEvents(events, next, done) + return events, nil +} + +func (c *calendar) handleEvents(events chan *v1alpha1.Event, next Next, done <-chan struct{}) { + defer close(events) + eventTimer := c.getEventTimer(next, done) + for t := range eventTimer { + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventID: t.String(), + EventType: EventType, + CloudEventsVersion: sdk.CloudEventsVersion, + EventTime: metav1.Time{Time: t}, + Extensions: make(map[string]string), + }, + } + events <- event + } +} + +func (c *calendar) getEventTimer(next Next, done <-chan struct{}) <-chan time.Time { + lastT := time.Now() + eventTimer := make(chan time.Time) + go func() { + defer close(eventTimer) + for { + t := next(lastT) + timer := time.After(time.Until(t)) + log.Printf("expected next calendar event %s", t) + select { + case tx := <-timer: + eventTimer <- tx + lastT = tx + case <-done: + return + } + } + }() + return eventTimer +} + +func resolveSchedule(cal *v1alpha1.CalendarSignal) (cronlib.Schedule, error) { + if cal.Schedule != "" { + schedule, err := cronlib.Parse(cal.Schedule) + if err != nil { + return nil, fmt.Errorf("failed to parse schedule %s from calendar signal. Cause: %+v", cal.Schedule, err.Error()) + } + return schedule, nil + } else if cal.Interval != "" { + intervalDuration, err := time.ParseDuration(cal.Interval) + if err != nil { + return nil, fmt.Errorf("failed to parse interval %s from calendar signal. Cause: %+v", cal.Interval, err.Error()) + } + schedule := cronlib.ConstantDelaySchedule{Delay: intervalDuration} + return schedule, nil + } else { + return nil, fmt.Errorf("calendar signal must contain either a schedule or interval") + } +} diff --git a/signals/calendar/signal_test.go b/signals/calendar/calendar_test.go similarity index 69% rename from signals/calendar/signal_test.go rename to signals/calendar/calendar_test.go index 6ff14160b173..924bae714d2a 100644 --- a/signals/calendar/signal_test.go +++ b/signals/calendar/calendar_test.go @@ -21,11 +21,9 @@ import ( "time" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - cronlib "github.com/robfig/cron" - "github.com/stretchr/testify/assert" ) -func TestCalendarStartFailures(t *testing.T) { +func TestCalendarListenFailures(t *testing.T) { signal := v1alpha1.Signal{ Name: "nats-test", Calendar: &v1alpha1.CalendarSignal{ @@ -33,9 +31,10 @@ func TestCalendarStartFailures(t *testing.T) { }, } cal := New() + done := make(chan struct{}) // test unknown signal - _, err := cal.Start(&signal) + _, err := cal.Listen(&signal, done) if err == nil { t.Errorf("expected a non nil error for an unknown calendar signal") } @@ -47,7 +46,7 @@ func TestCalendarStartFailures(t *testing.T) { Schedule: "this is not a schedule", }, } - _, err = cal.Start(&signal) + _, err = cal.Listen(&signal, done) if err == nil { t.Errorf("expected a non nil error for invalid parsing of schedule") } @@ -59,14 +58,18 @@ func TestCalendarStartFailures(t *testing.T) { Interval: "this is not a schedule", }, } - _, err = cal.Start(&signal) + _, err = cal.Listen(&signal, done) if err == nil { t.Errorf("expected a non nil error for invalid parsing of interval") } + + close(done) } func TestScheduleCalendar(t *testing.T) { cal := New() + done := make(chan struct{}) + signal := v1alpha1.Signal{ Name: "nats-test", Calendar: &v1alpha1.CalendarSignal{ @@ -74,7 +77,7 @@ func TestScheduleCalendar(t *testing.T) { }, } - events, err := cal.Start(&signal) + events, err := cal.Listen(&signal, done) if err != nil { t.Error(err) } @@ -85,10 +88,7 @@ func TestScheduleCalendar(t *testing.T) { t.Errorf("expected an event but found none") } - err = cal.Stop() - if err != nil { - t.Error(err) - } + close(done) // ensure the event was correct if event.Context.EventType != EventType { @@ -98,6 +98,8 @@ func TestScheduleCalendar(t *testing.T) { func TestIntervalCalendar(t *testing.T) { cal := New() + done := make(chan struct{}) + signal := v1alpha1.Signal{ Name: "nats-test", Calendar: &v1alpha1.CalendarSignal{ @@ -105,7 +107,7 @@ func TestIntervalCalendar(t *testing.T) { }, } - events, err := cal.Start(&signal) + events, err := cal.Listen(&signal, done) if err != nil { t.Error(err) } @@ -116,10 +118,7 @@ func TestIntervalCalendar(t *testing.T) { t.Errorf("expected an event but found none") } - err = cal.Stop() - if err != nil { - t.Error(err) - } + close(done) // ensure the event was correct // ensure the event was correct @@ -127,19 +126,3 @@ func TestIntervalCalendar(t *testing.T) { t.Errorf("event context EventType\nexpected: %s\nactual: %s", EventType, event.Context.EventType) } } - -func TestGetNextTime(t *testing.T) { - c := calendar{ - schedule: cronlib.ConstantDelaySchedule{Delay: time.Minute}, - exclusionDates: []time.Time{time.Date(2018, time.May, 10, 0, 0, 0, 0, time.UTC)}, - } - lastEventTime := time.Date(2018, time.May, 9, 23, 59, 0, 0, time.UTC) - nextEventTime := c.getNextTime(lastEventTime) - expectedNextEventTime := time.Date(2018, time.May, 11, 0, 0, 0, 0, time.UTC) - assert.Equal(t, expectedNextEventTime, nextEventTime) - - lastEventTime = time.Date(2018, time.May, 9, 23, 58, 59, 0, time.UTC) - nextEventTime = c.getNextTime(lastEventTime) - expectedNextEventTime = time.Date(2018, time.May, 9, 23, 59, 59, 0, time.UTC) - assert.Equal(t, expectedNextEventTime, nextEventTime) -} diff --git a/signals/calendar/micro/Dockerfile b/signals/calendar/micro/Dockerfile new file mode 100644 index 000000000000..ffa0c865953b --- /dev/null +++ b/signals/calendar/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/calendar-signal / +CMD [ "/calendar-signal" ] diff --git a/signals/calendar/micro/calendar_service.go b/signals/calendar/micro/calendar_service.go new file mode 100644 index 000000000000..11dab329ee12 --- /dev/null +++ b/signals/calendar/micro/calendar_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/calendar" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("calendar")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(calendar.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/calendar/signal.go b/signals/calendar/signal.go deleted file mode 100644 index 2eed87690090..000000000000 --- a/signals/calendar/signal.go +++ /dev/null @@ -1,136 +0,0 @@ -/* -Copyright 2018 BlackRock, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package calendar - -import ( - "fmt" - "log" - "time" - - "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" - cronlib "github.com/robfig/cron" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - EventType = "com.github.argoproj.calendar" -) - -const ( - tickMethodSchedule = iota - tickMethodInterval -) - -type calendar struct { - tickMethod int - schedule cronlib.Schedule - exclusionDates []time.Time - stop chan struct{} -} - -// New creates a new calendar signaler -func New() shared.Signaler { - return &calendar{ - stop: make(chan struct{}), - } -} - -func (c *calendar) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - // parse out the calendar configurations - var err error - if signal.Calendar.Schedule != "" { - c.tickMethod = tickMethodSchedule - c.schedule, err = cronlib.Parse(signal.Calendar.Schedule) - if err != nil { - return nil, fmt.Errorf("failed to parse schedule %s from calendar signal. Cause: %+v", signal.Calendar.Schedule, err.Error()) - } - } else if signal.Calendar.Interval != "" { - c.tickMethod = tickMethodInterval - intervalDuration, err := time.ParseDuration(signal.Calendar.Interval) - if err != nil { - return nil, fmt.Errorf("failed to parse interval %s from calendar signal. Cause: %+v", signal.Calendar.Interval, err.Error()) - } - c.schedule = cronlib.ConstantDelaySchedule{Delay: intervalDuration} - } else { - return nil, fmt.Errorf("calendar signal must contain either a schedule or interval") - } - - c.exclusionDates = parseExclusionDates(signal.Calendar.Recurrence) - - events := make(chan *v1alpha1.Event) - go c.handleEvents(events) - return events, nil -} - -func (c *calendar) Stop() error { - c.stop <- struct{}{} - close(c.stop) - return nil -} - -func (c *calendar) handleEvents(events chan *v1alpha1.Event) { - defer close(events) - eventTimer := c.getEventTimer() - for t := range eventTimer { - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventID: t.String(), - EventType: EventType, - CloudEventsVersion: shared.CloudEventsVersion, - EventTime: metav1.Time{Time: t}, - Extensions: make(map[string]string), - }, - } - events <- event - } -} - -func (c *calendar) getEventTimer() <-chan time.Time { - lastT := time.Now() - eventTimer := make(chan time.Time) - go func() { - defer close(eventTimer) - for { - t := c.getNextTime(lastT) - timer := time.After(time.Until(t)) - log.Printf("expected next calendar event %s", t) - select { - case tx := <-timer: - eventTimer <- tx - lastT = tx - case <-c.stop: - return - } - } - }() - return eventTimer -} - -func (c *calendar) getNextTime(lastEventTime time.Time) time.Time { - nextEventTime := c.schedule.Next(lastEventTime) - nextYear := nextEventTime.Year() - nextMonth := nextEventTime.Month() - nextDay := nextEventTime.Day() - for _, exDate := range c.exclusionDates { - // if exDate == nextEvent, then we need to skip this and get the next - if exDate.Year() == nextYear && exDate.Month() == nextMonth && exDate.Day() == nextDay { - return c.getNextTime(nextEventTime) - } - } - return nextEventTime -} diff --git a/signals/resource/micro/Dockerfile b/signals/resource/micro/Dockerfile new file mode 100644 index 000000000000..e6c14072725e --- /dev/null +++ b/signals/resource/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/resource-signal / +CMD [ "/resource-signal" ] diff --git a/signals/resource/micro/resource_signal.go b/signals/resource/micro/resource_signal.go new file mode 100644 index 000000000000..a56a86a12555 --- /dev/null +++ b/signals/resource/micro/resource_signal.go @@ -0,0 +1,45 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/argoproj/argo-events/common" + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/resource" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("artifact")) + svc.Init() + + // kubernetes configuration + kubeConfig, _ := os.LookupEnv(common.EnvVarKubeConfig) + rest, err := common.GetClientConfig(kubeConfig) + if err != nil { + panic(err) + } + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(resource.New(rest))) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/resource/signal.go b/signals/resource/resource.go similarity index 76% rename from signals/resource/signal.go rename to signals/resource/resource.go index 1295ed87a60b..265581bf77cc 100644 --- a/signals/resource/signal.go +++ b/signals/resource/resource.go @@ -33,43 +33,84 @@ import ( "k8s.io/client-go/rest" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" ) +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +// Listen() methods CAN retrieve the kubeConfig from the resource struct. type resource struct { kubeConfig *rest.Config - watches []watch.Interface } // New creates a new resource signaler -func New(kubeConfig *rest.Config) shared.Signaler { +func New(kubeConfig *rest.Config) sdk.Listener { return &resource{kubeConfig: kubeConfig} } -func (r *resource) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - var err error +func (r *resource) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + resources, err := r.discoverResources(signal.Resource) + if err != nil { + return nil, err + } + + options := metav1.ListOptions{Watch: true} + if signal.Resource.Filter != nil { + options.LabelSelector = labels.Set(signal.Resource.Filter.Labels).AsSelector().String() + } + + wg := sync.WaitGroup{} + watches := make([]watch.Interface, 0) + events := make(chan *v1alpha1.Event) + + // start up listeners + for i := 0; i < len(resources); i++ { + resource := resources[i] + watch, err := resource.Watch(options) + if err != nil { + return nil, err + } + watches = append(watches, watch) + + wg.Add(1) + go r.listen(events, watch, signal.Resource.Filter, &wg) + } + + // wait for stop signal + go func() { + <-done + for _, watch := range watches { + watch.Stop() + } + }() + + // wait for all watches to complete, then close the events channel + go func() { + wg.Wait() + close(events) + }() + + return events, nil +} + +func (r *resource) discoverResources(obj *v1alpha1.ResourceSignal) ([]dynamic.ResourceInterface, error) { dynClientPool := dynamic.NewDynamicClientPool(r.kubeConfig) disco, err := discovery.NewDiscoveryClientForConfig(r.kubeConfig) if err != nil { return nil, err } - var groupVersion string - if signal.Resource.Version == "v1" { - groupVersion = "v1" - } else { - groupVersion = signal.Resource.Group + "/" + signal.Resource.Version - } + + groupVersion := resolveGroupVersion(obj) resourceInterfaces, err := disco.ServerResourcesForGroupVersion(groupVersion) if err != nil { return nil, err } - resources := make([]dynamic.ResourceInterface, 0) for i := range resourceInterfaces.APIResources { apiResource := resourceInterfaces.APIResources[i] gvk := schema.FromAPIVersionAndKind(resourceInterfaces.GroupVersion, apiResource.Kind) log.Printf("found API Resource %s", gvk.String()) - if apiResource.Kind != signal.Resource.Kind { + if apiResource.Kind != obj.Kind { continue } canWatch := false @@ -84,42 +125,17 @@ func (r *resource) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error if err != nil { return nil, err } - resources = append(resources, client.Resource(&apiResource, signal.Resource.Namespace)) + resources = append(resources, client.Resource(&apiResource, obj.Namespace)) } } - - options := metav1.ListOptions{Watch: true} - if signal.Resource.Filter != nil { - options.LabelSelector = labels.Set(signal.Resource.Filter.Labels).AsSelector().String() - } - - wg := sync.WaitGroup{} - events := make(chan *v1alpha1.Event) - for i := 0; i < len(resources); i++ { - resource := resources[i] - watch, err := resource.Watch(options) - if err != nil { - return nil, err - } - r.watches = append(r.watches, watch) - - wg.Add(1) - go r.listen(events, watch, signal.Resource.Filter, &wg) - } - - go func() { - wg.Wait() - close(events) - }() - - return events, nil + return resources, nil } -func (r *resource) Stop() error { - for _, watch := range r.watches { - watch.Stop() +func resolveGroupVersion(obj *v1alpha1.ResourceSignal) string { + if obj.Version == "v1" { + return obj.Version } - return nil + return obj.Group + "/" + obj.Version } func (r *resource) listen(events chan *v1alpha1.Event, w watch.Interface, filter *v1alpha1.ResourceFilter, wg *sync.WaitGroup) { @@ -138,7 +154,7 @@ func (r *resource) listen(events chan *v1alpha1.Event, w watch.Interface, filter log.Panic(err) } - if r.passFilters(itemObj, filter) { + if passFilters(itemObj, filter) { events <- event } } @@ -146,7 +162,7 @@ func (r *resource) listen(events chan *v1alpha1.Event, w watch.Interface, filter } // helper method to return a flag indicating if the object passed the client side filters -func (r *resource) passFilters(obj *unstructured.Unstructured, filter *v1alpha1.ResourceFilter) bool { +func passFilters(obj *unstructured.Unstructured, filter *v1alpha1.ResourceFilter) bool { // check prefix if !strings.HasPrefix(obj.GetName(), filter.Prefix) { log.Printf("FILTERED: resource name '%s' does not match prefix '%s'", obj.GetName(), filter.Prefix) diff --git a/signals/stream/README.md b/signals/stream/README.md index 6605da589dc4..70b40453e4ed 100644 --- a/signals/stream/README.md +++ b/signals/stream/README.md @@ -1,12 +1,12 @@ -# Stream Signal Plugins -In today's ecosystem with the rise in popularity of real-time systems and microservices, there exists a plethora of message brokers and streaming platforms from which to choose from. This results in the problem of being able to support a wide variety of platforms (external and internal) depending on the technology stack. The current solution to this problem uses pluggable binaries that implement the `Signaler` interface, run as a separate process to the main `sensor-controller`, and communicate over `RPC` or `gRPC`. +# Signal Microservices +In today's ecosystem with the rise in popularity of real-time systems and microservices, there exists a plethora of message brokers and streaming platforms from which to choose from. This results in the problem of being able to support a wide variety of platforms (external and internal) depending on the technology stack. The current solution to this problem uses [micro](https://github.com/micro/go-micro) microservices that implement the `Signaler` interface, run as a separate deployment (multiple pods + service) to the `sensor-controller`, and communicate over `gRPC`. ## Use Cases 1. User wishes to listen to events from outside the currently default builtin stream functionality without having to change the code or add to the existing `Stream` configuration. 2. User wishes to leverage an existing platform from their stack as a source to trigger workflows. ## Design -- `Signaler` interface is defined within the `shared` package. All signals must implement this interface. +- `Signaler` interface is defined within the `sdk` package. All signals must implement this interface. ``` // Signaler is the interface for signaling type Signaler interface { @@ -14,21 +14,10 @@ type Signaler interface { Stop() error } ``` -- Use [go-plugin](https://github.com/hashicorp/go-plugin) to register/create all signals. -- Plugin must have a `main` package and a `main()` function and use `go-plugin` to serve up the plugin structure/functionality via RPC or gRPC. (One plugin per binary!) -- Binaries MUST be included in the sensor-controller's `Dockerfile` as: -``` -COPY dist/plugins/ /plugins/ -``` -where the plugin binaries are located in the `dist/plugins` directory. +- Use [micro k8s](https://github.com/micro/kubernetes) to register deployed microservices. Each pod is considered a node of the microservice. +- `Sensor controller` registers itself using the `micro k8s` as a client to the signal microservices. It then is able to ## Build your own -Building your own stream signal plugin is easy. All you need is a `struct` which implements the `Signaler` interface. You can write your plugin in a number of different languages, however using a language other than `Go` requires a bit more work and is not covered in this. - -If you're using Go, take a look at the `builtin` package for examples on writing your own custom plugin. You'll need to call the `github.com/hashicorp/go-plugin`.`Serve()` method in the main function of your program. You'll also need to append an entry to the `PluginMap` in the `shared` directory. Please, put the new plugin under the `custom` package. You'll then need to compile the program and output it to the following directory: `dist/plugins`. You can do this using: -``` -$ go build -o dist/plugins/{plugin-name} {path to main program} -``` +Building your own signal microservice is easy. All you need is a `struct` which implements the `Signaler` interface. You can write your service in a number of different languages, however using a language other than `Go` requires a bit more work and is not covered in this. -## Future work -It may be desirable to use 2 or more streaming services as signals. In the current implementation, only one stream plugin binary is included in the controller's `Dockerfile`. We can include the ability to add a directory for all the plugin binary files. These files can then be included in the Docker build or injected dynamically into the program through a `PVC`. \ No newline at end of file +If you're using Go, take a look at the `builtin` package for examples on writing your own custom services. You'll need to call the `sdk.RegisterSignalServiceHandler()` method in the main function of your program. Please, put the new plugin under the `custom` package. diff --git a/signals/stream/builtin/amqp/amqp.go b/signals/stream/builtin/amqp/amqp.go index 879ff5df10cd..a5ce656b3e80 100644 --- a/signals/stream/builtin/amqp/amqp.go +++ b/signals/stream/builtin/amqp/amqp.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package amqp import ( "fmt" + "log" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" - plugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-events/sdk" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" amqplib "github.com/streadway/amqp" @@ -35,98 +35,107 @@ const ( routingKeyKey = "routingKey" ) -type amqp struct { - conn *amqplib.Connection - delivery <-chan amqplib.Delivery -} +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type amqp struct{} -// New creates an amqp signaler -func New() shared.Signaler { - return &amqp{} +// New creates an amqp listener +func New() sdk.Listener { + return new(amqp) } -func (a *amqp) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - // parse out the attributes - exchangeName, ok := signal.Stream.Attributes[exchangeNameKey] - if !ok { - return nil, shared.ErrMissingRequiredAttribute - } - exchangeType, ok := signal.Stream.Attributes[exchangeTypeKey] - if !ok { - return nil, shared.ErrMissingRequiredAttribute - } - routingKey, ok := signal.Stream.Attributes[routingKeyKey] - if !ok { - return nil, shared.ErrMissingRequiredAttribute +func (*amqp) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + conn, err := amqplib.Dial(signal.Stream.URL) + if err != nil { + return nil, fmt.Errorf("failed to connect to server: %s", err) } - - var err error - a.conn, err = amqplib.Dial(signal.Stream.URL) + ch, err := conn.Channel() if err != nil { - return nil, fmt.Errorf("failed to connect to RabbitMQ. cause: %s", err) + return nil, fmt.Errorf("failed to open channel: %s", err) } - ch, err := a.conn.Channel() + delivery, err := getDelivery(ch, signal.Stream.Attributes) + events := make(chan *v1alpha1.Event) + + // start listening for messages + go func() { + defer close(events) + for { + select { + case msg := <-delivery: + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventID: msg.MessageId, + EventType: EventType, + EventTypeVersion: msg.ConsumerTag, + CloudEventsVersion: sdk.CloudEventsVersion, + Source: &v1alpha1.URI{}, + EventTime: metav1.Time{Time: msg.Timestamp}, + SchemaURL: &v1alpha1.URI{}, + ContentType: msg.ContentType, + Extensions: map[string]string{"content-encoding": msg.ContentEncoding}, + }, + Data: msg.Body, + } + events <- event + msg.Ack(false) + case <-done: + if err := ch.Close(); err != nil { + log.Printf("failed to close channel for signal '%s': %s", signal.Name, err) + } + if err := conn.Close(); err != nil { + log.Panicf("failed to close connection for signal '%s': %s", signal.Name, err) + } + log.Printf("shut down signal '%s'", signal.Name) + return + } + } + }() + + return events, nil +} + +func getDelivery(ch *amqplib.Channel, attr map[string]string) (<-chan amqplib.Delivery, error) { + exName, exType, rKey, err := parseAttributes(attr) if err != nil { - return nil, fmt.Errorf("failed to open channel. cause: %s", err) + return nil, err } - err = ch.ExchangeDeclare(exchangeName, exchangeType, true, false, false, false, nil) + err = ch.ExchangeDeclare(exName, exType, true, false, false, false, nil) if err != nil { - return nil, fmt.Errorf("failed to declare RabbitMQ %s exchange '%s'. cause: %s", exchangeType, exchangeName, err) + return nil, fmt.Errorf("failed to declare %s exchange '%s': %s", exType, exName, err) } q, err := ch.QueueDeclare("", false, false, true, false, nil) if err != nil { - return nil, fmt.Errorf("failed to declare RabbigMQ queue. cause: %s", err) + return nil, fmt.Errorf("failed to declare queue: %s", err) } - err = ch.QueueBind(q.Name, routingKey, exchangeName, false, nil) + err = ch.QueueBind(q.Name, rKey, exName, false, nil) if err != nil { - return nil, fmt.Errorf("failed to bind RabbitMQ %s exchange '%s' to queue with routingKey: %s. cause: %s", exchangeType, exchangeName, routingKey, err) + return nil, fmt.Errorf("failed to bind %s exchange '%s' to queue with routingKey: %s: %s", exType, exName, rKey, err) } - a.delivery, err = ch.Consume(q.Name, "", true, false, false, false, nil) + + delivery, err := ch.Consume(q.Name, "", true, false, false, false, nil) if err != nil { - return nil, fmt.Errorf("failed to begin consuming RabbitMQ messages. cause: %s", err) + return nil, fmt.Errorf("failed to begin consuming messages: %s", err) } - - events := make(chan *v1alpha1.Event) - go a.listen(events) - return events, nil + return delivery, nil } -func (a *amqp) Stop() error { - return a.conn.Close() -} - -func (a *amqp) listen(events chan *v1alpha1.Event) { - for msg := range a.delivery { - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventID: msg.MessageId, - EventType: EventType, - EventTypeVersion: msg.ConsumerTag, - CloudEventsVersion: shared.CloudEventsVersion, - Source: &v1alpha1.URI{}, - EventTime: metav1.Time{Time: msg.Timestamp}, - SchemaURL: &v1alpha1.URI{}, - ContentType: msg.ContentType, - Extensions: map[string]string{"content-encoding": msg.ContentEncoding}, - }, - Data: msg.Body, - } - events <- event - msg.Ack(false) +func parseAttributes(attr map[string]string) (string, string, string, error) { + // parse out the attributes + exchangeName, ok := attr[exchangeNameKey] + if !ok { + return "", "", "", sdk.ErrMissingRequiredAttribute } -} - -func main() { - amqp := New() - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: shared.Handshake, - Plugins: map[string]plugin.Plugin{ - shared.SignalPluginName: shared.NewPlugin(amqp), - }, - GRPCServer: plugin.DefaultGRPCServer, - }) + exchangeType, ok := attr[exchangeTypeKey] + if !ok { + return exchangeName, "", "", sdk.ErrMissingRequiredAttribute + } + routingKey, ok := attr[routingKeyKey] + if !ok { + return exchangeName, "", "", sdk.ErrMissingRequiredAttribute + } + return exchangeName, exchangeType, routingKey, nil } diff --git a/signals/stream/builtin/amqp/micro/Dockerfile b/signals/stream/builtin/amqp/micro/Dockerfile new file mode 100644 index 000000000000..c4177254490b --- /dev/null +++ b/signals/stream/builtin/amqp/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/amqp-signal / +CMD [ "/amqp-signal" ] diff --git a/signals/stream/builtin/amqp/micro/amqp_service.go b/signals/stream/builtin/amqp/micro/amqp_service.go new file mode 100644 index 000000000000..9e27d08bcb6d --- /dev/null +++ b/signals/stream/builtin/amqp/micro/amqp_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/stream/builtin/amqp" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("amqp")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(amqp.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/stream/builtin/doc.go b/signals/stream/builtin/doc.go index f21ffe97df53..57b71ca23915 100644 --- a/signals/stream/builtin/doc.go +++ b/signals/stream/builtin/doc.go @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package builtin contains builtin stream plugins +// Package builtin contains builtin stream microservices package builtin diff --git a/signals/stream/builtin/kafka/kafka.go b/signals/stream/builtin/kafka/kafka.go index 90bf1ddc19ff..35dd3d5d3c33 100644 --- a/signals/stream/builtin/kafka/kafka.go +++ b/signals/stream/builtin/kafka/kafka.go @@ -14,16 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package kafka import ( "fmt" + "log" "strconv" "github.com/Shopify/sarama" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" - plugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-events/sdk" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,104 +33,105 @@ const ( EventType = "org.apache.kafka.pub" ) -type kafka struct { - consumer sarama.Consumer - partitionConsumer sarama.PartitionConsumer - stop chan struct{} -} +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type kafka struct{} // New creates a new kafka signaler -func New() shared.Signaler { - return &kafka{ - stop: make(chan struct{}), - } +func New() sdk.Listener { + return new(kafka) } -func (k *kafka) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { +func (*kafka) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { // parse out the attributes - topic, ok := signal.Stream.Attributes["topic"] - if !ok { - return nil, shared.ErrMissingRequiredAttribute - } - var pString string - if pString, ok = signal.Stream.Attributes[partitionKey]; !ok { - return nil, fmt.Errorf(shared.ErrMissingAttribute, partitionKey) - } - pInt, err := strconv.ParseInt(pString, 10, 32) + topic, partition, err := parseAttributes(signal.Stream.Attributes) if err != nil { return nil, err } - partition := int32(pInt) - k.consumer, err = sarama.NewConsumer([]string{signal.Stream.URL}, nil) + consumer, err := sarama.NewConsumer([]string{signal.Stream.URL}, nil) if err != nil { - return nil, fmt.Errorf("failed to connect to kafka cluster at %s", signal.Stream.URL) + return nil, fmt.Errorf("failed to connect to cluster at %s", signal.Stream.URL) } - availablePartitions, err := k.consumer.Partitions(topic) + availablePartitions, err := consumer.Partitions(topic) if err != nil { return nil, fmt.Errorf("unable to get available partitions for kafka topic '%s'. cause: %s", topic, err.Error()) } - - if ok := k.verifyPartitionAvailable(partition, availablePartitions); !ok { + if ok := verifyPartitionAvailable(partition, availablePartitions); !ok { return nil, fmt.Errorf("partition %v does not exist for topic '%s'", partition, topic) } - k.partitionConsumer, err = k.consumer.ConsumePartition(topic, partition, sarama.OffsetNewest) + partitionConsumer, err := consumer.ConsumePartition(topic, partition, sarama.OffsetNewest) if err != nil { return nil, fmt.Errorf("failed to create partition consumer for topic '%s' and partition: %v. cause: %s", topic, partition, err.Error()) } + events := make(chan *v1alpha1.Event) - go k.listen(events) + + // start listening for messages + go func() { + defer close(events) + for { + select { + case msg := <-partitionConsumer.Messages(): + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventID: fmt.Sprintf("partition-%v-offset-%v", msg.Partition, msg.Offset), + EventType: EventType, + EventTime: metav1.Time{Time: msg.Timestamp}, + CloudEventsVersion: sdk.CloudEventsVersion, + Extensions: make(map[string]string), + }, + Data: msg.Value, + } + log.Printf("signal '%s' received msg", signal.Name) + events <- event + case err := <-partitionConsumer.Errors(): + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventID: fmt.Sprintf("partition-%v-", err.Partition), + EventType: EventType, + CloudEventsVersion: sdk.CloudEventsVersion, + Extensions: map[string]string{sdk.ContextExtensionErrorKey: err.Err.Error()}, + }, + } + log.Printf("signal '%s' received error", signal.Name) + events <- event + case <-done: + if err := partitionConsumer.Close(); err != nil { + log.Printf("failed to close partition consumer for signal '%s': %s", signal.Name, err) + } + if err := consumer.Close(); err != nil { + log.Panicf("failed to close consumer for signal '%s': %s", signal.Name, err) + } + log.Printf("shut down signal '%s'", signal.Name) + return + } + } + }() + return events, nil } -func (k *kafka) Stop() error { - k.stop <- struct{}{} - err := k.partitionConsumer.Close() - if err != nil { - return err +func parseAttributes(attr map[string]string) (string, int32, error) { + topic, ok := attr["topic"] + if !ok { + return "", 0, sdk.ErrMissingRequiredAttribute } - err = k.consumer.Close() - if err != nil { - return err + var pString string + if pString, ok = attr[partitionKey]; !ok { + return topic, 0, fmt.Errorf(sdk.ErrMissingAttribute, partitionKey) } - return nil -} - -func (k *kafka) listen(events chan *v1alpha1.Event) { - defer close(events) - for { - select { - case msg := <-k.partitionConsumer.Messages(): - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventID: fmt.Sprintf("partition-%v-offset-%v", msg.Partition, msg.Offset), - EventType: EventType, - EventTime: metav1.Time{Time: msg.Timestamp}, - CloudEventsVersion: shared.CloudEventsVersion, - Extensions: make(map[string]string), - }, - Data: msg.Value, - } - events <- event - case err := <-k.partitionConsumer.Errors(): - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventID: fmt.Sprintf("partition-%v-", err.Partition), - EventType: EventType, - CloudEventsVersion: shared.CloudEventsVersion, - Extensions: map[string]string{shared.ContextExtensionErrorKey: err.Err.Error()}, - }, - } - events <- event - case <-k.stop: - return - } + pInt, err := strconv.ParseInt(pString, 10, 32) + if err != nil { + return topic, 0, err } + partition := int32(pInt) + return topic, partition, nil } -func (k *kafka) verifyPartitionAvailable(part int32, partitions []int32) bool { +func verifyPartitionAvailable(part int32, partitions []int32) bool { for _, p := range partitions { if part == p { return true @@ -138,14 +139,3 @@ func (k *kafka) verifyPartitionAvailable(part int32, partitions []int32) bool { } return false } - -func main() { - kafka := New() - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: shared.Handshake, - Plugins: map[string]plugin.Plugin{ - shared.SignalPluginName: shared.NewPlugin(kafka), - }, - GRPCServer: plugin.DefaultGRPCServer, - }) -} diff --git a/signals/stream/builtin/kafka/kafka_test.go b/signals/stream/builtin/kafka/kafka_test.go index ec77365c1464..0bf130acdc8b 100644 --- a/signals/stream/builtin/kafka/kafka_test.go +++ b/signals/stream/builtin/kafka/kafka_test.go @@ -14,66 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package kafka -/* -func TestSignal(t *testing.T) { - consumer := mocks.NewConsumer(t, sarama.NewConfig()) - consumer.SetTopicMetadata(map[string][]int32{"test": []int32{0}}) - expectedPartitionConsumer := consumer.ExpectConsumePartition("test", 0, -1) - expectedPartitionConsumer.ExpectMessagesDrainedOnClose() - expectedPartitionConsumer.ExpectErrorsDrainedOnClose() - - kafka := &kafka{ - consumer: consumer, - stop: make(chan struct{}), - } - testCh := make(chan job.Event) - - // unable to get available partitions - err := signal.Start(testCh) - assert.NotNil(t, err) - - // partition not available - consumer.SetTopicMetadata(map[string][]int32{"unknown": []int32{1}}) - err = signal.Start(testCh) - assert.NotNil(t, err) - - // success - consumer.SetTopicMetadata(map[string][]int32{"test": []int32{0}}) - signal.topic = "test" - signal.partition = 0 - - err = signal.Start(testCh) - assert.Nil(t, err) - - // send message - testMsg := &sarama.ConsumerMessage{ - Topic: "test", - Partition: 0, - Offset: 1, - Timestamp: time.Now(), - Key: []byte("key"), - Value: []byte("hello, world"), - } - expectedPartitionConsumer.YieldMessage(testMsg) - - // verify the message - event := <-testCh - assert.Equal(t, "", event.GetID()) - assert.Equal(t, "test", event.GetSource()) - assert.Equal(t, []byte("hello, world"), event.GetBody()) - assert.Equal(t, signal, event.GetSignal()) - - // send an error - err = fmt.Errorf("this is a test error") - expectedPartitionConsumer.YieldError(err) - - // verify the error - event = <-testCh - assert.Equal(t, err, event.GetError()) - - err = signal.Stop() - assert.Nil(t, err) -} -*/ +// TODO: implement e2e test +// the github.com/Shopify/sarama/mocks doesn't work because we +// can't pass the mock Consumer to the kafka struct diff --git a/signals/stream/builtin/kafka/micro/Dockerfile b/signals/stream/builtin/kafka/micro/Dockerfile new file mode 100644 index 000000000000..80bae6ab8532 --- /dev/null +++ b/signals/stream/builtin/kafka/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/kafka-signal / +CMD [ "/kafka-signal" ] diff --git a/signals/stream/builtin/kafka/micro/kafka_service.go b/signals/stream/builtin/kafka/micro/kafka_service.go new file mode 100644 index 000000000000..5e4b4759170b --- /dev/null +++ b/signals/stream/builtin/kafka/micro/kafka_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/stream/builtin/kafka" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("kafka")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(kafka.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/stream/builtin/mqtt/micro/Dockerfile b/signals/stream/builtin/mqtt/micro/Dockerfile new file mode 100644 index 000000000000..d41b65fb0baf --- /dev/null +++ b/signals/stream/builtin/mqtt/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/mqtt-signal / +CMD [ "/mqtt-signal" ] diff --git a/signals/stream/builtin/mqtt/micro/mqtt_service.go b/signals/stream/builtin/mqtt/micro/mqtt_service.go new file mode 100644 index 000000000000..1065521bf36a --- /dev/null +++ b/signals/stream/builtin/mqtt/micro/mqtt_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/stream/builtin/mqtt" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("mqtt")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(mqtt.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/stream/builtin/mqtt/mqtt.go b/signals/stream/builtin/mqtt/mqtt.go index 9b2debe8db04..c1914c1ac6e0 100644 --- a/signals/stream/builtin/mqtt/mqtt.go +++ b/signals/stream/builtin/mqtt/mqtt.go @@ -14,17 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package mqtt import ( "fmt" + "log" "strconv" "time" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" MQTTlib "github.com/eclipse/paho.mqtt.golang" - plugin "github.com/hashicorp/go-plugin" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,92 +33,60 @@ const ( EventType = "mqtt.github.io.msg" ) -type mqtt struct { - client MQTTlib.Client - stop chan struct{} - msgCh chan MQTTlib.Message +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type mqtt struct{} - //attribute fields - topic string +// New creates a new mqtt listener +func New() sdk.Listener { + return new(mqtt) } -// New creates a new mqtt signaler -func New() shared.Signaler { - return &mqtt{ - msgCh: make(chan MQTTlib.Message), - stop: make(chan struct{}), - } -} - -func (m *mqtt) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { +func (*mqtt) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { // parse out the attributes - var ok bool - m.topic, ok = signal.Stream.Attributes[topicKey] + topic, ok := signal.Stream.Attributes[topicKey] if !ok { - return nil, shared.ErrMissingRequiredAttribute + return nil, sdk.ErrMissingRequiredAttribute } - opts := MQTTlib.NewClientOptions().AddBroker(signal.Stream.URL).SetClientID(signal.Name) - m.client = MQTTlib.NewClient(opts) - if token := m.client.Connect(); token.Wait() && token.Error() != nil { - return nil, fmt.Errorf("failed to connect to mqtt client. cause: %s", token.Error()) - } + events := make(chan *v1alpha1.Event) - // subscribe to the topic - if token := m.client.Subscribe(m.topic, 0, m.handleMsg); token.Wait() && token.Error() != nil { - return nil, fmt.Errorf("failed to subscribe to mqtt topic. cause: %s", token.Error()) + handler := func(c MQTTlib.Client, msg MQTTlib.Message) { + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventID: strconv.FormatUint(uint64(msg.MessageID()), 10), + EventType: EventType, + CloudEventsVersion: sdk.CloudEventsVersion, + EventTime: metav1.Time{Time: time.Now().UTC()}, + Extensions: make(map[string]string), + }, + Data: msg.Payload(), + } + log.Printf("signal '%s' received msg", signal.Name) + events <- event } - events := make(chan *v1alpha1.Event) - go m.listen(events) - return events, nil -} -func (m *mqtt) Stop() error { - defer close(m.msgCh) - m.stop <- struct{}{} - if token := m.client.Unsubscribe(m.topic); token.Wait() && token.Error() != nil { - return fmt.Errorf("failed to unsubscribe from mqtt topic. cause: %s", token.Error()) + opts := MQTTlib.NewClientOptions().AddBroker(signal.Stream.URL).SetClientID(signal.Name) + client := MQTTlib.NewClient(opts) + if token := client.Connect(); token.Wait() && token.Error() != nil { + return nil, fmt.Errorf("failed to connect to client: %s", token.Error()) } - m.client.Disconnect(0) - return nil -} -// callback message handler passes the message to the mqtt message channel -func (m *mqtt) handleMsg(client MQTTlib.Client, message MQTTlib.Message) { - if !message.Duplicate() { - m.msgCh <- message + if token := client.Subscribe(topic, 0, handler); token.Wait() && token.Error() != nil { + return nil, fmt.Errorf("failed to subscribe to topic: %s", token.Error()) } -} -func (m *mqtt) listen(events chan *v1alpha1.Event) { - defer close(events) - for { - select { - case msg := <-m.msgCh: - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventID: strconv.FormatUint(uint64(msg.MessageID()), 16), - EventType: EventType, - CloudEventsVersion: shared.CloudEventsVersion, - EventTime: metav1.Time{Time: time.Now().UTC()}, - Extensions: make(map[string]string), - }, - Data: msg.Payload(), - } - events <- event - case <-m.stop: - return + // wait for done signal + go func() { + defer close(events) + <-done + if token := client.Unsubscribe(topic); token.Wait() && token.Error() != nil { + log.Printf("failed to unsubscribe from topic: %s", token.Error()) } - } -} + client.Disconnect(0) + log.Printf("shut down signal '%s'", signal.Name) + }() -func main() { - mqtt := New() - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: shared.Handshake, - Plugins: map[string]plugin.Plugin{ - shared.SignalPluginName: shared.NewPlugin(mqtt), - }, - GRPCServer: plugin.DefaultGRPCServer, - }) + log.Printf("signal '%s' listening for mqtt msgs on topic [%s]...", signal.Name, topic) + return events, nil } diff --git a/signals/stream/builtin/mqtt/mqtt_test.go b/signals/stream/builtin/mqtt/mqtt_test.go index 2750b621f713..9bd4f352f275 100644 --- a/signals/stream/builtin/mqtt/mqtt_test.go +++ b/signals/stream/builtin/mqtt/mqtt_test.go @@ -1,3 +1,19 @@ -package main +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mqtt //TODO: implement unit test diff --git a/signals/stream/builtin/nats/micro/Dockerfile b/signals/stream/builtin/nats/micro/Dockerfile new file mode 100644 index 000000000000..45afccd60acf --- /dev/null +++ b/signals/stream/builtin/nats/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/nats-signal / +CMD [ "/nats-signal" ] diff --git a/signals/stream/builtin/nats/micro/nats_service.go b/signals/stream/builtin/nats/micro/nats_service.go new file mode 100644 index 000000000000..e51942fa7704 --- /dev/null +++ b/signals/stream/builtin/nats/micro/nats_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/stream/builtin/nats" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("nats")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(nats.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/stream/builtin/nats/nats.go b/signals/stream/builtin/nats/nats.go index bd2e54c2dce1..db37d78a5bc5 100644 --- a/signals/stream/builtin/nats/nats.go +++ b/signals/stream/builtin/nats/nats.go @@ -14,17 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package nats import ( "fmt" "log" "strconv" + "sync/atomic" "time" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" - plugin "github.com/hashicorp/go-plugin" + "github.com/argoproj/argo-events/sdk" natsio "github.com/nats-io/go-nats" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,83 +34,61 @@ const ( EventType = "com.github.nats-io.pub" ) -// nats is a plugin for a stream signal -type nats struct { - natsConn *natsio.Conn - natsSubscription *natsio.Subscription - msgCh chan *natsio.Msg - stop chan struct{} -} +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type nats struct{} // New creates a new nats signaler -func New() shared.Signaler { - return &nats{ - msgCh: make(chan *natsio.Msg), - stop: make(chan struct{}), - } +func New() sdk.Listener { + return new(nats) } -// Start nats signal -func (n *nats) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { +func (*nats) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { // parse out the attributes subject, ok := signal.Stream.Attributes[subjectKey] if !ok { - return nil, shared.ErrMissingRequiredAttribute + return nil, sdk.ErrMissingRequiredAttribute } - var err error - n.natsConn, err = natsio.Connect(signal.Stream.URL) + + events := make(chan *v1alpha1.Event) + + var id uint64 + handler := func(msg *natsio.Msg) { + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventType: EventType, + CloudEventsVersion: sdk.CloudEventsVersion, + EventID: msg.Subject + "-" + strconv.FormatUint(atomic.AddUint64(&id, 1), 10), + EventTime: metav1.Time{Time: time.Now().UTC()}, + Extensions: make(map[string]string), + }, + Data: msg.Data, + } + log.Printf("signal '%s' received msg", signal.Name) + events <- event + } + + conn, err := natsio.Connect(signal.Stream.URL) if err != nil { - return nil, fmt.Errorf("failed to connect to nats cluster url %s. Cause: %+v", signal.Stream.URL, err.Error()) + return nil, fmt.Errorf("failed to connect to cluster url %s: %+v", signal.Stream.URL, err.Error()) } - n.natsSubscription, err = n.natsConn.ChanSubscribe(subject, n.msgCh) + sub, err := conn.Subscribe(subject, handler) if err != nil { - return nil, fmt.Errorf("failed to subscribe to nats subject %s. Cause: %+v", subject, err.Error()) + return nil, fmt.Errorf("failed to subscribe to subject %s: %+v", subject, err.Error()) } - events := make(chan *v1alpha1.Event) - go n.listen(events) - return events, nil -} - -// Stop nats signal -func (n *nats) Stop() error { - defer n.natsConn.Close() - defer close(n.msgCh) - log.Printf("stopping signal") - n.stop <- struct{}{} - return n.natsSubscription.Unsubscribe() -} -func (n *nats) listen(events chan *v1alpha1.Event) { - defer close(events) - id := 0 - for { - select { - case natsMsg := <-n.msgCh: - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventType: EventType, - CloudEventsVersion: shared.CloudEventsVersion, - EventID: natsMsg.Subject + "-" + strconv.Itoa(id), - EventTime: metav1.Time{Time: time.Now().UTC()}, - Extensions: make(map[string]string), - }, - Data: natsMsg.Data, - } - log.Printf("sending nat event") - events <- event - case <-n.stop: - return - } - } -} + // wait for done signal + go func() { + defer close(events) + <-done + del, _ := sub.Delivered() + drop, _ := sub.Dropped() + queue, _ := sub.QueuedMsgs() + sub.Unsubscribe() + conn.Close() + log.Printf("shut down signal '%s'\nSubscription Stats:\nDelivered:%v\nDropped:%v\nQueued:%v", signal.Name, del, drop, queue) + }() -func main() { - nats := New() - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: shared.Handshake, - Plugins: map[string]plugin.Plugin{ - shared.SignalPluginName: shared.NewPlugin(nats), - }, - GRPCServer: plugin.DefaultGRPCServer, - }) + log.Printf("signal '%s' listening for NATS msgs on subject [%s]...", signal.Name, subject) + return events, nil } diff --git a/signals/stream/builtin/nats/nats_test.go b/signals/stream/builtin/nats/nats_test.go index e9374c95324f..cf83eb4ff1b8 100644 --- a/signals/stream/builtin/nats/nats_test.go +++ b/signals/stream/builtin/nats/nats_test.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package nats import ( "strconv" "testing" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" + "github.com/argoproj/argo-events/sdk" "github.com/nats-io/gnatsd/server" "github.com/nats-io/gnatsd/test" natsio "github.com/nats-io/go-nats" @@ -45,16 +45,18 @@ func TestSignal(t *testing.T) { }, } + done := make(chan struct{}) + // start the signal - expect ErrMissingRequiredAttribute - _, err := nats.Start(&signal) - if err != shared.ErrMissingRequiredAttribute { - t.Errorf("expected: %s\n found: %s", shared.ErrMissingRequiredAttribute, err) + _, err := nats.Listen(&signal, done) + if err != sdk.ErrMissingRequiredAttribute { + t.Errorf("expected: %s\n found: %s", sdk.ErrMissingRequiredAttribute, err) } // add required attributes subject := "test" signal.Stream.Attributes = map[string]string{"subject": subject} - _, err = nats.Start(&signal) + _, err = nats.Listen(&signal, done) if err == nil { t.Errorf("expected: failed to connect to nats cluster\nfound: %s", err) } @@ -62,7 +64,7 @@ func TestSignal(t *testing.T) { // run an embedded gnats server testServer := test.RunServer(&natsEmbeddedServerOpts) defer testServer.Shutdown() - events, err := nats.Start(&signal) + events, err := nats.Listen(&signal, done) if err != nil { t.Error(err) } @@ -79,25 +81,22 @@ func TestSignal(t *testing.T) { } // now lets get the event - nextMsg, ok := <-events + e, ok := <-events if !ok { t.Errorf("failed to receive msg from events channel") } - if nextMsg.Context.EventID != "test-0" { - t.Errorf("event context eventID:\nexpected: %s\nactual: %s", "test-0", nextMsg.Context.EventID) + if e.Context.EventID != "test-1" { + t.Errorf("event context eventID:\nexpected: %s\nactual: %s", "test-1", e.Context.EventID) } - if nextMsg.Context.EventType != EventType { - t.Errorf("event context EventType:\nexpected: %s\nactual: %s", EventType, nextMsg.Context.EventID) + if e.Context.EventType != EventType { + t.Errorf("event context EventType:\nexpected: %s\nactual: %s", EventType, e.Context.EventID) } - if nextMsg.Context.CloudEventsVersion != shared.CloudEventsVersion { - t.Errorf("event context CloudEventsVersion:\nexpected: %s\nactual: %s", shared.CloudEventsVersion, nextMsg.Context.CloudEventsVersion) + if e.Context.CloudEventsVersion != sdk.CloudEventsVersion { + t.Errorf("event context CloudEventsVersion:\nexpected: %s\nactual: %s", sdk.CloudEventsVersion, e.Context.CloudEventsVersion) } // stop the signal - err = nats.Stop() - if err != nil { - t.Errorf("failed to stop signal. cause: %s", err) - } + close(done) // ensure events channel is closed if _, ok := <-events; ok { diff --git a/signals/stream/custom/doc.go b/signals/stream/custom/doc.go index 8ebcc9b5b71d..f5214710f43f 100644 --- a/signals/stream/custom/doc.go +++ b/signals/stream/custom/doc.go @@ -14,5 +14,5 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package custom contains custom user built stream plugins +// Package custom contains custom user built stream microservices package custom diff --git a/signals/webhook/micro/Dockerfile b/signals/webhook/micro/Dockerfile new file mode 100644 index 000000000000..c85affc91d29 --- /dev/null +++ b/signals/webhook/micro/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/webhook-signal / +CMD [ "/webhook-signal" ] diff --git a/signals/webhook/micro/client/Dockerfile b/signals/webhook/micro/client/Dockerfile new file mode 100644 index 000000000000..7cf0ae61dbad --- /dev/null +++ b/signals/webhook/micro/client/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +COPY dist/webhook-signal-client / +CMD [ "/webhook-signal-client" ] diff --git a/signals/webhook/micro/client/main.go b/signals/webhook/micro/client/main.go new file mode 100644 index 000000000000..c3f01aeaad5d --- /dev/null +++ b/signals/webhook/micro/client/main.go @@ -0,0 +1,82 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "io" + "log" + "net/http" + + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/shared" +) + +func main() { + webhook := shared.NewMicroSignalClient().NewSignalService("webhook") + + signal := &v1alpha1.Signal{ + Name: "webhook-1", + Webhook: &v1alpha1.WebhookSignal{ + Endpoint: "/app", + Port: 7070, + Method: "POST", + }, + } + + stream, err := webhook.Listen(context.Background(), signal) + if err != nil { + log.Panicf("failed to listen to webhook: %s", err) + } + + waitc := make(chan struct{}) + go func() { + defer close(waitc) + for { + event, err := stream.Recv() + if err == io.EOF { + return + } + if err != nil { + log.Panicf("error during processing: %s", err) + } + log.Printf("received event: %v", *event) + } + }() + + client := &http.Client{} + req, _ := http.NewRequest("POST", "http://webhook:7070/app", bytes.NewBuffer([]byte(`{"title":"Buy cheese and bread for breakfast."}`))) + _, err = client.Do(req) + if err != nil { + log.Panicf("failed to post http: %s", err) + } + + // now terminate the signal + err = stream.Send(sdk.Terminate) + if err != nil { + log.Printf("failed to send stop signal: %s", err) + } + + err = stream.Close() + if err != nil { + log.Printf("failed to close stream: %s", err) + } + <-waitc + log.Printf("exiting signal client") +} diff --git a/signals/webhook/micro/webhook_service.go b/signals/webhook/micro/webhook_service.go new file mode 100644 index 000000000000..f3215788a99e --- /dev/null +++ b/signals/webhook/micro/webhook_service.go @@ -0,0 +1,35 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/argoproj/argo-events/sdk" + "github.com/argoproj/argo-events/signals/webhook" + "github.com/micro/go-micro" + k8s "github.com/micro/kubernetes/go/micro" +) + +func main() { + svc := k8s.NewService(micro.Name("webhook")) + svc.Init() + + sdk.RegisterSignalServiceHandler(svc.Server(), sdk.NewMicroSignalServer(webhook.New())) + + if err := svc.Run(); err != nil { + panic(err) + } +} diff --git a/signals/webhook/signal.go b/signals/webhook/signal.go deleted file mode 100644 index 4d2c414ca823..000000000000 --- a/signals/webhook/signal.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright 2018 BlackRock, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package webhook - -import ( - "log" - "net/http" - "time" - - "fmt" - "io/ioutil" - "strconv" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" - "github.com/argoproj/argo-events/shared" -) - -const ( - EventType = "Webhook" -) - -type webhook struct { - method string - events chan *v1alpha1.Event - server *http.Server -} - -// New creates a new webhook signaler -func New() shared.Signaler { - return &webhook{} -} - -// Handler for the http rest endpoint -func (w *webhook) handler(writer http.ResponseWriter, request *http.Request) { - log.Printf("received a request from '%s'", request.Host) - if request.Method == w.method { - payload, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("unable to process request payload. Cause: %s", err) - writer.WriteHeader(http.StatusInternalServerError) - return - } - event := &v1alpha1.Event{ - Context: v1alpha1.EventContext{ - EventType: EventType, - EventTypeVersion: request.Proto, - CloudEventsVersion: shared.CloudEventsVersion, - Source: &v1alpha1.URI{ - Scheme: request.RequestURI, - Host: request.Host, - }, - EventTime: metav1.Time{Time: time.Now().UTC()}, - }, - Data: payload, - } - w.events <- event - writer.WriteHeader(http.StatusOK) - } else { - log.Printf("HTTP method of request '%s' does not match expected '%s'", request.Method, w.method) - writer.WriteHeader(http.StatusBadRequest) - } -} - -// Start signal -func (w *webhook) Start(signal *v1alpha1.Signal) (<-chan *v1alpha1.Event, error) { - w.method = signal.Webhook.Method - port := strconv.Itoa(int(signal.Webhook.Port)) - endpoint := signal.Webhook.Endpoint - // Attach handler - http.HandleFunc(endpoint, w.handler) - w.server = &http.Server{Addr: fmt.Sprintf(":%s", port)} - - w.events = make(chan *v1alpha1.Event) - // Start http server - go func() { - err := w.server.ListenAndServe() - if err == http.ErrServerClosed { - log.Print("server successfully shutdown") - } else { - panic(fmt.Errorf("error occurred while server listening. Cause: %v", err)) - } - }() - return w.events, nil -} - -// Stop signal -func (w *webhook) Stop() error { - close(w.events) - err := w.server.Shutdown(nil) - if err != nil { - return fmt.Errorf("unable to shutdown server. Cause: %v", err) - } - return nil -} diff --git a/signals/webhook/webhook.go b/signals/webhook/webhook.go new file mode 100644 index 000000000000..6ea02bdf8ea0 --- /dev/null +++ b/signals/webhook/webhook.go @@ -0,0 +1,117 @@ +/* +Copyright 2018 BlackRock, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "net/http" + "strconv" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" + "github.com/argoproj/argo-events/sdk" +) + +const ( + EventType = "Webhook" +) + +// Note: micro requires stateless operation so the Listen() method should not use the +// receive struct to save or modify state. +type webhook struct{} + +// New creates a new webhook listener +func New() sdk.Listener { + return new(webhook) +} + +func (*webhook) Listen(signal *v1alpha1.Signal, done <-chan struct{}) (<-chan *v1alpha1.Event, error) { + method := signal.Webhook.Method + port := strconv.Itoa(int(signal.Webhook.Port)) + endpoint := signal.Webhook.Endpoint + + events := make(chan *v1alpha1.Event) + + handler := func(w http.ResponseWriter, req *http.Request) { + log.Printf("signal '%s' received a request from '%s'", signal.Name, req.Host) + if req.Method == method { + payload, err := ioutil.ReadAll(req.Body) + if err != nil { + log.Printf("unable to process request payload: %s", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + event := &v1alpha1.Event{ + Context: v1alpha1.EventContext{ + EventType: EventType, + EventTypeVersion: req.Proto, + CloudEventsVersion: sdk.CloudEventsVersion, + Source: &v1alpha1.URI{ + Scheme: req.RequestURI, + Host: req.Host, + }, + EventTime: metav1.Time{Time: time.Now().UTC()}, + }, + Data: payload, + } + events <- event + w.WriteHeader(http.StatusOK) + } else { + log.Printf("http method '%s' does not match expected method '%s' for signal '%s'", req.Method, method, signal.Name) + w.WriteHeader(http.StatusBadRequest) + } + } + + // Attach new mux handler + // TODO: explore in tests how listening on multiple webhook signals fares... + // TODO: use github.com/gorilla/mux + // - pattern matching specifics + // - same endpoint resolution - are both signals resolved? + mux := http.NewServeMux() + mux.HandleFunc(endpoint, handler) + srv := &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: mux, + } + + // Start http server + go func() { + err := srv.ListenAndServe() + if err == http.ErrServerClosed { + log.Printf("successfully shutdown http server for signal '%s'", signal.Name) + } else { + log.Panicf("http server encountered error listening for signal '%s': %v", signal.Name, err) + } + }() + + // wait for stop signal + go func() { + defer close(events) + <-done + err := srv.Shutdown(context.TODO()) + if err != nil { + log.Panicf("failed to gracefully shutdown http server for signal '%s': %s", signal.Name, err) + } + }() + log.Printf("signal '%s' listening for webhooks at [%s]...", signal.Name, signal.Webhook.Endpoint) + return events, nil +} diff --git a/signals/webhook/signal_test.go b/signals/webhook/webhook_test.go similarity index 78% rename from signals/webhook/signal_test.go rename to signals/webhook/webhook_test.go index 407d4223bdc8..8a141e8e99ab 100644 --- a/signals/webhook/signal_test.go +++ b/signals/webhook/webhook_test.go @@ -22,20 +22,20 @@ import ( "strings" "testing" - "github.com/argoproj/argo-events/common" "github.com/argoproj/argo-events/pkg/apis/sensor/v1alpha1" ) var ( - client = &http.Client{} - payload = "{name: x}" + testingTargetPort = 45677 + client = &http.Client{} + payload = "{name: x}" ) func handleEvent(t *testing.T, testEventChan <-chan *v1alpha1.Event) { event := <-testEventChan - if event.Context.Source.Host != fmt.Sprintf("localhost:%d", common.WebhookServicePort) { - t.Errorf("event Context SourceHost:\nexpected: %s\nactual: %s", fmt.Sprintf("localhost:%d", common.WebhookServicePort), event.Context.Source.Host) + if event.Context.Source.Host != fmt.Sprintf("localhost:%d", testingTargetPort) { + t.Errorf("event Context SourceHost:\nexpected: %s\nactual: %s", fmt.Sprintf("localhost:%d", testingTargetPort), event.Context.Source.Host) } if string(event.Data) != payload { t.Errorf("event Data:\nexpected: %s\nactual: %s", payload, string(event.Data)) @@ -46,16 +46,19 @@ func makeAPIRequest(t *testing.T, httpMethod string, endpoint string) { web := New() signal := v1alpha1.Signal{ Webhook: &v1alpha1.WebhookSignal{ - Port: common.WebhookServiceTargetPort, + Port: int32(testingTargetPort), Endpoint: endpoint, Method: httpMethod, }, } - events, err := web.Start(&signal) + done := make(chan struct{}) + // stop listening and ensure the events channel is closed on exit + defer func() { done <- struct{}{} }() + events, err := web.Listen(&signal, done) go handleEvent(t, events) - request, err := http.NewRequest(httpMethod, fmt.Sprintf("http://localhost:%d%s", common.WebhookServicePort, endpoint), strings.NewReader(payload)) + request, err := http.NewRequest(httpMethod, fmt.Sprintf("http://localhost:%d%s", testingTargetPort, endpoint), strings.NewReader(payload)) if err != nil { t.Fatalf("unable to create http request. cause: %s", err) } @@ -67,10 +70,6 @@ func makeAPIRequest(t *testing.T, httpMethod string, endpoint string) { if resp.Status != "200 OK" { t.Errorf("response status expected: '200 OK' actual: '%s'", resp.Status) } - err = web.Stop() - if err != nil { - t.Error(err) - } } func testPostRequest(t *testing.T) {