diff --git a/Dockerfile.asserts b/Dockerfile.asserts index 7b9ab9478..5c5f7528d 100644 --- a/Dockerfile.asserts +++ b/Dockerfile.asserts @@ -16,9 +16,11 @@ COPY . /go/src/github.com/jaegertracing/jaeger-operator/ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./uiconfig -a ./tests/assert-jobs/uiconfig/main.go RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./reporter -a ./tests/assert-jobs/reporter/main.go RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./query -a ./tests/assert-jobs/query/main.go +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./index -a ./tests/assert-jobs/index/main.go FROM scratch WORKDIR / COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/uiconfig . COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/reporter . COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/query . +COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/index . diff --git a/Makefile b/Makefile index f5d3a73fd..a2720ff33 100644 --- a/Makefile +++ b/Makefile @@ -489,6 +489,7 @@ prepare-e2e-kuttl-tests: build docker build-assert-job @cp deploy/crds/jaegertracing.io_jaegers_crd.yaml tests/_build/crds/jaegertracing.io_jaegers_crd.yaml docker pull jaegertracing/vertx-create-span:operator-e2e-tests + docker pull docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6 # end-to-tests .PHONY: kuttl-e2e @@ -503,6 +504,7 @@ start-kind: kind load docker-image local/jaeger-operator:e2e kind load docker-image local/asserts:e2e kind load docker-image jaegertracing/vertx-create-span:operator-e2e-tests + kind load docker-image docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6 .PHONY: build-assert-job build-assert-job: diff --git a/kuttl-test.yaml b/kuttl-test.yaml index 771efc03d..17d8a2bab 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -7,7 +7,7 @@ kindContainers: - local/asserts:e2e - jaegertracing/vertx-create-span:operator-e2e-tests commands: - - command: kubectl create namespace jaeger-operator-system + - script: kubectl create namespace jaeger-operator-system 2>&1 | grep -v "already exists" || true - command: kubectl apply -f ./tests/_build/manifests/01-jaeger-operator.yaml -n jaeger-operator-system - command: kubectl wait --timeout=5m --for=condition=available deployment jaeger-operator -n jaeger-operator-system - command: kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.0.1/deploy/static/provider/kind/deploy.yaml diff --git a/tests/assert-jobs/index/main.go b/tests/assert-jobs/index/main.go new file mode 100644 index 000000000..d94283c13 --- /dev/null +++ b/tests/assert-jobs/index/main.go @@ -0,0 +1,185 @@ +package main + +import ( + "flag" + "fmt" + "os" + "regexp" + "strings" + + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils" + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/elasticsearch" + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/logger" +) + +var log logrus.Logger + +const ( + flagEsNamespace = "es-namespace" + flagEsPort = "es-port" + flagEsURL = "es-url" + flagPattern = "pattern" + flagName = "name" + flagExist = "assert-exist" + flagAssertCountIndices = "assert-count-indices" + flagAssertCountDocs = "assert-count-docs" + flagJaegerService = "jaeger-service" + flagVerbose = "verbose" +) + +func filterIndices(indices *[]elasticsearch.EsIndex, pattern string) ([]elasticsearch.EsIndex, error) { + regexPattern, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("There was a problem with the pattern: %s", err)) + } + + var matchingIndices []elasticsearch.EsIndex + + for _, index := range *indices { + if regexPattern.MatchString(index.Index) { + log.Debugf("Index '%s' matched", index.Index) + matchingIndices = append(matchingIndices, index) + } + } + + log.Debugf("%d indices matches the pattern '%s'", len(matchingIndices), pattern) + + return matchingIndices, nil + +} + +// Init the CMD and return error if something didn't go properly +func initCmd() error { + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() + + viper.SetDefault(flagEsNamespace, "default") + flag.String(flagEsNamespace, "", "ElasticSearch namespace to use") + + viper.SetDefault(flagEsPort, "9200") + flag.String(flagEsPort, "", "ElasticSearch port") + + viper.SetDefault(flagEsURL, "http://localhost") + flag.String(flagEsURL, "", "ElasticSearch URL") + + viper.SetDefault(flagVerbose, false) + flag.Bool(flagVerbose, false, "Enable verbosity") + + viper.SetDefault(flagExist, false) + flag.Bool(flagExist, false, "Assert the pattern matches something") + + viper.SetDefault(flagPattern, "") + flag.String(flagPattern, "", "Pattern to use to match indices") + + viper.SetDefault(flagName, "") + flag.String(flagName, "", "Name of the desired index (needed for aliases)") + + viper.SetDefault(flagJaegerService, "") + flag.String(flagJaegerService, "", "Name of the Jaeger Service") + + viper.SetDefault(flagAssertCountIndices, "-1") + flag.Int(flagAssertCountIndices, -1, "Assert the number of matched indices") + + viper.SetDefault(flagAssertCountDocs, "-1") + flag.Int(flagAssertCountDocs, -1, "Assert the number of documents") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + err := viper.BindPFlags(pflag.CommandLine) + if err != nil { + return err + } + params := utils.NewParameters() + params.Parse() + + if viper.GetString(flagName) != "" && viper.GetString(flagPattern) != "" { + return fmt.Errorf(fmt.Sprintf("--%s and --%s provided. Provide just one", flagName, flagPattern)) + } else if viper.GetString(flagName) == "" && viper.GetString(flagPattern) == "" { + return fmt.Errorf(fmt.Sprintf("--%s nor --%s provided. Provide one at least", flagName, flagPattern)) + } else if viper.GetBool(flagAssertCountDocs) && viper.GetString(flagJaegerService) == "" { + return fmt.Errorf(fmt.Sprintf("--%s provided. Provide --%s", flagAssertCountDocs, flagJaegerService)) + } + + return nil +} + +func main() { + err := initCmd() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + log = *logger.InitLog(viper.GetBool(flagVerbose)) + + connection := elasticsearch.EsConnection{ + Port: viper.GetString(flagEsPort), + Namespace: viper.GetString(flagEsNamespace), + URL: viper.GetString(flagEsURL), + } + connection.PrettyString(log.Debug) + + err = elasticsearch.CheckESConnection(connection) + if err != nil { + log.Fatalln(err) + log.Exit(1) + } + + var matchingIndices []elasticsearch.EsIndex + if viper.GetString(flagPattern) != "" { + indices, err := elasticsearch.GetEsIndices(connection) + if err != nil { + log.Fatalln("There was an error while getting the ES indices: ", err) + log.Exit(1) + } + + matchingIndices, err = filterIndices(&indices, viper.GetString(flagPattern)) + if err != nil { + log.Fatalln(err) + os.Exit(1) + } + } else { + index := elasticsearch.GetEsIndex(connection, viper.GetString(flagName)) + matchingIndices = []elasticsearch.EsIndex{index} + } + + if viper.GetBool(flagExist) { + if len(matchingIndices) == 0 { + log.Fatalln("No indices match the pattern") + os.Exit(1) + } + } + + if viper.GetString(flagName) != "" && viper.GetString(flagAssertCountIndices) != "" { + log.Warnln("Ignoring parameter", flagAssertCountIndices, "because we are checking the info for one index") + } else if viper.GetString(flagPattern) != "" && viper.GetInt(flagAssertCountIndices) > -1 { + if len(matchingIndices) != viper.GetInt(flagAssertCountIndices) { + log.Fatalln(len(matchingIndices), "indices found.", viper.GetInt(flagAssertCountIndices), "expected") + os.Exit(1) + } + } + + if viper.GetInt(flagAssertCountDocs) > -1 { + foundDocs := 0 + jaegerServiceName := viper.GetString(flagJaegerService) + for _, index := range matchingIndices { + spans, err := index.GetServiceIndexSpans(jaegerServiceName) + if err != nil { + log.Errorln("Something failed while getting the index spans:", err) + } + foundDocs += len(spans) + } + log.Debug(foundDocs, " in ", len(matchingIndices), " indices") + + if foundDocs != viper.GetInt(flagAssertCountDocs) { + log.Fatalln(foundDocs, "docs found.", viper.GetInt(flagAssertCountDocs), "expected") + os.Exit(1) + } + } + +} diff --git a/tests/assert-jobs/reporter/main.go b/tests/assert-jobs/reporter/main.go index 4e9f61067..3dce082c9 100644 --- a/tests/assert-jobs/reporter/main.go +++ b/tests/assert-jobs/reporter/main.go @@ -1,22 +1,49 @@ package main import ( + "encoding/json" + "flag" "fmt" "io" + "os" + "strings" + "time" + + "net/http" "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/uber/jaeger-client-go" "github.com/uber/jaeger-client-go/config" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils" + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/logger" ) +var log logrus.Logger + const ( - envOperationName = "OPERATION_NAME" + flagJaegerServiceName = "jaeger-service-name" + flagJaegerOperationName = "operation-name" + flagDays = "days" + flagVerbose = "verbose" + flagServices = "services" + envVarJaegerEndpoint = "jaeger_endpoint" + enVarJaegerQuery = "jaeger_query" ) -func initTracer() (opentracing.Tracer, io.Closer) { +// Init the Jaeger tracer. Returns the tracer and the closer. +// serviceName: name of the service to report spans +func initTracer(serviceName string) (opentracing.Tracer, io.Closer) { cfg, err := config.FromEnv() - cfg.Reporter.LogSpans = true + if serviceName != "" { + cfg.ServiceName = serviceName + } + + cfg.Reporter.LogSpans = viper.GetBool(flagVerbose) cfg.Sampler = &config.SamplerConfig{ Type: "const", Param: 1, @@ -32,13 +59,194 @@ func initTracer() (opentracing.Tracer, io.Closer) { return tracer, closer } -func main() { - viper.AutomaticEnv() - operationName := viper.GetString(envOperationName) +// Assert the span was reported properly +// spanDate: start date of the reported span +// serviceName: name of the span service +func assertSpanWasCreated(spanDate time.Time, serviceName string) bool { + startQueryTime := spanDate.Add(time.Minute * -1) + finishQueryTime := spanDate.Add(time.Minute) + + jaegerCollectorEndpoint := viper.GetString(enVarJaegerQuery) + + url := fmt.Sprintf( + "%s?lookback=custom&service=%s&limit=200&start=%d&end=%d", + jaegerCollectorEndpoint, + serviceName, + startQueryTime.UnixNano()/1000, + finishQueryTime.UnixNano()/1000, + ) + params := utils.TestParams{Timeout: time.Second * 20, RetryInterval: time.Second * 5} + + err := utils.TestGetHTTP(url, ¶ms, func(response *http.Response, body []byte) (done bool, err error) { + resp := struct { + Data []struct { + Spans []struct { + StartTime int64 `json:"startTime"` + } `json:"spans"` + } `json:"data"` + }{} + + err = json.Unmarshal(body, &resp) + if err != nil { + return false, err + } + + for _, reportedTrace := range resp.Data { + for _, reportedSpan := range reportedTrace.Spans { + if reportedSpan.StartTime == spanDate.UnixNano()/1000 { + return true, nil + } + } + } + + return false, nil + }) + if err == nil { + logrus.Info("Span asserted properly") + return true + } + logrus.Error("There was a problem reporting the information: ", err) + return false +} + +// Generate spans for the given service +// serviceName: name of the service to generate spans +// operationName: name of the operation for the spans +// days: number of days to generate spans +func generateSpansHistoryService(serviceName, operationName string, days int) { + if days < 1 { + log.Warn("days parameter for generateSpansHistory is less than 1. Doing nothing") + return + } + + log.Info("Generating spans for the last ", days, " days for service ", serviceName) - tracer, closer := initTracer() + currentDate := time.Now() + tracer, closer := initTracer(serviceName) defer closer.Close() - span := tracer.StartSpan(operationName) - span.Finish() + generatedSpans := 0 + + for day := 0; day < days; day++ { + spanDate := currentDate.AddDate(0, 0, -1*day) + spanOperationName := fmt.Sprintf("%s-%d", operationName, day) + + generateSpan(spanDate, spanOperationName, &tracer) + + jaegerQueryEndpoint := viper.GetString(enVarJaegerQuery) + if jaegerQueryEndpoint != "" { + for !assertSpanWasCreated(spanDate, serviceName) { + generateSpan(spanDate, spanOperationName, &tracer) + } + generatedSpans++ + logrus.Info(generatedSpans, " spans reported properly") + } + } +} + +func generateSpan(spanDate time.Time, operationName string, tracer *opentracing.Tracer) { + stringDate := spanDate.Format("2 Jan 2006 15:04:05") + span := (*tracer).StartSpan(operationName, opentracing.StartTime(spanDate)) + span.SetTag("string-date", stringDate) + span.FinishWithOptions(opentracing.FinishOptions{FinishTime: spanDate.Add(time.Hour * 2)}) +} + +// Generate spans for multiple services +// serviceName: prefix name name of the services to generate spans +// operationName: name of the operation for the spans +// days: number of days to generate spans +// services: number of services to generate +func generateSpansHistory(serviceName, operationName string, days, services int) { + for service := 0; service < services; service++ { + reportedServiceName := serviceName + if services > 1 { + reportedServiceName = fmt.Sprintf("%s-%d", serviceName, service) + } + generateSpansHistoryService(reportedServiceName, operationName, days) + } +} + +// Block the execution until the Jaeger REST API is available (or timeout) +func waitUntilRestAPIAvailable(jaegerEndpoint string) error { + transport := &http.Transport{} + client := http.Client{Transport: transport} + + err := wait.Poll(time.Second*5, time.Minute*5, func() (done bool, err error) { + req, err := http.NewRequest(http.MethodGet, jaegerEndpoint, nil) + if err != nil { + return false, err + } + + resp, err := client.Do(req) + + // The GET HTTP verb is not supported by the Jaeger Collector REST API enpoint. An error 405 + // means the REST API is there + if resp != nil && resp.StatusCode == 405 { + return true, nil + } + if err != nil { + return false, err + } + + log.Warningln(jaegerEndpoint, "is not available. Is", envVarJaegerEndpoint, "environment variable properly set?") + return false, nil + }) + return err +} + +// Init the CMD and return error if something didn't go properly +func initCmd() error { + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() + + viper.SetDefault(flagJaegerServiceName, "jaeger-service") + flag.String(flagJaegerServiceName, "", "Jaeger service name") + + viper.SetDefault(flagDays, 1) + flag.Int(flagDays, 1, "History days") + + viper.SetDefault(flagServices, 1) + flag.Int(flagServices, 1, "Number of services") + + viper.SetDefault(flagVerbose, false) + flag.Bool(flagVerbose, false, "Enable verbosity") + + viper.SetDefault(flagJaegerOperationName, "jaeger-operation") + flag.String(flagJaegerOperationName, "", "Jaeger operation name") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + err := viper.BindPFlags(pflag.CommandLine) + return err +} + +func main() { + err := initCmd() + if err != nil { + log.Error(err) + os.Exit(1) + } + + log = *logger.InitLog(viper.GetBool(flagVerbose)) + + jaegerEndpoint := viper.GetString(envVarJaegerEndpoint) + if jaegerEndpoint == "" { + log.Errorln("Please, specify a Jaeger Collector endpoint") + os.Exit(1) + } + + // Sometimes, Kubernetes reports the Jaeger service is there but there is an interval where the service is up but the + // REST API is not operative yet + err = waitUntilRestAPIAvailable(jaegerEndpoint) + if err != nil { + log.Fatalln(err) + os.Exit(1) + } + + generateSpansHistory(viper.GetString(flagJaegerServiceName), viper.GetString(flagJaegerOperationName), viper.GetInt(flagDays), viper.GetInt(flagServices)) + + // After reporting the spans, we wait some seconds to ensure the spans were reported and + // stored in the final storage (ES or other) + time.Sleep(time.Second * 10) } diff --git a/tests/assert-jobs/utils/elasticsearch/elasticsearch.go b/tests/assert-jobs/utils/elasticsearch/elasticsearch.go new file mode 100644 index 000000000..f31f26dae --- /dev/null +++ b/tests/assert-jobs/utils/elasticsearch/elasticsearch.go @@ -0,0 +1,198 @@ +package elasticsearch + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +// EsConnection details to the ElasticSearch database +type EsConnection struct { + Port string + URL string + Namespace string +} + +// EsSpan maps spans data from ES REST API response +// API endpoint: //_search?format=json +type EsSpan struct { + ID string + ServiceName string + OperationName string +} + +// PrettyString prints the ES connection details in a nice way +// callback: function to use to print the information +func (connection *EsConnection) PrettyString(callback func(args ...interface{})) { + callback("ElasticSearch connection details:") + callback(fmt.Sprintf("\t * Port: %s", connection.Port)) + callback(fmt.Sprintf("\t * URL: %s", connection.URL)) + callback(fmt.Sprintf("\t * Namespace: %s", connection.Namespace)) +} + +// EsIndex maps indices data from ES REST API response +// API endpoint: /_cat/indices?format=json +type EsIndex struct { + Index string `json:"index"` + es EsConnection +} + +// GetServiceIndexSpans gets the spans associated to one index and one service +// serviceName: name of the Jaeger service +func (index *EsIndex) GetServiceIndexSpans(serviceName string) ([]EsSpan, error) { + spans, err := index.GetIndexSpans() + if err != nil { + return []EsSpan{}, err + } + + filteredSpans := []EsSpan{} + + for _, span := range spans { + if span.ServiceName == serviceName { + filteredSpans = append(filteredSpans, span) + } + } + return filteredSpans, nil +} + +// GetIndexSpans gets the spans associated to one index +func (index *EsIndex) GetIndexSpans() ([]EsSpan, error) { + searchResponse := struct { + Hits struct { + Hits []struct { + ID string `json:"_id"` + Source struct { + Process struct { + ServiceName string `json:"serviceName"` + } `json:"process"` + OperationName string `json:"operationName"` + } `json:"_source"` + } `json:"hits"` + } `json:"hits"` + }{} + + body := struct { + Query struct { + QueryString struct { + Query string `json:"query"` + } `json:"query_string"` + } `json:"query"` + Size int `json:"size"` + From int `json:"from"` + }{} + body.From = 0 + body.Size = 10000 + body.Query.QueryString.Query = "*" + + bodyReq, _ := json.Marshal(body) + + bodyBytes, err := executeEsRequest(index.es, http.MethodPost, fmt.Sprintf("/%s/_search?format=json", index.Index), bodyReq) + if err != nil { + return []EsSpan{}, fmt.Errorf(fmt.Sprintf("Something failed while quering the ES REST API: %s", err)) + } + + err = json.Unmarshal(bodyBytes, &searchResponse) + if err != nil { + return []EsSpan{}, fmt.Errorf(fmt.Sprintf("Something failed while unmarshalling API response: %s", err)) + } + + spans := []EsSpan{} + for _, jsonSpan := range searchResponse.Hits.Hits { + span := EsSpan{ID: jsonSpan.ID, ServiceName: jsonSpan.Source.Process.ServiceName, OperationName: jsonSpan.Source.OperationName} + spans = append(spans, span) + } + return spans, nil +} + +// CheckESConnection checs if the connection to ElasticSearch can be done +// es: connection details to the ElasticSearch database +func CheckESConnection(es EsConnection) error { + _, err := executeEsRequest(es, http.MethodGet, "/", nil) + if err != nil { + return fmt.Errorf(fmt.Sprint("There was a problem while connecting to the ES instance: ", err)) + } + return nil +} + +// FormatEsIndices formats the ES Indices information to print it or something +// esIndices: indices to format +// prefix: a prefix for each ES index +// postfix: a postfix for each ES index +func FormatEsIndices(esIndices []EsIndex, prefix, postfix string) string { + output := "" + for _, index := range esIndices { + output = fmt.Sprintf("%s%s%s%s", output, prefix, index.Index, postfix) + } + return output +} + +// GetEsIndex gets information from an specific ElasticSearch index +// es: connection details to the ElasticSearch database +// indexName: name of the index +func GetEsIndex(es EsConnection, indexName string) EsIndex { + return EsIndex{indexName, es} +} + +// GetEsIndices returns the indices from the ElasticSearch node +// es: connection details to the ElasticSearch database +func GetEsIndices(es EsConnection) ([]EsIndex, error) { + bodyBytes, err := executeEsRequest(es, http.MethodGet, "/_cat/indices?format=json", nil) + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("Something failed while quering the ES REST API: %s", err)) + } + + // Convert JSON data to struct format + esIndices := make([]EsIndex, 0) + err = json.Unmarshal(bodyBytes, &esIndices) + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("Something failed while unmarshalling API response: %s", err)) + } + + for i := range esIndices { + esIndices[i].es = es + } + + return esIndices, nil +} + +// Executes a REST API ElasticSearch request +// es: connection details to the ElasticSearch database +// httpMethod: HTTP method to use for the query +// api: API endpoint to query +func executeEsRequest(es EsConnection, httpMethod, api string, body []byte) ([]byte, error) { + esURL := fmt.Sprintf("%s:%s%s", es.URL, es.Port, api) + + // Create the HTTP client to interact with the API + transport := &http.Transport{} + client := http.Client{Transport: transport} + + var bodyReq []byte + var err error + + if body == nil { + bodyReq = nil + } else { + bodyReq = body + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("Something failed while marshalling the body: %s", err)) + } + } + + req, err := http.NewRequest(httpMethod, esURL, bytes.NewBuffer(bodyReq)) + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("The HTTP client creation failed: %s", err)) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf(fmt.Sprintf("The HTTP request failed: %s", err)) + } + + defer resp.Body.Close() + + return ioutil.ReadAll(resp.Body) +} diff --git a/tests/assert-jobs/utils/logger/logger.go b/tests/assert-jobs/utils/logger/logger.go new file mode 100644 index 000000000..3d1c87585 --- /dev/null +++ b/tests/assert-jobs/utils/logger/logger.go @@ -0,0 +1,21 @@ +package logger + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +// InitLog creates a logger and enables the verbose level if specified +// enableVerbose: verbose logging level when true +func InitLog(enableVerbose bool) *logrus.Logger { + log := logrus.New() + log.Out = os.Stdout + if enableVerbose { + log.SetLevel(logrus.DebugLevel) + } else { + log.SetLevel(logrus.InfoLevel) + } + + return log +} diff --git a/tests/assert-jobs/utils/testhttp.go b/tests/assert-jobs/utils/testhttp.go index 2b05261bb..187235312 100644 --- a/tests/assert-jobs/utils/testhttp.go +++ b/tests/assert-jobs/utils/testhttp.go @@ -21,7 +21,7 @@ func TestGetHTTP(url string, params *TestParams, testFn func(response *http.Resp client := http.Client{Timeout: 30 * time.Second} - logrus.Info("Polling to %s", url) + logrus.Info("Polling to ", url) return wait.Poll(params.RetryInterval, params.Timeout, func() (done bool, err error) { logrus.Info("Doing request..") diff --git a/tests/cmd-utils/wait-cronjob/main.go b/tests/cmd-utils/wait-cronjob/main.go new file mode 100644 index 000000000..97c85cd81 --- /dev/null +++ b/tests/cmd-utils/wait-cronjob/main.go @@ -0,0 +1,204 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "strings" + + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + + "github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/logger" +) + +const ( + flagcronJobName = "cronjob" + flagVerbose = "verbose" + flagNamespace = "namespace" + flagKubeconfig = "kubeconfig" + flagRetryInterval = "retry-interval" + flagTimeout = "timeout" +) + +var log logrus.Logger + +// Check if a CronJob exists in the given Kubernetes context +// clientset: Kubernetes API client +func checkCronJobExists(clientset *kubernetes.Clientset) error { + cronjobName := viper.GetString(flagcronJobName) + namespace := viper.GetString(flagNamespace) + retryInterval := viper.GetDuration(flagRetryInterval) + timeout := viper.GetDuration(flagTimeout) + + log.Debugln("Checking if the", cronjobName, "CronJob exists") + + err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cronjobs, err := clientset.BatchV1beta1().CronJobs(namespace).List(ctxWithTimeout, metav1.ListOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + log.Debug("No cronjobs were found") + return false, nil + } + return false, err + } + + for _, cronjob := range cronjobs.Items { + if cronjob.Name == cronjobName { + return true, nil + } + } + + return false, fmt.Errorf(fmt.Sprintf("The %s CronJob was not found", cronjobName)) + }) + + log.Debugln("Cronjob", cronjobName, "found successfully") + return err +} + +// Wait for the next job from the given CronJob +// clientset: Kubernetes API client +func waitForNextJob(clientset *kubernetes.Clientset) error { + cronjobName := viper.GetString(flagcronJobName) + namespace := viper.GetString(flagNamespace) + retryInterval := viper.GetDuration(flagRetryInterval) + timeout := viper.GetDuration(flagTimeout) + start := time.Now() + + log.Debugln("Waiting for the next scheduled job from", cronjobName, "cronjob") + err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { + ctxWithTimeout, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + jobList, err := clientset.BatchV1().Jobs(namespace).List(ctxWithTimeout, metav1.ListOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + log.Debugf("No jobs provided by the Kubernetes API") + return false, nil + } + return false, err + } + + for _, j := range jobList.Items { + for _, r := range j.OwnerReferences { + // Check if this job is related to the desired CronJob + if cronjobName != r.Name { + continue + } + + // Check if the job has finished properly + if j.Status.Succeeded == 0 || j.Status.Failed != 0 || j.Status.Active != 0 { + continue + } + + timeSinceCompleted := j.Status.CompletionTime.Sub(start) + + // The job finished before this program started. We are interested in a newer execution + if timeSinceCompleted <= 0 { + continue + } + + return true, nil + + } + } + + log.Debugln("Waiting for next job from", cronjobName, "to succeed") + return false, nil + }) + log.Debugln("Job of owner", cronjobName, "succeeded after", cronjobName, time.Since(start)) + return err +} + +/// Get the Kubernetes client from the environment configuration +func getKubernetesClient() *kubernetes.Clientset { + // Use the current context + config, err := clientcmd.BuildConfigFromFlags("", viper.GetString(flagKubeconfig)) + if err != nil { + panic(err.Error()) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + return clientset +} + +// Init the CMD and return error if something didn't go properly +func initCmd() error { + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() + + viper.SetDefault(flagcronJobName, "") + flag.String(flagcronJobName, "", "Cronjob name") + + viper.SetDefault(flagRetryInterval, time.Second*5) + flag.Duration(flagRetryInterval, time.Second*5, "Retry interval") + + viper.SetDefault(flagTimeout, time.Minute) + flag.Duration(flagTimeout, time.Minute, "Timeout") + + viper.SetDefault(flagNamespace, "default") + flag.String(flagNamespace, "", "Kubernetes namespace") + + viper.SetDefault(flagVerbose, false) + flag.Bool(flagVerbose, false, "Enable verbosity") + + viper.SetDefault(flagKubeconfig, filepath.Join(homedir.HomeDir(), ".kube", "config")) + flag.String("kubeconfig", "", "absolute path to the kubeconfig file") + + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + err := viper.BindPFlags(pflag.CommandLine) + if err != nil { + return err + } + + if viper.GetString(flagcronJobName) == "" { + return fmt.Errorf(fmt.Sprintf("Parameter --%s must be set", flagcronJobName)) + } + + if _, err := os.Stat(viper.GetString(flagKubeconfig)); err != nil { + return fmt.Errorf(fmt.Sprintf("%s file does not exists. Point to the correct one using the --%s flag", viper.GetString(flagKubeconfig), flagKubeconfig)) + } + + return nil +} + +func main() { + err := initCmd() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + log = *logger.InitLog(viper.GetBool(flagVerbose)) + clientset := getKubernetesClient() + + err = checkCronJobExists(clientset) + if err != nil { + log.Errorln(err) + os.Exit(1) + } + + err = waitForNextJob(clientset) + if err != nil { + log.Fatal(err) + } +} diff --git a/tests/e2e/allinone-ingress/01-report-span.yaml b/tests/e2e/allinone-ingress/01-report-span.yaml index 34925a05a..3ac53067a 100644 --- a/tests/e2e/allinone-ingress/01-report-span.yaml +++ b/tests/e2e/allinone-ingress/01-report-span.yaml @@ -16,5 +16,7 @@ spec: value: "my-little-op" - name: JAEGER_ENDPOINT value: "http://my-jaeger-collector-headless:14268/api/traces" + - name: JAEGER_QUERY + value: "http://my-jaeger-query:16686/api/traces" restartPolicy: OnFailure backoffLimit: 10 \ No newline at end of file diff --git a/tests/e2e/es-rollover/00-assert.yaml b/tests/e2e/es-rollover/00-assert.yaml new file mode 100644 index 000000000..d19c71ee1 --- /dev/null +++ b/tests/e2e/es-rollover/00-assert.yaml @@ -0,0 +1,27 @@ +# Assert the Jaeger collector is up and running +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-jaeger-collector +spec: + replicas: 1 +status: + readyReplicas: 1 +--- +# Assert the Jaeger query is up and running +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-jaeger-query +spec: + replicas: 1 +status: + readyReplicas: 1 +--- +# Assert the ElasticSearch is up and running +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch +status: + readyReplicas: 1 \ No newline at end of file diff --git a/tests/e2e/es-rollover/00-install.yaml b/tests/e2e/es-rollover/00-install.yaml new file mode 100644 index 000000000..5340a834d --- /dev/null +++ b/tests/e2e/es-rollover/00-install.yaml @@ -0,0 +1,84 @@ +# Create create the StatefulSet for the ElasticSearch database +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch + labels: + app: jaeger + jaeger-infra: elasticsearch-statefulset +spec: + selector: + matchLabels: + app: jaeger-elasticsearch + serviceName: elasticsearch + replicas: 1 + template: + metadata: + labels: + app: jaeger-elasticsearch + jaeger-infra: elasticsearch-replica + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6 + imagePullPolicy: Always + env: + - name: "http.host" + value: "0.0.0.0" + - name: "transport.host" + value: "127.0.0.1" + - name: "cluster.routing.allocation.disk.threshold_enabled" + value: "false" + ports: + - name: elasticsearch + containerPort: 9200 + - name: transport + containerPort: 9300 + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + readinessProbe: + httpGet: + path: / + port: 9200 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 4 + securityContext: + capabilities: + add: ["SYS_CHROOT"] + volumes: + - name: data + emptyDir: {} +--- +# Deploy the ElasticSearch service +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + app: jaeger + jaeger-infra: elasticsearch-service +spec: + clusterIP: None + selector: + app: jaeger-elasticsearch + ports: + - port: 9200 + name: elasticsearch + - port: 9300 + name: transport +--- +# Deploy the Jaeger instance +apiVersion: jaegertracing.io/v1 +kind: Jaeger +metadata: + name: my-jaeger +spec: + strategy: production + storage: + type: elasticsearch + options: + es: + server-urls: http://elasticsearch:9200 + use-aliases: false diff --git a/tests/e2e/es-rollover/01-assert.yaml b/tests/e2e/es-rollover/01-assert.yaml new file mode 100644 index 000000000..d36c6533b --- /dev/null +++ b/tests/e2e/es-rollover/01-assert.yaml @@ -0,0 +1,7 @@ +# Assert the spans are reported +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-1 +status: + succeeded: 1 \ No newline at end of file diff --git a/tests/e2e/es-rollover/01-generate-spans.yaml b/tests/e2e/es-rollover/01-generate-spans.yaml new file mode 100644 index 000000000..03cd5114d --- /dev/null +++ b/tests/e2e/es-rollover/01-generate-spans.yaml @@ -0,0 +1,27 @@ +# Generate some traces in the Jaeger instance +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-1 +spec: + template: + spec: + containers: + - name: report-span-1 + image: local/asserts:e2e + command: ["./reporter", + "--days", "2", + "--services", "1", + "--verbose" + ] + env: + - name: JAEGER_SERVICE_NAME + value: "my-test-service" + - name: OPERATION_NAME + value: "my-little-op" + - name: JAEGER_ENDPOINT + value: "http://my-jaeger-collector-headless:14268/api/traces" + - name: JAEGER_QUERY + value: "http://my-jaeger-query:16686/api/traces" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/es-rollover/02-assert.yaml b/tests/e2e/es-rollover/02-assert.yaml new file mode 100644 index 000000000..998d7a661 --- /dev/null +++ b/tests/e2e/es-rollover/02-assert.yaml @@ -0,0 +1,14 @@ +# Check the two assert jobs were run succesfully +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-before-0 +status: + succeeded: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-before-1 +status: + succeeded: 1 \ No newline at end of file diff --git a/tests/e2e/es-rollover/02-check-es-status.yaml b/tests/e2e/es-rollover/02-check-es-status.yaml new file mode 100644 index 000000000..adbe3082c --- /dev/null +++ b/tests/e2e/es-rollover/02-check-es-status.yaml @@ -0,0 +1,45 @@ +# Check the indices are generated +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-before-0 +spec: + template: + spec: + containers: + - name: assert-indexes-before-0 + image: local/asserts:e2e + command: ["./index", + "--pattern", "jaeger-span-\\d{4}-\\d{2}-\\d{2}", + "--assert-count-indices", "2", + "--assert-count-docs", "2", + "--jaeger-service", "my-test-service", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 +--- +# Check there are not indices related to the ES rollover feature +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-before-1 +spec: + template: + spec: + containers: + - name: assert-indexes-before-1 + image: local/asserts:e2e + command: ["./index", + "--pattern", "jaeger-span-\\d{6}", + "--assert-count-indices", "0", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/es-rollover/03-assert.yaml b/tests/e2e/es-rollover/03-assert.yaml new file mode 100644 index 000000000..1fa4d7ec4 --- /dev/null +++ b/tests/e2e/es-rollover/03-assert.yaml @@ -0,0 +1,27 @@ +# Assert the Jaeger collector is up and running +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-jaeger-collector +spec: + replicas: 1 +status: + readyReplicas: 1 +--- +# Assert the Jaeger collector is up and running +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-jaeger-query +spec: + replicas: 1 +status: + readyReplicas: 1 +--- +# Assert the ES Rollover mapping is run +apiVersion: batch/v1 +kind: Job +metadata: + name: my-jaeger-es-rollover-create-mapping +status: + succeeded: 1 diff --git a/tests/e2e/es-rollover/03-install.yaml b/tests/e2e/es-rollover/03-install.yaml new file mode 100644 index 000000000..a1948bc1a --- /dev/null +++ b/tests/e2e/es-rollover/03-install.yaml @@ -0,0 +1,18 @@ +# Enable the ES Rollover feature +apiVersion: jaegertracing.io/v1 +kind: Jaeger +metadata: + name: my-jaeger +spec: + strategy: production + storage: + type: elasticsearch + options: + es: + server-urls: http://elasticsearch:9200 + use-aliases: true + esRollover: + enabled: true + conditions: "{\"max_docs\": \"20\"}" + readTTL: 72h + schedule: "* * * * *" \ No newline at end of file diff --git a/tests/e2e/es-rollover/04-assert.yaml b/tests/e2e/es-rollover/04-assert.yaml new file mode 100644 index 000000000..42a865154 --- /dev/null +++ b/tests/e2e/es-rollover/04-assert.yaml @@ -0,0 +1,7 @@ +# Assert the spans are reported +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-2 +status: + succeeded: 1 \ No newline at end of file diff --git a/tests/e2e/es-rollover/04-generate-spans.yaml b/tests/e2e/es-rollover/04-generate-spans.yaml new file mode 100644 index 000000000..f500cfe7c --- /dev/null +++ b/tests/e2e/es-rollover/04-generate-spans.yaml @@ -0,0 +1,27 @@ +# Generate some traces in the Jaeger instance +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-2 +spec: + template: + spec: + containers: + - name: report-span-2 + image: local/asserts:e2e + command: ["./reporter", + "--days", "2", + "--services", "1", + "--verbose" + ] + env: + - name: JAEGER_SERVICE_NAME + value: "my-test-service" + - name: OPERATION_NAME + value: "my-little-op" + - name: JAEGER_ENDPOINT + value: "http://my-jaeger-collector-headless:14268/api/traces" + - name: JAEGER_QUERY + value: "http://my-jaeger-query:16686/api/traces" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/es-rollover/05-assert.yaml b/tests/e2e/es-rollover/05-assert.yaml new file mode 100644 index 000000000..191adec71 --- /dev/null +++ b/tests/e2e/es-rollover/05-assert.yaml @@ -0,0 +1,21 @@ +# Check the assert jobs were run succesfully +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-0 +status: + succeeded: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-1 +status: + succeeded: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-2 +status: + succeeded: 1 \ No newline at end of file diff --git a/tests/e2e/es-rollover/05-check-indices.yaml b/tests/e2e/es-rollover/05-check-indices.yaml new file mode 100644 index 000000000..f9fabc604 --- /dev/null +++ b/tests/e2e/es-rollover/05-check-indices.yaml @@ -0,0 +1,67 @@ +# Assert the previous indices are still there +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-0 +spec: + template: + spec: + containers: + - name: assert-indexes-after-0 + image: local/asserts:e2e + command: ["./index", + "--pattern", "jaeger-span-\\d{4}-\\d{2}-\\d{2}", + "--assert-count-indices", "2", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 +--- +# Assert there is a new index using the format `jaeger-span-000001` +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-1 +spec: + template: + spec: + containers: + - name: assert-indexes-after-1 + image: local/asserts:e2e + command: ["./index", + "--pattern", "jaeger-span-\\d{6}", + "--assert-count-docs", "2", + "--jaeger-service", "my-test-service", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 +--- +# Assert the alias was created +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-2 +spec: + template: + spec: + containers: + - name: assert-indexes-after-2 + image: local/asserts:e2e + command: ["./index", + "--name", "jaeger-span-read", + "--assert-count-docs", "2", + "--jaeger-service", "my-test-service", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/es-rollover/06-assert.yaml b/tests/e2e/es-rollover/06-assert.yaml new file mode 100644 index 000000000..a22b5ab16 --- /dev/null +++ b/tests/e2e/es-rollover/06-assert.yaml @@ -0,0 +1,7 @@ +# Assert the spans are reported +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-3 +status: + succeeded: 1 diff --git a/tests/e2e/es-rollover/06-generate-spans.yaml b/tests/e2e/es-rollover/06-generate-spans.yaml new file mode 100644 index 000000000..f9c2760fb --- /dev/null +++ b/tests/e2e/es-rollover/06-generate-spans.yaml @@ -0,0 +1,27 @@ +# Generate some traces in the Jaeger instance +apiVersion: batch/v1 +kind: Job +metadata: + name: report-span-3 +spec: + template: + spec: + containers: + - name: report-span-3 + image: local/asserts:e2e + command: ["./reporter", + "--days", "2", + "--services", "1", + "--verbose" + ] + env: + - name: JAEGER_SERVICE_NAME + value: "my-test-service" + - name: OPERATION_NAME + value: "my-little-op" + - name: JAEGER_ENDPOINT + value: "http://my-jaeger-collector-headless:14268/api/traces" + - name: JAEGER_QUERY + value: "http://my-jaeger-query:16686/api/traces" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/es-rollover/07-wait-rollover.yaml b/tests/e2e/es-rollover/07-wait-rollover.yaml new file mode 100644 index 000000000..51a252b4c --- /dev/null +++ b/tests/e2e/es-rollover/07-wait-rollover.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: "go run ../../cmd-utils/wait-cronjob/main.go --cronjob my-jaeger-es-rollover --namespace $NAMESPACE" \ No newline at end of file diff --git a/tests/e2e/es-rollover/08-assert.yaml b/tests/e2e/es-rollover/08-assert.yaml new file mode 100644 index 000000000..ebae7de02 --- /dev/null +++ b/tests/e2e/es-rollover/08-assert.yaml @@ -0,0 +1,14 @@ +# Check the assert jobs were run succesfully +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-3 +status: + succeeded: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-4 +status: + succeeded: 1 diff --git a/tests/e2e/es-rollover/08-check-indices.yaml b/tests/e2e/es-rollover/08-check-indices.yaml new file mode 100644 index 000000000..bece7fb27 --- /dev/null +++ b/tests/e2e/es-rollover/08-check-indices.yaml @@ -0,0 +1,44 @@ +--- +# Assert the index rollover +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-3 +spec: + template: + spec: + containers: + - name: assert-indexes-after-3 + image: local/asserts:e2e + command: ["./index", + "--name", "jaeger-span-000002", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 +--- +# Assert the information can be accessed after the rollover +apiVersion: batch/v1 +kind: Job +metadata: + name: assert-indexes-after-4 +spec: + template: + spec: + containers: + - name: assert-indexes-after-4 + image: local/asserts:e2e + command: ["./index", + "--name", "jaeger-span-read", + "--assert-count-docs", "4", + "--jaeger-service", "my-test-service", + "--verbose" + ] + env: + - name: ES_URL + value: "http://elasticsearch" + restartPolicy: OnFailure + backoffLimit: 10 diff --git a/tests/e2e/outside-cluster/01-check-collector.yaml b/tests/e2e/outside-cluster/01-check-collector.yaml index a42970f95..24d8d6ad1 100644 --- a/tests/e2e/outside-cluster/01-check-collector.yaml +++ b/tests/e2e/outside-cluster/01-check-collector.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: "JAEGER_SERVICE_NAME=my-test-service OPERATION_NAME=my-little-op JAEGER_ENDPOINT=http://localhost/collector/api/traces go run ../../assert-jobs/reporter/main.go" + - script: "JAEGER_SERVICE_NAME=my-test-service OPERATION_NAME=my-little-op JAEGER_ENDPOINT=http://localhost/collector/api/traces go run ../../assert-jobs/reporter/main.go --verbose" - script: "sleep 5" # This sleep is needed to ensure the traces from the previous step are available - script: "go run ../../assert-jobs/query/main.go --service-name my-test-service --query-host localhost/query"