Add E2E testing for ES Rollover feature

Dockerfile.asserts

@@ -12,9 +12,11 @@ RUN go mod download
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./uiconfig -a ./tests/assert-jobs/uiconfig/main.go
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./reporter -a ./tests/assert-jobs/reporter/main.go
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./query -a ./tests/assert-jobs/query/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o ./index -a ./tests/assert-jobs/index/main.go
 
 FROM scratch
 WORKDIR /
 COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/uiconfig .
 COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/reporter .
 COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/query .
+COPY --from=builder /go/src/github.com/jaegertracing/jaeger-operator/index .

Makefile

@@ -489,6 +489,7 @@ prepare-e2e-kuttl-tests: build docker build-assert-job
 
 	@cp deploy/crds/jaegertracing.io_jaegers_crd.yaml tests/_build/crds/jaegertracing.io_jaegers_crd.yaml
 	docker pull jaegertracing/vertx-create-span:operator-e2e-tests
+	docker pull docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6
 
 # end-to-tests
 .PHONY: kuttl-e2e
@@ -503,6 +504,7 @@ start-kind:
 	kind load docker-image local/jaeger-operator:e2e
 	kind load docker-image local/asserts:e2e
 	kind load docker-image jaegertracing/vertx-create-span:operator-e2e-tests
+	kind load docker-image docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.6
 
 .PHONY: build-assert-job
 build-assert-job:

kuttl-test.yaml

@@ -7,7 +7,7 @@ kindContainers:
   - local/asserts:e2e
   - jaegertracing/vertx-create-span:operator-e2e-tests
 commands:
-  - command: kubectl create namespace jaeger-operator-system
+  - script: kubectl create namespace jaeger-operator-system 2>&1 | grep -v "already exists" || true
   - command: kubectl apply -f ./tests/_build/manifests/01-jaeger-operator.yaml -n jaeger-operator-system
   - command: kubectl wait --timeout=5m --for=condition=available deployment jaeger-operator -n jaeger-operator-system
   - command: kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.0.1/deploy/static/provider/kind/deploy.yaml

tests/assert-jobs/index/main.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	"github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils"
+	"github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/elasticsearch"
+	"github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/logger"
+)
+
+var log logrus.Logger
+
+const (
+	flagEsNamespace        = "es-namespace"
+	flagEsPort             = "es-port"
+	flagEsURL              = "es-url"
+	flagPattern            = "pattern"
+	flagName               = "name"
+	flagIsAlias            = "is-alias"
+	flagExist              = "assert-exist"
+	flagAssertCountIndices = "assert-count-indices"
+	flagAssertCountDocs    = "assert-count-docs"
+	flagVerbose            = "verbose"
+)
+
+func filterIndices(indices *[]elasticsearch.EsIndex, pattern string) ([]elasticsearch.EsIndex, error) {
+	regexPattern, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, fmt.Errorf(fmt.Sprintf("There was a problem with the pattern: %s", err))
+	}
+
+	var matchingIndices []elasticsearch.EsIndex
+
+	for _, index := range *indices {
+		if regexPattern.MatchString(index.Index) {
+			log.Debugf("Index '%s' matched", index.Index)
+			matchingIndices = append(matchingIndices, index)
+		}
+	}
+
+	log.Debugf("%d indices matches the pattern '%s'", len(matchingIndices), pattern)
+
+	return matchingIndices, nil
+
+}
+
+// Init the CMD and return error if something didn't go properly
+func initCmd() error {
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	viper.AutomaticEnv()
+
+	viper.SetDefault(flagEsNamespace, "default")
+	flag.String(flagEsNamespace, "", "ElasticSearch namespace to use")
+
+	viper.SetDefault(flagEsPort, "9200")
+	flag.String(flagEsPort, "", "ElasticSearch port")
+
+	viper.SetDefault(flagEsURL, "http://localhost")
+	flag.String(flagEsURL, "", "ElasticSearch URL")
+
+	viper.SetDefault(flagVerbose, false)
+	flag.Bool(flagVerbose, false, "Enable verbosity")
+
+	viper.SetDefault(flagExist, false)
+	flag.Bool(flagExist, false, "Assert the pattern matches something")
+
+	viper.SetDefault(flagPattern, "")
+	flag.String(flagPattern, "", "Pattern to use to match indices")
+
+	viper.SetDefault(flagName, "")
+	flag.String(flagName, "", "Name of the desired index (needed for aliases)")
+
+	viper.SetDefault(flagAssertCountIndices, "-1")
+	flag.Int(flagAssertCountIndices, -1, "Assert the number of matched indices")
+
+	viper.SetDefault(flagAssertCountDocs, "-1")
+	flag.Int(flagAssertCountDocs, -1, "Assert the number of documents")
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+
+	err := viper.BindPFlags(pflag.CommandLine)
+	if err != nil {
+		return err
+	}
+	params := utils.NewParameters()
+	params.Parse()
+
+	if viper.GetString(flagName) != "" && viper.GetString(flagPattern) != "" {
+		return fmt.Errorf(fmt.Sprintf("--%s and --%s provided. Provide just one", flagName, flagPattern))
+	} else if viper.GetString(flagName) == "" && viper.GetString(flagPattern) == "" {
+		return fmt.Errorf(fmt.Sprintf("--%s nor --%s provided. Provide one at least", flagName, flagPattern))
+	}
+
+	return nil
+}
+
+func main() {
+	err := initCmd()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log = *logger.InitLog(viper.GetBool(flagVerbose))
+
+	connection := elasticsearch.EsConnection{
+		Port:      viper.GetString(flagEsPort),
+		Namespace: viper.GetString(flagEsNamespace),
+		URL:       viper.GetString(flagEsURL),
+	}
+	connection.PrettyString(log.Debug)
+
+	err = elasticsearch.CheckESConnection(connection)
+	if err != nil {
+		log.Fatalln(err)
+		log.Exit(1)
+	}
+
+	var matchingIndices []elasticsearch.EsIndex
+	if viper.GetString(flagPattern) != "" {
+		indices, err := elasticsearch.GetEsIndices(connection)
+		if err != nil {
+			log.Fatalln("There was an error while getting the ES indices: ", err)
+			log.Exit(1)
+		}
+
+		matchingIndices, err = filterIndices(&indices, viper.GetString(flagPattern))
+	} else {
+		var index elasticsearch.EsIndex
+		index, err = elasticsearch.GetEsIndex(connection, viper.GetString(flagName))
+		matchingIndices = []elasticsearch.EsIndex{index}
+	}
+
+	if err != nil {
+		log.Fatalln(err)
+		os.Exit(1)
+	}
+
+	if viper.GetBool(flagExist) {
+		if len(matchingIndices) == 0 {
+			log.Fatalln("No indices match the pattern")
+			os.Exit(1)
+		}
+	}
+
+	if viper.GetString(flagName) != "" && viper.GetString(flagAssertCountIndices) != "" {
+		log.Warnln("Ignoring parameter", flagAssertCountIndices, "because we are checking the info for one index")
+	} else if viper.GetString(flagPattern) != "" && viper.GetInt(flagAssertCountIndices) > -1 {
+		if len(matchingIndices) != viper.GetInt(flagAssertCountIndices) {
+			log.Fatalln(len(matchingIndices), "indices found.", viper.GetInt(flagAssertCountIndices), "expected")
+			os.Exit(1)
+		}
+	}
+
+	if viper.GetInt(flagAssertCountDocs) > -1 {
+		foundDocs := 0
+		for _, index := range matchingIndices {
+			foundDocs += index.RealDocCount
+		}
+		log.Debug(foundDocs, " in ", len(matchingIndices), " indices")
+
+		if foundDocs != viper.GetInt(flagAssertCountDocs) {
+			log.Fatalln(foundDocs, "docs found.", viper.GetInt(flagAssertCountDocs), "expected")
+			os.Exit(1)
+		}
+	}
+
+}

tests/assert-jobs/reporter/main.go

@@ -1,22 +1,46 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"io"
+	"os"
+	"strings"
+	"time"
+
+	"net/http"
 
 	"github.com/opentracing/opentracing-go"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"github.com/uber/jaeger-client-go"
 	"github.com/uber/jaeger-client-go/config"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/jaegertracing/jaeger-operator/tests/assert-jobs/utils/logger"
 )
 
+var log logrus.Logger
+
 const (
-	envOperationName = "OPERATION_NAME"
+	flagJaegerServiceName   = "jaeger-service-name"
+	flagJaegerOperationName = "operation-name"
+	flagDays                = "days"
+	flagVerbose             = "verbose"
+	flagServices            = "services"
+	envVarJaegerEndpoint    = "jaeger_endpoint"
 )
 
-func initTracer() (opentracing.Tracer, io.Closer) {
+// Init the Jaeger tracer. Returns the tracer and the closer.
+// serviceName: name of the service to report spans
+func initTracer(serviceName string) (opentracing.Tracer, io.Closer) {
 	cfg, err := config.FromEnv()
-	cfg.Reporter.LogSpans = true
+	if serviceName != "" {
+		cfg.ServiceName = serviceName
+	}
+
+	cfg.Reporter.LogSpans = viper.GetBool(flagVerbose)
 	cfg.Sampler = &config.SamplerConfig{
 		Type:  "const",
 		Param: 1,
@@ -32,13 +56,128 @@ func initTracer() (opentracing.Tracer, io.Closer) {
 	return tracer, closer
 }
 
-func main() {
-	viper.AutomaticEnv()
-	operationName := viper.GetString(envOperationName)
+// Generate spans for the given service
+// serviceName: name of the service to generate spans
+// operationName: name of the operation for the spans
+// days: number of days to generate spans
+func generateSpansHistoryService(serviceName, operationName string, days int) {
+	if days < 1 {
+		log.Warn("days parameter for generateSpansHistory is less than 1. Doing nothing")
+		return
+	}
+
+	log.Info("Generating spans for the last ", days, " days for service ", serviceName)
 
-	tracer, closer := initTracer()
+	currentDate := time.Now()
+	tracer, closer := initTracer(serviceName)
 	defer closer.Close()
 
-	span := tracer.StartSpan(operationName)
-	span.Finish()
+	for day := 0; day < days; day++ {
+		spanDate := currentDate.AddDate(0, 0, -1*day)
+		stringDate := spanDate.Format("2006-01-02")
+		span := tracer.StartSpan(fmt.Sprintf("%s-%d", operationName, day), opentracing.StartTime(spanDate))
+		span.SetTag("string-date", stringDate)
+		span.FinishWithOptions(opentracing.FinishOptions{FinishTime: spanDate.Add(time.Hour * 2)})
+	}
+}
+
+// Generate spans for multiple services
+// serviceName: prefix name name of the services to generate spans
+// operationName: name of the operation for the spans
+// days: number of days to generate spans
+// services: number of services to generate
+func generateSpansHistory(serviceName, operationName string, days, services int) {
+	for service := 0; service < services; service++ {
+		reportedServiceName := serviceName
+		if services > 1 {
+			reportedServiceName = fmt.Sprintf("%s-%d", serviceName, service)
+		}
+		generateSpansHistoryService(reportedServiceName, operationName, days)
+	}
+}
+
+// Block the execution until the Jaeger REST API is available (or timeout)
+func waitUntilRestAPIAvailable(jaegerEndpoint string) error {
+	transport := &http.Transport{}
+	client := http.Client{Transport: transport}
+
+	err := wait.Poll(time.Second*5, time.Minute*5, func() (done bool, err error) {
+		req, err := http.NewRequest(http.MethodGet, jaegerEndpoint, nil)
+		if err != nil {
+			return false, err
+		}
+
+		resp, err := client.Do(req)
+
+		// The GET HTTP verb is not supported by the Jaeger Collector REST API enpoint. An error 405
+		// means the REST API is there
+		if resp != nil && resp.StatusCode == 405 {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+
+		log.Warningln(jaegerEndpoint, "is not available. Is", envVarJaegerEndpoint, "environment variable properly set?")
+		return false, nil
+	})
+	return err
+}
+
+// Init the CMD and return error if something didn't go properly
+func initCmd() error {
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	viper.AutomaticEnv()
+
+	viper.SetDefault(flagJaegerServiceName, "jaeger-service")
+	flag.String(flagJaegerServiceName, "", "Jaeger service name")
+
+	viper.SetDefault(flagDays, 1)
+	flag.Int(flagDays, 1, "History days")
+
+	viper.SetDefault(flagServices, 1)
+	flag.Int(flagServices, 1, "Number of services")
+
+	viper.SetDefault(flagVerbose, false)
+	flag.Bool(flagVerbose, false, "Enable verbosity")
+
+	viper.SetDefault(flagJaegerOperationName, "jaeger-operation")
+	flag.String(flagJaegerOperationName, "", "Jaeger operation name")
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+
+	err := viper.BindPFlags(pflag.CommandLine)
+	return err
+}
+
+func main() {
+	err := initCmd()
+	if err != nil {
+		log.Error(err)
+		os.Exit(1)
+	}
+
+	log = *logger.InitLog(viper.GetBool(flagVerbose))
+
+	jaegerEndpoint := viper.GetString(envVarJaegerEndpoint)
+
+	if jaegerEndpoint == "" {
+		log.Fatalln("Please, specify a Jaeger endpoint")
+		os.Exit(1)
+	}
+
+	// Sometimes, Kubernetes reports the Jaeger service is there but there is an interval where the service is up but the
+	// REST API is not operative yet
+	err = waitUntilRestAPIAvailable(jaegerEndpoint)
+	if err != nil {
+		log.Fatalln(err)
+		os.Exit(1)
+	}
+
+	generateSpansHistory(viper.GetString(flagJaegerServiceName), viper.GetString(flagJaegerOperationName), viper.GetInt(flagDays), viper.GetInt(flagServices))
+
+	// After reporting the spans, we wait some seconds to ensure the spans were reported and
+	// stored in the final storage (ES or other)
+	time.Sleep(time.Second * 10)
 }