diff --git a/go.mod b/go.mod index ba99f13a9..749b881b4 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/googleapis/gnostic v0.3.1 github.com/mitchellh/go-homedir v1.1.0 github.com/openshift/api v0.0.0-20200701144905-de5b010b2b38 + github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e github.com/operator-framework/operator-sdk v0.18.2 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.5.0 diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index 67bce3222..01301c896 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -79,7 +79,7 @@ func (suite *AutoscaleTestSuite) TestAutoScaleCollector() { jaegerInstanceName := "simple-prod" var jaegerInstance *v1.Jaeger if skipESExternal { - jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, int32(1)) + jaegerInstance = GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, namespace, int32(1)) createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) } else { waitForElasticSearch() diff --git a/test/e2e/elasticsearch_index_test.go b/test/e2e/elasticsearch_index_test.go new file mode 100644 index 000000000..7af8cafbd --- /dev/null +++ b/test/e2e/elasticsearch_index_test.go @@ -0,0 +1,301 @@ +// +build elasticsearch + +package e2e + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "testing" + "time" + + "github.com/opentracing/opentracing-go" + framework "github.com/operator-framework/operator-sdk/pkg/test" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + v1 "github.com/jaegertracing/jaeger-operator/pkg/apis/jaegertracing/v1" +) + +type ElasticSearchIndexTestSuite struct { + suite.Suite + esIndexCleanerHistoryDays int // generate spans and services history + esNamespace string // default storage namespace location +} + +const ElasticSearchIndexDateLayout = "2006-01-02" // date layout in elasticsearch indices, example: + +// esIndexData struct is used to keep index data in simple format +// will be useful for the validations +type esIndexData struct { + IndexName string // original index name + Type string // index type. span or service? + Prefix string // prefix of the index + Date time.Time // index day/date +} + +func TestElasticSearchIndexSuite(t *testing.T) { + indexSuite := new(ElasticSearchIndexTestSuite) + // update default values + indexSuite.esIndexCleanerHistoryDays = 45 + // storage namespace + if skipESExternal { + indexSuite.esNamespace = namespace + } else { + indexSuite.esNamespace = storageNamespace + } + suite.Run(t, indexSuite) +} + +func (suite *ElasticSearchIndexTestSuite) SetupSuite() { + t = suite.T() + var err error + ctx, err = prepare(t) + if err != nil { + if ctx != nil { + ctx.Cleanup() + } + require.FailNow(t, "Failed in prepare") + } + fw = framework.Global + namespace = ctx.GetID() + require.NotNil(t, namespace, "GetID failed") + + addToFrameworkSchemeForSmokeTests(t) +} + +func (suite *ElasticSearchIndexTestSuite) TearDownSuite() { + handleSuiteTearDown() +} + +func (suite *ElasticSearchIndexTestSuite) SetupTest() { + t = suite.T() + // delete indices from external elasticsearch node + if !skipESExternal { + DeleteEsIndices(suite.esNamespace) + } +} + +func (suite *ElasticSearchIndexTestSuite) AfterTest(suiteName, testName string) { + handleTestFailure() +} + +// executes es index cleaner with default index prefix +func (suite *ElasticSearchIndexTestSuite) TestEsIndexCleaner() { + suite.runIndexCleaner("", []int{45, 30, 7, 1, 0}) +} + +// executes es index cleaner tests with custom index prefix +func (suite *ElasticSearchIndexTestSuite) TestEsIndexCleanerWithIndexPrefix() { + suite.runIndexCleaner("my-custom_prefix", []int{3, 1, 0}) +} + +// executes index cleaner tests +func (suite *ElasticSearchIndexTestSuite) runIndexCleaner(esIndexPrefix string, daysRange []int) { + logrus.Infof("index cleaner test started. daysRange=%v, prefix=%s", daysRange, esIndexPrefix) + jaegerInstanceName := "test-es-index-cleaner" + if esIndexPrefix != "" { + jaegerInstanceName = "test-es-index-cleaner-with-prefix" + } + // get jaeger CR to create jaeger services + jaegerInstance := GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, namespace, 1) + + // If there is an external es deployment use it instead of creating a self provision one + if !skipESExternal { + if isOpenShift(t) { + esServerUrls = "http://elasticsearch." + storageNamespace + ".svc.cluster.local:9200" + } + jaegerInstance.Spec.Storage = v1.JaegerStorageSpec{ + Type: v1.JaegerESStorage, + Options: v1.NewOptions(map[string]interface{}{ + "es.server-urls": esServerUrls, + }), + } + } + + // update jaeger CR with index cleaner specifications + // initially disable es index cleaner job + esIndexCleanerEnabled := false + esIndexCleanerNumberOfDays := suite.esIndexCleanerHistoryDays + jaegerInstance.Spec.Storage.EsIndexCleaner.Enabled = &esIndexCleanerEnabled + jaegerInstance.Spec.Storage.EsIndexCleaner.NumberOfDays = &esIndexCleanerNumberOfDays + jaegerInstance.Spec.Storage.EsIndexCleaner.Schedule = "*/1 * * * *" + // update es.index-prefix, if supplied + if esIndexPrefix != "" { + if jaegerInstance.Spec.Storage.Options.Map() == nil { + jaegerInstance.Spec.Storage.Options = v1.NewOptions(map[string]interface{}{}) + } + jaegerInstance.Spec.Storage.Options.Map()["es.index-prefix"] = esIndexPrefix + } + + logrus.Infof("Creating jaeger services for es index cleaner test: %s", jaegerInstanceName) + createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) + defer undeployJaegerInstance(jaegerInstance) + + suite.generateSpansHistoy(namespace, jaegerInstanceName) + + suite.triggerIndexCleanerAndVerifyIndices(jaegerInstance, esIndexPrefix, daysRange) + +} + +func (suite *ElasticSearchIndexTestSuite) generateSpansHistoy(namespace, jaegerInstanceName string) { + logrus.Info("Enabling collector port forward") + fwdPortColl, closeChanColl := CreatePortForward(namespace, jaegerInstanceName+"-collector", "collector", []string{fmt.Sprintf(":%d", jaegerCollectorPort)}, fw.KubeConfig) + defer fwdPortColl.Close() + defer close(closeChanColl) + // get localhost collector port + colPorts, err := fwdPortColl.GetPorts() + require.NoError(t, err) + localPortColl := colPorts[0].Local + logrus.Infof("Generating spans and services for the last %d days", suite.esIndexCleanerHistoryDays) + currentDate := time.Now() + for day := 0; day < suite.esIndexCleanerHistoryDays; day++ { + spanDate := currentDate.AddDate(0, 0, -1*day) + stringDate := spanDate.Format(ElasticSearchIndexDateLayout) + // get tracing client + serviceName := fmt.Sprintf("%s_%s", jaegerInstanceName, stringDate) + tracer, closer, err := getTracingClientWithCollectorEndpoint(serviceName, fmt.Sprintf("http://localhost:%d/api/traces", localPortColl)) + require.NoError(t, err) + // generate span + tracer.StartSpan("span-index-cleaner", opentracing.StartTime(spanDate)). + SetTag("jaeger-instance", jaegerInstanceName). + SetTag("test-case", t.Name()). + SetTag("string-date", stringDate). + FinishWithOptions(opentracing.FinishOptions{FinishTime: spanDate.Add(time.Second)}) + closer.Close() + } +} + +// function to get indices +// returns in order: serviceIndices, spansIndices +func (suite *ElasticSearchIndexTestSuite) getIndices() ([]esIndexData, []esIndexData) { + // get indices from es node + esIndices, err := GetEsIndices(suite.esNamespace) + require.NoError(t, err) + logrus.Infof("Number of indices found on rest api response:%d", len(esIndices)) + + servicesIndices := make([]esIndexData, 0) + spansIndices := make([]esIndexData, 0) + + // parse date, prefix, type from index + re := regexp.MustCompile(`\d{4}-\d{2}-\d{2}`) + for _, esIndex := range esIndices { + indexName := esIndex.Index + dateString := re.FindString(indexName) + if dateString == "" { // assume this index not belongs to jaeger + continue + } + + indexName = strings.Replace(indexName, dateString, "", 1) + + indexDate, err := time.Parse(ElasticSearchIndexDateLayout, dateString) + require.NoError(t, err) + + esData := esIndexData{ + IndexName: esIndex.Index, + Date: indexDate, + } + + // reference + // https://github.com/jaegertracing/jaeger/blob/6c2be456ca41cdb98ac4b81cb8d9a9a9044463cd/plugin/storage/es/spanstore/reader.go#L40 + if strings.Contains(indexName, "jaeger-span-") { + esData.Type = "span" + prefix := strings.Replace(indexName, "jaeger-span-", "", 1) + if len(prefix) > 0 { + esData.Prefix = prefix[:len(prefix)-1] // removes "-" at end + } + spansIndices = append(spansIndices, esData) + } else if strings.Contains(indexName, "jaeger-service-") { + esData.Type = "service" + prefix := strings.Replace(indexName, "jaeger-service-", "", 1) + if len(prefix) > 0 { + esData.Prefix = prefix[:len(prefix)-1] // removes "-" at end + } + servicesIndices = append(servicesIndices, esData) + } + } + return servicesIndices, spansIndices +} + +// function to validate indices +func (suite *ElasticSearchIndexTestSuite) assertIndex(esIndexPrefix string, indices []esIndexData, verifyDateAfter time.Time, count int) { + // sort and print indices + sort.Slice(indices, func(i, j int) bool { + return indices[i].Date.After(indices[j].Date) + }) + indicesSlice := make([]string, 0) + for _, ind := range indices { + indicesSlice = append(indicesSlice, ind.IndexName) + } + logrus.Infof("indices should be after %v, indices list: %v", verifyDateAfter, indicesSlice) + require.Equal(t, count, len(indices), "number of available indices not matching, %v", indices) + for _, index := range indices { + require.True(t, index.Date.After(verifyDateAfter), "this index must removed by index cleaner job: %v", index) + require.Equal(t, esIndexPrefix, index.Prefix, "index prefix not matching") + } +} + +// trigger the index cleaner job for the given day range and verifies the indices availability +func (suite *ElasticSearchIndexTestSuite) triggerIndexCleanerAndVerifyIndices(jaegerInstance *v1.Jaeger, esIndexPrefix string, daysRange []int) { + for _, verifyDays := range daysRange { + logrus.Infof("Scheduling index cleaner job for %d days", verifyDays) + // update and trigger index cleaner job + suite.turnOnEsIndexCleaner(jaegerInstance, verifyDays) + + // get services and spans indices + servicesIndices, spanIndices := suite.getIndices() + // set valid index start date + indexDateReference := time.Now().AddDate(0, 0, -1*verifyDays) + // set hours, minutes, seconds, etc.. to 0 + indexDateReference = time.Date(indexDateReference.Year(), indexDateReference.Month(), indexDateReference.Day(), 0, 0, 0, 0, indexDateReference.Location()) + logrus.Infof("indices status on es node={numberOfDays:%d, services:%d, spans:%d}", verifyDays, len(servicesIndices), len(spanIndices)) + suite.assertIndex(esIndexPrefix, servicesIndices, indexDateReference, verifyDays) + suite.assertIndex(esIndexPrefix, spanIndices, indexDateReference, verifyDays) + } +} + +func (suite *ElasticSearchIndexTestSuite) turnOnEsIndexCleaner(jaegerInstance *v1.Jaeger, indexCleanerNumOfDays int) { + // enable index cleaner job + suite.updateJaegerCR(jaegerInstance, indexCleanerNumOfDays, true) + + // wait till the cron job created + err := WaitForCronJob(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout+1*time.Minute) + require.NoError(t, err, "Error waiting for Cron Job") + + // wait for the first successful cron job pod + err = WaitForJobOfAnOwner(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout) + require.NoError(t, err, "Error waiting for Cron Job") + + // disable index cleaner job + suite.updateJaegerCR(jaegerInstance, indexCleanerNumOfDays, false) + + // seeing inconsistency in minikube when immediately disabling and enabling index cleaner job + // as a result index clear job is not triggering, so sleep for a while + time.Sleep(time.Second * 5) + + // delete completed job pods + err = fw.KubeClient.CoreV1().Pods(namespace).DeleteCollection( + context.Background(), + metav1.DeleteOptions{}, + metav1.ListOptions{LabelSelector: "app.kubernetes.io/component=cronjob-es-index-cleaner"}) + require.NoError(t, err, "Error on delete index cleaner pods") +} + +// function to update jaeger CR +func (suite *ElasticSearchIndexTestSuite) updateJaegerCR(jaegerInstance *v1.Jaeger, indexCleanerNumOfDays int, indexCleanerEnabled bool) { + // get existing values + key := types.NamespacedName{Name: jaegerInstance.Name, Namespace: jaegerInstance.GetNamespace()} + err := fw.Client.Get(context.Background(), key, jaegerInstance) + require.NoError(t, err) + + // update values + jaegerInstance.Spec.Storage.EsIndexCleaner.Enabled = &indexCleanerEnabled + jaegerInstance.Spec.Storage.EsIndexCleaner.NumberOfDays = &indexCleanerNumOfDays + err = fw.Client.Update(context.Background(), jaegerInstance) + require.NoError(t, err) +} diff --git a/test/e2e/elasticsearch_test.go b/test/e2e/elasticsearch_test.go index 5e7a3a8c7..4756c6f28 100644 --- a/test/e2e/elasticsearch_test.go +++ b/test/e2e/elasticsearch_test.go @@ -4,39 +4,21 @@ package e2e import ( "context" - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "net/http" - "strconv" - "strings" "testing" - "time" framework "github.com/operator-framework/operator-sdk/pkg/test" "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/portforward" v1 "github.com/jaegertracing/jaeger-operator/pkg/apis/jaegertracing/v1" ) -type ElasticSearchTestSuite struct { +type ElasticSearchBasicTestSuite struct { suite.Suite } -var esIndexCleanerEnabled = false -var esUrl string -var esNamespace = storageNamespace - -func (suite *ElasticSearchTestSuite) SetupSuite() { +func (suite *ElasticSearchBasicTestSuite) SetupSuite() { t = suite.T() var err error ctx, err = prepare(t) @@ -57,23 +39,23 @@ func (suite *ElasticSearchTestSuite) SetupSuite() { } } -func (suite *ElasticSearchTestSuite) TearDownSuite() { +func (suite *ElasticSearchBasicTestSuite) TearDownSuite() { handleSuiteTearDown() } func TestElasticSearchSuite(t *testing.T) { - suite.Run(t, new(ElasticSearchTestSuite)) + suite.Run(t, new(ElasticSearchBasicTestSuite)) } -func (suite *ElasticSearchTestSuite) SetupTest() { +func (suite *ElasticSearchBasicTestSuite) SetupTest() { t = suite.T() } -func (suite *ElasticSearchTestSuite) AfterTest(suiteName, testName string) { +func (suite *ElasticSearchBasicTestSuite) AfterTest(suiteName, testName string) { handleTestFailure() } -func (suite *ElasticSearchTestSuite) TestSparkDependenciesES() { +func (suite *ElasticSearchBasicTestSuite) TestSparkDependenciesES() { if skipESExternal { t.Skip("This test requires an insecure ElasticSearch instance") } @@ -87,7 +69,7 @@ func (suite *ElasticSearchTestSuite) TestSparkDependenciesES() { require.NoError(t, err, "SparkTest failed") } -func (suite *ElasticSearchTestSuite) TestSimpleProd() { +func (suite *ElasticSearchBasicTestSuite) TestSimpleProd() { if skipESExternal { t.Skip("This case is covered by the self_provisioned_elasticsearch_test") } @@ -96,7 +78,7 @@ func (suite *ElasticSearchTestSuite) TestSimpleProd() { // create jaeger custom resource name := "simple-prod" - exampleJaeger := getJaegerSimpleProdWithServerUrls(name) + exampleJaeger := GetJaegerSimpleProdWithServerUrlsCR(name, esServerUrls) err = fw.Client.Create(context.TODO(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) require.NoError(t, err, "Error deploying example Jaeger") defer undeployJaegerInstance(exampleJaeger) @@ -112,239 +94,3 @@ func (suite *ElasticSearchTestSuite) TestSimpleProd() { // Make sure we were using the correct collector image verifyCollectorImage(name, namespace, specifyOtelImages) } - -func (suite *ElasticSearchTestSuite) TestEsIndexCleanerWithIndexPrefix() { - esIndexCleanerEnabled = false - esIndexPrefix := "prefix" - jaegerInstanceName := "test-es-index-prefixes" - jaegerInstance := &v1.Jaeger{} - - if skipESExternal { - esNamespace = namespace - numberOfDays := 0 - indexCleanerSpec := v1.JaegerEsIndexCleanerSpec{ - Enabled: &esIndexCleanerEnabled, - Schedule: "*/1 * * * *", - NumberOfDays: &numberOfDays, - } - - jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) - jaegerInstance.Spec.Storage.EsIndexCleaner = indexCleanerSpec - addIndexPrefix(jaegerInstance, esIndexPrefix) - - createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) - defer undeployJaegerInstance(jaegerInstance) - - ProductionSmokeTest(jaegerInstanceName) - } else { - esNamespace = storageNamespace - jaegerInstance = getJaegerAllInOne(jaegerInstanceName) - addIndexPrefix(jaegerInstance, esIndexPrefix) - - err := fw.Client.Create(context.Background(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) - require.NoError(t, err, "Error deploying Jaeger") - defer undeployJaegerInstance(jaegerInstance) - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName, 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for deployment") - - // Run the smoke test so indices will be created - AllInOneSmokeTest(jaegerInstanceName) - } - // Now verify that we have indices with the prefix we want - indexWithPrefixExists(esIndexPrefix+"-jaeger-", true, esNamespace) - - // Turn on index clean and make sure we clean up - turnOnEsIndexCleaner(jaegerInstance) - indexWithPrefixExists(esIndexPrefix+"-jaeger-", false, esNamespace) - -} - -func addIndexPrefix(jaegerInstance *v1.Jaeger, esIndexPrefix string) { - // Add an index prefix to the CR before creating this Jaeger instance - options := jaegerInstance.Spec.Storage.Options.Map() - updateOptions := make(map[string]interface{}) - for key, value := range options { - updateOptions[key] = value - } - updateOptions["es.index-prefix"] = esIndexPrefix - jaegerInstance.Spec.Storage.Options = v1.NewOptions(updateOptions) -} - -func (suite *ElasticSearchTestSuite) TestEsIndexCleaner() { - esIndexCleanerEnabled = false - jaegerInstanceName := "test-es-index-cleaner" - jaegerInstance := &v1.Jaeger{} - - if skipESExternal { - esNamespace = namespace - numberOfDays := 0 - indexCleanerSpec := v1.JaegerEsIndexCleanerSpec{ - Enabled: &esIndexCleanerEnabled, - Schedule: "*/1 * * * *", - NumberOfDays: &numberOfDays, - } - - jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) - jaegerInstance.Spec.Storage.EsIndexCleaner = indexCleanerSpec - createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) - defer undeployJaegerInstance(jaegerInstance) - - ProductionSmokeTest(jaegerInstanceName) - } else { - esNamespace = storageNamespace - jaegerInstance = getJaegerAllInOne(jaegerInstanceName) - - err := fw.Client.Create(context.Background(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) - require.NoError(t, err, "Error deploying Jaeger") - defer undeployJaegerInstance(jaegerInstance) - - err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName, 1, retryInterval, timeout) - require.NoError(t, err, "Error waiting for deployment") - - // create span, then make sure indices have been created - AllInOneSmokeTest(jaegerInstanceName) - } - indexWithPrefixExists("jaeger-", true, esNamespace) - - // Once we've created a span with the smoke test, enable the index cleaner - turnOnEsIndexCleaner(jaegerInstance) - - // Now make sure indices have been deleted - indexWithPrefixExists("jaeger-", false, esNamespace) -} - -func getJaegerSimpleProdWithServerUrls(name string) *v1.Jaeger { - ingressEnabled := true - exampleJaeger := &v1.Jaeger{ - TypeMeta: metav1.TypeMeta{ - Kind: "Jaeger", - APIVersion: "jaegertracing.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: v1.JaegerSpec{ - Ingress: v1.JaegerIngressSpec{ - Enabled: &ingressEnabled, - Security: v1.IngressSecurityNoneExplicit, - }, - Strategy: v1.DeploymentStrategyProduction, - Storage: v1.JaegerStorageSpec{ - Type: v1.JaegerESStorage, - Options: v1.NewOptions(map[string]interface{}{ - "es.server-urls": esServerUrls, - }), - }, - }, - } - - if specifyOtelImages { - logrus.Infof("Using OTEL collector for %s", name) - exampleJaeger.Spec.Collector.Image = otelCollectorImage - exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269")) - } - - return exampleJaeger -} - -func getJaegerAllInOne(name string) *v1.Jaeger { - numberOfDays := 0 - ingressEnabled := true - j := &v1.Jaeger{ - TypeMeta: v12.TypeMeta{ - Kind: "Jaeger", - APIVersion: "jaegertracing.io/v1", - }, - ObjectMeta: v12.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: v1.JaegerSpec{ - Ingress: v1.JaegerIngressSpec{ - Enabled: &ingressEnabled, - Security: v1.IngressSecurityNoneExplicit, - }, - Strategy: v1.DeploymentStrategyAllInOne, - Storage: v1.JaegerStorageSpec{ - Type: v1.JaegerESStorage, - Options: v1.NewOptions(map[string]interface{}{ - "es.server-urls": esServerUrls, - }), - EsIndexCleaner: v1.JaegerEsIndexCleanerSpec{ - Enabled: &esIndexCleanerEnabled, - Schedule: "*/1 * * * *", - NumberOfDays: &numberOfDays, - }, - }, - }, - } - return j -} - -func hasIndexWithPrefix(prefix string, esPort string) (bool, error) { - transport := &http.Transport{} - if skipESExternal { - esUrl = "https://localhost:" + esPort + "/_cat/indices" - esSecret, err := fw.KubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{}) - require.NoError(t, err) - pool := x509.NewCertPool() - pool.AppendCertsFromPEM(esSecret.Data["admin-ca"]) - - clientCert, err := tls.X509KeyPair(esSecret.Data["admin-cert"], esSecret.Data["admin-key"]) - require.NoError(t, err) - - transport.TLSClientConfig = &tls.Config{ - RootCAs: pool, - Certificates: []tls.Certificate{clientCert}, - } - } else { - esUrl = "http://localhost:" + esPort + "/_cat/indices" - } - client := http.Client{Transport: transport} - - req, err := http.NewRequest(http.MethodGet, esUrl, nil) - require.NoError(t, err) - - resp, err := client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - bodyBytes, err := ioutil.ReadAll(resp.Body) - bodyString := string(bodyBytes) - - return strings.Contains(bodyString, prefix), nil -} - -func createEsPortForward(esNamespace string) (portForwES *portforward.PortForwarder, closeChanES chan struct{}, esPort string) { - portForwES, closeChanES = CreatePortForward(esNamespace, string(v1.JaegerESStorage), string(v1.JaegerESStorage), []string{"0:9200"}, fw.KubeConfig) - forwardedPorts, err := portForwES.GetPorts() - require.NoError(t, err) - return portForwES, closeChanES, strconv.Itoa(int(forwardedPorts[0].Local)) -} - -func turnOnEsIndexCleaner(jaegerInstance *v1.Jaeger) { - key := types.NamespacedName{Name: jaegerInstance.Name, Namespace: jaegerInstance.GetNamespace()} - err := fw.Client.Get(context.Background(), key, jaegerInstance) - require.NoError(t, err) - esIndexCleanerEnabled = true - err = fw.Client.Update(context.Background(), jaegerInstance) - require.NoError(t, err) - - err = WaitForCronJob(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout+1*time.Minute) - require.NoError(t, err, "Error waiting for Cron Job") - - err = WaitForJobOfAnOwner(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout) - require.NoError(t, err, "Error waiting for Cron Job") -} - -func indexWithPrefixExists(prefix string, condition bool, esNamespace string) { - portForwES, closeChanES, esPort := createEsPortForward(esNamespace) - defer portForwES.Close() - defer close(closeChanES) - err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { - flag, err := hasIndexWithPrefix(prefix, esPort) - return flag == condition, err - }) - require.NoError(t, err) -} diff --git a/test/e2e/multiple_instances_test.go b/test/e2e/multiple_instances_test.go index 20ec117cf..92e3f9ce6 100644 --- a/test/e2e/multiple_instances_test.go +++ b/test/e2e/multiple_instances_test.go @@ -60,7 +60,7 @@ func (suite *MultipleInstanceTestSuite) TestVerifySecrets() { jaegerInstanceName := "simple-prod" // In production we'd use 3 nodes but 1 is sufficient for this test. - jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + jaegerInstance := GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, namespace, 1) createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) defer undeployJaegerInstance(jaegerInstance) @@ -68,7 +68,7 @@ func (suite *MultipleInstanceTestSuite) TestVerifySecrets() { secondContext, err := createNewTestContext() defer secondContext.Cleanup() secondNamespace := secondContext.GetID() - secondJaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, secondNamespace, 1) + secondJaegerInstance := GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, secondNamespace, 1) createESSelfProvDeployment(secondJaegerInstance, jaegerInstanceName, secondNamespace) defer undeployJaegerInstance(secondJaegerInstance) diff --git a/test/e2e/self_provisioned_elasticsearch_test.go b/test/e2e/self_provisioned_elasticsearch_test.go index c7f86f41d..a74c15705 100644 --- a/test/e2e/self_provisioned_elasticsearch_test.go +++ b/test/e2e/self_provisioned_elasticsearch_test.go @@ -80,7 +80,7 @@ func (suite *SelfProvisionedTestSuite) AfterTest(suiteName, testName string) { func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() { // create jaeger custom resource jaegerInstanceName := "simple-prod" - jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + jaegerInstance := GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, namespace, 1) createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) defer undeployJaegerInstance(jaegerInstance) @@ -92,7 +92,7 @@ func (suite *SelfProvisionedTestSuite) TestSelfProvisionedESSmokeTest() { func (suite *SelfProvisionedTestSuite) TestIncreasingReplicas() { jaegerInstanceName := "simple-prod2" - jaegerInstance := getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1) + jaegerInstance := GetJaegerSelfProvSimpleProdCR(jaegerInstanceName, namespace, 1) createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace) defer undeployJaegerInstance(jaegerInstance) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index d84410ab3..a3dcf5c88 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "fmt" + "io" "io/ioutil" "net/http" "os" @@ -14,6 +15,9 @@ import ( "testing" "time" + "github.com/opentracing/opentracing-go" + "github.com/uber/jaeger-client-go/config" + "github.com/jaegertracing/jaeger-operator/pkg/apis/kafka/v1beta1" osv1 "github.com/openshift/api/route/v1" @@ -57,6 +61,7 @@ var ( cassandraServiceName = "cassandra." + storageNamespace + ".svc" cassandraKeyspace = "jaeger_v1_datacenter1" cassandraDatacenter = "datacenter1" + jaegerCollectorPort = 14268 otelCollectorImage = "jaegertracing/jaeger-opentelemetry-collector:latest" otelIngesterImage = "jaegertracing/jaeger-opentelemetry-ingester:latest" otelAgentImage = "jaegertracing/jaeger-opentelemetry-agent:latest" @@ -733,45 +738,6 @@ func waitForElasticSearch() { require.NoError(t, err, "Error waiting for elasticsearch") } -func getJaegerSelfProvSimpleProd(instanceName, namespace string, nodeCount int32) *v1.Jaeger { - ingressEnabled := true - exampleJaeger := &v1.Jaeger{ - TypeMeta: metav1.TypeMeta{ - Kind: "Jaeger", - APIVersion: "jaegertracing.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: instanceName, - Namespace: namespace, - }, - Spec: v1.JaegerSpec{ - Ingress: v1.JaegerIngressSpec{ - Enabled: &ingressEnabled, - Security: v1.IngressSecurityNoneExplicit, - }, - Strategy: v1.DeploymentStrategyProduction, - Storage: v1.JaegerStorageSpec{ - Type: v1.JaegerESStorage, - Elasticsearch: v1.ElasticsearchSpec{ - NodeCount: nodeCount, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, - Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, - }, - }, - }, - }, - } - - if specifyOtelImages { - logrus.Infof("Using OTEL collector for %s", instanceName) - exampleJaeger.Spec.Collector.Image = otelCollectorImage - exampleJaeger.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269")) - } - - return exampleJaeger -} - func createESSelfProvDeployment(jaegerInstance *v1.Jaeger, jaegerInstanceName, jaegerNamespace string) { err := fw.Client.Create(context.TODO(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval}) require.NoError(t, err, "Error deploying example Jaeger") @@ -883,3 +849,15 @@ func getJaegerSelfProvisionedESAndKafka(instanceName string) *v1.Jaeger { return jaegerInstance } + +func getTracingClientWithCollectorEndpoint(serviceName, collectorEndpoint string) (opentracing.Tracer, io.Closer, error) { + if collectorEndpoint == "" { + collectorEndpoint = fmt.Sprintf("http://localhost:%d/api/traces", jaegerCollectorPort) + } + cfg := config.Configuration{ + Reporter: &config.ReporterConfig{CollectorEndpoint: collectorEndpoint}, + Sampler: &config.SamplerConfig{Type: "const", Param: 1}, + ServiceName: serviceName, + } + return cfg.NewTracer() +} diff --git a/test/e2e/utils_cr.go b/test/e2e/utils_cr.go new file mode 100644 index 000000000..574ac4a3c --- /dev/null +++ b/test/e2e/utils_cr.go @@ -0,0 +1,86 @@ +package e2e + +import ( + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1 "github.com/jaegertracing/jaeger-operator/pkg/apis/jaegertracing/v1" +) + +func updateOtelImages(jaegerInstance *v1.Jaeger) { + if specifyOtelImages { + logrus.Infof("Using OTEL collector for %s", jaegerInstance.Name) + jaegerInstance.Spec.Collector.Image = otelCollectorImage + jaegerInstance.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269")) + } +} + +// GetJaegerSimpleProdWithServerUrlsCR returns simple production CR with external es server urls +func GetJaegerSimpleProdWithServerUrlsCR(name, esServerUrls string) *v1.Jaeger { + ingressEnabled := true + simpleProdCR := &v1.Jaeger{ + TypeMeta: metav1.TypeMeta{ + Kind: "Jaeger", + APIVersion: "jaegertracing.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1.JaegerSpec{ + Ingress: v1.JaegerIngressSpec{ + Enabled: &ingressEnabled, + Security: v1.IngressSecurityNoneExplicit, + }, + Strategy: v1.DeploymentStrategyProduction, + Storage: v1.JaegerStorageSpec{ + Type: v1.JaegerESStorage, + Options: v1.NewOptions(map[string]interface{}{ + "es.server-urls": esServerUrls, + }), + }, + }, + } + + updateOtelImages(simpleProdCR) + + return simpleProdCR +} + +// GetJaegerSelfProvSimpleProdCR returns self provisioned production simple CR +func GetJaegerSelfProvSimpleProdCR(instanceName, namespace string, nodeCount int32) *v1.Jaeger { + ingressEnabled := true + selfProvSimpleProdCR := &v1.Jaeger{ + TypeMeta: metav1.TypeMeta{ + Kind: "Jaeger", + APIVersion: "jaegertracing.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: instanceName, + Namespace: namespace, + }, + Spec: v1.JaegerSpec{ + Ingress: v1.JaegerIngressSpec{ + Enabled: &ingressEnabled, + Security: v1.IngressSecurityNoneExplicit, + }, + Strategy: v1.DeploymentStrategyProduction, + Storage: v1.JaegerStorageSpec{ + Type: v1.JaegerESStorage, + Elasticsearch: v1.ElasticsearchSpec{ + NodeCount: nodeCount, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}, + }, + }, + }, + }, + } + + updateOtelImages(selfProvSimpleProdCR) + + return selfProvSimpleProdCR +} diff --git a/test/e2e/utils_elasticsearch.go b/test/e2e/utils_elasticsearch.go new file mode 100644 index 000000000..06aa22544 --- /dev/null +++ b/test/e2e/utils_elasticsearch.go @@ -0,0 +1,105 @@ +package e2e + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/portforward" + + v1 "github.com/jaegertracing/jaeger-operator/pkg/apis/jaegertracing/v1" +) + +// EsIndex struct to map indices data from es rest api response +// es api: /_cat/indices?format=json +type EsIndex struct { + UUID string `json:"uuid"` + Status string `json:"status"` + Index string `json:"index"` + Health string `json:"health"` + DocsCount string `json:"docs.count"` + DocsDeleted string `json:"docs.deleted"` + StoreSize string `json:"store.size"` +} + +// GetEsIndices return indices from es node +func GetEsIndices(esNamespace string) ([]EsIndex, error) { + bodyBytes, err := ExecuteEsRequest(esNamespace, http.MethodGet, "/_cat/indices?format=json") + require.NoError(t, err) + + // convert json data to struct format + esIndices := make([]EsIndex, 0) + err = json.Unmarshal(bodyBytes, &esIndices) + require.NoError(t, err) + + return esIndices, nil +} + +// DeleteEsIndices deletes all the indices on es node +func DeleteEsIndices(esNamespace string) { + logrus.Info("deleting all es node indices") + _, err := ExecuteEsRequest(esNamespace, http.MethodDelete, "/_all?format=json") + require.NoError(t, err) +} + +// ExecuteEsRequest executes rest api request on es node +func ExecuteEsRequest(esNamespace, httpMethod, api string) ([]byte, error) { + // enable port forward + fwdPortES, closeChanES, esPort := CreateEsPortForward(esNamespace) + defer fwdPortES.Close() + defer close(closeChanES) + + // update es node url + urlScheme := "http" + if skipESExternal { + urlScheme = "https" + } + esURL := fmt.Sprintf("%s://localhost:%s%s", urlScheme, esPort, api) + + // create rest client to access es node rest API + transport := &http.Transport{} + client := http.Client{Transport: transport} + + // update certificates, if the es node provided by jaeger-operator + if skipESExternal { + esSecret, err := fw.KubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{}) + require.NoError(t, err) + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(esSecret.Data["admin-ca"]) + + clientCert, err := tls.X509KeyPair(esSecret.Data["admin-cert"], esSecret.Data["admin-key"]) + require.NoError(t, err) + + transport.TLSClientConfig = &tls.Config{ + RootCAs: pool, + Certificates: []tls.Certificate{clientCert}, + } + } + + req, err := http.NewRequest(httpMethod, esURL, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.EqualValues(t, 200, resp.StatusCode) + + return ioutil.ReadAll(resp.Body) +} + +// CreateEsPortForward creates local port forwarding +func CreateEsPortForward(esNamespace string) (portForwES *portforward.PortForwarder, closeChanES chan struct{}, esPort string) { + portForwES, closeChanES = CreatePortForward(esNamespace, string(v1.JaegerESStorage), string(v1.JaegerESStorage), []string{"0:9200"}, fw.KubeConfig) + forwardedPorts, err := portForwES.GetPorts() + require.NoError(t, err) + return portForwES, closeChanES, strconv.Itoa(int(forwardedPorts[0].Local)) +}