Skip to content

Commit

Permalink
Enable running tests without an external ES instance (jaegertracing#1316
Browse files Browse the repository at this point in the history 
)

Signed-off-by: Kevin Earls <kearls@redhat.com>
  • Loading branch information
@kevinearls
kevinearls authored Dec 2, 2020
1 parent 18d32a0 commit 172683f
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 107 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,11 @@ endif

.PHONY: es
es: storage
ifeq ($(SKIP_ES_EXTERNAL),true)
@echo Skipping creation of external Elasticsearch instance
else
@kubectl create -f ./test/elasticsearch.yml --namespace $(STORAGE_NAMESPACE) 2>&1 | grep -v "already exists" || true
endif

.PHONY: cassandra
cassandra: storage
Expand Down
27 changes: 20 additions & 7 deletions test/e2e/autoscale_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,17 @@ func (suite *AutoscaleTestSuite) TestAutoScaleCollector() {
if !isOpenShift(t) {
t.Skip("This test is currently only supported on OpenShift")
}
waitForElasticSearch()

jaegerInstanceName := "simple-prod"
jaegerInstance := getSimpleProd(jaegerInstanceName, namespace, cpuResourceLimit, memoryResourceLimit)
createAndWaitFor(jaegerInstance, jaegerInstanceName, false)
var jaegerInstance *v1.Jaeger
if skipESExternal {
jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, int32(1))
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
} else {
waitForElasticSearch()
jaegerInstance = getSimpleProd(jaegerInstanceName, namespace, cpuResourceLimit, memoryResourceLimit)
createAndWaitFor(jaegerInstance, jaegerInstanceName, false)
}
defer undeployJaegerInstance(jaegerInstance)

tracegen := createTracegenDeployment(jaegerInstanceName, namespace, tracegenDurationInMinutes, replicas)
Expand All @@ -92,12 +98,19 @@ func (suite *AutoscaleTestSuite) TestAutoScaleIngester() {
if !isOpenShift(t) {
t.Skip("This test is currently only supported on OpenShift")
}
waitForElasticSearch()
waitForKafkaInstance()

jaegerInstanceName := "simple-streaming"
jaegerInstance := getSimpleStreaming(jaegerInstanceName, namespace)
createAndWaitFor(jaegerInstance, jaegerInstanceName, true)
var jaegerInstance *v1.Jaeger
if skipESExternal {
jaegerInstance = getJaegerSelfProvisionedESAndKafka(jaegerInstanceName)
createESKafkaSelfProvDeployment(jaegerInstance)
} else {
waitForElasticSearch()
waitForKafkaInstance()
jaegerInstance := getSimpleStreaming(jaegerInstanceName, namespace)
createAndWaitFor(jaegerInstance, jaegerInstanceName, true)
}

defer undeployJaegerInstance(jaegerInstance)

tracegenReplicas := int32(1)
Expand Down
170 changes: 120 additions & 50 deletions test/e2e/elasticsearch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ package e2e

import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net/http"
Expand Down Expand Up @@ -31,6 +33,8 @@ type ElasticSearchTestSuite struct {
}

var esIndexCleanerEnabled = false
var esUrl string
var esNamespace = storageNamespace

func (suite *ElasticSearchTestSuite) SetupSuite() {
t = suite.T()
Expand Down Expand Up @@ -70,6 +74,9 @@ func (suite *ElasticSearchTestSuite) AfterTest(suiteName, testName string) {
}

func (suite *ElasticSearchTestSuite) TestSparkDependenciesES() {
if skipESExternal {
t.Skip("This test requires an insecure ElasticSearch instance")
}
storage := v1.JaegerStorageSpec{
Type: v1.JaegerESStorage,
Options: v1.NewOptions(map[string]interface{}{
Expand All @@ -81,6 +88,9 @@ func (suite *ElasticSearchTestSuite) TestSparkDependenciesES() {
}

func (suite *ElasticSearchTestSuite) TestSimpleProd() {
if skipESExternal {
t.Skip("This case is covered by the self_provisioned_elasticsearch_test")
}
err := WaitForStatefulset(t, fw.KubeClient, storageNamespace, string(v1.JaegerESStorage), retryInterval, timeout)
require.NoError(t, err, "Error waiting for elasticsearch")

Expand All @@ -106,57 +116,101 @@ func (suite *ElasticSearchTestSuite) TestSimpleProd() {
func (suite *ElasticSearchTestSuite) TestEsIndexCleanerWithIndexPrefix() {
esIndexCleanerEnabled = false
esIndexPrefix := "prefix"
name := "test-es-index-prefixes"
jaegerInstanceName := "test-es-index-prefixes"
jaegerInstance := &v1.Jaeger{}

if skipESExternal {
esNamespace = namespace
numberOfDays := 0
indexCleanerSpec := v1.JaegerEsIndexCleanerSpec{
Enabled: &esIndexCleanerEnabled,
Schedule: "*/1 * * * *",
NumberOfDays: &numberOfDays,
}

exampleJaeger := getJaegerAllInOne(name)
jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1)
jaegerInstance.Spec.Storage.EsIndexCleaner = indexCleanerSpec
addIndexPrefix(jaegerInstance, esIndexPrefix)

// Add an index prefix to the CR before creating this Jaeger instance
options := exampleJaeger.Spec.Storage.Options.Map()
updateOptions := make(map[string]interface{})
for key, value := range options {
updateOptions[key] = value
}
updateOptions["es.index-prefix"] = esIndexPrefix
exampleJaeger.Spec.Storage.Options = v1.NewOptions(updateOptions)
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
defer undeployJaegerInstance(jaegerInstance)

err := fw.Client.Create(context.Background(), exampleJaeger, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying Jaeger")
defer undeployJaegerInstance(exampleJaeger)
err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, name, 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for deployment")
ProductionSmokeTest(jaegerInstanceName)
} else {
esNamespace = storageNamespace
jaegerInstance = getJaegerAllInOne(jaegerInstanceName)
addIndexPrefix(jaegerInstance, esIndexPrefix)

// Run the smoke test so indices will be created
AllInOneSmokeTest(name)
err := fw.Client.Create(context.Background(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying Jaeger")
defer undeployJaegerInstance(jaegerInstance)
err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName, 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for deployment")

// Run the smoke test so indices will be created
AllInOneSmokeTest(jaegerInstanceName)
}
// Now verify that we have indices with the prefix we want
indexWithPrefixExists(esIndexPrefix+"-jaeger-", true)
indexWithPrefixExists(esIndexPrefix+"-jaeger-", true, esNamespace)

// Turn on index clean and make sure we clean up
turnOnEsIndexCleaner(name, exampleJaeger)
indexWithPrefixExists(esIndexPrefix+"-jaeger-", false)
turnOnEsIndexCleaner(jaegerInstance)
indexWithPrefixExists(esIndexPrefix+"-jaeger-", false, esNamespace)

}

func addIndexPrefix(jaegerInstance *v1.Jaeger, esIndexPrefix string) {
// Add an index prefix to the CR before creating this Jaeger instance
options := jaegerInstance.Spec.Storage.Options.Map()
updateOptions := make(map[string]interface{})
for key, value := range options {
updateOptions[key] = value
}
updateOptions["es.index-prefix"] = esIndexPrefix
jaegerInstance.Spec.Storage.Options = v1.NewOptions(updateOptions)
}

func (suite *ElasticSearchTestSuite) TestEsIndexCleaner() {
esIndexCleanerEnabled = false
name := "test-es-index-cleaner"
j := getJaegerAllInOne(name)
jaegerInstanceName := "test-es-index-cleaner"
jaegerInstance := &v1.Jaeger{}

if skipESExternal {
esNamespace = namespace
numberOfDays := 0
indexCleanerSpec := v1.JaegerEsIndexCleanerSpec{
Enabled: &esIndexCleanerEnabled,
Schedule: "*/1 * * * *",
NumberOfDays: &numberOfDays,
}

jaegerInstance = getJaegerSelfProvSimpleProd(jaegerInstanceName, namespace, 1)
jaegerInstance.Spec.Storage.EsIndexCleaner = indexCleanerSpec
createESSelfProvDeployment(jaegerInstance, jaegerInstanceName, namespace)
defer undeployJaegerInstance(jaegerInstance)

err := fw.Client.Create(context.Background(), j, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying Jaeger")
defer undeployJaegerInstance(j)
ProductionSmokeTest(jaegerInstanceName)
} else {
esNamespace = storageNamespace
jaegerInstance = getJaegerAllInOne(jaegerInstanceName)

err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, name, 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for deployment")
err := fw.Client.Create(context.Background(), jaegerInstance, &framework.CleanupOptions{TestContext: ctx, Timeout: timeout, RetryInterval: retryInterval})
require.NoError(t, err, "Error deploying Jaeger")
defer undeployJaegerInstance(jaegerInstance)

// create span, then make sure indices have been created
AllInOneSmokeTest(name)
indexWithPrefixExists("jaeger-", true)
err = e2eutil.WaitForDeployment(t, fw.KubeClient, namespace, jaegerInstanceName, 1, retryInterval, timeout)
require.NoError(t, err, "Error waiting for deployment")

// create span, then make sure indices have been created
AllInOneSmokeTest(jaegerInstanceName)
}
indexWithPrefixExists("jaeger-", true, esNamespace)

// Once we've created a span with the smoke test, enable the index cleaner
turnOnEsIndexCleaner(name, j)
turnOnEsIndexCleaner(jaegerInstance)

// Now make sure indices have been deleted
indexWithPrefixExists("jaeger-", false)
indexWithPrefixExists("jaeger-", false, esNamespace)
}

func getJaegerSimpleProdWithServerUrls(name string) *v1.Jaeger {
Expand Down Expand Up @@ -229,15 +283,31 @@ func getJaegerAllInOne(name string) *v1.Jaeger {
}

func hasIndexWithPrefix(prefix string, esPort string) (bool, error) {
c := http.Client{}
req, err := http.NewRequest(http.MethodGet, "http://localhost:"+esPort+"/_cat/indices", nil)
if err != nil {
return false, err
}
resp, err := c.Do(req)
if err != nil {
return false, err
transport := &http.Transport{}
if skipESExternal {
esUrl = "https://localhost:" + esPort + "/_cat/indices"
esSecret, err := fw.KubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "elasticsearch", metav1.GetOptions{})
require.NoError(t, err)
pool := x509.NewCertPool()
pool.AppendCertsFromPEM(esSecret.Data["admin-ca"])

clientCert, err := tls.X509KeyPair(esSecret.Data["admin-cert"], esSecret.Data["admin-key"])
require.NoError(t, err)

transport.TLSClientConfig = &tls.Config{
RootCAs: pool,
Certificates: []tls.Certificate{clientCert},
}
} else {
esUrl = "http://localhost:" + esPort + "/_cat/indices"
}
client := http.Client{Transport: transport}

req, err := http.NewRequest(http.MethodGet, esUrl, nil)
require.NoError(t, err)

resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()

bodyBytes, err := ioutil.ReadAll(resp.Body)
Expand All @@ -246,30 +316,30 @@ func hasIndexWithPrefix(prefix string, esPort string) (bool, error) {
return strings.Contains(bodyString, prefix), nil
}

func createEsPortForward() (portForwES *portforward.PortForwarder, closeChanES chan struct{}, esPort string) {
portForwES, closeChanES = CreatePortForward(storageNamespace, string(v1.JaegerESStorage), string(v1.JaegerESStorage), []string{"0:9200"}, fw.KubeConfig)
func createEsPortForward(esNamespace string) (portForwES *portforward.PortForwarder, closeChanES chan struct{}, esPort string) {
portForwES, closeChanES = CreatePortForward(esNamespace, string(v1.JaegerESStorage), string(v1.JaegerESStorage), []string{"0:9200"}, fw.KubeConfig)
forwardedPorts, err := portForwES.GetPorts()
require.NoError(t, err)
return portForwES, closeChanES, strconv.Itoa(int(forwardedPorts[0].Local))
}

func turnOnEsIndexCleaner(name string, exampleJaeger *v1.Jaeger) {
key := types.NamespacedName{Name: name, Namespace: namespace}
err := fw.Client.Get(context.Background(), key, exampleJaeger)
func turnOnEsIndexCleaner(jaegerInstance *v1.Jaeger) {
key := types.NamespacedName{Name: jaegerInstance.Name, Namespace: jaegerInstance.GetNamespace()}
err := fw.Client.Get(context.Background(), key, jaegerInstance)
require.NoError(t, err)
esIndexCleanerEnabled = true
err = fw.Client.Update(context.Background(), exampleJaeger)
err = fw.Client.Update(context.Background(), jaegerInstance)
require.NoError(t, err)

err = WaitForCronJob(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", name), retryInterval, timeout+1*time.Minute)
err = WaitForCronJob(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout+1*time.Minute)
require.NoError(t, err, "Error waiting for Cron Job")

err = WaitForJobOfAnOwner(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", name), retryInterval, timeout)
err = WaitForJobOfAnOwner(t, fw.KubeClient, namespace, fmt.Sprintf("%s-es-index-cleaner", jaegerInstance.Name), retryInterval, timeout)
require.NoError(t, err, "Error waiting for Cron Job")
}

func indexWithPrefixExists(prefix string, condition bool) {
portForwES, closeChanES, esPort := createEsPortForward()
func indexWithPrefixExists(prefix string, condition bool, esNamespace string) {
portForwES, closeChanES, esPort := createEsPortForward(esNamespace)
defer portForwES.Close()
defer close(closeChanES)
err := wait.Poll(retryInterval, timeout, func() (done bool, err error) {
Expand Down
9 changes: 9 additions & 0 deletions test/e2e/examples2_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,25 @@ func (suite *ExamplesTestSuite2) TestServiceTypesExample() {
}

func (suite *ExamplesTestSuite2) TestSimpleProdWithVolumes() {
if skipESExternal {
t.Skip("This example requires an external ElasticSearch instance")
}
yamlFileName := "../../examples/simple-prod-with-volumes.yaml"
smokeTestProductionExample("simple-prod", yamlFileName)
}

func (suite *ExamplesTestSuite2) TestSimpleProdExample() {
if skipESExternal {
t.Skip("This example requires an external ElasticSearch instance")
}
yamlFileName := "../../examples/simple-prod.yaml"
smokeTestProductionExample("simple-prod", yamlFileName)
}

func (suite *ExamplesTestSuite2) TestSimpleStreamingExample() {
if skipESExternal {
t.Skip("This example requires an external ElasticSearch instance")
}
yamlFileName := "../../examples/simple-streaming.yaml"
smokeTestProductionExample("simple-streaming", yamlFileName)
}
Expand Down
42 changes: 0 additions & 42 deletions test/e2e/self_provisioned_elasticsearch_kafka_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,9 @@ import (

framework "github.com/operator-framework/operator-sdk/pkg/test"
"github.com/operator-framework/operator-sdk/pkg/test/e2eutil"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/jaegertracing/jaeger-operator/pkg/apis"
Expand Down Expand Up @@ -114,42 +111,3 @@ func (suite *SelfProvisionedESWithKafkaTestSuite) TestSelfProvisionedESAndKafkaS
// Make sure we were using the correct collector image
verifyCollectorImage(jaegerInstanceName, namespace, specifyOtelImages)
}

func getJaegerSelfProvisionedESAndKafka(instanceName string) *v1.Jaeger {
ingressEnabled := true
jaegerInstance := &v1.Jaeger{
TypeMeta: metav1.TypeMeta{
Kind: "Jaeger",
APIVersion: "jaegertracing.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: instanceName,
Namespace: namespace,
},
Spec: v1.JaegerSpec{
Ingress: v1.JaegerIngressSpec{
Enabled: &ingressEnabled,
Security: v1.IngressSecurityNoneExplicit,
},
Strategy: v1.DeploymentStrategyStreaming,
Storage: v1.JaegerStorageSpec{
Type: v1.JaegerESStorage,
Elasticsearch: v1.ElasticsearchSpec{
NodeCount: 1,
Resources: &corev1.ResourceRequirements{
Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")},
},
},
},
},
}

if specifyOtelImages {
logrus.Infof("Using OTEL collector for %s", instanceName)
jaegerInstance.Spec.Collector.Image = otelCollectorImage
jaegerInstance.Spec.Collector.Config = v1.NewFreeForm(getOtelConfigForHealthCheckPort("14269"))
}

return jaegerInstance
}
Loading

0 comments on commit 172683f

Please sign in to comment.