From 27f690ef8f4b80dd5cd2f5e97a431f9671ea886d Mon Sep 17 00:00:00 2001 From: Povilas Versockas Date: Thu, 19 Sep 2024 00:10:30 -0700 Subject: [PATCH 1/3] feat: [TKC-2551] add support for advanced label filtering (#5853) --- pkg/repository/testworkflow/filter.go | 30 +++-- pkg/repository/testworkflow/interface.go | 12 ++ pkg/repository/testworkflow/mongo.go | 14 ++ .../testworkflow/mongo_integration_test.go | 127 ++++++++++++++++++ 4 files changed, 173 insertions(+), 10 deletions(-) diff --git a/pkg/repository/testworkflow/filter.go b/pkg/repository/testworkflow/filter.go index 41d4c4d277b..c293926dd9e 100644 --- a/pkg/repository/testworkflow/filter.go +++ b/pkg/repository/testworkflow/filter.go @@ -7,16 +7,17 @@ import ( ) type FilterImpl struct { - FName string - FLastNDays int - FStartDate *time.Time - FEndDate *time.Time - FStatuses []testkube.TestWorkflowStatus - FPage int - FPageSize int - FTextSearch string - FSelector string - FTagSelector string + FName string + FLastNDays int + FStartDate *time.Time + FEndDate *time.Time + FStatuses []testkube.TestWorkflowStatus + FPage int + FPageSize int + FTextSearch string + FSelector string + FTagSelector string + FLabelSelector *LabelSelector } func NewExecutionsFilter() *FilterImpl { @@ -77,6 +78,11 @@ func (f *FilterImpl) WithTagSelector(tagSelector string) *FilterImpl { return f } +func (f *FilterImpl) WithLabelSelector(selector *LabelSelector) *FilterImpl { + f.FLabelSelector = selector + return f +} + func (f FilterImpl) Name() string { return f.FName } @@ -140,3 +146,7 @@ func (f FilterImpl) Selector() string { func (f FilterImpl) TagSelector() string { return f.FTagSelector } + +func (f FilterImpl) LabelSelector() *LabelSelector { + return f.FLabelSelector +} diff --git a/pkg/repository/testworkflow/interface.go b/pkg/repository/testworkflow/interface.go index 9a786d3c85f..99ffcc15476 100644 --- a/pkg/repository/testworkflow/interface.go +++ b/pkg/repository/testworkflow/interface.go @@ -8,6 +8,17 @@ import ( "github.com/kubeshop/testkube/pkg/api/v1/testkube" ) +type Label struct { + Key string + Value *string + // If value is nil, we check if key exists / not exists + Exists *bool +} + +type LabelSelector struct { + Or []Label +} + const PageDefaultLimit int = 100 type Filter interface { @@ -27,6 +38,7 @@ type Filter interface { TextSearch() string Selector() string TagSelector() string + LabelSelector() *LabelSelector } //go:generate mockgen -destination=./mock_repository.go -package=testworkflow "github.com/kubeshop/testkube/pkg/repository/testworkflow" Repository diff --git a/pkg/repository/testworkflow/mongo.go b/pkg/repository/testworkflow/mongo.go index 7721eecb5a2..e7c7ff704a0 100644 --- a/pkg/repository/testworkflow/mongo.go +++ b/pkg/repository/testworkflow/mongo.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kubeshop/testkube/pkg/repository/common" + "github.com/kubeshop/testkube/pkg/utils" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -372,6 +373,19 @@ func composeQueryAndOpts(filter Filter) (bson.M, *options.FindOptions) { } } + if filter.LabelSelector() != nil && len(filter.LabelSelector().Or) > 0 { + subquery := bson.A{} + for _, label := range filter.LabelSelector().Or { + if label.Value != nil { + subquery = append(subquery, bson.M{"workflow.labels." + utils.EscapeDots(label.Key): *label.Value}) + } else if label.Exists != nil { + subquery = append(subquery, + bson.M{"workflow.labels." + utils.EscapeDots(label.Key): bson.M{"$exists": *label.Exists}}) + } + } + query["$or"] = subquery + } + opts.SetSkip(int64(filter.Page() * filter.PageSize())) opts.SetLimit(int64(filter.PageSize())) opts.SetSort(bson.D{{Key: "scheduledat", Value: -1}}) diff --git a/pkg/repository/testworkflow/mongo_integration_test.go b/pkg/repository/testworkflow/mongo_integration_test.go index f233041c1ac..e5023d1258b 100644 --- a/pkg/repository/testworkflow/mongo_integration_test.go +++ b/pkg/repository/testworkflow/mongo_integration_test.go @@ -86,3 +86,130 @@ func TestNewMongoRepository_UpdateReport_Integration(t *testing.T) { assert.Equal(t, *report1, fresh.Reports[0]) assert.Equal(t, *report2, fresh.Reports[1]) } + +func TestNewMongoRepository_Executions_Integration(t *testing.T) { + test.IntegrationTest(t) + + ctx := context.Background() + + client, err := mongo.Connect(ctx, options.Client().ApplyURI(cfg.APIMongoDSN)) + if err != nil { + t.Fatalf("error connecting to mongo: %v", err) + } + db := client.Database("testworkflow-executions-mongo-repository-test") + t.Cleanup(func() { + db.Drop(ctx) + }) + + repo := NewMongoRepository(db, false) + + execution := testkube.TestWorkflowExecution{ + Id: "test-id", + Name: "test-name", + Workflow: &testkube.TestWorkflow{ + Name: "test-name", + Labels: map[string]string{ + "workflow.labels.testkube.io/group": "grp1", + }, + Spec: &testkube.TestWorkflowSpec{}, + }, + } + if err := repo.Insert(ctx, execution); err != nil { + t.Fatalf("error inserting execution: %v", err) + } + + execution = testkube.TestWorkflowExecution{ + Id: "test-no-group", + Name: "test-no-group-name", + Workflow: &testkube.TestWorkflow{ + Name: "test-no-group-name", + Labels: map[string]string{}, + Spec: &testkube.TestWorkflowSpec{}, + }, + } + if err := repo.Insert(ctx, execution); err != nil { + t.Fatalf("error inserting execution: %v", err) + } + execution = testkube.TestWorkflowExecution{ + Id: "test-group2-id", + Name: "test-group2-name", + Workflow: &testkube.TestWorkflow{ + Name: "test-group2-name", + Labels: map[string]string{ + "workflow.labels.testkube.io/group": "grp2", + }, + Spec: &testkube.TestWorkflowSpec{}, + }, + } + if err := repo.Insert(ctx, execution); err != nil { + t.Fatalf("error inserting execution: %v", err) + } + + res, err := repo.GetExecutions(ctx, NewExecutionsFilter()) + if err != nil { + t.Fatalf("error getting executions: %v", err) + } + + assert.Len(t, res, 3) + + labelSelector := LabelSelector{ + Or: []Label{ + {Key: "workflow.labels.testkube.io/group", Value: strPtr("grp2")}, + }, + } + res, err = repo.GetExecutions(ctx, NewExecutionsFilter().WithLabelSelector(&labelSelector)) + if err != nil { + t.Fatalf("error getting executions: %v", err) + } + + assert.Len(t, res, 1) + assert.Equal(t, "test-group2-name", res[0].Name) + + labelSelector = LabelSelector{ + Or: []Label{ + {Key: "workflow.labels.testkube.io/group", Exists: boolPtr(false)}, + }, + } + res, err = repo.GetExecutions(ctx, NewExecutionsFilter().WithLabelSelector(&labelSelector)) + if err != nil { + t.Fatalf("error getting executions: %v", err) + } + + assert.Len(t, res, 1) + assert.Equal(t, "test-no-group-name", res[0].Name) + + labelSelector = LabelSelector{ + Or: []Label{ + {Key: "workflow.labels.testkube.io/group", Exists: boolPtr(false)}, + {Key: "workflow.labels.testkube.io/group", Value: strPtr("grp2")}, + }, + } + res, err = repo.GetExecutions(ctx, NewExecutionsFilter().WithLabelSelector(&labelSelector)) + if err != nil { + t.Fatalf("error getting executions: %v", err) + } + + assert.Len(t, res, 2) + + labelSelector = LabelSelector{ + Or: []Label{ + {Key: "workflow.labels.testkube.io/group", Exists: boolPtr(false)}, + {Key: "workflow.labels.testkube.io/group", Value: strPtr("grp1")}, + {Key: "workflow.labels.testkube.io/group", Value: strPtr("grp2")}, + }, + } + res, err = repo.GetExecutions(ctx, NewExecutionsFilter().WithLabelSelector(&labelSelector)) + if err != nil { + t.Fatalf("error getting executions: %v", err) + } + + assert.Len(t, res, 3) +} + +func strPtr(s string) *string { + return &s +} + +func boolPtr(b bool) *bool { + return &b +} From f6c7de6102d6fe8be7affab20ede1b1b215ca4b7 Mon Sep 17 00:00:00 2001 From: Tomasz Konieczny Date: Thu, 19 Sep 2024 12:32:43 +0200 Subject: [PATCH 2/3] feat: Status page - example workflows (#5857) * curl workflow tests updated * curl - status-page - example workflows * container image --- .../executor-tests/crd-workflow/smoke.yaml | 6 +-- test/examples/status-page/crd.yaml | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 test/examples/status-page/crd.yaml diff --git a/test/curl/executor-tests/crd-workflow/smoke.yaml b/test/curl/executor-tests/crd-workflow/smoke.yaml index 7c81cbed835..d4d13869eeb 100644 --- a/test/curl/executor-tests/crd-workflow/smoke.yaml +++ b/test/curl/executor-tests/crd-workflow/smoke.yaml @@ -12,7 +12,7 @@ spec: memory: 32Mi steps: - name: Run tests - shell: curl https://testkube.io + shell: curl -f -LI https://testkube.io container: image: curlimages/curl:8.7.1 --- @@ -33,7 +33,7 @@ spec: memory: 32Mi steps: - name: Run tests - shell: curl https://testkube.io && sleep 10 + shell: curl -f -LI https://testkube.io && sleep 10 container: image: tkoniecznykubeshop/example-private-repo:curl-8.7.1 --- @@ -54,6 +54,6 @@ spec: parallel: matrix: url: ['https://testkube.io', 'https://docs.testkube.io'] - shell: curl '{{ matrix.url }}' + shell: curl -f -LI '{{ matrix.url }}' container: image: curlimages/curl:8.7.1 diff --git a/test/examples/status-page/crd.yaml b/test/examples/status-page/crd.yaml new file mode 100644 index 00000000000..3b4500c939e --- /dev/null +++ b/test/examples/status-page/crd.yaml @@ -0,0 +1,53 @@ +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-website-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://testkube.io +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-docs-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://docs.testkube.io +--- +apiVersion: testworkflows.testkube.io/v1 +kind: TestWorkflow +metadata: + name: tw-testkube-blog-5m +spec: + events: + - cronjob: + cron: '*/5 * * * *' + container: + image: curlimages/curl:8.7.1 + resources: + requests: + cpu: 32m + memory: 32Mi + steps: + - name: Run tests + shell: curl -f -LI https://testkube.io/blog From 7ab27282cef940768f2fad8b02197af846c810ec Mon Sep 17 00:00:00 2001 From: ypoplavs <45286051+ypoplavs@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:32:02 +0300 Subject: [PATCH 3/3] Update sandbox.yaml --- .github/workflows/sandbox.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sandbox.yaml b/.github/workflows/sandbox.yaml index 424ee545e96..a613058565e 100644 --- a/.github/workflows/sandbox.yaml +++ b/.github/workflows/sandbox.yaml @@ -846,6 +846,9 @@ jobs: pr_creation: runs-on: ubuntu-latest steps: + - name: 'Checkout' + uses: 'actions/checkout@v4' + - name: Get a branch name if PR is created if: startsWith(github.event.pull_request.head.ref, 'sandbox/') run: |