From 94c6286448eec874a9b8b7dae7b58684b09d4f56 Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 11 Dec 2024 09:36:04 -0500 Subject: [PATCH] make it possible to mark tests as flaky and exclude them from normal runs --- pkg/testsuites/flaky.go | 8 ++++++++ pkg/testsuites/standard_suites.go | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 pkg/testsuites/flaky.go diff --git a/pkg/testsuites/flaky.go b/pkg/testsuites/flaky.go new file mode 100644 index 000000000000..5052a3c86122 --- /dev/null +++ b/pkg/testsuites/flaky.go @@ -0,0 +1,8 @@ +package testsuites + +// This file is not yet updated from an official query +var ( + flakyTestNames = map[string]struct{}{ + //`[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L3 primary UDN [Suite:openshift/conformance/parallel]`: {}, + } +) diff --git a/pkg/testsuites/standard_suites.go b/pkg/testsuites/standard_suites.go index 13894410bcb5..3e6cb3373eac 100644 --- a/pkg/testsuites/standard_suites.go +++ b/pkg/testsuites/standard_suites.go @@ -48,6 +48,10 @@ var staticSuites = []ginkgo.TestSuite{ Only the portion of the openshift/conformance test suite that run in parallel. `), Matches: func(name string) bool { + _, isFlaky := flakyTestNames[name] + if isFlaky { + return false + } if isDisabled(name) { return false } @@ -56,6 +60,21 @@ var staticSuites = []ginkgo.TestSuite{ Parallelism: 30, MaximumAllowedFlakes: 15, }, + { + Name: "openshift/conformance/ocp-flaky", + Description: templates.LongDesc(` + Our flaky tests plus the portion of the openshift/conformance test suite that run in parallel. + `), + // same as Matches from "openshift/conformance/parallel" without the flakyTestName check + Matches: func(name string) bool { + if isDisabled(name) { + return false + } + return strings.Contains(name, "[Suite:openshift/conformance/parallel") + }, + Parallelism: 30, + MaximumAllowedFlakes: 0, + }, { Name: "openshift/conformance/serial", Description: templates.LongDesc(`