Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 8 additions & 14 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ make build

# Set API URL and run tests
export HYPERFLEET_API_URL=https://api.hyperfleet.example.com
./bin/hyperfleet-e2e test --label-filter=critical
./bin/hyperfleet-e2e test --label-filter=tier0
```

**Done!** The framework created a cluster, validated adapters, and cleaned up resources.
Expand All @@ -29,17 +29,11 @@ export HYPERFLEET_API_URL=https://api.hyperfleet.example.com
### Filter by Labels

```bash
# Core critical path tests (PR gate)
./bin/hyperfleet-e2e test --label-filter="tier0 && stable"
# Critical severity tests (Release gate)
./bin/hyperfleet-e2e test --label-filter=tier0

# All stable tests (daily regression)
./bin/hyperfleet-e2e test --label-filter=stable

# Lifecycle tests only
./bin/hyperfleet-e2e test --label-filter=lifecycle

# Combined filters: Tier0/Tier1 stable tests
./bin/hyperfleet-e2e test --label-filter="(tier0 || tier1) && stable"
# Exclude slow tests
./bin/hyperfleet-e2e test --label-filter="!slow"
```

### Common Options
Expand Down Expand Up @@ -117,9 +111,9 @@ hyperfleet-e2e/
# Set API URL
export HYPERFLEET_API_URL=$CI_API_URL

# Run critical tests with JUnit output
# Run critical severity tests with JUnit output
./bin/hyperfleet-e2e test \
--label-filter=critical \
--label-filter=tier0 \
--junit-report=results.xml \
--log-format=json
```
Expand All @@ -131,7 +125,7 @@ make image
podman run --rm \
-e HYPERFLEET_API_URL=https://api.example.com \
quay.io/openshift-hyperfleet/hyperfleet-e2e:latest \
test --label-filter=critical
test --label-filter=tier0
```

## Development
Expand Down
69 changes: 37 additions & 32 deletions docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,44 +72,39 @@ import (
. "github.com/onsi/gomega"

"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi"
"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client"
"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/config"
"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
)

var testName = "[Suite: cluster] Create Cluster via API"

var _ = ginkgo.Describe(testName,
ginkgo.Label(labels.Lifecycle, labels.Critical, labels.HappyPath),
ginkgo.Label(labels.Tier0),
func() {
var h *helper.Helper
var clusterID string

ginkgo.BeforeEach(func() {
cfg, err := config.Load()
Expect(err).NotTo(HaveOccurred())
h, err = helper.New(cfg)
Expect(err).NotTo(HaveOccurred())
h = helper.New()
})

ginkgo.It("should create cluster successfully", func(ctx context.Context) {
ginkgo.By("submitting cluster creation request")
cluster, err := h.Client.CreateClusterFromPayload(ctx, "testdata/payloads/clusters/gcp.json")
Expect(err).NotTo(HaveOccurred())
clusterID = cluster.ID
clusterID = *cluster.Id

ginkgo.By("waiting for cluster to become Ready")
Eventually(func(g Gomega) {
cluster, err := h.Client.GetCluster(ctx, clusterID)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(cluster.Status.Phase).To(Equal(openapi.Ready))
}, h.Cfg.Timeouts.ClusterReady, h.Cfg.Timeouts.PollInterval).Should(Succeed())
err = h.WaitForClusterPhase(ctx, clusterID, openapi.Ready, h.Cfg.Timeouts.Cluster.Ready)
Expect(err).NotTo(HaveOccurred())
})

ginkgo.AfterEach(func(ctx context.Context) {
if !h.Cfg.KeepResources && clusterID != "" {
_ = h.Client.DeleteCluster(ctx, clusterID)
if h == nil || clusterID == "" {
return
}
if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
}
})
},
Expand All @@ -132,14 +127,13 @@ var lifecycleTestName = "[Suite: cluster] Full Cluster Creation Flow on GCP"

All tests must use labels for categorization. See `pkg/labels/labels.go` for complete definitions.

**Required labels (3)**:
- **Priority**: `Tier0` | `Tier1` | `Tier2`
- **Stability**: `Stable` | `Informing` | `Flaky`
- **Scenario**: `HappyPath` | `Negative` | `Scale`
**Required labels (1)**:
- **Severity**: `Tier0` | `Tier1` | `Tier2`

**Optional labels**:
- **Functionality**: `Lifecycle` | `Upgrade`
- **Constraint**: `Serial` | `Disruptive` | `Slow`
- **Scenario**: `Negative` | `Performance`
- **Functionality**: `Upgrade`
- **Constraint**: `Disruptive` | `Slow`

**Example**:

Expand All @@ -148,7 +142,17 @@ import "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"

var testName = "[Suite: cluster] Full Cluster Creation Flow on GCP"
var _ = ginkgo.Describe(testName,
ginkgo.Label(labels.Tier0, labels.Stable, labels.HappyPath, labels.Lifecycle),
ginkgo.Label(labels.Tier0),
func() { ... }
)
```

**Example with optional labels**:

```go
// Negative test case with slow execution
var _ = ginkgo.Describe(testName,
ginkgo.Label(labels.Tier1, labels.Negative, labels.Slow),
func() { ... }
)
```
Expand All @@ -157,15 +161,11 @@ var _ = ginkgo.Describe(testName,

```go
ginkgo.BeforeEach(func() {
cfg, err := config.Load()
Expect(err).NotTo(HaveOccurred())
h, err = helper.New(cfg)
Expect(err).NotTo(HaveOccurred())
h = helper.New()
})
```

- Load configuration
- Create Helper instance
- Create Helper instance (automatically loads configuration)
- Initialize test context

### 4. Test Steps with ginkgo.By
Expand All @@ -189,13 +189,18 @@ ginkgo.By("verifying adapter conditions")

```go
ginkgo.AfterEach(func(ctx context.Context) {
if !h.Cfg.KeepResources && clusterID != "" {
_ = h.Client.DeleteCluster(ctx, clusterID)
if h == nil || clusterID == "" {
return
}
if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
}
})
```

- Clean up resources after test
- Skip cleanup if helper not initialized or no cluster created
- Log cleanup failures as warnings

## Writing Assertions

Expand All @@ -212,7 +217,7 @@ Eventually(func(g Gomega) {
cluster, err := h.Client.GetCluster(ctx, clusterID)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(cluster.Status.Phase).To(Equal(openapi.Ready))
}, h.Cfg.Timeouts.ClusterReady, h.Cfg.Timeouts.PollInterval).Should(Succeed())
}, h.Cfg.Timeouts.Cluster.Ready, h.Cfg.Polling.Interval).Should(Succeed())
```

**Important**: Inside `Eventually` closures, use `g.Expect()` instead of `Expect()`
Expand All @@ -222,7 +227,7 @@ Eventually(func(g Gomega) {
### Wait for Cluster Ready

```go
err = h.WaitForClusterPhase(ctx, clusterID, openapi.Ready, h.Cfg.Timeouts.ClusterReady)
err = h.WaitForClusterPhase(ctx, clusterID, openapi.Ready, h.Cfg.Timeouts.Cluster.Ready)
Expect(err).NotTo(HaveOccurred())
```

Expand Down
2 changes: 1 addition & 1 deletion e2e/cluster/creation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
var lifecycleTestName = "[Suite: cluster][baseline] Full Cluster Creation Flow on GCP"

var _ = ginkgo.Describe(lifecycleTestName,
ginkgo.Label(labels.Tier0, labels.Stable, labels.HappyPath, labels.Lifecycle),
ginkgo.Label(labels.Tier0),
func() {
var h *helper.Helper
var clusterID string
Expand Down
2 changes: 1 addition & 1 deletion e2e/nodepool/creation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
var lifecycleTestName = "[Suite: nodepool] Full NodePool Creation Flow"

var _ = ginkgo.Describe(lifecycleTestName,
ginkgo.Label(labels.Tier0, labels.Stable, labels.HappyPath, labels.Lifecycle),
ginkgo.Label(labels.Tier0),
func() {
var h *helper.Helper
var clusterID string
Expand Down
28 changes: 13 additions & 15 deletions pkg/labels/labels.go
Original file line number Diff line number Diff line change
@@ -1,35 +1,33 @@
package labels

// Priority labels - Business value dimension: determines failure handling priority
// Severity labels - Business Impact Dimension: Describes the severity of failure if the functionality is broken
const (
Tier0 = "tier0" // Critical path: fix immediately, blocks merge
Tier1 = "tier1" // Important: non-critical but frequently used, fix within 24h
Tier2 = "tier2" // Edge case: low-frequency scenarios, run in scheduled jobs only
Tier0 = "tier0" // Critical: Core user journey broken, fix immediately, blocks release
Tier1 = "tier1" // Major: Important features affected, should be addressed
Tier2 = "tier2" // Minor: Edge cases or low-frequency scenarios, can be deferred
)

// Stability labels - Test quality dimension: determines CI gate policy
const (
Stable = "stable" // Production-ready: stable and reliable, must pass to merge (Blocking)
Informing = "informing" // Observation period: new test onboarding (Non-blocking)
Flaky = "flaky" // Known unstable: quarantined for investigation
)
// const (
// Stable = "stable" // Production-ready: stable and reliable, must pass to merge (Blocking)
// Informing = "informing" // Observation period: new test onboarding (Non-blocking)
// Flaky = "flaky" // Known unstable: quarantined for investigation
// )

// Scenario labels - Test path dimension: describes test design intent
const (
HappyPath = "happy-path" // Normal workflow: ideal path
Negative = "negative" // Error handling: edge cases and failure scenarios
Scale = "scale" // Performance: stress tests or large-scale resource scenarios
// HappyPath = "happy-path" // Normal workflow: ideal path
Negative = "negative" // Error handling: edge cases and failure scenarios
Performance = "perf" // Performance: stress tests or large-scale resource scenarios
)

// Functionality labels - Feature category dimension: describes test coverage target
const (
Lifecycle = "lifecycle" // Full lifecycle: Create -> Ready -> Delete
Upgrade = "upgrade" // Version compatibility: smooth upgrades
Upgrade = "upgrade" // Version compatibility: smooth upgrades
)

// Constraint labels - Execution constraint dimension: determines scheduling strategy
const (
Serial = "serial" // Must run serially: cannot run in parallel
Disruptive = "disruptive" // Destructive testing: fault injection
Slow = "slow" // Long-running: execution time exceeds 5-10 minutes
)
31 changes: 10 additions & 21 deletions pkg/labels/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,38 +4,27 @@ import "fmt"

// ValidateLabels verifies that tests contain all required label dimensions
func ValidateLabels(testLabels []string) error {
hasPriority := false
hasStability := false
hasScenario := false
hasSeverity := false

for _, label := range testLabels {
switch label {
// Priority dimension (required)
// Severity dimension (required)
case Tier0, Tier1, Tier2:
hasPriority = true
// Stability dimension (required)
case Stable, Informing, Flaky:
hasStability = true
// Scenario dimension (required)
case HappyPath, Negative, Scale:
hasScenario = true
hasSeverity = true
// Scenario dimension (optional)
case Negative, Performance:
// Optional, no validation needed
// Functionality dimension (optional)
case Lifecycle, Upgrade:
case Upgrade:
// Optional, no validation needed
// Constraint dimension (optional)
case Serial, Disruptive, Slow:
case Disruptive, Slow:
// Optional, no validation needed
}
}

if !hasPriority {
return fmt.Errorf("missing priority label (tier0/tier1/tier2)")
}
if !hasStability {
return fmt.Errorf("missing stability label (stable/informing/flaky)")
}
if !hasScenario {
return fmt.Errorf("missing scenario label (happy-path/negative/scale)")
if !hasSeverity {
return fmt.Errorf("missing severity label (%s/%s/%s)", Tier0, Tier1, Tier2)
}

return nil
Expand Down
17 changes: 5 additions & 12 deletions pkg/labels/validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestAllE2ETestsHaveRequiredLabels(t *testing.T) {
t.Errorf("Found %d test specs with missing required labels:\n - %s",
len(failures), strings.Join(failures, "\n - "))
} else {
t.Logf("✓ All %d test specs have required labels (Priority, Stability, Scenario)", len(specs))
t.Logf("✓ All %d test specs have required labels (Severity)", len(specs))
}
}

Expand Down Expand Up @@ -207,23 +207,16 @@ func constantToLabelValue(constName string) string {
// Map constant names to their string values
// This must match the constants in pkg/labels/labels.go
mapping := map[string]string{
// Priority
// Severity
"Tier0": labels.Tier0,
"Tier1": labels.Tier1,
"Tier2": labels.Tier2,
// Stability
"Stable": labels.Stable,
"Informing": labels.Informing,
"Flaky": labels.Flaky,
// Scenario
"HappyPath": labels.HappyPath,
"Negative": labels.Negative,
"Scale": labels.Scale,
"Negative": labels.Negative,
"Performance": labels.Performance,
// Functionality
"Lifecycle": labels.Lifecycle,
"Upgrade": labels.Upgrade,
"Upgrade": labels.Upgrade,
// Constraint
"Serial": labels.Serial,
"Disruptive": labels.Disruptive,
"Slow": labels.Slow,
}
Expand Down