feat(e2e): improve scorecard tests with real backend integration

.ibm/pipelines/auth/secrets-rhdh-secrets.yaml

@@ -42,4 +42,5 @@ data:
   GITHUB_OAUTH_APP_ID: $GITHUB_OAUTH_APP_ID_ENCODED
   GITHUB_OAUTH_APP_SECRET: $GITHUB_OAUTH_APP_SECRET_ENCODED
   BACKEND_SECRET: $BACKEND_SECRET
+  JIRA_TOKEN: $JIRA_TOKEN
 type: Opaque

.ibm/pipelines/env_variables.sh

@@ -90,6 +90,7 @@ QE_USER8_ID=$(cat /tmp/secrets/QE_USER8_ID)
 QE_USER8_PASS=$(cat /tmp/secrets/QE_USER8_PASS)
 QE_USER9_ID=$(cat /tmp/secrets/QE_USER9_ID)
 QE_USER9_PASS=$(cat /tmp/secrets/QE_USER9_PASS)
+JIRA_TOKEN=$(cat /tmp/secrets/jira_token)
 
 K8S_CLUSTER_TOKEN_TEMPORARY=$(cat /tmp/secrets/K8S_CLUSTER_TOKEN_TEMPORARY)
 

.ibm/pipelines/resources/config_map/app-config-rhdh-rbac.yaml

@@ -77,6 +77,25 @@ catalog:
   rules:
     - allow: [API, Component, Group, Location, Resource, System, Template]
   locations:
+    # Scorecard test entities
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/all-scorecards.yaml
+
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/jira-scorecard-only.yaml
+
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/github-scorecard-only.yaml
+
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/no-scorecards.yaml
+
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/metrics-unavailable.yaml
+
+    - type: url
+      target: https://github.com/janus-qe/RHDH-scorecard-plugin-test/blob/main/invalid-threshold.yaml
+
     - type: url
       target: https://github.com/redhat-developer/rhdh/blob/main/catalog-entities/all.yaml
     - type: url

.ibm/pipelines/value_files/values_showcase-rbac.yaml

@@ -257,39 +257,52 @@ global:
       - package: ./dynamic-plugins/dist/backstage-community-plugin-analytics-provider-segment
         disabled: true
       #Enable Scorecard plugin.
-      - disabled: false
-        package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard
+      - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard:bs_1.45.3__2.3.4!red-hat-developer-hub-backstage-plugin-scorecard
+        disabled: false
         pluginConfig:
           dynamicPlugins:
             frontend:
               red-hat-developer-hub.backstage-plugin-scorecard:
                 entityTabs:
                   - path: "/scorecard"
                     title: Scorecard
-                    titleKey: catalog.entityPage.scorecard.title
                     mountPoint: entity.page.scorecard
                 mountPoints:
                   - mountPoint: entity.page.scorecard/cards
                     importName: EntityScorecardContent
                     config:
                       layout:
                         gridColumn: 1 / -1
-      - disabled: false
-        package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend
-      - disabled: false
-        package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github
+                      if:
+                        allOf:
+                          - isKind: component
+      - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend:bs_1.45.3__2.3.4!red-hat-developer-hub-backstage-plugin-scorecard-backend
+        disabled: false
         pluginConfig:
-          integrations:
-            github:
-              - host: github.com
-                token: "{gh-token}"
-      - disabled: false
-        package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira:pr_1499__0.1.0!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira
+          scorecard:
+            plugins:
+              jira:
+                open_issues:
+                  schedule:
+                    frequency: { hours: 1 }
+                    timeout: { minutes: 15 }
+                    initialDelay: { minutes: 1 }
+              github:
+                open_prs:
+                  schedule:
+                    frequency: { hours: 1 }
+                    timeout: { minutes: 15 }
+                    initialDelay: { minutes: 1 }
+      - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github:bs_1.45.3__2.3.4!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-github
+        disabled: false
+      - package: oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira:bs_1.45.3__2.3.4!red-hat-developer-hub-backstage-plugin-scorecard-backend-module-jira
         pluginConfig:
           jira:
-            baseUrl: "{jira-base-url}"
-            token: "{jira-api-token}"
-            product: datacenter
+            baseUrl: https://redhat-team-f703ynt3.atlassian.net
+            token: ${JIRA_TOKEN}
+            product: cloud
+        disabled: false
+
       # Enable orchestrator plugins - Official release (Backstage 1.45.3)
       - package: "oci://ghcr.io/redhat-developer/rhdh-plugin-export-overlays/red-hat-developer-hub-backstage-plugin-orchestrator:bs_1.45.3__5.1.0!red-hat-developer-hub-backstage-plugin-orchestrator"
         disabled: false

e2e-tests/playwright/e2e/plugins/scorecard/scorecard.spec.ts

@@ -14,24 +14,16 @@
  * limitations under the License.
  */
 
-import { test, expect } from "@playwright/test";
+import { test } from "@playwright/test";
 import { Common } from "../../../utils/common";
-import { mockScorecardResponse } from "../../../utils/scorecard-utils";
-import { ComponentImportPage } from "../../../support/page-objects/scorecard/component-import-page";
 import { Catalog } from "../../../support/pages/catalog";
 import { ScorecardPage } from "../../../support/page-objects/scorecard/scorecard-page";
-import {
-  CUSTOM_SCORECARD_RESPONSE,
-  EMPTY_SCORECARD_RESPONSE,
-  UNAVAILABLE_METRIC_RESPONSE,
-  INVALID_THRESHOLD_RESPONSE,
-} from "../../../utils/scorecard-response-utils";
+import type { BrowserContext, Page } from "@playwright/test";
 
 test.describe.serial("Scorecard Plugin Tests", () => {
-  let context;
-  let page;
+  let context: BrowserContext;
+  let page: Page;
   let catalog: Catalog;
-  let importPage: ComponentImportPage;
   let scorecardPage: ScorecardPage;
 
   test.beforeAll(async ({ browser }, testInfo) => {
@@ -43,120 +35,78 @@ test.describe.serial("Scorecard Plugin Tests", () => {
     context = await browser.newContext();
     page = await context.newPage();
     catalog = new Catalog(page);
-    importPage = new ComponentImportPage(page);
     scorecardPage = new ScorecardPage(page);
     await new Common(page).loginAsKeycloakUser();
-
-    // Import the component here instead of the first tests so that they can re-run.
-    // It would be great if this would detect if the component is already imported.
-    await catalog.go();
-    await importPage.startComponentImport();
-    await importPage.analyzeComponent(
-      "https://github.com/rhdh-pai-qe/backstage-catalog/blob/main/catalog-info.yaml",
-    );
-    await importPage.viewImportedComponent();
   });
 
   test.afterAll(async () => {
     await context?.close();
   });
 
-  test("Import component and validate scorecard tabs for GitHub PRs and Jira tickets", async () => {
-    await mockScorecardResponse(page, CUSTOM_SCORECARD_RESPONSE);
-
+  test("Validate scorecard tabs for GitHub PRs and Jira tickets", async () => {
     await catalog.go();
-    await catalog.goToByName("rhdh-app");
+    await catalog.goToByName("all-scorecards");
     await scorecardPage.openTab();
 
-    await scorecardPage.verifyScorecardValues({
-      "GitHub open PRs": "9",
-      "Jira open blocking tickets": "8",
-    });
-
     for (const metric of scorecardPage.scorecardMetrics) {
       await scorecardPage.validateScorecardAriaFor(metric);
     }
   });
 
-  test("Display empty state when scorecard API returns no metrics", async () => {
-    await mockScorecardResponse(page, EMPTY_SCORECARD_RESPONSE);
-
+  test("Validate empty scorecard state", async () => {
     await catalog.go();
-    await catalog.goToByName("rhdh-app");
+    await catalog.goToByName("no-scorecards");
     await scorecardPage.openTab();
-
     await scorecardPage.expectEmptyState();
   });
 
   test("Displays error state for unavailable data while rendering metrics", async () => {
-    await mockScorecardResponse(page, UNAVAILABLE_METRIC_RESPONSE);
-
     await catalog.go();
-    await catalog.goToByName("rhdh-app");
+    await catalog.goToByName("unavailable-metric-service");
     await scorecardPage.openTab();
 
-    const jiraMetric = scorecardPage.scorecardMetrics[1];
-    const githubMetric = scorecardPage.scorecardMetrics[0];
+    const [githubMetric, jiraMetric] = scorecardPage.scorecardMetrics;
+
+    await scorecardPage.expectScorecardVisible(githubMetric.title);
+    await scorecardPage.expectScorecardVisible(jiraMetric.title);
+    await scorecardPage.expectErrorHeading("Metric data unavailable");
+    await scorecardPage.validateScorecardAriaFor(jiraMetric);
+  });
 
-    const isJiraVisible = await scorecardPage.isScorecardVisible(
-      jiraMetric.title,
-    );
-    expect(isJiraVisible).toBe(true);
+  test("Validate only GitHub scorecard is displayed", async () => {
+    await catalog.go();
+    await catalog.goToByName("github-scorecard-only");
+    await scorecardPage.openTab();
 
-    const isGithubVisible = await scorecardPage.isScorecardVisible(
-      githubMetric.title,
-    );
-    expect(isGithubVisible).toBe(true);
+    const [githubMetric, jiraMetric] = scorecardPage.scorecardMetrics;
 
-    const errorLocator = page.getByRole("heading", {
-      name: "Metric data unavailable",
-    });
-    await expect(errorLocator).toBeVisible();
+    await scorecardPage.expectScorecardVisible(githubMetric.title);
+    await scorecardPage.expectScorecardHidden(jiraMetric.title);
+    await scorecardPage.validateScorecardAriaFor(githubMetric);
+  });
 
-    await errorLocator.hover();
-    const errorTooltip = UNAVAILABLE_METRIC_RESPONSE.find(
-      (metric) => metric.id === "github.open-prs",
-    )?.error;
+  test("Validate only Jira scorecard is displayed", async () => {
+    await catalog.go();
+    await catalog.goToByName("jira-scorecard-only");
+    await scorecardPage.openTab();
 
-    expect(errorTooltip).toBeTruthy();
-    await expect(page.getByText(errorTooltip!)).toBeVisible();
+    const [githubMetric, jiraMetric] = scorecardPage.scorecardMetrics;
 
+    await scorecardPage.expectScorecardHidden(githubMetric.title);
+    await scorecardPage.expectScorecardVisible(jiraMetric.title);
     await scorecardPage.validateScorecardAriaFor(jiraMetric);
   });
 
   test("Display error state for invalid threshold config while rendering metrics", async () => {
-    await mockScorecardResponse(page, INVALID_THRESHOLD_RESPONSE);
-
     await catalog.go();
-    await catalog.goToByName("rhdh-app");
+    await catalog.goToByName("invalid-threshold");
     await scorecardPage.openTab();
 
-    const githubMetric = scorecardPage.scorecardMetrics[0];
-    const jiraMetric = scorecardPage.scorecardMetrics[1];
-
-    const isGithubVisible = await scorecardPage.isScorecardVisible(
-      githubMetric.title,
-    );
-    expect(isGithubVisible).toBe(true);
-
-    const isJiraVisible = await scorecardPage.isScorecardVisible(
-      jiraMetric.title,
-    );
-    expect(isJiraVisible).toBe(true);
-
-    const errorLocator = page.getByRole("heading", {
-      name: "Invalid thresholds",
-    });
-    await expect(errorLocator).toBeVisible();
-
-    await errorLocator.hover();
-    const errorTooltip = INVALID_THRESHOLD_RESPONSE.find(
-      (metric) => metric.id === "github.open-prs",
-    )?.result?.thresholdResult?.error;
-
-    expect(errorTooltip).toBeTruthy();
-    await expect(page.getByText(errorTooltip!)).toBeVisible();
+    const [githubMetric, jiraMetric] = scorecardPage.scorecardMetrics;
 
+    await scorecardPage.expectScorecardVisible(githubMetric.title);
+    await scorecardPage.expectScorecardVisible(jiraMetric.title);
+    await scorecardPage.expectErrorHeading("Invalid thresholds");
     await scorecardPage.validateScorecardAriaFor(jiraMetric);
   });
 });