[CDAPI-100]: Integrate CI Tests with APIM proxy

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "pathology-api-build-container",
-  "build":{
+  "build": {
     "dockerfile": "../infrastructure/images/build-container/Dockerfile",
     "args": {
       "INCLUDE_DEV_CERTS": "true"
@@ -38,13 +38,17 @@
       "omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions.git https://github.com/zsh-users/zsh-syntax-highlighting.git"
     }
   },
-  "postCreateCommand": "doas apk add --no-cache openjdk21-jre && scripts/devcontainer/configure-zsh.sh && bash -c 'source ~/.bashrc && make config && pyenv activate pathology && make dependencies'",
   "initializeCommand": "scripts/devcontainer/create-docker-network-if-required.sh && cp .tool-versions ${localWorkspaceFolder}/infrastructure/images/build-container/resources/.tool-versions",
+  "postCreateCommand": "doas apk add --no-cache openjdk21-jre && scripts/devcontainer/configure-zsh.sh && bash -c 'source ~/.bashrc && make config && pyenv activate pathology && make dependencies'",
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=cached"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=cached",
+    "source=/home/${localEnv:USER}/.proxygen,target=/home/pathology-dev/.proxygen,type=bind,consistency=cached"
   ],
   "remoteUser": "pathology-dev",
   "containerUser": "pathology-dev",
+  "containerEnv": {
+    "REQUESTS_CA_BUNDLE": "/etc/ssl/cert.pem"
+  },
   "portsAttributes": {
     // Disabling automatic port forwarding as the devcontainer should already have access to any required ports.
     "default": {

.github/actions/proxy/generate-apigee-token/action.yaml

@@ -0,0 +1,43 @@
+name: Generate Apigee Token
+description: Generate the Apigee Token needed to run the tests
+
+inputs:
+  proxygen-key-secret:
+    description: 'Proxygen private key secret'
+    required: true
+  proxygen-key-id:
+    description: 'Proxygen key ID'
+    required: true
+  proxygen-client-id:
+    description: 'Proxygen client ID'
+    required: true
+  proxygen-api-name:
+    description: 'Proxygen API name'
+    required: true
+outputs:
+  apigee-access-token:
+    description: "The token needed to use apigee in our testing"
+    value: ${{ steps.retrieve_apigee_token.outputs.apigee-access-token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Configure Proxygen
+      uses: ./.github/actions/proxy/configure-proxygen
+      with:
+        proxygen-key-secret: ${{ inputs.proxygen-key-secret }}
+        proxygen-key-id: ${{ inputs.proxygen-key-id }}
+        proxygen-client-id: ${{ inputs.proxygen-client-id }}
+        proxygen-api-name: ${{ inputs.proxygen-api-name }}
+
+    - id: retrieve_apigee_token
+      name: Retrieve token
+      shell: bash
+      run: |
+        TOKEN=$(proxygen pytest-nhsd-apim get-token | jq -r '.pytest_nhsd_apim_token')
+        if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
+          echo "::error::Failed to retrieve Apigee token"
+          exit 1
+        fi
+        echo "apigee-access-token=$TOKEN" >> $GITHUB_OUTPUT
+        echo "Token retrieved successfully (length: ${#TOKEN})"

.github/actions/retrieve-apigee-token/action.yaml

@@ -0,0 +1,49 @@
+name: Retrieve Apigee Token from Secrets Manager
+description: Configure AWS credentials and retrieve the Apigee token from AWS Secrets Manager
+
+inputs:
+  secret-name:
+    description: 'The name of the secret in AWS Secrets Manager'
+    required: true
+  aws-role-arn:
+    description: 'AWS IAM role ARN to assume'
+    required: true
+  aws-region:
+    description: 'AWS region'
+    required: true
+    default: 'eu-west-2'
+
+outputs:
+  apigee-access-token:
+    description: "The Apigee access token retrieved from Secrets Manager"
+    value: ${{ steps.set-token.outputs.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@a7a2c1125c67f40a1e95768f4e4a7d8f019f87af
+      with:
+        role-to-assume: ${{ inputs.aws-role-arn }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Retrieve token from Secrets Manager
+      uses: aws-actions/aws-secretsmanager-get-secrets@a9a7eb4e2f2871d30dc5b892576fde60a2ecc802
+      with:
+        secret-ids: ${{ inputs.secret-name }}
+        parse-json-secrets: false
+
+    - name: Set token output
+      id: set-token
+      shell: bash
+      run: |
+        SECRET_NAME="${{ inputs.secret-name }}"
+        # AWS action creates env var with secret name (hyphens→underscores, lowercase→uppercase)
+        ENV_VAR_NAME=$(echo "$SECRET_NAME" | tr '-' '_' | tr '[:lower:]' '[:upper:]')
+        TOKEN_VALUE="${!ENV_VAR_NAME}"
+        if [ -z "$TOKEN_VALUE" ]; then
+          echo "::error::Failed to retrieve token from Secrets Manager"
+          exit 1
+        fi
+        echo "token=$TOKEN_VALUE" >> $GITHUB_OUTPUT
+        echo "Token retrieved successfully from Secrets Manager"

.github/actions/setup-python-project/action.yaml

@@ -7,6 +7,11 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Install system dependencies
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libxml2-dev libxslt-dev
     - name: "Set up Python"
       uses: actions/setup-python@v6
       with:

.github/workflows/preview-env.yaml

@@ -23,6 +23,9 @@ env:
   PROXYGEN_KEY_ID: ${{ vars.PREVIEW_ENV_PROXYGEN_KEY_ID }}
   PROXYGEN_CLIENT_ID: ${{ vars.PREVIEW_ENV_PROXYGEN_CLIENT_ID }}
   PROXYGEN_API_NAME: ${{ vars.PROXYGEN_API_NAME }}
+  BASE_URL: 'https://internal-dev.api.service.nhs.uk/${{ vars.PROXYGEN_API_NAME }}-pr-${{ github.event.pull_request.number }}'
+  ENV: "remote"
+  PR_NUMBER: ${{ github.event.pull_request.number }}
 
 jobs:
   pr-preview:
@@ -236,14 +239,135 @@ jobs:
           proxygen-client-id: ${{ env.PROXYGEN_CLIENT_ID }}
           proxygen-api-name: ${{ env.PROXYGEN_API_NAME }}
 
+      - name: Retrieve Apigee Token
+        id: apigee-token
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          APIGEE_TOKEN="$(proxygen pytest-nhsd-apim get-token | jq -r '.pytest_nhsd_apim_token' 2>/dev/null)"
+          if [ -z "$APIGEE_TOKEN" ] || [ "$APIGEE_TOKEN" = "null" ]; then
+            echo "::error::Failed to retrieve Apigee token"
+            exit 1
+          fi
+
+          echo "::add-mask::$APIGEE_TOKEN"
+          echo "mask-test: $APIGEE_TOKEN"
+          printf 'apigee-access-token=%s\n' "$APIGEE_TOKEN" >> "$GITHUB_OUTPUT"
+          echo "Token retrieved successfully (length: ${#APIGEE_TOKEN})"
+
+      - name: "Create coverage artefact name"
+        id: create-name
+        uses: ./.github/actions/create-artefact-name
+        with:
+          prefix: coverage
+
+      - name: "Run contract tests"
+        run: make test-contract
+      - name: "Upload contract test results"
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: contract-test-results
+          path: pathology-api/test-artefacts/
+          retention-days: 30
+      - name: "Publish contract test results to summary"
+        if: always()
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
+        with:
+          paths: pathology-api/test-artefacts/contract-tests.xml
+
+      - name: "Run schema validation tests"
+        run: make test-schema
+      - name: "Upload schema test results"
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: schema-test-results
+          path: pathology-api/test-artefacts/
+          retention-days: 30
+      - name: "Publish schema test results to summary"
+        if: always()
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
+        with:
+          paths: pathology-api/test-artefacts/schema-tests.xml
+
+      - name: "Run integration test"
+        run: make test-integration
+      - name: "Upload integration test results"
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: integration-test-results
+          path: pathology-api/test-artefacts/
+          retention-days: 30
+      - name: "Publish integration test results to summary"
+        if: always()
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
+        with:
+          paths: pathology-api/test-artefacts/integration-tests.xml
+
+      - name: "Run acceptance test"
+        run: make test-acceptance
+      - name: "Upload acceptance test results"
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: acceptance-test-results
+          path: pathology-api/test-artefacts/
+          retention-days: 30
+      - name: "Publish acceptance test results to summary"
+        if: always()
+        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
+        with:
+          paths: pathology-api/test-artefacts/acceptance-tests.xml
+
+      - name: "Download all test coverage artefacts"
+        uses: actions/download-artifact@v7
+        with:
+          path: pathology-api/test-artefacts/
+          merge-multiple: false
+      - name: "Merge coverage data"
+        run: make test-coverage
+      - name: "Rename coverage XML with unique name"
+        run: |
+          cd pathology-api/test-artefacts
+          mv coverage-merged.xml ${{ steps.create-name.outputs.artefact-name }}.xml
+      - name: "Upload combined coverage report"
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: ${{ steps.create-name.outputs.artefact-name }}
+          path: pathology-api/test-artefacts
+          retention-days: 30
+
+      - name: "Checkout code"
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          fetch-depth: 0 # Full history is needed for better analysis
+      - name: "Download merged coverage report"
+        uses: actions/download-artifact@v7
+        with:
+          name: ${{ steps.create-name.outputs.artefact-name }}
+          path: coverage-reports/
+      - name: "SonarCloud Scan"
+        uses: SonarSource/sonarqube-scan-action@v7
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          args: >
+            -Dsonar.organization=${{ vars.SONAR_ORGANISATION_KEY }}
+            -Dsonar.projectKey=${{ vars.SONAR_PROJECT_KEY }}
+            -Dsonar.python.coverage.reportPaths=coverage-reports/${{ steps.create-name.outputs.artefact-name }}.xml
+
       - name: Comment function name on PR
         if: github.event_name == 'pull_request' && github.event.action != 'closed'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd
         with:
           script: |
             const fn = '${{ steps.names.outputs.function_name }}';
             const url = '${{ steps.names.outputs.preview_url }}';
-            const proxy_url = 'https://internal-dev.api.service.nhs.uk/${{ env.PROXYGEN_API_NAME }}-pr-${{ github.event.pull_request.number }}';
+            const proxy_url = '${{ env.BASE_URL }}';
             const owner = context.repo.owner;
             const repo = context.repo.repo;
             const issueNumber = context.issue.number;

.github/workflows/stage-1-commit.yaml

@@ -88,15 +88,6 @@ jobs:
           fetch-depth: 0 # Full history is needed to compare branches
       - name: "Check English usage"
         uses: ./.github/actions/check-english-usage
-  lint-terraform:
-    name: "Lint Terraform"
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@v6
-      - name: "Lint Terraform"
-        uses: ./.github/actions/lint-terraform
   count-lines-of-code:
     name: "Count lines of code"
     runs-on: ubuntu-latest

.github/workflows/stage-2-test.yaml

@@ -3,6 +3,7 @@ name: "Test stage"
 env:
   BASE_URL: "http://localhost:5002"
   HOST: "localhost"
+  ENV: "local"
 
 on:
   workflow_call:
@@ -143,40 +144,9 @@ jobs:
         with:
           paths: pathology-api/test-artefacts/integration-tests.xml
 
-  test-acceptance:
-    name: "Acceptance tests"
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - name: "Checkout code"
-        uses: actions/checkout@v6
-      - name: "Setup Python project"
-        uses: ./.github/actions/setup-python-project
-        with:
-          python-version: ${{ inputs.python_version }}
-      - name: "Start local Lambda"
-        uses: ./.github/actions/start-local-lambda
-        with:
-          python-version: ${{ inputs.python_version }}
-          max-seconds: 90
-      - name: "Run acceptance test"
-        run: make test-acceptance
-      - name: "Upload acceptance test results"
-        if: always()
-        uses: actions/upload-artifact@v6
-        with:
-          name: acceptance-test-results
-          path: pathology-api/test-artefacts/
-          retention-days: 30
-      - name: "Publish acceptance test results to summary"
-        if: always()
-        uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
-        with:
-          paths: pathology-api/test-artefacts/acceptance-tests.xml
-
   merge-test-coverage:
     name: "Merge test coverage"
-    needs: [create-coverage-name, test-unit, test-contract, test-schema, test-integration, test-acceptance]
+    needs: [create-coverage-name, test-unit, test-contract, test-schema, test-integration]
     runs-on: ubuntu-latest
     timeout-minutes: 2
     steps:

.gitleaksignore

@@ -6,3 +6,5 @@ cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:
 
 pathology-api/pyproject.toml:ipv4:51
 pathology-api/pyproject.toml:ipv4:50
+mocks/pyproject.toml:ipv4:54
+mocks/pyproject.toml:ipv4:55

.tool-versions

@@ -1,6 +1,5 @@
 # This file is for you! Please, updated to the versions agreed by your team.
 
-terraform 1.7.0
 pre-commit 3.6.0
 gitleaks 8.18.4
 
@@ -15,7 +14,6 @@ gitleaks 8.18.4
 # docker/ghcr.io/make-ops-tools/gocloc latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 # SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc
 # docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image
 # docker/hadolint/hadolint 2.12.0-alpine@sha256:7dba9a9f1a0350f6d021fb2f6f88900998a4fb0aaf8e4330aa8c38544f04db42 # SEE: https://hub.docker.com/r/hadolint/hadolint/tags
-# docker/hashicorp/terraform 1.12.2@sha256:b3d13c9037d2bd858fe10060999aa7ca56d30daafe067d7715b29b3d4f5b162f # SEE: https://hub.docker.com/r/hashicorp/terraform/tags
 # docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c # SEE: https://hub.docker.com/r/koalaman/shellcheck/tags
 # docker/mstruebing/editorconfig-checker 2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24 # SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags
 # docker/sonarsource/sonar-scanner-cli 10.0@sha256:0bc49076468d2955948867620b2d98d67f0d59c0fd4a5ef1f0afc55cf86f2079 # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags