Switch test environments to cfd bosh lites [v8]

* Switch environment to cfd-bosh-lite
* Turn off set_kernel_parameters to use cfd-bosh-lite
* Expose shepherd configuration via repo variables
* Update secrets path to bosh-lite
* Expose TEST_FLAKE_ATTEMPTS and TEST_NODES
* Set default FLAKE_ATTEMPTS
* Bump gha test suit timeout from 60 to 120m
* Enter nodes as workflow_dispatch input
* Skip steps if SHEPHERD_LEASE_ID defined
* Skip scaling tests that are testing CAPI
* bosh-light environment default memory is 256Mb
* Fixed the issue with force update of bosh
* Delete created orgs during tests
* Ensure that the orgName is empty for each test
* Prevent double deletion of org
* Lower number of tests running using client credentials to avoid flakes
* Allow user to define namespace of the leased environment
* Pass namespace of the lease to the reusable workflow

Co-authored-by: João Pereira <joao.pereira@broadcom.com>

Author: a-b

.github/ops-files/diego-cell-instances.yml

@@ -2,3 +2,7 @@
 - type: replace
   path: /instance_groups/name=diego-cell/instances
   value: 4
+
+- type: replace
+  path: /instance_groups/name=isolated-diego-cell/jobs/name=rep/properties?/set_kernel_parameters
+  value: false

.github/workflows/tests-integration-reusable.yml

@@ -3,7 +3,7 @@
 # secrets.CLIENT_SECRET
 # secrets.GITHUB_TOKEN
 # secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN
-# vars.SHEPHERD_POOL_NAME
+# vars.TEST_FLAKE_ATTEMPTS
 
 name: "pvt: run integration tests"
 
@@ -22,17 +22,29 @@ on:
       lease-id:
         required: true
         type: string
+      lease-namespace:
+        required: false
+        type: string
+        default: 'tas-devex'
       gitRef:
         type: string
         default: ${{github.event.workflow_run.head_sha}}
-        
+      nodes:
+        type: string
+        default: "16"
+
+env:
+  NODES: ${{ inputs.nodes }}
+  FLAKE_ATTEMPTS: ${{ vars.TEST_FLAKE_ATTEMPTS || '4' }}
+
 jobs:
   run-integration-tests:
     defaults:
       run:
         shell: bash
-    runs-on: ${{ inputs.os }} 
+    runs-on: ${{ inputs.os }}
     container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest
+    timeout-minutes: 120
     steps:
     - name: Checkout cli
       uses: actions/checkout@v4
@@ -61,7 +73,9 @@ jobs:
 
     - name: Install Tools
       env:
-        account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
+        account_token:    ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
+        lease_namespace:  ${{ inputs.lease-namespace }}
+        lease_id:         ${{ inputs.lease-id }}
       run: |
         go version
 
@@ -95,22 +109,22 @@ jobs:
         apt-get install -y build-essential unzip
 
         shepherd login service-account ${account_token}
-        shepherd get lease ${{ inputs.lease-id }} --namespace tas-devex --json | jq .output > metadata.json
+        shepherd get lease ${lease_id} --namespace ${lease_namespace} --json | jq .output > metadata.json
 
     - name: Add CATS config
       if: ${{ inputs.name == 'cats' }}
       run: |
         set -eu
-        ENV=$(jq -r .name metadata.json)
+        env_name=$(jq -r .name metadata.json)
         API="$(jq -r .cf.api_url metadata.json)"
         DOMAIN=$(echo $API | sed "s/^api\.//")
         CF_INT_USERNAME="admin"
 
-        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv
+        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
         eval "$(bbl print-env --metadata-file ./metadata.json)"
 
         credhub login
-        CF_INT_PASSWORD=$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)
+        CF_INT_PASSWORD=$(credhub get -n /bosh-lite/cf/cf_admin_password | bosh interpolate --path /value -)
 
         cat << EOF | jq -S . > cats_config.json
         {
@@ -184,26 +198,24 @@ jobs:
           --keep-going \
           --randomize-all \
           --skip-package=helpers \
-          --nodes="12" \
-          --flake-attempts=2 \
+          --nodes="${NODES}" \
+          --flake-attempts=${FLAKE_ATTEMPTS} \
           --timeout="2h" \
           --no-color
 
     - name: Run Integration Tests
       if: ${{ !inputs.run-with-client-creds  &&  inputs.name != 'cats' }}
       run: |
-        ENV=$(cat metadata.json | jq -r '.name')
-        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv
+        env_name=$(cat metadata.json | jq -r '.name')
+        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
         eval "$(bbl print-env --metadata-file ./metadata.json)"
-        export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)"
+        export CF_INT_PASSWORD="$(credhub get -n /bosh-lite/cf/cf_admin_password | bosh interpolate --path /value -)"
         export CF_INT_OIDC_USERNAME="admin-oidc"
-        export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
+        export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-lite/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
         API_URL="$(jq -r .cf.api_url metadata.json)"
         export CF_INT_API="https://$API_URL"
         export CF_DIAL_TIMEOUT=15
         export CF_USERNAME=admin
-        export FLAKE_ATTEMPTS=2
-        export NODES=16
         export GOPATH=$PWD/go
         export PATH="$GOPATH/bin:$PATH"
         export PATH="$PWD/out:$PATH"
@@ -223,18 +235,16 @@ jobs:
         CF_INT_CLIENT_ID: 'potato-face'
         CF_INT_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
       run: |
-        ENV=$(cat metadata.json | jq -r '.name')
-        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${ENV}.priv
+        env_name=$(cat metadata.json | jq -r '.name')
+        jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
         eval "$(bbl print-env --metadata-file ./metadata.json)"
-        export CF_INT_PASSWORD="$(credhub get -n /bosh-$ENV/cf/cf_admin_password | bosh interpolate --path /value -)"
+        export CF_INT_PASSWORD="$(credhub get -n /bosh-lite/cf/cf_admin_password | bosh interpolate --path /value -)"
         export CF_INT_OIDC_USERNAME="admin-oidc"
-        export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-$ENV/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
+        export CF_INT_OIDC_PASSWORD=$(credhub get -n /bosh-lite/cf/uaa_oidc_admin_password | bosh interpolate --path /value -)
         API_URL="$(jq -r .cf.api_url metadata.json)"
         export CF_INT_API="https://$API_URL"
         export CF_DIAL_TIMEOUT=15
         export CF_USERNAME=admin
-        export FLAKE_ATTEMPTS=2
-        export NODES=16
         export GOPATH=$PWD/go
         export PATH="$GOPATH/bin:$PATH"
         export PATH="$PWD/out:$PATH"
@@ -246,4 +256,4 @@ jobs:
         cf api ${CF_INT_API} --skip-ssl-validation
         cf auth
 
-        make integration-tests-full-ci
+        make integration-tests-ci-client-creds

.github/workflows/tests-integration.yml

@@ -1,3 +1,14 @@
+# GitHub repo level Secrets and Variables
+
+# secrets.CLIENT_SECRET
+# secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN
+# vars.CAPI_RELEASE_VERSION
+# vars.SHEPHERD_LEASE_DURATION
+# vars.SHEPHERD_LEASE_NAMESPACE
+# vars.SHEPHERD_TEMPLATE_ARGUMENT
+# vars.SHEPHERD_TEMPLATE_NAME
+# vars.SHEPHERD_TEMPLATE_NAMESPACE
+
 name: "Tests: Integration"
 
 run-name: "Integration [${{ github.event_name }}: ${{ github.event.pull_request.head.sha || github.event.push.after || github.event.workflow_run.head_sha}}]: ${{ github.event.workflow_run.head_commit.message }}"
@@ -14,6 +25,30 @@ on:
           - run-integration-tests-cf-env
           - run-integration-tests-cf-env-with-client-creds
           - run-cats-cf-env
+      nodes:
+        description: Number of test nodes
+        required: false
+        type: string
+        default: "12"
+      lease_id:
+        description: Pre-provisioned environment lease-id to use in tests
+        required: false
+        type: string
+      lease_namespace:
+        description: Pre-provisioned environment lease namespace to use in tests
+        required: false
+        type: string
+      run_unit_tests:
+        description: Run unit tests
+        required: false
+        type: boolean
+        default: true
+      reinstall_cfd:
+        description: Force re-installation of CFD
+        required: false
+        type: boolean
+        default: true
+
   push:
     tags:
       - "v8.*"
@@ -27,6 +62,10 @@ on:
       - "doc/**"
       - ".gitpod.yml"
       - "README.md"
+
+env:
+  SHEPHERD_LEASE_ID: ${{ inputs.lease_id }}
+
 jobs:
   get-sha:
     runs-on: ubuntu-latest
@@ -53,14 +92,17 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        if: ${{ inputs.run_unit_tests == 'true' }}
         with:
           ref: ${{needs.get-sha.outputs.gitRef}}
       - name: Set Up Go
         uses: actions/setup-go@v5
+        if: ${{ inputs.run_unit_tests == 'true' }}
         with:
           go-version-file: go.mod
           check-latest: true
       - name: Run Units
+        if: ${{ inputs.run_unit_tests == 'true' }}
         run: make units
 
   claim-env:
@@ -84,23 +126,57 @@ jobs:
       - name: claim
         id: claim
         env:
-          account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
-          pool_name: ${{ vars.SHEPHERD_POOL_NAME }}
-          pool_namespace: official
+          account_token:       ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
+          template_argument:   ${{ vars.SHEPHERD_TEMPLATE_ARGUMENT        }}
+          template_name:       ${{ vars.SHEPHERD_TEMPLATE_NAME       || 'cfd-bosh-lite@1.0' }}
+          template_namespace:  ${{ vars.SHEPHERD_TEMPLATE_NAMESPACE  || 'official'          }}
+          lease_duration:      ${{ vars.SHEPHERD_LEASE_DURATION      || '8h'                }}
+          lease_namespace:     ${{ inputs.lease_namespace  || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
         run: |
           shepherd login service-account ${account_token}
 
-          echo "shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --description 'CLI GHA'"
-          lease_id=$(shepherd create lease --duration 8h --pool ${pool_name} --pool-namespace ${pool_namespace} --namespace tas-devex --json | jq -r .id)
-          
+          if [[ -z $SHEPHERD_LEASE_ID ]]; then
+
+            if [ -z "$template_argument" ]; then
+              export template_argument='{"gcp_region":    "us-west2",
+                                         "vm_type":       "n1-standard-8",
+                                         "root_disk_gb":  32,
+                                         "disk_pool_gb":  150,
+                                         "cfd_version":   "",
+                                         "additional_opsfiles_b64": ""}'
+            fi
+
+            lease_id=$( shepherd create lease                           \
+                          --template-argument  "$template_argument"     \
+                          --template-namespace "${template_namespace}"  \
+                          --template           "${template_name}"       \
+                          --namespace          "${lease_namespace}"     \
+                          --duration           "${lease_duration}"      \
+                          --description "Claimed by CF CLI workflow ${{ github.workflow_run.url }}" \
+                          --json \
+                        | jq -r .id
+                      )
+          else
+            lease_id=$SHEPHERD_LEASE_ID
+          fi
+
+          echo "Shepherd lease ID: ${lease_id}"
+
           # Give sometime for the lease to complete. Shepherd may take upto an 3 hours to create an env
           # if the pool is empty.
           count=0
           while [ $count -lt 360 ] ; do
             sleep 30
-            status=$(shepherd get lease ${lease_id} --namespace tas-devex --json | jq -r .status)
+            status=$( shepherd get lease ${lease_id} \
+                        --namespace ${lease_namespace} \
+                        --json \
+                      | jq -r .status
+                    )
             if [ $status == "LEASED" ] ; then
-              shepherd get lease ${lease_id} --namespace tas-devex --json | jq .output > metadata.json
+              shepherd get lease ${lease_id} \
+                --namespace ${lease_namespace} \
+                --json \
+              | jq .output > metadata.json
               break
             elif [ $status == "FAILED" -o $status == "EXPIRED" ] ; then
               echo "There was an error obtaining the lease. Lease status is ${status}."
@@ -115,7 +191,7 @@ jobs:
           echo "env name is ${env_name}"
           echo "leaseid=${lease_id}" >> "${GITHUB_OUTPUT}"
 
-          cf_deployment_version=$(jq -r '."cf-deployment_version"' metadata.json)
+          cf_deployment_version=$(jq -r '."cf_deployment_version"' metadata.json)
           echo "cf_deployment_version is ${cf_deployment_version}"
           echo "cf_deployment_version=${cf_deployment_version}" >> "${GITHUB_OUTPUT}"
 
@@ -126,6 +202,7 @@ jobs:
           check-latest: true
 
       - name: Install Tools
+        if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
         run: |
           go version
 
@@ -145,11 +222,11 @@ jobs:
           apt-get install -y build-essential unzip
 
       - name: Upload latest CAPI release
+        if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
         env:
           capi_release_version: ${{ vars.CAPI_RELEASE_VERSION }}
         run: |
-          if [ -z "$capi_release_version" ]
-          then
+          if [ -z "$capi_release_version" ]; then
             capi_release_version=$(curl -s https://api.github.com/repos/cloudfoundry/capi-release/releases/latest | jq -r .tag_name)
           fi
           
@@ -162,19 +239,20 @@ jobs:
           bosh upload-release "https://bosh.io/d/github.com/cloudfoundry/capi-release?v=$capi_release_version"
 
       - name: Checkout cf-deployment
+        if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
         uses: actions/checkout@v4
         with:
           repository: cloudfoundry/cf-deployment
           path: cf-deployment
           ref: ${{steps.claim.outputs.cf_deployment_version}}    
 
       - name: Deploy Isolation Segment and OIDC Provider
+        if: ${{ (inputs.lease_id == '') || (inputs.reinstall_cfd == true) }}
         run: |
           env_name=$(jq -r .name metadata.json)
           jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv
           eval "$(bbl print-env --metadata-file metadata.json)"
 
-          # deploy
           bosh -d cf manifest > /tmp/manifest.yml
           bosh interpolate /tmp/manifest.yml \
             -o cf-deployment/operations/use-internal-lookup-for-route-services.yml \
@@ -204,6 +282,8 @@ jobs:
       name: Integration
       gitRef: ${{needs.get-sha.outputs.gitRef}}
       lease-id: ${{ needs.claim-env.outputs.leaseid }}
+      lease-namespace: ${{ inputs.lease_namespace  || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
+      nodes: ${{ inputs.nodes }}
     secrets: inherit
 
   run-integration-tests-cf-env-with-client-creds:
@@ -220,6 +300,7 @@ jobs:
       name: Integration client creds
       gitRef: ${{needs.get-sha.outputs.gitRef}}
       lease-id: ${{ needs.claim-env.outputs.leaseid }}
+      lease-namespace: ${{ inputs.lease_namespace  || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
     secrets: inherit
 
   run-cats-cf-env:
@@ -237,21 +318,24 @@ jobs:
       name: cats
       gitRef: ${{needs.get-sha.outputs.gitRef}}
       lease-id: ${{ needs.claim-env.outputs.leaseid }}
+      lease-namespace: ${{ inputs.lease_namespace  || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
     secrets: inherit
 
   unclaim-env:
     name: Unclaim environment
+    if: ${{ inputs.lease_id == '' }}
     runs-on: ubuntu-latest
     container: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource:latest
     needs:
       - claim-env
       - run-cats-cf-env
-    if: always()
     steps:
       - name: unclaim
         env:
-          account_token: ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
+          account_token:    ${{ secrets.SHEPHERD_SERVICE_ACCOUNT_TOKEN }}
+          lease_namespace:  ${{ inputs.lease_namespace || vars.SHEPHERD_LEASE_NAMESPACE || 'tas-devex' }}
         run: |
           shepherd login service-account ${account_token}
           set -x
-          shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} --namespace tas-devex
+          shepherd delete lease ${{ needs.claim-env.outputs.leaseid }} \
+            --namespace ${lease_namespace}

.gitignore

@@ -68,3 +68,5 @@ integration/assets/test_plugin/test_plugin
 
 ### VisualStudioCode ###
 .vscode
+.secrets
+.vars