Building and testing #91
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Test | |
| run-name: Building and testing | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release: | |
| required: false | |
| type: boolean | |
| default: false | |
| release-to-pypi: | |
| required: false | |
| type: boolean | |
| default: false | |
| push: | |
| jobs: | |
| params: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| oci_repository: ${{ steps.params.outputs.oci_repository }} | |
| oci_platforms: ${{ steps.params.outputs.oci_platforms }} | |
| gcp_project: ${{ steps.params.outputs.gcp_project }} | |
| gcp_service_account: ${{ steps.params.outputs.service_account }} | |
| workload_identity_provider: ${{ steps.params.outputs.idp }} | |
| steps: | |
| - name: params | |
| id: params | |
| run: | | |
| repo_base=europe-docker.pkg.dev/gardener-project | |
| snapshots_repo="${repo_base}/snapshots" | |
| releases_repo="${repo_base}/releases" | |
| if ${{ inputs.release || false }}; then | |
| # "erroneously" still publish to snapshots for testing purporses | |
| # XXX this is a debugging / testing artefact | |
| ocm_repository=${releases_repo} | |
| oci_repository=${releases_repo} | |
| else | |
| ocm_repository=${snapshots_repo} | |
| oci_repository=${snapshots_repo} | |
| fi | |
| echo "ocm_repository=${ocm_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_repository=${oci_repository}" >> "${GITHUB_OUTPUT}" | |
| echo "oci_platforms=linux/amd64,linux/arm64" >> "${GITHUB_OUTPUT}" | |
| echo "gcp_project=gardener-project" >> "${GITHUB_OUTPUT}" | |
| echo "service_account=ocm-ci-opensource@gardener-project.iam.gserviceaccount.com" \ | |
| >> "${GITHUB_OUTPUT}" | |
| echo "idp=projects/694386720375/locations/global/workloadIdentityPools/github-actions/providers/github-actions-provider" \ | |
| >> "${GITHUB_OUTPUT}" | |
| version: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| effective_version: ${{ steps.version.outputs.effective_version }} | |
| next_version: ${{ steps.version.outputs.next_version }} | |
| repo_version: ${{ steps.version.outputs.repo_version }} | |
| setuptools_version: ${{ steps.version.outputs.setuptools_version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: calculate-effective-version | |
| id: version | |
| run: | | |
| src_version=$(.ci/read-version) | |
| commit=${{ github.sha }} | |
| echo "commit-digest: ${commit}" | |
| major="$(echo ${src_version} | cut -d. -f1)" | |
| minor="$(echo ${src_version} | cut -d. -f2)" | |
| patch="$(echo ${src_version} | cut -d. -f3 | cut -d- -f1)" | |
| if ${{ inputs.release || false }}; then | |
| effective_version=${major}.${minor}.${patch} | |
| setuptools_version=${effective_version} | |
| # hardcode to bumping "minor" for now | |
| next_minor=$(( $minor + 1 )) | |
| next_version="${major}.${next_minor}.${patch}-dev" | |
| echo "next_version=${next_version}" >> "${GITHUB_OUTPUT}" | |
| echo "next_version=${next_version}" | |
| else | |
| effective_version=${major}.${minor}.${patch}-${commit} | |
| setuptools_version=${src_version} | |
| fi | |
| echo "effective-version: ${effective_version}" | |
| echo "effective_version=${effective_version}" >> "${GITHUB_OUTPUT}" | |
| echo "repo_version=${src_version}" >> "${GITHUB_OUTPUT}" | |
| echo "setuptools_version=${setuptools_version}" >> "${GITHUB_OUTPUT}" | |
| package: | |
| runs-on: ubuntu-latest | |
| environment: build | |
| outputs: | |
| ocm_resources: ${{ steps.package.outputs.ocm_resources }} | |
| needs: version | |
| container: | |
| image: python:alpine | |
| steps: | |
| - name: Install git, setuptools, node (for upload-artifact) | |
| run: | | |
| apk add \ | |
| bash \ | |
| file \ | |
| git \ | |
| nodejs \ | |
| xz | |
| pip3 install --root-user-action ignore \ | |
| setuptools \ | |
| pyyaml | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 # required by upload-artifact / `act` | |
| # see: https://github.com/nektos/act/issues/973 | |
| - name: create distribution package | |
| id: package | |
| run: | | |
| set -eu | |
| version=${{ needs.version.outputs.setuptools_version }} | |
| echo "version: ${version}" | |
| echo "${version}" | .ci/write-version | |
| pkg_dir=dist | |
| mkdir -p ${pkg_dir} | |
| pkg_dir="$(readlink -f dist)" | |
| echo "pkgdir: ${pkg_dir}" | |
| for path in \ | |
| setup.py \ | |
| setup.oci.py \ | |
| setup.whd.py \ | |
| ; do | |
| echo "building distribution package from ${path}" | |
| python3 ${path} \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| set -x | |
| rm -rf build | |
| set +x | |
| done | |
| # special-case: cli-package (need to chdir in order to not confuse setuptools) | |
| ( | |
| cd cli | |
| python3 setup.py \ | |
| bdist_wheel \ | |
| --dist-dir ${pkg_dir} | |
| ) | |
| echo "Built packages" | |
| ls "${pkg_dir}" | |
| blobs_dir="${pkg_dir}/blobs.d" | |
| mkdir ${blobs_dir} | |
| resources_file=resources.yaml | |
| for package in gardener-oci gardener-cicd-whd gardener-cicd-cli gardener-cicd-libs; do | |
| path="$(echo $package | tr - _)-*" | |
| prefix=dist | |
| access_type='localBlob' | |
| outf="${pkg_dir}/${package}.tar.gz" | |
| tar cJf ${outf} -C ${pkg_dir} $(cd ${pkg_dir}; ls ${path}) | |
| mimetype=$(file -i ${outf} | cut -d: -f2 | cut -d' ' -f2-) | |
| leng=$(stat -c"%s" ${outf}) | |
| digest="$(sha256sum ${outf} | cut -d' ' -f1)" | |
| echo "\ | |
| - name: ${package} | |
| version: ${version} | |
| type: ${mimetype} | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${digest} | |
| size: ${leng} | |
| mediaType: ${mimetype}" \ | |
| >> ${resources_file} | |
| mv ${outf} ${blobs_dir}/${digest} | |
| done | |
| cp "${resources_file}" dist/ocm_resources.yaml | |
| echo "ocm_resources=dist/ocm_resources.yaml" >> "${GITHUB_OUTPUT}" | |
| find "${pkg_dir}" | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: dist/ | |
| component_descriptor: | |
| name: OCM + Release (only on manual triggering) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| permissions: | |
| contents: write | |
| id-token: write | |
| needs: | |
| - version | |
| - params | |
| - package | |
| - images | |
| - lint | |
| - unittests | |
| outputs: | |
| release_commit_digest: ${{ steps.releasecommit.commitdigest }} | |
| ocm_repository: ${{ steps.params.outputs.ocm_repository }} | |
| steps: | |
| - name: Install Packages | |
| run: | | |
| apk add --no-cache \ | |
| bash \ | |
| git | |
| git config --global --add safe.directory /__w/cc-utils/cc-utils | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: Retrieve Linting Logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linting-logs # targetpath: bandit.tar.gz | |
| path: /tmp/linting-logs | |
| - name: Google-Auth | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| token_format: access_token | |
| project_id: ${{ needs.params.outputs.gcp_project }} | |
| service_account: ${{ needs.params.outputs.gcp_service_account }} | |
| workload_identity_provider: ${{ needs.params.outputs.workload_identity_provider }} | |
| - name: Create Release and Bump-Commits | |
| id: releasecommit | |
| if: ${{ inputs.release }} | |
| run: | | |
| echo "gha-creds*" >> .git/info/exclude | |
| git config --global --add safe.directory /__w/cc-utils/cc-utils | |
| git config user.name 'Gardener-CICD Bot' | |
| git config user.email 'gardener.ci.user@gmail.com' | |
| effective_version=${{ needs.version.outputs.effective_version }} | |
| echo "${effective_version}" | .ci/write-version | |
| git add . | |
| git commit -m "Release ${effective_version}" | |
| git show | |
| commit_digest=$(git rev-parse @) | |
| echo "release_commit_digest=${commit_digest}" >> "${GITHUB_OUTPUT}" | |
| tgt_ref="refs/tags/${effective_version}" | |
| echo "pushing release-commit ${commit_digest} to ${tgt_ref}" | |
| git push origin "@:${tgt_ref}" | |
| next_version=${{ needs.version.outputs.next_version }} | |
| echo "next version: ${next_version}" | |
| git reset --hard @~ | |
| echo "${next_version}" | .ci/write-version | |
| git add . | |
| git commit -m"Prepare next Dev-Cycle" | |
| git pull --rebase | |
| git push origin | |
| - name: component-descriptor | |
| run: | | |
| set -eu | |
| pip3 install --no-cache \ | |
| aiohttp \ | |
| dacite \ | |
| deprecated \ | |
| pyaml \ | |
| python-dateutil \ | |
| requests \ | |
| www-authenticate \ | |
| &>/dev/null | |
| python -c "import oci" | |
| version=${{ needs.version.outputs.effective_version }} | |
| ocm_repo=${{ needs.params.outputs.ocm_repository }} | |
| echo "generating component-descriptor" | |
| python3 -m ocm create \ | |
| --name github.com/${{ github.repository }} \ | |
| --version ${version} \ | |
| --ocm-repo ${ocm_repo} \ | |
| --provider sap-se \ | |
| --label '{ | |
| "name": "cloud.gardener.cnudie/responsibles", | |
| "value": [ | |
| { | |
| "type": "githubTeam", | |
| "teamname": "gardener/ci-maintainers", | |
| "github_hostname": "github.com" | |
| } | |
| ] | |
| }' \ | |
| > component-descriptor.yaml | |
| if ${{ inputs.release || false }}; then | |
| commit_digest=${{ steps.releasecommit.outputs.release_commit_digest || '' }} | |
| else | |
| commit_digest=${{ github.sha }} | |
| fi | |
| echo "adding main source" | |
| echo "\ | |
| name: main-source | |
| version: ${version} | |
| type: git | |
| labels: | |
| - name: cloud.gardener/cicd/source | |
| value: | |
| repository-classification: main | |
| access: | |
| type: github | |
| repoUrl: github.com/${{ github.repository }} | |
| version: ${version} | |
| commit: ${commit_digest} | |
| ref: ${{ github.ref }} | |
| " \ | |
| | python3 -m ocm append source \ | |
| --file component-descriptor.yaml | |
| echo "adding resources" | |
| ocm_resources=${{ needs.package.outputs.ocm_resources }} | |
| echo "ocm-resources-file: /tmp/${ocm_resources}" | |
| cat "/tmp/${ocm_resources}" | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "adding linting-evidence resource" | |
| linting_evidence=/tmp/linting-logs/bandit.tar.gz | |
| linting_digest=$(sha256sum ${linting_evidence} | cut -d' ' -f1) | |
| cp ${linting_evidence} /tmp/dist/blobs.d/${linting_digest} | |
| cat << EOF > linting_evidence.ocm-resource | |
| name: sast-linting-evidence | |
| version: ${version} | |
| type: application/gzip | |
| relation: local | |
| access: | |
| type: localBlob | |
| localReference: sha256:${linting_digest} | |
| size: $(stat -c"%s" ${linting_evidence}) | |
| labels: | |
| - name: gardener.cloud/purposes | |
| value: | |
| - lint | |
| - sast | |
| - pybandit | |
| - name: gardener.cloud/comment | |
| value: | | |
| we use bandit (linter) for SAST scans | |
| see: https://bandit.readthedocs.io/en/latest/ | |
| EOF | |
| cat linting_evidence.ocm-resource | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "${{ needs.images.outputs.ocm_resources }}" | base64 -d > oci_ocm_resources.yaml | |
| cat oci_ocm_resources.yaml | \ | |
| python3 -m ocm append resource \ | |
| --file component-descriptor.yaml | |
| echo "component-descriptor to be uploaded:" | |
| cat component-descriptor.yaml | |
| # from google-auth | |
| username=oauth2accesstoken | |
| password=${{ steps.auth.outputs.access_token }} | |
| token=$(echo -n ${username}:${password} | base64 -w0) | |
| mkdir $HOME/.docker | |
| docker_cfg=$HOME/.docker/config.json | |
| registry_hostname=$(echo ${{ needs.params.outputs.ocm_repository }} | cut -d/ -f1) | |
| cat << EOF > ${docker_cfg} | |
| { | |
| "auths": { | |
| "${registry_hostname}": { | |
| "auth": "${token}" | |
| } | |
| } | |
| } | |
| EOF | |
| echo "uploading component-descriptor" | |
| python -m ocm upload \ | |
| --file component-descriptor.yaml \ | |
| --blobs-dir /tmp/dist/blobs.d | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - package | |
| container: | |
| image: python:alpine | |
| steps: | |
| - name: install git | |
| run: | | |
| apk add --no-cache git | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: lint | |
| run: | | |
| echo "install dependencies for python-packages" | |
| if ! apk add --no-cache $(cat gardener-cicd-libs.apk-packages) >/tmp/apk.log; then | |
| echo "error while trying to install apk-packages:" | |
| cat /tmp/apk.log | |
| exit 1 | |
| fi | |
| echo "installing linters" | |
| if ! pip3 install --upgrade --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| bandit \ | |
| flake8 \ | |
| setuptools \ | |
| pylama \ | |
| pylint \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| echo "running linters" | |
| bandit_logfile=bandit.log | |
| bandit_evidence=bandit.tar.gz | |
| bandit_extra_args="-f txt -o ${bandit_logfile}" \ | |
| .ci/lint | |
| # pass bandit.log + used cfg (pyproject.toml) as evidence | |
| tar czf $bandit_evidence $bandit_logfile pyproject.toml | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: linting-logs | |
| path: bandit.tar.gz | |
| unittests: | |
| needs: | |
| - package | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| container: | |
| image: python:alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: run-tests | |
| run: | | |
| set -eu | |
| echo "install dependencies for python-packages" | |
| apk add --no-cache $(cat gardener-cicd-libs.apk-packages) | |
| echo "install packages" | |
| if ! pip3 install --break-system-packages \ | |
| --find-links /tmp/dist \ | |
| gardener-cicd-libs \ | |
| gardener-cicd-cli \ | |
| gardener-oci \ | |
| pytest \ | |
| setuptools \ | |
| > /tmp/pip3-install.log; then | |
| echo "error while trying to install packages:" | |
| cat /tmp/pip3-install.log | |
| fi | |
| pip3 list | |
| echo "running tests" | |
| mkdir /tmp/fake-cfg.d | |
| touch /tmp/fake-cfg.d/config_types.yaml | |
| export CC_CONFIG_DIR=/tmp/fake-cfg.d | |
| .ci/test | |
| pypi: | |
| if: ${{ inputs.release-to-pypi }} | |
| runs-on: ubuntu-latest | |
| name: Publish to PYPI | |
| needs: | |
| - version | |
| - package | |
| - params | |
| - lint | |
| - unittests | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: prepare build-filesystem | |
| id: prepare | |
| run: | | |
| cp -r /tmp/dist . | |
| ls -lta dist/ | |
| rm -rf dist/blobs.d dist/ocm_resources.yaml | |
| ls -lta dist/ | |
| - name: publish to pypi | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| images: | |
| name: Build OCI Images | |
| needs: | |
| - version | |
| - package | |
| - params | |
| outputs: | |
| oci_image_ref: ${{ steps.prepare.outputs.oci_image_ref }} | |
| ocm_resources: ${{ steps.prepare.outputs.ocm_resources }} | |
| runs-on: ubuntu-latest | |
| environment: build | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Docker-Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Google-Auth | |
| id: auth | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| token_format: access_token | |
| project_id: ${{ needs.params.outputs.gcp_project }} | |
| service_account: ${{ needs.params.outputs.gcp_service_account }} | |
| workload_identity_provider: ${{ needs.params.outputs.workload_identity_provider }} | |
| - name: docker-auth | |
| id: docker-auth | |
| uses: docker/login-action@v3 | |
| with: | |
| username: oauth2accesstoken | |
| password: ${{ steps.auth.outputs.access_token }} | |
| registry: europe-docker.pkg.dev | |
| - name: Retrieve Distribution Packages | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: distribution-packages | |
| path: /tmp/dist | |
| - name: prepare build-filesystem | |
| id: prepare | |
| run: | | |
| cp -r /tmp/dist . | |
| ls -lta | |
| setuptools_version=${{ needs.version.outputs.setuptools_version }} | |
| # workaround: set repository-version to setuptools-version so installation of | |
| # packages will succeed | |
| echo "${setuptools_version}" | .ci/write-version | |
| oci_repo=${{ needs.params.outputs.oci_repository }} | |
| image_tag=${{ needs.version.outputs.effective_version }} | |
| image_ref=${oci_repo}/cicd/job-image:${image_tag} | |
| echo "oci_image_ref=${image_ref}" >> ${GITHUB_OUTPUT} | |
| cat << EOF > ocm_resources.yaml | |
| name: job-image | |
| version: ${image_tag} | |
| type: ociImage | |
| access: | |
| type: ociRegistry | |
| imageReference: ${image_ref} | |
| relation: local | |
| labels: | |
| - name: cloud.gardener.cnudie/dso/scanning-hints/package-versions | |
| value: | |
| - name: containerd | |
| version: v1.6.15 | |
| - name: gardener.cloud/cve-categorisation | |
| value: | |
| authentication_enforced: true | |
| availability_requirement: low | |
| confidentiality_requirement: high | |
| integrity_requirement: high | |
| network_exposure: protected | |
| user_interaction: gardener-operator | |
| EOF | |
| echo "ocm_resources=$(cat ocm_resources.yaml | base64 -w0)" >> ${GITHUB_OUTPUT} | |
| - name: Build OCI Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| push: true | |
| platforms: ${{ needs.params.outputs.oci_platforms }} | |
| tags: ${{ steps.prepare.outputs.oci_image_ref }} | |
| context: . # pass modified path rather than clean checkout |