From 3b35412c397011fd3979e49e88b803034963a1fe Mon Sep 17 00:00:00 2001 From: Steven Kreitzer Date: Mon, 25 Dec 2023 10:37:22 -0600 Subject: [PATCH] chore(github): update workflows for the 500x time this week --- .github/scripts/extract-images.mjs | 97 ----------------- .github/workflows/flux-diff.yaml | 39 ++++--- .github/workflows/flux-hr-image-test.yaml | 120 --------------------- .github/workflows/flux-hr-sync.yaml | 6 +- .github/workflows/flux-image-test.yaml | 122 ++++++++++++++++++++++ .github/workflows/label-sync.yaml | 2 +- .github/workflows/publish-terraform.yaml | 8 +- .github/workflows/release.yaml | 2 +- .github/workflows/renovate.yaml | 4 +- 9 files changed, 161 insertions(+), 239 deletions(-) delete mode 100644 .github/scripts/extract-images.mjs delete mode 100644 .github/workflows/flux-hr-image-test.yaml create mode 100644 .github/workflows/flux-image-test.yaml diff --git a/.github/scripts/extract-images.mjs b/.github/scripts/extract-images.mjs deleted file mode 100644 index 4c47eb41ad..0000000000 --- a/.github/scripts/extract-images.mjs +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env zx -$.verbose = false - -/** - * * extract-images.mjs - * * Extracts all container images from a HelmRelease and renders them as a JSON object - * @param --helmrelease : The source Flux HelmRelease to compare against the target - * @param --kubernetes-dir : The directory containing your Flux manifests including the HelmRepository manifests - */ -const HelmRelease = argv['helmrelease'] -const KubernetesDir = argv['kubernetes-dir'] - -const helm = await which('helm') -const kustomize = await which('kustomize') - -function extractImageValues(data) { - const imageValues = []; - function extractValues(obj) { - for (const key in obj) { - if (typeof obj[key] === 'object') { - extractValues(obj[key]); - } else if (key === 'image') { - imageValues.push(obj[key]); - } - } - } - extractValues(data); - return imageValues; -} - -async function parseHelmRelease(releaseFile) { - const helmRelease = await fs.readFile(releaseFile, 'utf8') - const doc = YAML.parseAllDocuments(helmRelease).map((item) => item.toJS()) - const release = doc.filter((item) => - item.apiVersion === 'helm.toolkit.fluxcd.io/v2beta2' - && item.kind === 'HelmRelease' - ) - return release[0] -} - -async function parseHelmRepository(kubernetesDir, releaseName) { - const files = await globby([`${kubernetesDir}/**/*.yaml`]) - for await (const file of files) { - const contents = await fs.readFile(file, 'utf8') - const repository = YAML.parseAllDocuments(contents).map((item) => item.toJS()) - if (repository[0] && 'apiVersion' in repository[0] && repository[0].apiVersion === 'source.toolkit.fluxcd.io/v1beta2' - && 'kind' in repository[0] && repository[0].kind === 'HelmRepository' - && 'metadata' in repository[0] && 'name' in repository[0].metadata && repository[0].metadata.name === releaseName) - { - return repository[0] - } - } -} - -async function renderKustomize(releaseBaseDir, releaseName) { - const build = await $`${kustomize} build --load-restrictor=LoadRestrictionsNone ${releaseBaseDir}` - const docs = YAML.parseAllDocuments(build.stdout).map((item) => item.toJS()) - const release = docs.filter((item) => - item.apiVersion === 'helm.toolkit.fluxcd.io/v2beta2' - && item.kind === 'HelmRelease' - && item.metadata.name === releaseName - ) - return release[0] -} - -async function helmTemplate(release, repository) { - const values = new YAML.Document() - values.contents = release.spec.values - const valuesFile = await $`mktemp` - await fs.writeFile(valuesFile.stdout.trim(), values.toString()) - - // Template out helm values into Kubernetes manifests - let manifests - if ('type' in repository.spec && repository.spec.type == 'oci') { - manifests = await $`${helm} template --kube-version 1.28.0 --release-name ${release.metadata.name} --include-crds=false --skip-tests ${repository.spec.url}/${release.spec.chart.spec.chart} --version ${release.spec.chart.spec.version} --values ${valuesFile.stdout.trim()}` - } else { - await $`${helm} repo add ${release.spec.chart.spec.sourceRef.name} ${repository.spec.url}` - manifests = await $`${helm} template --kube-version 1.28.0 --release-name ${release.metadata.name} --include-crds=false --skip-tests ${release.spec.chart.spec.sourceRef.name}/${release.spec.chart.spec.chart} --version ${release.spec.chart.spec.version} --values ${valuesFile.stdout.trim()}` - } - - let documents = YAML.parseAllDocuments(manifests.stdout.trim()).map((item) => item.toJS()) - - const images = []; - documents.forEach((doc) => { - const docImageValues = extractImageValues(doc); - images.push(...docImageValues); - }); - - return Array.from(new Set(images)); -} - -const helmRelease = await parseHelmRelease(HelmRelease) -const kustomizeBuild = await renderKustomize(path.dirname(HelmRelease), helmRelease.metadata.name) -const helmRepository = await parseHelmRepository(KubernetesDir, kustomizeBuild.spec.chart.spec.sourceRef.name) -const images = await helmTemplate(kustomizeBuild, helmRepository) - -echo(JSON.stringify(images)) diff --git a/.github/workflows/flux-diff.yaml b/.github/workflows/flux-diff.yaml index e3623885b4..5eec1d22b7 100644 --- a/.github/workflows/flux-diff.yaml +++ b/.github/workflows/flux-diff.yaml @@ -4,7 +4,7 @@ name: Flux Diff on: pull_request: branches: ["master"] - paths: ["kubernetes/**.yaml"] + paths: ["kubernetes/**"] concurrency: group: ${{ github.workflow }}-${{ github.event.number || github.ref }} @@ -18,25 +18,27 @@ jobs: pull-requests: write strategy: matrix: - path: ["kubernetes"] + paths: ["kubernetes"] resources: ["helmrelease", "kustomization"] + max-parallel: 4 + fail-fast: false steps: - name: Generate Token - uses: actions/create-github-app-token@2986852ad836768dfea7781f31828eb3e17990fa # v1.6.2 + uses: actions/create-github-app-token@v1 id: app-token with: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout Live Branch - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - name: Checkout Default Branch + uses: actions/checkout@v4 with: token: "${{ steps.app-token.outputs.token }}" - ref: master + ref: "${{ github.event.repository.default_branch }}" path: default - - name: Checkout PR branch - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - name: Checkout Pull Request Branch + uses: actions/checkout@v4 with: token: "${{ steps.app-token.outputs.token }}" path: pull @@ -48,8 +50,8 @@ jobs: --log-level DEBUG diff ${{ matrix.resources }} --unified 6 - --path-orig /github/workspace/default/${{ matrix.path }} - --path /github/workspace/pull/${{ matrix.path }} + --path-orig /github/workspace/default/${{ matrix.paths }} + --path /github/workspace/pull/${{ matrix.paths }} --strip-attrs "helm.sh/chart,checksum/config,app.kubernetes.io/version,chart" --limit-bytes 10000 --all-namespaces @@ -65,13 +67,24 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - if: ${{ steps.diff.outputs.diff != '' }} - name: Add comment - uses: mshick/add-pr-comment@7c0890544fb33b0bdd2e59467fbacb62e028a096 # v2.8.1 + name: Add Comment + uses: mshick/add-pr-comment@v2.8.1 with: repo-token: "${{ steps.app-token.outputs.token }}" - message-id: "${{ github.event.pull_request.number }}/${{ matrix.path }}/${{ matrix.resources }}" + message-id: "${{ github.event.pull_request.number }}/${{ matrix.paths }}/${{ matrix.resources }}" message-failure: Diff was not successful message: | ```diff ${{ steps.diff.outputs.diff }} ``` + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + flux-diff-success: + if: ${{ always() }} + needs: ["flux-diff"] + name: Flux Diff Successful + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + name: Check matrix status + run: exit 1 diff --git a/.github/workflows/flux-hr-image-test.yaml b/.github/workflows/flux-hr-image-test.yaml deleted file mode 100644 index f2d8861697..0000000000 --- a/.github/workflows/flux-hr-image-test.yaml +++ /dev/null @@ -1,120 +0,0 @@ ---- -name: Flux HelmRelease Image Test - -on: - pull_request: - branches: ["master"] - paths: ["kubernetes/**/helmrelease.yaml"] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.number || github.ref }} - cancel-in-progress: true - -env: - KUBERNETES_DIR: ./kubernetes - -jobs: - changed-files: - name: Get Changed Files - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.changed-files.outputs.all_changed_and_modified_files }} - steps: - - name: Generate Token - uses: actions/create-github-app-token@2986852ad836768dfea7781f31828eb3e17990fa # v1.6.2 - id: app-token - with: - app-id: "${{ secrets.BOT_APP_ID }}" - private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - token: "${{ steps.app-token.outputs.token }}" - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@716b1e13042866565e00e85fd4ec490e186c4a2f # v41.0.1 - with: - escape_json: false - files: kubernetes/**/helmrelease.yaml - json: true - - - name: List all changed files - run: echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" - - extract-images: - name: Extract images from HelmRelease - runs-on: ubuntu-latest - needs: ["changed-files"] - strategy: - matrix: - files: ${{ fromJSON(needs.changed-files.outputs.matrix) }} - max-parallel: 4 - fail-fast: false - outputs: - matrix: ${{ steps.extract-images.outputs.images }} - steps: - - name: Generate Token - uses: actions/create-github-app-token@2986852ad836768dfea7781f31828eb3e17990fa # v1.6.2 - id: app-token - with: - app-id: "${{ secrets.BOT_APP_ID }}" - private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - token: "${{ steps.app-token.outputs.token }}" - - - name: Setup System Tools - shell: bash - run: sudo apt-get -qq update && sudo apt-get -qq install --no-install-recommends -y curl git - - - name: Setup Workflow Tools - uses: jdx/rtx-action@61c1bfbed63c70a82ff0ce3301bde41875b17844 # v1 - with: - install: true - cache: true - rtx_toml: | - [tools] - helm = "latest" - kustomize = "latest" - - - name: Extract Images from HelmRelease - id: extract-images - run: | - images=$( - npx zx .github/scripts/extract-images.mjs \ - --kubernetes-dir "${{ env.KUBERNETES_DIR }}" \ - --helmrelease "${{ matrix.files }}" - ) - echo "images=${images}" >> $GITHUB_OUTPUT - echo "${images}" - - test-images: - name: Test images from HelmRelease - runs-on: ubuntu-latest - needs: ["extract-images"] - strategy: - matrix: - images: ${{ fromJSON(needs.extract-images.outputs.matrix) }} - max-parallel: 4 - fail-fast: false - steps: - - name: Generate Token - uses: actions/create-github-app-token@2986852ad836768dfea7781f31828eb3e17990fa # v1.6.2 - id: app-token - with: - app-id: "${{ secrets.BOT_APP_ID }}" - private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - - name: Login to GitHub Container Registry - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - registry: ghcr.io - username: "${{ secrets.BOT_APP_ID }}" - password: "${{ steps.app-token.outputs.token }}" - - - name: Test Images from HelmRelease - run: docker pull ${{ matrix.images }} diff --git a/.github/workflows/flux-hr-sync.yaml b/.github/workflows/flux-hr-sync.yaml index 4624ce9b67..761c37ec36 100644 --- a/.github/workflows/flux-hr-sync.yaml +++ b/.github/workflows/flux-hr-sync.yaml @@ -27,7 +27,7 @@ jobs: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout + - name: Checkout Default Branch uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: "${{ steps.app-token.outputs.token }}" @@ -61,6 +61,10 @@ jobs: files: kubernetes/**/helmrelease.yaml safe_output: false + - if: ${{ github.event.inputs.helmRepoNamespace == '' && github.event.inputs.helmRepoName == '' }} + name: List All Changed Files + run: echo "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" + - if: ${{ github.event.inputs.helmRepoNamespace == '' && github.event.inputs.helmRepoName == '' }} name: Sync HelmRepository env: diff --git a/.github/workflows/flux-image-test.yaml b/.github/workflows/flux-image-test.yaml new file mode 100644 index 0000000000..aa5e0013f0 --- /dev/null +++ b/.github/workflows/flux-image-test.yaml @@ -0,0 +1,122 @@ +--- +name: Flux Image Test + +on: + pull_request: + branches: ["master"] + paths: ["kubernetes/**"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +jobs: + extract-images: + name: Extract Images + runs-on: ubuntu-latest + permissions: + pull-requests: write + strategy: + matrix: + paths: ["kubernetes"] + outputs: + matrix: ${{ steps.extract-images.outputs.images }} + steps: + - name: Generate Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: "${{ secrets.BOT_APP_ID }}" + private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + + - name: Setup System Tools + shell: bash + run: sudo apt-get -qq update && sudo apt-get -qq install --no-install-recommends -y curl git jo + + - name: Setup Workflow Tools + uses: jdx/rtx-action@v1 + with: + install: true + cache: true + rtx_toml: | + [tools] + flux2 = "latest" + yq = "latest" + + - name: Checkout Default Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + ref: "${{ github.event.repository.default_branch }}" + path: default + + - name: Checkout Pull Request Branch + uses: actions/checkout@v4 + with: + token: "${{ steps.app-token.outputs.token }}" + path: pull + + - name: Gather Images in Default Branch + uses: docker://ghcr.io/allenporter/flux-local:pr-472 + with: + args: >- + get cluster + --path /github/workspace/default/${{ matrix.paths }} + --enable-images + --output yaml + --output-file default.yaml + + - name: Gather Images in Pull Request Branch + uses: docker://ghcr.io/allenporter/flux-local:pr-472 + with: + args: >- + get cluster + --path /github/workspace/pull/${{ matrix.paths }} + --enable-images + --output yaml + --output-file pull.yaml + + - name: Filter Default Branch Results + shell: bash + run: | + yq -r '[.. | .images? | select(. != null)] | flatten | sort | unique | .[]' \ + default.yaml > default.txt + + - name: Filter Pull Request Branch Results + shell: bash + run: | + yq -r '[.. | .images? | select(. != null)] | flatten | sort | unique | .[]' \ + pull.yaml > pull.txt + + - name: Compare Default and Pull Request Images + id: extract-images + shell: bash + run: | + images=$(jo -a $(grep -vf default.txt pull.txt)) + echo "images=${images}" >> $GITHUB_OUTPUT + echo "${images}" + + test-images: + if: ${{ needs.extract-images.outputs.matrix != '[]' }} + name: Test images + runs-on: ubuntu-latest + needs: ["extract-images"] + strategy: + matrix: + images: ${{ fromJSON(needs.extract-images.outputs.matrix) }} + max-parallel: 4 + fail-fast: false + steps: + - name: Test Images + run: docker pull ${{ matrix.images }} + + # Summarize matrix https://github.community/t/status-check-for-a-matrix-jobs/127354/7 + test-images-success: + if: ${{ always() }} + needs: ["test-images"] + name: Test Images Successful + runs-on: ubuntu-latest + steps: + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} + name: Check matrix status + run: exit 1 diff --git a/.github/workflows/label-sync.yaml b/.github/workflows/label-sync.yaml index 7fc6699ae7..3ec9fe68f4 100644 --- a/.github/workflows/label-sync.yaml +++ b/.github/workflows/label-sync.yaml @@ -19,7 +19,7 @@ jobs: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout + - name: Checkout Default Branch uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: "${{ steps.app-token.outputs.token }}" diff --git a/.github/workflows/publish-terraform.yaml b/.github/workflows/publish-terraform.yaml index 7c448b8b95..6461b23ac7 100644 --- a/.github/workflows/publish-terraform.yaml +++ b/.github/workflows/publish-terraform.yaml @@ -25,7 +25,7 @@ jobs: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout + - name: Checkout Default Branch uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: "${{ steps.app-token.outputs.token }}" @@ -53,12 +53,12 @@ jobs: # username: "${{ secrets.BOT_APP_ID }}" # password: "${{ steps.app-token.outputs.token }}" - - name: Generate tag + - name: Generate Tag id: generate-tag shell: bash run: echo "tag=ghcr.io/${{ github.repository_owner }}/manifests/terraform:$(git rev-parse --short HEAD)" >> "${GITHUB_OUTPUT}" - - name: Publish manifests + - name: Publish OCI Artifacts shell: bash run: | flux push artifact oci://${{ steps.generate-tag.outputs.tag }} \ @@ -66,6 +66,6 @@ jobs: --source="$(git config --get remote.origin.url)" \ --revision="$(git branch --show-current)/$(git rev-parse HEAD)" - - name: Tag manifests + - name: Tag OCI Artifact shell: bash run: flux tag artifact oci://${{ steps.generate-tag.outputs.tag }} --tag master diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c2d53d6c0c..2081eae161 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,7 +18,7 @@ jobs: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout + - name: Checkout Default Branch uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: "${{ steps.app-token.outputs.token }}" diff --git a/.github/workflows/renovate.yaml b/.github/workflows/renovate.yaml index b92cd69d05..d1db864d4e 100644 --- a/.github/workflows/renovate.yaml +++ b/.github/workflows/renovate.yaml @@ -52,12 +52,12 @@ jobs: app-id: "${{ secrets.BOT_APP_ID }}" private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - - name: Checkout + - name: Checkout Default Branch uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: "${{ steps.app-token.outputs.token }}" - - name: Override default config from dispatch variables + - name: Override Default Config shell: bash run: | echo "RENOVATE_DRY_RUN=${{ github.event.inputs.dryRun || env.WORKFLOW_DRY_RUN }}" >> "${GITHUB_ENV}"