fetch.yml

#
# This workflow was created automatically from the `package-template.yml` by running `make -C .github workflows`
# DO NOT EDIT THIS WORKFLOW, changes will be lost on the next update.
#

name: "fetch"
concurrency:
  group: ${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }} (${{ inputs.package_version_override || 'LATEST' }}_r${{ inputs.release_number_override || '0' }})
  cancel-in-progress: true
on:
  push:
    branches:
      - main

    paths:
      - apk/**
      - deb/**
      - rpm/**
      - tasks/**
      - vendor/fetch/**
      - .github/workflows/fetch.yml


  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - apk/**
      - deb/**
      - rpm/**
      - tasks/**
      - vendor/fetch/**
      - .github/workflows/fetch.yml

  #bridgecrew:skip=BC_REPO_GITHUB_ACTION_7:The whole point of the workflow dispatch is to feed in a version
  workflow_dispatch:
    inputs:
      package_version_override:
        description: 'Version of fetch package to build. Defaults to vendor/fetch/VERSION.'
        required: false
        type: string
      release_number_override:
        description: 'Zero-based release number of fetch package to publish. Defaults to 0 (zero) when version is specified, ignored if not.'
        required: false
        type: string

env:
  fetch_VERSION: ${{ inputs.package_version_override }}
  fetch_RELEASE: ${{ inputs.release_number_override }}

jobs:
  # Mergify cannot distinguish between 2 jobs with the same name run from different workflows,
  # so each job must have a unique name for the rules to work properly.
  # See https://github.com/Mergifyio/mergify/discussions/5082
  # and https://github.com/Mergifyio/mergify/issues/5083
  matrix-fetch:
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    outputs:
      package-enabled: ${{ steps.info.outputs.package_enabled }}
      package-matrix:  ${{steps.info.outputs.package_matrix}}
      arch-matrix:  ${{steps.info.outputs.arch_matrix}}
      apk-enabled: ${{ steps.info.outputs.package_enabled == 'true' && steps.info.outputs.apk_package_enabled == 'true' }}
    steps:
    - uses: actions/checkout@v4

    - name: Export package build matrix
      shell: bash
      id: info
      env:
        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
      run: |
        echo setting ouputs
        make -C vendor/fetch info/github
        echo
        echo outputs set


  # Build for alpine linux
  # Kept separate because it is old and slightly different than the other package builds
  alpine-fetch:
    needs: matrix-fetch
    if: github.event_name != 'schedule' && needs.matrix-fetch.outputs.apk-enabled != 'false'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # These versions must be strings. E.g. Otherwise `3.10` -> `3.1`
        alpine:
          # Now that we are just building 1 binary for all distributions, we do not
          # need to track which distribution we are building on.
          - 'alpine'
    env:
      APK_KEY_RSA: "${{ secrets.APK_KEY_RSA }}"
      APK_PACKAGES_PATH: ${{github.workspace}}/artifacts/${{matrix.alpine}}
      PACKAGER: ops@cloudposse.com
      PACKAGER_PRIVKEY: /dev/shm/ops@cloudposse.com.rsa
      PACKAGER_PUBKEY: ${{github.workspace}}/artifacts/ops@cloudposse.com.rsa.pub

    container:
      image: cloudposse/packages-apkbuild:${{matrix.alpine}}
      credentials:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    steps:
      # Checkout the packages repo so we can build the packages as a monorepo
      - name: "Checkout source code at current commit"
        uses: actions/checkout@v4

      # Export the apk keys as files from secrets
      - name: "Export keys"
        run: "make -C .github/ export"

      # Build the alpine packages for the matrix version of alpine
      - name: "Build alpine packages"
        run: "make -C vendor/${{github.workflow}} apk"

      # Verify the packages were built or error
      - name: "List packages"
        run: 'find ${APK_PACKAGES_PATH} -type f -name \*.apk | xargs --no-run-if-empty ls -l | grep .'

      # Export the artifact filename including path
      # Path must be relative to workdir for Cloudsmith action to be able to find it
      - name: "Set output path to artifact"
        id: artifact
        shell: bash
        run: |
          artifact=$(find artifacts/${{matrix.alpine}} -type f -name \*.apk)
          echo "path=$artifact" | tee -a $GITHUB_OUTPUT
          echo creating '"pip"' cache directory for Cloudsmith
          mkdir -p $HOME/.cache/pip && chmod -R 777 $HOME/.cache || echo Ignoring error creating '"pip"' cache directory


      # Determine which package organization we should use (e.g. dev or prod)
      - name: "Determine package repo"
        shell: bash
        id: repo
        run: |
          if [[ ${GITHUB_REF} == 'refs/heads/main' ]]; then
              echo "org=${{github.repository_owner}}" | tee -a $GITHUB_OUTPUT
          else
              echo "org=${{github.repository_owner}}-dev" | tee -a $GITHUB_OUTPUT
          fi
        env:
          GITHUB_REF: ${{ github.ref }}

      # Publish the artifacts
      - name: "Push artifact to package repository"
        uses: cloudsmith-io/action@v0.5.4
        with:
          api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
          command: 'push'
          format: 'alpine'
          owner: '${{steps.repo.outputs.org}}'              # Your Cloudsmith account name or org name (namespace)
          repo: 'packages'                                  # Your Cloudsmith Repository name (slug)
          distro: 'alpine'                                  # Your Distribution  (i.e debian, ubuntu, alpine)
          release: 'any-version'                            # Use "any-version" if your package is compatible with more than one version of alpine linux
          republish: 'true'                                 # Needed if version is not changing
          file: '${{steps.artifact.outputs.path}}'          # Package filename (including path)
          no-wait-for-sync: 'true'                          # Skip the waiting for package synchronisation (i.e. upload only)

  # Build packages with fpm package manager
  package-fetch:
    needs: matrix-fetch
    # Should not be needed, but without these conditions, this job would fail with an error if the matrix is []
    # and would run with package-type empty if matrix is ["apk"]
    if: >
      github.event_name != 'schedule' && needs.matrix-fetch.outputs.package-enabled != 'false'
      && needs.matrix-fetch.outputs.package-matrix != '[]' && needs.matrix-fetch.outputs.package-matrix != '["apk"]'

    strategy:
      matrix:
        package-type: ${{ fromJSON(needs.matrix-fetch.outputs.package-matrix) }}
        arch: ${{ fromJSON(needs.matrix-fetch.outputs.arch-matrix) }}
        exclude:
        - package-type: 'apk'
        include:
        # Default value for runs-on. Original matrix values will not be overridden, but added ones (like runs-on) can be.
        # See https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#expanding-or-adding-matrix-configurations
        - runs-on: "self-hosted-arm64-large"
        # By including `arch: amd64` here, we override the `runs-on` value when the matrix `arch` is `amd64`.
        # This also forces the matrix to include `arch: amd64` even if it is not in the original matrix.
        # This is why we do not default for amd64 and then override for arm64. (Because it would force arm64 to be included, and some tools are not available for arm64.)
        - arch: amd64
          runs-on: "ubuntu-latest"
    runs-on: ${{ matrix.runs-on }}
    env:
      # We are in a bit of a bind here because of how GitHub actions work as of 2020-11-19
      # Although the "workspace" is mounted to the container, it is not mounted
      # at `/github/workspace` or ${{github.workspace}}, although through some
      # mechanism, an environment variable whose value starts with ${{github.workspace}}
      # will have ${{github.workspace}} replaced with the correct mount point.
      #
      # We need an absolute path for the package build system, since every build happens
      # in a different directory, but because the mount point changes, we also
      # need a path relative to the initial working directory to communicate between
      # the package building container and the cloudsmith action.
      PACKAGES_PATH: ${{github.workspace}}/artifacts/${{matrix.package-type}}/any-version
      PACKAGE_RELPATH: artifacts/${{matrix.package-type}}/any-version

    # Unfortunately, there is no reasonable way to configure the docker image tag based on the package-type
    container:
      image: cloudposse/packages-${{matrix.package-type}}build:latest
      credentials:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}

    steps:
      # Checkout the packages repo so we can build the packages as a monorepo
      - name: "Checkout source code at current commit"
        uses: actions/checkout@v4

      # Build the packages for the matrix version
      - name: "Build ${{matrix.package-type}} packages"
        shell: bash
        run: |
          echo Current directory is $(pwd)
          [[ $PACKAGES_PATH =~ ^$(pwd) ]] || { echo Package dir \"$PACKAGES_PATH\" not beneath workdir \"$(pwd)\" >&2; exit 1; }
          make -C vendor/${{github.workflow}} ${{matrix.package-type}}

      # Export the artifact filename including path
      - name: "Set output path to artifact"
        id: artifact
        shell: bash
        run: |
          [[ -n $PACKAGE_RELPATH ]] || { echo Error: PACKAGE_RELPATH is not set >&2; exit 1; }
          packages=($(find ${PACKAGE_RELPATH} -type f -name \*.${{matrix.package-type}}))
          echo List packages found:
          printf "%s\n" "${packages[@]}" | xargs --no-run-if-empty ls -l
          echo Error if not exactly 1 package found
          (( ${#packages[@]} == 1 )) || { echo "Error: other than 1 package found (${#packages[@]})" >&2; exit 1; }

          echo "setting output"
          echo "path=$packages" | tee -a $GITHUB_OUTPUT

          echo creating '"pip"' cache directory for Cloudsmith
          mkdir -p $HOME/.cache/pip && chmod -R 777 $HOME/.cache || echo Ignoring error creating '"pip"' cache directory

      # Determine which package organization we should use (e.g. dev or prod)
      - name: "Determine package repo"
        shell: bash
        id: repo
        run: |
          if [[ ${GITHUB_REF} == 'refs/heads/main' ]]; then
              echo "org=${{github.repository_owner}}" | tee -a $GITHUB_OUTPUT
          else
              echo "org=${{github.repository_owner}}-dev" | tee -a $GITHUB_OUTPUT
          fi
        env:
          GITHUB_REF: ${{ github.ref }}

      # Publish the artifacts
      - name: "Push artifact to package repository"
        uses: cloudsmith-io/action@v0.5.4
        with:
          api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
          command: 'push'
          format: '${{matrix.package-type}}'
          owner: '${{steps.repo.outputs.org}}'              # Your Cloudsmith account name or org name (namespace)
          repo: 'packages'                                  # Your Cloudsmith Repository name (slug)
          distro: 'any-distro'                              # Use "any-distro" since our package is compatible with more than more distribution
          release: 'any-version'                            # Use "any-version" since our package is compatible with more than more version
          republish: 'true'                                 # Needed if version is not changing
          file: '${{steps.artifact.outputs.path}}'          # Package filename (including path)
          no-wait-for-sync: 'true'                          # Skip the waiting for package synchronisation (i.e. upload only)