Continuous Integration #662
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Integration | |
| env: | |
| DOCKER_IMAGE: wyrihaximusnet/redirect | |
| DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING: '{"ghcr.io":"GHCR_TOKEN","docker.io":"HUB_PASSCODE"}' | |
| on: | |
| push: | |
| schedule: | |
| - cron: '0 0 * * 0' | |
| jobs: | |
| supported-arch-matrix: | |
| name: Supported processor architectures | |
| runs-on: ubuntu-latest | |
| needs: | |
| - lint-dockerfile | |
| outputs: | |
| arch: ${{ steps.supported-arch-matrix.outputs.arch }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: supported-arch-matrix | |
| name: Generate Arch | |
| run: | | |
| echo "arch=[\"linux/amd64\",\"linux/arm64\"]" >> $GITHUB_OUTPUT | |
| registry-matrix: | |
| name: Extract registries from registry secret mapping | |
| if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master' | |
| runs-on: ubuntu-latest | |
| needs: | |
| - tests | |
| outputs: | |
| registry: ${{ steps.registry-matrix.outputs.registry }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: registry-matrix | |
| name: Extract registries from registry secret mapping | |
| run: | | |
| echo "registry=$(printenv DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING | jq -c 'keys')" >> $GITHUB_OUTPUT | |
| generate-image-strategy: | |
| name: Generate Image Strategy | |
| runs-on: ubuntu-latest | |
| outputs: | |
| images: ${{ steps.generate-image-strategy.outputs.images }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: generate-image-strategy | |
| name: Generate Jobs | |
| run: | | |
| ls images | jq -csR '. + "random" | split("\n")' > images.list | |
| cat images.list | |
| echo "images=$(cat images.list)" >> $GITHUB_OUTPUT | |
| generate-rule-strategy: | |
| name: Generate Rules Strategy | |
| runs-on: ubuntu-latest | |
| needs: | |
| - scan-vulnerability | |
| outputs: | |
| rules: ${{ steps.generate-rule-strategy.outputs.rules }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: generate-rule-strategy | |
| name: Generate Jobs | |
| run: | | |
| ls tests/rules | jq -csR '. | rtrimstr("\n") | split("\n")' > rules.list | |
| cat rules.list | |
| echo "rules=$(cat rules.list)" >> $GITHUB_OUTPUT | |
| lint-dockerfile: | |
| name: Lint ${{ matrix.image }}'s Dockerfile | |
| needs: | |
| - generate-image-strategy | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: ${{ fromJson(needs.generate-image-strategy.outputs.images) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| if: matrix.image != 'random' | |
| - name: Lint Dockerfile | |
| if: matrix.image != 'random' | |
| uses: docker://hadolint/hadolint:latest-debian | |
| with: | |
| entrypoint: hadolint | |
| args: ./images/${{ matrix.image }}/Dockerfile | |
| build-docker-image: | |
| name: Build ${{ matrix.image }} Docker (${{ matrix.platform }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }} | |
| image: ${{ fromJson(needs.generate-image-strategy.outputs.images) }} | |
| needs: | |
| - supported-arch-matrix | |
| - generate-image-strategy | |
| - lint-dockerfile | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - uses: actions/checkout@v4 | |
| - run: cp -R $(echo -e "./images/$(ls ./images/ | shuf -n 1)") ./images/random | |
| if: matrix.image == 'random' | |
| - run: mkdir ./docker-image | |
| - run: docker image build --platform=${{ matrix.platform }} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` -t "${DOCKER_IMAGE}:${{ matrix.image }}-${{ env.PLATFORM_PAIR }}" --no-cache --build-arg VERSION=$TAG_VERSION ./images/${{ matrix.image }}/ | |
| - run: docker save "${DOCKER_IMAGE}:${{ matrix.image }}-${{ env.PLATFORM_PAIR }}" -o ./docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }} | |
| path: ./docker-image | |
| scan-vulnerability: | |
| name: Scan ${{ matrix.image }} for vulnerabilities (${{ matrix.platform }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }} | |
| image: ${{ fromJson(needs.generate-image-strategy.outputs.images) }} | |
| needs: | |
| - supported-arch-matrix | |
| - generate-image-strategy | |
| - build-docker-image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }} | |
| path: /tmp/docker-image | |
| - run: docker load --input /tmp/docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar | |
| - run: rm -Rf /tmp/docker-image/ | |
| - run: echo -e "${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-${{ env.PLATFORM_PAIR }}" | xargs -I % sh -c 'docker run -v /tmp/trivy:/var/lib/trivy -v /var/run/docker.sock:/var/run/docker.sock -t aquasec/trivy:latest --cache-dir /var/lib/trivy image --exit-code 1 --no-progress --format table %' | |
| tests: | |
| name: Test ${{ matrix.image }} against ${{ matrix.rule }} (${{ matrix.platform }}) | |
| needs: | |
| - supported-arch-matrix | |
| - generate-image-strategy | |
| - generate-rule-strategy | |
| - scan-vulnerability | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: ${{ fromJson(needs.supported-arch-matrix.outputs.arch) }} | |
| image: ${{ fromJson(needs.generate-image-strategy.outputs.images) }} | |
| rule: ${{ fromJson(needs.generate-rule-strategy.outputs.rules) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Prepare | |
| run: | | |
| platform=${{ matrix.platform }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: docker-image-${{ matrix.image }}-${{ env.PLATFORM_PAIR }} | |
| path: /tmp/docker-image | |
| - run: docker load --input /tmp/docker-image/docker_image-${{ env.PLATFORM_PAIR }}.tar | |
| - run: docker image ls -a | |
| - name: Start image ${{ matrix.image }} (${{ matrix.platform }}) | |
| run: docker run -d --platform=${{ matrix.platform }} --rm -v ${GITHUB_WORKSPACE}/${REDIRECT_CONFIG_FILE}:/etc/redirect/config.yaml ${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-${{ env.PLATFORM_PAIR }} | |
| env: | |
| IMAGE: ${{ steps.build.outputs.tag }} | |
| REDIRECT_CONFIG_FILE: tests/rules/${{ matrix.rule }}/config.yaml | |
| - name: Get running image ID | |
| id: ps | |
| run: printf "id=%s" $(docker ps --format "{{.ID}}") >> $GITHUB_OUTPUT | |
| env: | |
| IMAGE: ${{ steps.build.outputs.tag }} | |
| - name: Get running image IP | |
| id: inspect | |
| run: printf "ip=%s" $(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${IMAGE_ID}) >> $GITHUB_OUTPUT | |
| env: | |
| IMAGE_ID: ${{ steps.ps.outputs.id }} | |
| - name: Sleep 13 seconds before attempting to connect | |
| run: sleep 13 | |
| - name: Test that redirect port is reachable | |
| run: curl http://${IMAGE_IP}:7132/ | |
| env: | |
| IMAGE_IP: ${{ steps.inspect.outputs.ip }} | |
| - name: Test that metrics port is reachable | |
| run: curl http://${IMAGE_IP}:7133/ | |
| env: | |
| IMAGE_IP: ${{ steps.inspect.outputs.ip }} | |
| - name: Run unit tests ${{ matrix.rule }} against ${{ matrix.image }} | |
| run: docker run -i loadimpact/k6 run -u 100 -d 30s -e IMAGE_IP=${IMAGE_IP} -< ${SCRIPT_FILEIMAGE} | |
| env: | |
| IMAGE: ${{ matrix.image }} | |
| IMAGE_IP: ${{ steps.inspect.outputs.ip }} | |
| IMAGE_ID: ${{ steps.ps.outputs.id }} | |
| SCRIPT_FILEIMAGE: ./tests/rules/${{ matrix.rule }}/unit.js | |
| - name: Run metrics tests ${{ matrix.rule }} against ${{ matrix.image }} | |
| run: docker run -i loadimpact/k6 run -u 1 -d 10s -e IMAGE_IP=${IMAGE_IP} -< ${SCRIPT_FILEIMAGE} | |
| env: | |
| IMAGE: ${{ matrix.image }} | |
| IMAGE_IP: ${{ steps.inspect.outputs.ip }} | |
| IMAGE_ID: ${{ steps.ps.outputs.id }} | |
| SCRIPT_FILEIMAGE: ./tests/rules/${{ matrix.rule }}/metrics.js | |
| - name: Docker logs for image ${{ matrix.image }} | |
| run: docker logs ${IMAGE_ID} | |
| env: | |
| IMAGE_ID: ${{ steps.ps.outputs.id }} | |
| push-image: | |
| if: (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/master' | |
| name: Push ${{ matrix.image }} to ${{ matrix.registry }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| registry: ${{ fromJson(needs.registry-matrix.outputs.registry) }} | |
| image: ${{ fromJson(needs.generate-image-strategy.outputs.images) }} | |
| needs: | |
| - supported-arch-matrix | |
| - generate-image-strategy | |
| - tests | |
| - registry-matrix | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: network=host | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: docker-image-${{ matrix.image }}-* | |
| path: /tmp/docker-image | |
| merge-multiple: true | |
| - run: ls -lasth /tmp/docker-image/ | |
| - run: | | |
| for f in /tmp/docker-image/docker_image-*.tar; do | |
| docker load --input $f | |
| done | |
| - run: rm -Rf /tmp/docker-image/ | |
| - run: docker images | |
| - run: | | |
| archs=${{ join(fromJson(needs.supported-arch-matrix.outputs.arch), ',') }} | |
| for arch in ${archs//,/ } | |
| do | |
| docker tag "${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-${arch//\//-}" "localhost:5000/${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-${arch//\//-}" | |
| docker push "localhost:5000/${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-${arch//\//-}" | |
| done | |
| - run: docker images | |
| - name: Login to ${{ matrix.registry }} | |
| run: | | |
| echo "${{ env.DOCKER_PASSWORD }}" | \ | |
| docker login ${{ matrix.registry }} \ | |
| --username "${{ env.DOCKER_USER }}" \ | |
| --password-stdin | |
| env: | |
| DOCKER_USER: ${{ secrets.HUB_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets[fromJson(env.DOCKER_IMAGE_REGISTRIES_SECRET_MAPPING)[matrix.registry]] }} | |
| - name: Docker info | |
| run: docker info | |
| - name: Create merge Dockerfile | |
| run: echo "FROM localhost:5000/${{ env.DOCKER_IMAGE }}:${{ matrix.image }}-\${TARGETOS}-\${TARGETARCH}" >> docker-file-${{ matrix.registry }}-wyrihaximusnet-redirect-${{ matrix.image }} | |
| - run: cat docker-file-${{ matrix.registry }}-wyrihaximusnet-redirect-${{ matrix.image }} | |
| - name: Merged different arch imags into one | |
| run: docker buildx build -f docker-file-${{ matrix.registry }}-wyrihaximusnet-redirect-${{ matrix.image }} --platform=${{ join(fromJson(needs.supported-arch-matrix.outputs.arch), ',') }} -t ${{ matrix.registry }}/${{ env.DOCKER_IMAGE }}:${{ matrix.image }} . --push |