Go and Python standalone image (#6415)

.github/workflows/eco-branch-images.yml

@@ -0,0 +1,85 @@
+name: Eco branch images
+env:
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+on:
+  pull_request:
+    branches:
+      - main
+    types:
+      - synchronize
+  pull_request_review:
+    types:
+      - submitted
+
+jobs:
+  push-updater-images:
+    name: Deploy
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - { name: go_modules, ecosystem: gomod }
+          - { name: python, ecosystem: pip }
+    permissions:
+      contents: read
+      packages: write
+    env:
+      TAG: ${{ github.sha }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+        # sets DECISION to the PR's review decision, handling the push-after-approval case
+      - name: Check if pull request is approved
+        run: |
+          DECISION=$(gh pr view ${{ github.event.pull_request.number }} --json reviewDecision,state -t '{{.reviewDecision}}:{{.state}}')
+          echo "Review decision is: $DECISION"
+          echo "DECISION=$DECISION" >> $GITHUB_ENV
+
+      # overwrite the previous result if this is a fork since forks can't publish to GHCR
+      - name: Skip forks
+        if: github.event.pull_request.head.repo.fork
+        run: echo "DECISION=FORK" >> $GITHUB_ENV
+
+      - name: Log in to GHCR
+        if: env.DECISION == 'APPROVED:OPEN'
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Build updater core image
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "ghcr.io/dependabot/dependabot-updater-core:latest" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-updater-core \
+            -f Dockerfile.updater-core \
+            .
+
+      - name: Build ecosystem image
+        if: env.DECISION == 'APPROVED:OPEN'
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "ghcr.io/dependabot/dependabot-updater-${{ matrix.suite.ecosystem }}:$TAG" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-updater-core \
+            --cache-from ghcr.io/dependabot/dependabot-updater-${{ matrix.suite.ecosystem }} \
+            -f Dockerfile.${{ matrix.suite.name }} \
+            .
+
+      - name: Push branch image
+        if: env.DECISION == 'APPROVED:OPEN'
+        run: |
+          docker push "ghcr.io/dependabot/dependabot-updater-${{ matrix.suite.ecosystem }}:$TAG"
+
+      - name: Set summary
+        if: env.DECISION == 'APPROVED:OPEN'
+        run: |
+          echo "updater uploaded with tag \`$TAG\`" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "ghcr.io/dependabot/dependabot-updater-${{ matrix.suite.ecosystem }}:$TAG" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

.github/workflows/eco-smoke.yml

@@ -0,0 +1,126 @@
+# Runs all ecosystems cached and concurrently.
+name: Eco Smoke
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    paths-ignore:
+      - docs/**
+      - README.md
+env:
+  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - { path: go_modules, name: go, ecosystem: gomod }
+          - { path: python, name: pip, ecosystem: pip }
+          - { path: python, name: pipenv, ecosystem: pip}
+          - { path: python, name: pip-compile, ecosystem: pip }
+          - { path: python, name: poetry, ecosystem: pip }
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dorny/paths-filter@v2
+      if: github.event_name != 'workflow_dispatch'
+      id: changes
+      with:
+        filters: |
+          go:
+            - Dockerfile.go_modules
+            - Dockerfile.updater-core
+            - 'common/**'
+            - 'updater/**'
+            - 'go_modules/**'
+          pip:
+            - Dockerfile.python
+            - Dockerfile.updater-core
+            - 'common/**'
+            - 'updater/**'
+            - 'python/**'
+          'pip-compile':
+            - Dockerfile.python
+            - Dockerfile.updater-core
+            - 'common/**'
+            - 'updater/**'
+            - 'python/**'
+          pipenv:
+            - Dockerfile.python
+            - Dockerfile.updater-core
+            - 'common/**'
+            - 'updater/**'
+            - 'python/**'
+          poetry:
+            - Dockerfile.python
+            - Dockerfile.updater-core
+            - 'common/**'
+            - 'updater/**'
+            - 'python/**'
+
+    - name: Download CLI and test
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      run: |
+        gh release download --repo dependabot/cli -p "*linux-amd64.tar.gz"
+        tar xzvf *.tar.gz >/dev/null 2>&1
+        ./dependabot --version
+        URL=https://api.github.com/repos/dependabot/smoke-tests/contents/tests/smoke-${{ matrix.suite.name }}.yaml
+        curl $(gh api $URL --jq .download_url) -o smoke.yaml
+
+    # Download the Proxy cache. The job is ideally 100% cached so no real calls are made.
+    - name: Download cache
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      run: |
+        mkdir cache
+        cd cache
+        gh run download --repo dependabot/smoke-tests --name cache-${{ matrix.suite.name }}
+
+    - name: Build updater core image
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      env:
+        DOCKER_BUILDKIT: 1
+      run: |
+        docker build \
+          -t "dependabot/dependabot-updater-core:latest" \
+          --build-arg BUILDKIT_INLINE_CACHE=1 \
+          --cache-from ghcr.io/dependabot/dependabot-updater-core \
+          -f Dockerfile.updater-core \
+          .
+
+    - name: Build ecosystem image
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      env:
+        DOCKER_BUILDKIT: 1
+        OMNIBUS_VERSION: latest
+      run: |
+        docker build \
+          -t "dependabot/updater:latest" \
+          --build-arg BUILDKIT_INLINE_CACHE=1 \
+          --cache-from ghcr.io/dependabot/dependabot-updater-core \
+          --cache-from ghcr.io/dependabot/dependabot-updater-gomod \
+          -f Dockerfile.${{ matrix.suite.path }} \
+          .
+
+    - name: ${{ matrix.suite.name }}
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      env:
+        LOCAL_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -o pipefail
+        ./dependabot test -f=smoke.yaml -o=result.yaml --cache=cache --timeout=20m --updater-image=dependabot/updater:latest 2>&1 | tee -a log.txt
+
+    - name: Diff
+      if: always()
+      continue-on-error: true
+      run: diff --ignore-space-change smoke.yaml result.yaml && echo "Contents are identical"
+
+    - name: Create summary
+      if: steps.changes.outputs[matrix.suite.name] == 'true'
+      run: tail -n100 log.txt | grep -P '\d+/\d+ calls cached \(\d+%\)' >> $GITHUB_STEP_SUMMARY
+
+    # No upload at the end:
+    # - If a test is uncachable in some regard, the cache would grow unbound.
+    # - We might want to consider erroring if the cache is changed.

.github/workflows/ecosystem-ci.yml

@@ -0,0 +1,87 @@
+name: CI-Eco
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - { path: go_modules, name: go_module, ci_node_total: 2, ci_node_index: 0, ecosystem: gomod }
+          - { path: go_modules, name: go_module, ci_node_total: 2, ci_node_index: 1, ecosystem: gomod }
+          - { path: python, name: python, ci_node_total: 2, ci_node_index: 0, ecosystem: pip }
+          - { path: python, name: python, ci_node_total: 2, ci_node_index: 1, ecosystem: pip}
+          - { path: python, name: python_slow, ci_node_total: 2, ci_node_index: 0, ecosystem: pip }
+          - { path: python, name: python_slow, ci_node_total: 2, ci_node_index: 1, ecosystem: pip }
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - uses: dorny/paths-filter@v2
+        id: changes
+        with:
+          filters: |
+            python:
+              - Dockerfile.updater-core
+              - Dockerfile.python
+              - 'common/**'
+              - 'python/**'
+              - '.github/workflows/ecosystem-ci.yml'
+            go_modules:
+              - Dockerfile.updater-core
+              - Dockerfile.go_modules
+              - 'common/**'
+              - 'go_modules/**'
+              - '.github/workflows/ecosystem-ci.yml'
+
+      - name: Build updater core image
+        if: steps.changes.outputs[matrix.suite.path] == 'true'
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "dependabot/dependabot-updater-core:latest" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-updater-core \
+            -f Dockerfile.updater-core \
+            .
+
+      - name: Build ecosystem image
+        if: steps.changes.outputs[matrix.suite.path] == 'true'
+        env:
+          DOCKER_BUILDKIT: 1
+        run: |
+          docker build \
+            -t "dependabot/dependabot-updater-${{ matrix.suite.ecosystem }}:latest" \
+            --build-arg BUILDKIT_INLINE_CACHE=1 \
+            --cache-from ghcr.io/dependabot/dependabot-updater-${{ matrix.suite.ecosystem }} \
+            -f Dockerfile.${{ matrix.suite.path }} \
+            .
+
+      - name: Run ${{ matrix.suite.name }} tests
+        if: steps.changes.outputs[matrix.suite.path] == 'true'
+        run: |
+          docker run \
+            --env "CI=true" \
+            --env "RAISE_ON_WARNINGS=true" \
+            --env "DEPENDABOT_TEST_ACCESS_TOKEN=${{ secrets.GITHUB_TOKEN }}" \
+            --env "SUITE_NAME=${{ matrix.suite.name }}" \
+            --env "CI_NODE_TOTAL=${{ matrix.suite.ci_node_total }}" \
+            --env "CI_NODE_INDEX=${{ matrix.suite.ci_node_index }}" \
+            --rm dependabot/dependabot-updater-${{ matrix.suite.ecosystem }} bash -c \
+            "cd /home/dependabot/${{ matrix.suite.path }} && ./script/ci-test"
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: ./bin/lint

Dockerfile.go_modules

@@ -0,0 +1,28 @@
+FROM ghcr.io/dependabot/dependabot-updater-core
+ARG TARGETARCH=amd64
+
+USER root
+RUN apt-get update \
+  && apt-get upgrade -y \
+  && apt-get install -y --no-install-recommends curl
+
+# Install Go
+ARG GOLANG_VERSION=1.19
+# You can find the sha here: https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz.sha256
+ARG GOLANG_AMD64_CHECKSUM=464b6b66591f6cf055bc5df90a9750bf5fbc9d038722bb84a9d56a2bea974be6
+ARG GOLANG_ARM64_CHECKSUM=efa97fac9574fc6ef6c9ff3e3758fb85f1439b046573bf434cccb5e012bd00c8
+
+ENV PATH=/opt/go/bin:$PATH
+RUN cd /tmp \
+  && curl --http1.1 -o go-${TARGETARCH}.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-${TARGETARCH}.tar.gz \
+  && printf "$GOLANG_AMD64_CHECKSUM go-amd64.tar.gz\n$GOLANG_ARM64_CHECKSUM go-arm64.tar.gz\n" | sha256sum -c --ignore-missing - \
+  && tar -xzf go-${TARGETARCH}.tar.gz -C /opt \
+  && rm go-${TARGETARCH}.tar.gz
+
+ENV DEPENDABOT_NATIVE_HELPERS_PATH="/opt"
+
+COPY go_modules/helpers /opt/go_modules/helpers
+RUN bash /opt/go_modules/helpers/build
+
+USER dependabot
+COPY --chown=dependabot:dependabot go_modules /home/dependabot/go_modules