Sync v3-0-stable with v3-0-test to release 3.0.1

.asf.yaml

@@ -134,6 +134,11 @@ github:
       required_linear_history: true
       required_conversation_resolution: true
       required_signatures: false
+    v3-0-stable:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+      required_linear_history: true
+      required_signatures: false
     providers-fab/v1-5:
       required_pull_request_reviews:
         required_approving_review_count: 1

.github/actions/install-pre-commit/action.yml

@@ -24,7 +24,7 @@ inputs:
     default: "3.9"
   uv-version:
     description: 'uv version to use'
-    default: "0.6.13"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.7.2"  # Keep this comment to allow automatic replacement of uv version
   pre-commit-version:
     description: 'pre-commit version to use'
     default: "4.2.0"  # Keep this comment to allow automatic replacement of pre-commit version
@@ -61,6 +61,16 @@ runs:
         key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}
         path: /tmp/
       id: restore-pre-commit-cache
+    - name: "Check if pre-commit cache tarball exists"
+      shell: bash
+      run: |
+        if [ -f /tmp/cache-pre-commit.tar.gz ]; then
+          echo "✅ Cache tarball found: /tmp/cache-pre-commit.tar.gz"
+        else
+          echo "❌ Cache tarball missing. Expected /tmp/cache-pre-commit.tar.gz"
+          exit 1
+        fi
+      if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true'
     - name: "Restore .cache from the tar file"
       run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz
       shell: bash

.github/workflows/additional-ci-image-checks.yml

@@ -20,24 +20,20 @@ name: Additional CI image checks
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on-as-json-default:
-        description: "The array of labels (in json form) determining default runner used for the build."
+      runners:
+        description: "The array of labels (in json form) determining runners."
         required: true
         type: string
-      runs-on-as-json-public:
-        description: "The array of labels (in json form) determining public runners."
-        required: true
-        type: string
-      runs-on-as-json-self-hosted:
-        description: "The array of labels (in json form) determining self-hosted runners."
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
       python-versions:
         description: "The list of python versions (stringified JSON array) to run the tests on."
         required: true
         type: string
       branch:
-        description: "Branch used to run the CI jobs in (main/v2_*_test)."
+        description: "Branch used to run the CI jobs in (main/v*_*_test)."
         required: true
         type: string
       constraints-branch:
@@ -103,26 +99,27 @@ jobs:
       # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
       packages: write
     with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
+      runners: ${{ inputs.runners }}
       cache-type: "Early"
       include-prod-images: "false"
       push-latest-images: "false"
-      platform: "linux/amd64"
+      platform: ${{ inputs.platform }}
       python-versions: ${{ inputs.python-versions }}
       branch: ${{ inputs.branch }}
       constraints-branch: ${{ inputs.constraints-branch }}
       use-uv: ${{ inputs.use-uv }}
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
-    if: inputs.branch == 'main'
+    if: >
+      inputs.canary-run == 'true' &&
+      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
 
   # Check that after earlier cache push, breeze command will build quickly
   check-that-image-builds-quickly:
     timeout-minutes: 11
     name: Check that image builds quickly
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       UPGRADE_TO_NEWER_DEPENDENCIES: false
       PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
@@ -131,6 +128,7 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       GITHUB_USERNAME: ${{ github.actor }}
       VERBOSE: "true"
+      PLATFORM: ${{ inputs.platform }}
     if: inputs.branch == 'main'
     steps:
       - name: "Cleanup repo"
@@ -152,30 +150,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$actor" --password-stdin
       - name: "Check that image builds quickly"
-        run: breeze shell --max-time 600 --platform "linux/amd64"
-
-#  # This is only a check if ARM images are successfully building when committer runs PR from
-#  # Apache repository. This is needed in case you want to fix failing cache job in "canary" run
-#  # There is no point in running this one in "canary" run, because the above step is doing the
-#  # same build anyway.
-#  build-ci-arm-images:
-#    name: Build CI ARM images
-#    uses: ./.github/workflows/ci-image-build.yml
-#    permissions:
-#      contents: read
-#      packages: write
-#    with:
-#      platform: "linux/arm64"
-#      push-image: "false"
-#      upload-image-artifact: "true"
-#      upload-mount-cache-artifact:  ${{ inputs.canary-run }}
-#      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-#      runs-on-as-json-self-hosted: ${{ inputs.runs-on-as-json-self-hosted }}
-#      python-versions: ${{ inputs.python-versions }}
-#      branch: ${{ inputs.branch }}
-#      constraints-branch: ${{ inputs.constraints-branch }}
-#      use-uv: ${{ inputs.use-uv }}
-#      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-#      docker-cache: ${{ inputs.docker-cache }}
-#      disable-airflow-repo-cache:  ${{ inputs.disable-airflow-repo-cache }}
-#
+        run: breeze shell --max-time 600 --platform "${PLATFORM}"

.github/workflows/additional-prod-image-tests.yml

@@ -20,8 +20,12 @@ name: Additional PROD image tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on-as-json-public:
-        description: "The array of labels (in json form) determining public runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        required: true
+        type: string
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
         required: true
         type: string
       default-branch:
@@ -36,10 +40,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to upgrade to newer dependencies (true/false)"
         required: true
         type: string
-      chicken-egg-providers:
-        description: "Whether to build chicken-egg provider distributions in the same run (true/false)"
-        required: true
-        type: string
       docker-cache:
         description: "Docker cache specification to build the image (registry, local, disabled)."
         required: true
@@ -67,14 +67,13 @@ jobs:
     name: PROD image extra checks (main)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      build-provider-distributions: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -84,14 +83,13 @@ jobs:
     name: PROD image extra checks (release)
     uses: ./.github/workflows/prod-image-extra-checks.yml
     with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
       python-versions: "[ '${{ inputs.default-python-version }}' ]"
       default-python-version: ${{ inputs.default-python-version }}
       branch: ${{ inputs.default-branch }}
       use-uv: "false"
-      build-provider-distributions: ${{ inputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
       constraints-branch: ${{ inputs.constraints-branch }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
@@ -100,7 +98,7 @@ jobs:
   test-examples-of-prod-image-building:
     timeout-minutes: 60
     name: "Test examples of PROD image building"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -120,7 +118,7 @@ jobs:
       - name: "Prepare breeze & PROD image: ${{ inputs.default-python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           image-type: "prod"
           python: ${{ inputs.default-python-version }}
           use-uv: ${{ inputs.use-uv }}
@@ -138,7 +136,7 @@ jobs:
   test-docker-compose-quick-start:
     timeout-minutes: 60
     name: "Docker Compose quick start with PROD image verifying"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
@@ -157,7 +155,7 @@ jobs:
       - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           image-type: "prod"
           python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
           use-uv: ${{ inputs.use-uv }}

.github/workflows/airflow-distributions-tests.yml

@@ -21,6 +21,14 @@ on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
       # Static inputs defined to choose which distribution to test to run
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        required: true
+        type: string
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+        required: true
+        type: string
       distribution-name:
         description: "The name of the distribution to test"
         required: true
@@ -33,11 +41,6 @@ on:  # yamllint disable-line rule:truthy
         description: "distribution test type"  # eg task-sdk-tests
         required: true
         type: string
-      # Environment inputs
-      runs-on-as-json-default:
-        description: "The array of labels (in json form) determining default runner used for the build."
-        required: true
-        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -60,7 +63,7 @@ jobs:
   distributions-tests:
     timeout-minutes: 80
     name: ${{ inputs.distribution-name }}:P${{ matrix.python-version }} tests
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     strategy:
       fail-fast: false
       matrix:
@@ -83,7 +86,7 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ matrix.python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:
-          platform: "linux/amd64"
+          platform: ${{ inputs.platform }}
           python: ${{ matrix.python-version }}
           use-uv: ${{ inputs.use-uv }}
       - name: "Cleanup dist files"

.github/workflows/basic-tests.yml

@@ -20,8 +20,8 @@ name: Basic tests
 on:  # yamllint disable-line rule:truthy
   workflow_call:
     inputs:
-      runs-on-as-json-public:
-        description: "The array of labels (in json form) determining public runners."
+      runners:
+        description: "The array of labels (in json form) determining runners."
         required: true
         type: string
       run-ui-tests:
@@ -66,7 +66,7 @@ jobs:
   run-breeze-tests:
     timeout-minutes: 10
     name: Breeze unit tests
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -87,7 +87,7 @@ jobs:
   tests-ui:
     timeout-minutes: 15
     name: React UI tests
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     if: inputs.run-ui-tests == 'true'
     steps:
       - name: "Cleanup repo"
@@ -155,7 +155,7 @@ jobs:
   install-pre-commit:
     timeout-minutes: 5
     name: "Install pre-commit for cache"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
     steps:
@@ -183,7 +183,7 @@ jobs:
   static-checks-basic-checks-only:
     timeout-minutes: 30
     name: "Static checks: basic checks only"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     needs: install-pre-commit
     if: inputs.basic-checks-only == 'true'
     steps:
@@ -236,7 +236,7 @@ jobs:
   upgrade-check:
     timeout-minutes: 45
     name: "Upgrade checks"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     needs: install-pre-commit
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
@@ -306,7 +306,7 @@ jobs:
   test-airflow-release-commands:
     timeout-minutes: 80
     name: "Test Airflow release commands"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+    runs-on: ${{ fromJSON(inputs.runners) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       GITHUB_REPOSITORY: ${{ github.repository }}
@@ -337,24 +337,26 @@ jobs:
       - name: Install twine
         run: pip install twine
       - name: "Check Airflow create minor branch command"
-        run: |
-          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
-          release-management create-minor-branch --version-branch 2-8 --answer yes
+        run: >
+          breeze release-management create-minor-branch
+          --version-branch 3-1 --answer yes --dry-run
       - name: "Check Airflow RC process command"
-        run: |
-          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
-          release-management start-rc-process --version 2.8.3rc1 --previous-version 2.8.0 --answer yes
+        run: >
+          breeze release-management start-rc-process
+          --version 3.1.0rc1 --previous-version 3.0.0 --answer yes --dry-run
       - name: "Check Airflow release process command"
-        run: |
-          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
-          release-management start-release --release-candidate 2.8.3rc1 --previous-release 2.8.0 --answer yes
+        run: >
+          breeze release-management start-release --release-candidate 3.1.0rc1
+          --previous-release 3.0.0 --answer yes --dry-run
       - name: "Test providers metadata generation"
         run: |
-          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
-          release-management generate-providers-metadata --refresh-constraints
-      - name: "Fetch all git tags"
+          git remote add apache https://github.com/apache/airflow.git
+          git fetch apache --tags
+          breeze release-management generate-providers-metadata --refresh-constraints
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Fetch all git tags for origin"
         run: git fetch --tags >/dev/null 2>&1 || true
       - name: "Test airflow core issue generation automatically"
         run: |
-          ./scripts/ci/testing/run_breeze_command_with_retries.sh \
-          release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose
+          breeze release-management generate-issue-content-core --limit-pr-count 25 --latest --verbose