Move constraints generation to separate job in CI workflow

.github/actions/build-ci-images/action.yml

@@ -34,30 +34,6 @@ runs:
     - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Generate source constraints"
-      shell: bash
-      run: >
-        breeze release-management generate-constraints --run-in-parallel
-        --airflow-constraints-mode constraints-source-providers
-#    - name: "Generate PyPI constraints"
-#      shell: bash
-#      run: >
-#        breeze release-management generate-constraints --run-in-parallel
-#        --airflow-constraints-mode constraints
-#      if: ${{ inputs.build-provider-packages != 'true' }}
-    - name: "Print dependency upgrade summary"
-      shell: bash
-      run: |
-        for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
-          echo "Summarizing Python $PYTHON_VERSION"
-          cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true
-        done
-    - name: "Upload constraint artifacts"
-      uses: actions/upload-artifact@v3
-      with:
-        name: constraints
-        path: ./files/constraints-*/constraints-*.txt
-        retention-days: 7
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/workflows/ci.yml

@@ -326,6 +326,61 @@ jobs:
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           BUILD_TIMEOUT_MINUTES: 70
 
+  preview-constraints:
+    permissions:
+      contents: read
+    timeout-minutes: 20
+    continue-on-error: true
+    name: >
+      Preview constraints
+      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    runs-on: "${{needs.build-info.outputs.runs-on}}"
+    needs: [build-info, build-ci-images]
+    if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
+    env:
+      RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
+      PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+        run: breeze ci-image pull --run-in-parallel --tag-as-latest
+      - name: "Source constraints"
+        shell: bash
+        run: >
+          breeze release-management generate-constraints --run-in-parallel
+          --airflow-constraints-mode constraints-source-providers || true
+      - name: "PyPI constraints"
+        shell: bash
+        timeout-minutes: 25
+        run: >
+          breeze release-management generate-constraints --run-in-parallel
+          --airflow-constraints-mode constraints || true
+      - name: "Dependency upgrade summary"
+        shell: bash
+        run: |
+          for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
+            echo "Summarizing Python $PYTHON_VERSION"
+            cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true
+          done
+      - name: "Upload constraint artifacts"
+        uses: actions/upload-artifact@v3
+        with:
+          name: constraints
+          path: ./files/constraints-*/constraints-*.txt
+          retention-days: 7
+      - name: "Fix ownership"
+        run: breeze ci fix-ownership
+        if: always()
+
   build-prod-images:
     timeout-minutes: 80
     name: >
@@ -1662,6 +1717,7 @@ jobs:
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
     if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
     steps:
       - name: Cleanup repo
@@ -1677,16 +1733,18 @@ jobs:
         run: breeze ci-image pull --run-in-parallel --tag-as-latest
         env:
           PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
-      - name: "Generate constraints"
+      - name: "Generate source constraints"
+        run: |
+          breeze release-management generate-constraints \
+              --run-in-parallel --airflow-constraints-mode constraints-source-providers
+      - name: "Generate no-providers constraints"
         run: |
-          breeze release-management generate-constraints --run-in-parallel \
-              --airflow-constraints-mode constraints-source-providers
           breeze release-management generate-constraints \
               --run-in-parallel --airflow-constraints-mode constraints-no-providers
-#          breeze release-management generate-constraints \
-#              --run-in-parallel --airflow-constraints-mode constraints
-        env:
-          PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
+      - name: "Generate PyPi constraints"
+        run: |
+          breeze release-management generate-constraints \
+              --run-in-parallel --airflow-constraints-mode constraints
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}

Dockerfile.ci

@@ -1435,8 +1435,11 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
 # Those are additional constraints that are needed for some extras but we do not want to
 # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better
 # dependency resolution and we do not need to limit the versions of the dependencies
-# aiobotocore is limited temporarily until it stops backtracking pip
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
+#
+# Without grpcio-status limit, pip gets into very long backtracking
+# We should attempt to remove it in the future
+#
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="grpcio-status>=1.55.0"
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ARG VERSION_SUFFIX_FOR_PYPI=""
 

scripts/in_container/_in_container_utils.sh

@@ -333,9 +333,10 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
     # Installing it with Airflow makes sure that the version of package that matches current
     # Airflow requirements will be used.
     # shellcheck disable=SC2086
+    set -x
     pip install ".[${NO_PROVIDERS_EXTRAS}]" "${packages_to_install[@]}" ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} \
         --upgrade --upgrade-strategy eager
-
+    set +x
 }
 
 function install_all_provider_packages_from_wheels() {

scripts/in_container/run_generate_constraints.sh

@@ -152,5 +152,6 @@ echo
 echo "Constraints error markdown generated in ${CONSTRAINTS_MARKDOWN_DIFF}"
 echo
 
+ls "${CONSTRAINTS_MARKDOWN_DIFF}"
 
 exit 0

setup.py

@@ -1042,4 +1042,4 @@ def include_provider_namespace_packages_when_installing_from_sources() -> None:
 
 
 if __name__ == "__main__":
-    do_setup()  # comment
+    do_setup()  # comment to trigger upgrade to newer dependencies