apache · potiuk · Feb 28, 2025 · Feb 28, 2025
diff --git a/contributing-docs/11_provider_packages.rst b/contributing-docs/11_provider_packages.rst
@@ -99,9 +99,23 @@ How to manage provider's dependencies
 -------------------------------------
 
 If you want to add dependencies to the provider, you should add them to the corresponding ``pyproject.toml``
-file.
+file. For regular local virtualenv development you also need to run ``uv sync --extra PROVIDER`` after you
+do it or run ``pip install -e ./providers/PROVIDER`` to install the provider in development mode in your venv
+and IDE. If you are using breeze you should also run the below command to regenerate all files that
+needs to be updated after you change dependencies:
 
-Providers are not packaged together with the core when you build "apache-airflow" package.
+.. code:: bash
+
+    breeze static-checks --type update-providers-dependencies --all-files
+
+If you have ``pre-commit`` installed, this will be done automatically for you when you commit the changes and
+you should do it before you make a PR with such changed dependency changes
+
+Also, you should rebuild the image ``breeze ci-image build`` or answer ``y`` when you are asked to rebuild the
+image for the new dependencies to be used in the Breeze CI environment.
+
+Provider's cross-dependencies
+-----------------------------
 
 Some of the packages have cross-dependencies with other providers packages. This typically happens for
 transfer operators where operators use hooks from the other providers in case they are transferring
@@ -114,13 +128,6 @@ you need functionality from the other provider package you can install it adding
 ``pip install apache-airflow-providers-google[amazon]`` in case you want to use GCP
 transfer operators from Amazon ECS.
 
-If you add a new dependency between different providers packages, it will be detected automatically during
-and pre-commit will generate new entry in ``generated/provider_dependencies.json`` and update
-``pyproject.toml`` in the new providers so that the package extra dependencies are properly handled when
-package might be installed when breeze is restarted or by your IDE or by running ``uv sync --extra PROVIDER``
-or when you run ``pip install -e "./providers"`` or ``pip install -e "./providers/<PROVIDER>"`` for the new
-provider structure.
-
 How to reuse code between tests in different providers
 ------------------------------------------------------
 

@@ -70,8 +70,10 @@ classifiers = [
 ]
 requires-python = "{{ REQUIRES_PYTHON }}"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
 {{ INSTALL_REQUIREMENTS }}
 ]

diff --git a/providers/airbyte/provider.yaml b/providers/airbyte/provider.yaml
@@ -22,7 +22,7 @@ description: |
   `Airbyte <https://airbyte.com/>`__
 
 state: ready
-source-date-epoch: 1737371678
+source-date-epoch: 1740734068
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 5.0.0

diff --git a/providers/airbyte/pyproject.toml b/providers/airbyte/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "airbyte-api>=0.52.0",

diff --git a/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py b/providers/airbyte/src/airflow/providers/airbyte/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Airbyte",
         "description": "`Airbyte <https://airbyte.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1737371678,
+        "source-date-epoch": 1740734068,
         "versions": [
             "5.0.0",
             "4.0.0",

diff --git a/providers/alibaba/provider.yaml b/providers/alibaba/provider.yaml
@@ -22,7 +22,7 @@ description: |
     Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com/>`__).
 
 state: ready
-source-date-epoch: 1734526745
+source-date-epoch: 1740734069
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 3.0.0

diff --git a/providers/alibaba/pyproject.toml b/providers/alibaba/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "oss2>=2.14.0",

diff --git a/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py b/providers/alibaba/src/airflow/providers/alibaba/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Alibaba",
         "description": "Alibaba Cloud integration (including `Alibaba Cloud <https://www.alibabacloud.com/>`__).\n",
         "state": "ready",
-        "source-date-epoch": 1734526745,
+        "source-date-epoch": 1740734069,
         "versions": [
             "3.0.0",
             "2.9.1",

@@ -146,6 +146,7 @@ Dependent package
 `apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_      ``common.compat``
 `apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_            ``common.sql``
 `apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                    ``exasol``
+`apache-airflow-providers-fab <https://airflow.apache.org/docs/apache-airflow-providers-fab>`_                          ``fab``
 `apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                          ``ftp``
 `apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_                    ``google``
 `apache-airflow-providers-http <https://airflow.apache.org/docs/apache-airflow-providers-http>`_                        ``http``

@@ -22,7 +22,7 @@ description: |
   Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
 state: ready
-source-date-epoch: 1739958314
+source-date-epoch: 1740734072
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 9.4.0

@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "apache-airflow-providers-common-compat>=1.3.0",

@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Amazon",
         "description": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
         "state": "ready",
-        "source-date-epoch": 1739958314,
+        "source-date-epoch": 1740734072,
         "versions": [
             "9.4.0",
             "9.2.0",

diff --git a/providers/apache/beam/provider.yaml b/providers/apache/beam/provider.yaml
@@ -22,7 +22,7 @@ description: |
     `Apache Beam <https://beam.apache.org/>`__.
 
 state: ready
-source-date-epoch: 1739958460
+source-date-epoch: 1740734074
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 6.0.2

diff --git a/providers/apache/beam/pyproject.toml b/providers/apache/beam/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     # Apache Beam > 2.53.0 and pyarrow > 14.0.1 fix https://nvd.nist.gov/vuln/detail/CVE-2023-47248.

diff --git a/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py b/providers/apache/beam/src/airflow/providers/apache/beam/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Beam",
         "description": "`Apache Beam <https://beam.apache.org/>`__.\n",
         "state": "ready",
-        "source-date-epoch": 1739958460,
+        "source-date-epoch": 1740734074,
         "versions": [
             "6.0.2",
             "6.0.0",

diff --git a/providers/apache/cassandra/provider.yaml b/providers/apache/cassandra/provider.yaml
@@ -22,7 +22,7 @@ description: |
     `Apache Cassandra <https://cassandra.apache.org/>`__.
 
 state: ready
-source-date-epoch: 1734527695
+source-date-epoch: 1740734076
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 3.7.0

diff --git a/providers/apache/cassandra/pyproject.toml b/providers/apache/cassandra/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "cassandra-driver>=3.29.1",

diff --git a/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py b/providers/apache/cassandra/src/airflow/providers/apache/cassandra/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Cassandra",
         "description": "`Apache Cassandra <https://cassandra.apache.org/>`__.\n",
         "state": "ready",
-        "source-date-epoch": 1734527695,
+        "source-date-epoch": 1740734076,
         "versions": [
             "3.7.0",
             "3.6.0",

diff --git a/providers/apache/drill/docs/index.rst b/providers/apache/drill/docs/index.rst
@@ -96,13 +96,13 @@ Requirements
 
 The minimum Apache Airflow version supported by this provider package is ``2.9.0``.
 
-=======================================  ==================
+=======================================  ===========================
 PIP package                              Version required
-=======================================  ==================
+=======================================  ===========================
 ``apache-airflow``                       ``>=2.9.0``
 ``apache-airflow-providers-common-sql``  ``>=1.20.0``
-``sqlalchemy-drill``                     ``>=1.1.0``
-=======================================  ==================
+``sqlalchemy-drill``                     ``>=1.1.0,!=1.1.6,!=1.1.7``
+=======================================  ===========================
 
 Cross provider package dependencies
 -----------------------------------

diff --git a/providers/apache/drill/provider.yaml b/providers/apache/drill/provider.yaml
@@ -22,7 +22,7 @@ description: |
     `Apache Drill <https://drill.apache.org/>`__.
 
 state: ready
-source-date-epoch: 1734527755
+source-date-epoch: 1740734077
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 3.0.0

diff --git a/providers/apache/drill/pyproject.toml b/providers/apache/drill/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "apache-airflow-providers-common-sql>=1.20.0",

diff --git a/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py b/providers/apache/drill/src/airflow/providers/apache/drill/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Drill",
         "description": "`Apache Drill <https://drill.apache.org/>`__.\n",
         "state": "ready",
-        "source-date-epoch": 1734527755,
+        "source-date-epoch": 1740734077,
         "versions": [
             "3.0.0",
             "2.8.1",

diff --git a/providers/apache/druid/provider.yaml b/providers/apache/druid/provider.yaml
@@ -22,7 +22,7 @@ description: |
   `Apache Druid <https://druid.apache.org/>`__.
 
 state: ready
-source-date-epoch: 1734527841
+source-date-epoch: 1740734079
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 4.0.0

diff --git a/providers/apache/druid/pyproject.toml b/providers/apache/druid/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "apache-airflow-providers-common-sql>=1.20.0",

diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py b/providers/apache/druid/src/airflow/providers/apache/druid/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Druid",
         "description": "`Apache Druid <https://druid.apache.org/>`__.\n",
         "state": "ready",
-        "source-date-epoch": 1734527841,
+        "source-date-epoch": 1740734079,
         "versions": [
             "4.0.0",
             "3.12.1",

diff --git a/providers/apache/flink/provider.yaml b/providers/apache/flink/provider.yaml
@@ -22,7 +22,7 @@ description: |
     `Apache Flink <https://flink.apache.org/>`__
 
 state: ready
-source-date-epoch: 1734527925
+source-date-epoch: 1740734081
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 1.6.0

diff --git a/providers/apache/flink/pyproject.toml b/providers/apache/flink/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "cryptography>=41.0.0",

diff --git a/providers/apache/flink/src/airflow/providers/apache/flink/get_provider_info.py b/providers/apache/flink/src/airflow/providers/apache/flink/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Flink",
         "description": "`Apache Flink <https://flink.apache.org/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1734527925,
+        "source-date-epoch": 1740734081,
         "versions": [
             "1.6.0",
             "1.5.1",

diff --git a/providers/apache/hdfs/docs/index.rst b/providers/apache/hdfs/docs/index.rst
@@ -85,12 +85,11 @@ Requirements
 
 The minimum Apache Airflow version supported by this provider package is ``2.9.0``.
 
-=================================  =========================================
+=================================  =====================================
 PIP package                        Version required
-=================================  =========================================
+=================================  =====================================
 ``apache-airflow``                 ``>=2.9.0``
 ``hdfs[avro,dataframe,kerberos]``  ``>=2.5.4; python_version < "3.12"``
 ``hdfs[avro,dataframe,kerberos]``  ``>=2.7.3; python_version >= "3.12"``
-``pandas``                         ``>=2.1.2,<2.2; python_version >= "3.9"``
-``pandas``                         ``>=1.5.3,<2.2; python_version < "3.9"``
-=================================  =========================================
+``pandas``                         ``>=2.1.2,<2.2``
+=================================  =====================================
diff --git a/providers/apache/hdfs/provider.yaml b/providers/apache/hdfs/provider.yaml
@@ -23,7 +23,7 @@ description: |
   and `WebHDFS <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__.
 
 state: ready
-source-date-epoch: 1734527971
+source-date-epoch: 1740734082
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 4.7.0

diff --git a/providers/apache/hdfs/pyproject.toml b/providers/apache/hdfs/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     'hdfs[avro,dataframe,kerberos]>=2.5.4;python_version<"3.12"',

diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache HDFS",
         "description": "`Hadoop Distributed File System (HDFS) <https://hadoop.apache.org/docs/r1.2.1/hdfs_design.html>`__\nand `WebHDFS <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/WebHDFS.html>`__.\n",
         "state": "ready",
-        "source-date-epoch": 1734527971,
+        "source-date-epoch": 1740734082,
         "versions": [
             "4.7.0",
             "4.6.0",

diff --git a/providers/apache/hive/provider.yaml b/providers/apache/hive/provider.yaml
@@ -22,7 +22,7 @@ description: |
   `Apache Hive <https://hive.apache.org/>`__
 
 state: ready
-source-date-epoch: 1739958611
+source-date-epoch: 1740734084
 # note that those versions are maintained by release manager - do not update them manually
 versions:
   - 9.0.2

diff --git a/providers/apache/hive/pyproject.toml b/providers/apache/hive/pyproject.toml
@@ -52,8 +52,10 @@ classifiers = [
 ]
 requires-python = "~=3.9"
 
-# The dependencies should be modified in place in the generated file
+# The dependencies should be modified in place in the generated file.
 # Any change in the dependencies is preserved when the file is regenerated
+# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
+# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
     "apache-airflow>=2.9.0",
     "apache-airflow-providers-common-sql>=1.20.0",

diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py
@@ -27,7 +27,7 @@ def get_provider_info():
         "name": "Apache Hive",
         "description": "`Apache Hive <https://hive.apache.org/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1739958611,
+        "source-date-epoch": 1740734084,
         "versions": [
             "9.0.2",
             "9.0.0",