Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ body:
- fab
- facebook
- ftp
- git
- github
- google
- grpc
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-providers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ jobs:
run: |
for provider in ${REMOVE_PROVIDERS}; do
echo "Removing incompatible provider: ${provider}"
rm -vf dist/apache_airflow_providers_${provider/./_}*
rm -vf dist/apache_airflow_providers_${provider/./_}-*
done
if: matrix.remove-providers != ''
- name: "Download airflow package: wheel"
Expand Down
2 changes: 1 addition & 1 deletion INSTALL
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ airbyte, alibaba, amazon, apache.beam, apache.cassandra, apache.drill, apache.dr
apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka, apache.kylin, apache.livy,
apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana, atlassian.jira, celery, cloudant,
cncf.kubernetes, cohere, common.compat, common.io, common.messaging, common.sql, databricks,
datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp,
datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp, git,
github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure,
microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas,
openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector, pinecone, postgres,
Expand Down
2 changes: 2 additions & 0 deletions airflow-core/docs/extra-packages-ref.rst
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,8 @@ Some of those enable Airflow to use executors to run tasks with them - other tha
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| fab | ``pip install 'apache-airflow[fab]'`` | FAB auth manager | |
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| git | ``pip install 'apache-airflow[git]'`` | Git bundle and hook | |
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| github | ``pip install 'apache-airflow[github]'`` | GitHub operators and hook | |
+---------------------+-----------------------------------------------------+-----------------------------------------------------------------+----------------------------------------------+
| influxdb | ``pip install 'apache-airflow[influxdb]'`` | Influxdb operators and hook | |
Expand Down
2 changes: 1 addition & 1 deletion airflow-core/src/airflow/config_templates/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2478,7 +2478,7 @@ dag_processor:
[
{
"name": "my-git-repo",
"classpath": "airflow.dag_processing.bundles.git.GitDagBundle",
"classpath": "airflow.providers.git.bundles.git.GitDagBundle",
"kwargs": {
"subdir": "dags",
"tracking_ref": "main",
Expand Down
14 changes: 10 additions & 4 deletions airflow-core/src/airflow/dag_processing/bundles/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
# under the License.

---
package-name: apache-airflow-providers-bundles
package-name: apache-airflow-providers-git
name: GIT
description: |
`GIT <https://git-scm.com/>`__
`Distributed version control system (GIT) <https://git-scm.com/>`__

state: not-ready
source-date-epoch: 1726861127
Expand All @@ -29,13 +29,19 @@ versions:

integrations:
- integration-name: GIT (Git)
external-doc-url: https://git-scm.com/
tags: [software]

hooks:
- integration-name: GIT
python-modules:
- airflow.dag_processing.bundles.git
- airflow.providers.git.hooks.git
bundles:
- integration-name: GIT
python-modules:
- airflow.providers.git.bundles.git


connection-types:
- hook-class-name: airflow.dag_processing.bundles.git.GitHook
- hook-class-name: airflow.providers.git.hooks.git.GitHook
connection-type: git
24 changes: 24 additions & 0 deletions airflow-core/src/airflow/provider.yaml.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,30 @@
]
}
},
"bundles": {
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"type": "string",
"description": "Integration name. It must have a matching item in the 'integration' section of any provider."
},
"python-modules": {
"description": "List of python modules containing the bundles.",
"type": "array",
"items": {
"type": "string"
}
}
},
"additionalProperties": false,
"required": [
"integration-name",
"python-modules"
]
}
},
"connection-types": {
"type": "array",
"description": "Array of connection types mapped to hook class names",
Expand Down
24 changes: 24 additions & 0 deletions airflow-core/src/airflow/provider_info.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,30 @@
]
}
},
"bundles": {
"type": "array",
"items": {
"type": "object",
"properties": {
"integration-name": {
"type": "string",
"description": "Integration name. It must have a matching item in the 'integration' section of any provider."
},
"python-modules": {
"description": "List of python modules containing the bundles.",
"type": "array",
"items": {
"type": "string"
}
}
},
"additionalProperties": false,
"required": [
"integration-name",
"python-modules"
]
}
},
"connection-types": {
"type": "array",
"description": "Map of connection types mapped to hook class names.",
Expand Down
3 changes: 0 additions & 3 deletions airflow-core/src/airflow/providers_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,6 @@ def _create_customized_form_field_behaviours_schema_validator():


def _check_builtin_provider_prefix(provider_package: str, class_name: str) -> bool:
if "bundles" in provider_package:
# TODO: AIP-66: remove this when this package is moved to providers directory
return True
if provider_package.startswith("apache-airflow"):
provider_path = provider_package[len("apache-") :].replace("-", ".")
if not class_name.startswith(provider_path):
Expand Down
2 changes: 1 addition & 1 deletion airflow-core/tests/unit/api_fastapi/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def configure_git_connection_for_dag_bundle(session):
(
"dag_processor",
"dag_bundle_config_list",
): '[{ "name": "dag_maker", "classpath": "airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}, { "name": "another_bundle_name", "classpath": "airflow.dag_processing.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}]'
): '[{ "name": "dag_maker", "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}, { "name": "another_bundle_name", "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": {"subdir": "dags", "tracking_ref": "main", "refresh_interval": 0}}]'
}
):
yield
Expand Down
2 changes: 1 addition & 1 deletion contributing-docs/12_airflow_dependencies_and_extras.rst
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ airbyte, alibaba, amazon, apache.beam, apache.cassandra, apache.drill, apache.dr
apache.hdfs, apache.hive, apache.iceberg, apache.impala, apache.kafka, apache.kylin, apache.livy,
apache.pig, apache.pinot, apache.spark, apprise, arangodb, asana, atlassian.jira, celery, cloudant,
cncf.kubernetes, cohere, common.compat, common.io, common.messaging, common.sql, databricks,
datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp,
datadog, dbt.cloud, dingding, discord, docker, edge, elasticsearch, exasol, fab, facebook, ftp, git,
github, google, grpc, hashicorp, http, imap, influxdb, jdbc, jenkins, microsoft.azure,
microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mysql, neo4j, odbc, openai, openfaas,
openlineage, opensearch, opsgenie, oracle, pagerduty, papermill, pgvector, pinecone, postgres,
Expand Down
Loading