Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
138 commits
Select commit Hold shift + click to select a range
da7a9d7
feat(assets): rename airflow.datasets as airflow.assets
Lee-W Aug 9, 2024
035d5e6
feat: rename DatasetManager as AssetManger
Lee-W Aug 15, 2024
bad9ef1
feat(assets/manager): rename create_datasets as create_assets
Lee-W Aug 15, 2024
142fa99
feat(assets/manager): rename register_dataset_change as register_asse…
Lee-W Aug 15, 2024
ade73dc
feat(assets/manager): rename notify_dataset_created as notify_asset_c…
Lee-W Aug 15, 2024
c51f9b2
feat(assets/manager): rename notify_dataset_changed as notify_asset_c…
Lee-W Aug 21, 2024
56e382b
feat(assets/manager): rename dataset variables in _queue_dagrun as asset
Lee-W Aug 15, 2024
381bf3d
feat(assets/manager): rename resolve_dataset_manager as resolve_asset…
Lee-W Aug 15, 2024
4e1b0ee
feat(assets/manager): rename exported dataset_manager as asset_manager
Lee-W Aug 15, 2024
783382d
feat(providers/common/compat/assets): add common.compat.assets for pr…
Lee-W Aug 15, 2024
e03b924
feat(providers/amazon): use common.compat to import Dataset
Lee-W Aug 15, 2024
1ada946
feat(providers/common/io): use common.compat to import Dataset
Lee-W Aug 15, 2024
03aff94
feat(providers/openlineage): use common.compat to import Dataset
Lee-W Aug 15, 2024
807e607
fix(providers/common/compat/assets): import dataset alias only for ai…
Lee-W Aug 15, 2024
486381c
fix(providers/common/compat/assets): import dataset any, dataset all …
Lee-W Aug 15, 2024
eb1bc88
feat(liserners/spec/dataset): rename on_dataset_created as on_asset_c…
Lee-W Aug 15, 2024
5a9f294
feat(listeners/spec/dataset): rename on_dataset_changed as on_asset_c…
Lee-W Aug 15, 2024
860cfe6
feat(assets): rename BaseDataset as BaseAsset
Lee-W Aug 15, 2024
fa9bdc1
feat(assets): rename _DatasetBooleanCondition as _AssetBooleanCondition
Lee-W Aug 15, 2024
115fcfc
feat(assets): rename DatasetAny as AssetAny
Lee-W Aug 15, 2024
29215ed
feat(assets): rename DatasetAll as AssetAll
Lee-W Aug 15, 2024
035d880
feat(assets): rename method iter_datasets as iter_assets
Lee-W Aug 15, 2024
664345e
feat(assets): rename method iter_dataset_aliases as iter_asset_aliases
Lee-W Aug 15, 2024
c7ede69
feat(assets): replace part of the dataset log messages as asset in _s…
Lee-W Aug 16, 2024
c97e3f2
feat(assets): rename DatasetAliasEvent as AssetAliasEvent
Lee-W Aug 16, 2024
b807aca
feat(models/dataset): rename DatasetAliasModel as AssetAliasModel
Lee-W Aug 19, 2024
49c46dc
feat(assets): rename _DatasetAliasCondition as _AssetAliasCondition
Lee-W Aug 19, 2024
2663986
feat(assets): rename DatasetAlias as AssetAlias
Lee-W Aug 19, 2024
239f048
feat(models/dataset): rename DagScheduleDatasetAliasReference as DagS…
Lee-W Aug 19, 2024
7301033
feat(assets): rename DagScheduleDatasetReference with DagScheduleAsse…
Lee-W Aug 20, 2024
e66d9b2
feat(assets): rename TaskOutletDatasetReference as TaskOutletAssetRef…
Lee-W Aug 20, 2024
fba2f3a
feat(assets): rename DatasetDagRunQueue as AssetDagRunQueue
Lee-W Aug 20, 2024
07b4d78
feat(assets): rename DatasetOrTimeSchedule as AssetOrTimeSchedule
Lee-W Aug 20, 2024
c5b6ad3
feat(assets): rename DatasetTriggeredTimetable as AssetTriggeredTimet…
Lee-W Aug 20, 2024
e9d72b9
feat(timtables/base): rename _NullDataset as _NullAsset
Lee-W Aug 22, 2024
c7713e5
feat(assets): rename DatasetLineageInfo as AssetLineageInfo
Lee-W Aug 20, 2024
d6548eb
feat(providers/amazon): rename add_input_dataset as add_input_asset
Lee-W Aug 20, 2024
ba61299
feat(providers/openlineage): rename collected_datasets as collected_a…
Lee-W Aug 26, 2024
58c91b1
feat(assets): rename DatasetModel as AssetModel
Lee-W Aug 20, 2024
5437ff2
feat(assets): rename DatasetEvent as AssetEvent
Lee-W Aug 21, 2024
adefffa
feat(assets): rename LazyDatasetEventSelectSequence.from_select as La…
Lee-W Aug 21, 2024
51c5bea
feat(assets): rename dataset.updates as asset.updates in logging metrics
Lee-W Aug 21, 2024
96f4552
docs(airflow/example_dags): rename dataset example dags as asset exam…
Lee-W Aug 26, 2024
6f079ca
feat(models/asset): rename models.dataset as models.asset
Lee-W Aug 26, 2024
4ca7197
feat(api_connexion/schema/dataset_schema): rename DatasetAliasSchema …
Lee-W Aug 19, 2024
74f4860
feat(api_connexion/schema/dataset_schema): rename DatasetEventCollect…
Lee-W Aug 26, 2024
df37696
feat(api_connexion/schema/dataset_schema): rename CreateDatasetEventS…
Lee-W Aug 26, 2024
cbabb54
feat(api_connexion/schema/dataset_schema): rename DatasetEventCollect…
Lee-W Aug 26, 2024
1b64e51
feat(api_connexion/schema/dataset_schema): rename DatasetEventSchema …
Lee-W Aug 26, 2024
e2eec3c
feat(api_connexion/schema/dataset_schema): rename DatasetCollectionSc…
Lee-W Aug 26, 2024
73ef0e3
feat(api_connexion/schema/dataset_schema): rename DatasetEventCollect…
Lee-W Aug 26, 2024
d351fe7
feat(api_connexion/schema/dataset_schema): rename DatasetSchema as As…
Lee-W Aug 26, 2024
2b4a285
feat(api_connexion/schema/dataset_schema): rename dataset_collection_…
Lee-W Aug 26, 2024
7c80072
feat(api_connexion/schema/dataset_schema): rename create_dataset_even…
Lee-W Aug 26, 2024
242b694
feat(api_connexion/schema/dataset_schema): rename dataset_event_colle…
Lee-W Aug 26, 2024
bb7621f
feat(api_connexion/schema/dataset_schema): rename dataset_event_schem…
Lee-W Aug 26, 2024
e2ec81d
feat(api_connexion/schema/dataset_schema): rename dataset_schema as a…
Lee-W Aug 26, 2024
28d9d64
feat(api_connexion/schemas/asset_schema): rename module dataset_schem…
Lee-W Aug 26, 2024
a5d99cb
feat(timetables): rename airflow.timetables.datasets as airflow.timet…
Lee-W Aug 27, 2024
2a3ed0a
feat(models/dags): rename get_dataset_triggered_next_run_info as get_…
Lee-W Aug 27, 2024
393b908
feat(airflow/assets): rename Dataset as Asset
Lee-W Aug 27, 2024
1422260
feat(config_templates): rename strict_dataset_uri_validation as stric…
Lee-W Aug 28, 2024
6204b35
feat(airflow/serialization/pydantic): rename DatasetEventPydantic as …
Lee-W Aug 28, 2024
12fb384
feat(airflow/serialization/pydantic): rename DatasetPydantic as Asset…
Lee-W Aug 28, 2024
4e9471e
feat(airflow/serialization): rename DAT.DATA_SET as DAT.ASSET
Lee-W Aug 28, 2024
91faff6
feat(dag_dependency): rename DepDependency value dataset and dataset-…
Lee-W Aug 28, 2024
fed4780
feat(assets): rename DatasetDetails as AssetDetails
Lee-W Aug 22, 2024
33f1bb5
feat(providers/common/compat/assets): add AssetDetails to compat
Lee-W Aug 22, 2024
9fcbc42
feat(providers/amazon): rename DatasetDetails as AssetDetails
Lee-W Aug 22, 2024
115518f
feat(providers/fab): rename DatasetDetails as AssetDetails
Lee-W Aug 22, 2024
f1f7e0c
feat(providers/amazon): rename AvpEntities.DATASET as AvpEntities.ASSET
Lee-W Aug 22, 2024
e8816ca
feat(api_connexion/security): rename requires_access_dataset as requi…
Lee-W Aug 29, 2024
7ed71bb
feat(providers_manager): rename _dataset_uri_handlers as _asset_uri_h…
Lee-W Aug 29, 2024
fa66c4a
feat(providers_manager): rename dataset_factories as asset_factories
Lee-W Aug 29, 2024
47b0005
feat(providers_manager): rename dataset_to_openlineage_converters as …
Lee-W Aug 29, 2024
01bf88f
feat(providers_manager): rename initialize_providers_dataset_uri_reso…
Lee-W Aug 29, 2024
56794ab
feat(providers_manager): rename _discover_dataset_uri_resources as _d…
Lee-W Aug 29, 2024
27dff25
feat(jobs/scheduler_job_runner): rename _create_dag_runs_dataset_trig…
Lee-W Aug 29, 2024
8a48ea7
feat(jobs/scheduler_job_runner): rename _orphan_unreferenced_datasets…
Lee-W Aug 29, 2024
7c05c3c
docs(example_dags): rename dataset in example_outlet_event_extra.py a…
Lee-W Aug 29, 2024
6a65a80
feat(providers/common): rename convert_dataset_to_openlineage as conv…
Lee-W Aug 29, 2024
8cb7784
feat(providers/amazon): rename convert_dataset_to_openlineage as conv…
Lee-W Aug 29, 2024
cda6bb7
feat(providers/openlineage): rename translate_airflow_dataset as tran…
Lee-W Aug 29, 2024
b8d0702
feat(providers/amazon): rename create_dataset as create_asset
Lee-W Aug 29, 2024
ef2368b
feat(providers/amazon): rename airflow.providers.amazon.aws.datasets …
Lee-W Aug 29, 2024
0e0d2d2
feat(providers/common/io): rename create_dataset as create_asset
Lee-W Aug 29, 2024
ac5ae8b
feat(providers/common/io): rename airflow.providers.common.io.dataset…
Lee-W Aug 29, 2024
d7dc87e
feat(www): rename has_access_dataset as has_access_asset
Lee-W Aug 29, 2024
1392c20
feat(security): rename RESOURCE_DATASET as RESOURCE_ASSET
Lee-W Aug 29, 2024
5139403
feat(providers/fab): rename RESOURCE_DATASET as RESOURCE_ASSET
Lee-W Aug 29, 2024
2684991
feat(providers/amazon): rename RESOURCE_DATASET as RESOURCE_ASSET
Lee-W Aug 29, 2024
4c87c35
feat(security): rename is_authorized_dataset as is_authorized_asset
Lee-W Aug 29, 2024
718ca0b
feat(providers/amazon): rename is_authorized_dataset as is_authorized…
Lee-W Aug 29, 2024
11c171d
feat(providers/fab): rename is_authorized_dataset as is_authorized_asset
Lee-W Aug 29, 2024
813e63b
feat(provider_mangers): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
cf9c5fc
feat(providers/amazon): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
9a3ab9d
feat(providers/common/io): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
92dbc2c
feat(providers/google): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
3299de2
feat(providers/mysql): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
498329c
feat(providers/postgres): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
f6e6431
feat(providers/trino): rename dataset-uris as asset-uris
Lee-W Aug 29, 2024
81e817a
test(dags): rename test_datasets.py as test_assets.py
Lee-W Aug 29, 2024
a1a3a99
feat(jobs/scheduler_job_runner): rename dataset.orphaned as asset.orp…
Lee-W Aug 30, 2024
702e59d
feat(jobs/scheduler_job): rename dataset.triggered_dagruns as asset.t…
Lee-W Aug 30, 2024
474c8b5
feat(listener/spec): rename airflow.listeners.spec.dataset as airflow…
Lee-W Aug 30, 2024
5fe5ba8
feat(airflow/serialization/pydantic): rename airflow.serialization.py…
Lee-W Aug 30, 2024
41caffe
feat(providers/mysql): rename airflow.providers.mysql.datasets as air…
Lee-W Aug 30, 2024
1e7cdb0
feat(providers/postgres): rename airflow.providers.postgres.datasets …
Lee-W Aug 30, 2024
9458eea
feat(providers/trino): rename airflow.providers.trino.datasets as air…
Lee-W Aug 30, 2024
51e8983
feat(context): rename triggering_dataset_events as triggering_asset_e…
Lee-W Sep 2, 2024
52d5c1b
test(breeze): add common.compat to fab provider tests
Lee-W Sep 2, 2024
2f8a05e
feat(providers/fab): add common.compt to fab dependencies due to RESO…
Lee-W Sep 2, 2024
fc96ce6
feat(providers): add back dataset-uris for backward compatibility
Lee-W Sep 2, 2024
b4395fc
feat(api_ui): rename views.datasets as views.assets
Lee-W Sep 2, 2024
175d741
docs: update dataset as asset in documentation
Lee-W Aug 30, 2024
b22c33c
docs(newsfragements): initialize changes
Lee-W Aug 27, 2024
d6e321c
docs(newsfragments): add type info
Lee-W Sep 3, 2024
5bbd731
build: update auto gen files
Lee-W Sep 4, 2024
9095188
feat(provider/microsoft): rename Dataset as Asset
Lee-W Sep 5, 2024
7448244
test(assets): fix asset hash error
Lee-W Sep 6, 2024
baf5fce
tests(providers): fix asset extra default to dict
Lee-W Sep 6, 2024
cc8c0c2
build(fast_api): regen airflow/api_fastapi/openapi/v1-generated.yaml
Lee-W Sep 18, 2024
4736f9f
feat(providers/amazon): remove common.compat.asset dependency, we'll …
Lee-W Sep 19, 2024
f9d389f
feat(providers/fab): remove common.compat.asset dependency, we'll nee…
Lee-W Sep 19, 2024
c87f40d
feat(providers/openlineage): remove common.compat.asset dependency, w…
Lee-W Sep 19, 2024
d3b062a
test: remove common.compat deps from core tests
Lee-W Sep 19, 2024
9c1e800
feat(providers/common/io): remove common.compat.asset dependency, we'…
Lee-W Sep 19, 2024
13acb58
refactor(providers/common/compat): refactor asset compat get_hook_lin…
Lee-W Sep 19, 2024
795338f
feat(providers/amazon): remove common.compat.lineage.asset_compat_lin…
Lee-W Sep 19, 2024
f4752d1
feat(providers/openlineage): remove common.compat.lineage.asset_compa…
Lee-W Sep 19, 2024
a36e85d
style: fix mypy warnings
Lee-W Sep 19, 2024
782616e
feat(dag_processing): rename DatasetModelOperation as AssetModelOpera…
Lee-W Sep 20, 2024
0b03416
feat(assets): rename iter_dataset_aliases as iter_asset_aliases
Lee-W Sep 20, 2024
a02b572
feat(simple_auth_manager): rename is_authorized_dataset as is_authori…
Lee-W Sep 20, 2024
0770828
build(ui): regen openapi files
Lee-W Sep 23, 2024
27932ce
test: fix rebase error in api endpoints
Lee-W Sep 23, 2024
c2ccfa2
fix(api_fastapi): remove unused assets.py
Lee-W Sep 23, 2024
1c137e5
feat(asset/manager): rename iter_dataset_alias, notify_dataset_alias_…
Lee-W Sep 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions RELEASE_NOTES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -642,7 +642,7 @@ Dataset URIs are now validated on input (#37005)

Datasets must use a URI that conform to rules laid down in AIP-60, and the value
will be automatically normalized when the DAG file is parsed. See
`documentation on Datasets <https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/datasets.html>`_ for
`documentation on Datasets <https://airflow.apache.org/docs/apache-airflow/2.9.0/authoring-and-scheduling/datasets.html>`_ for
a more detailed description on the rules.

You may need to change your Dataset identifiers if they look like a URI, but are
Expand Down Expand Up @@ -3264,7 +3264,7 @@ If you have the producer and consumer in different files you do not need to use
Datasets represent the abstract concept of a dataset, and (for now) do not have any direct read or write
capability - in this release we are adding the foundational feature that we will build upon.

For more info on Datasets please see :doc:`/authoring-and-scheduling/datasets`.
For more info on Datasets please see `Datasets documentation <https://airflow.apache.org/docs/apache-airflow/2.4.0/authoring-and-scheduling/datasets.html>`_.

Expanded dynamic task mapping support
"""""""""""""""""""""""""""""""""""""
Expand Down
6 changes: 3 additions & 3 deletions airflow/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
__all__ = [
"__version__",
"DAG",
"Dataset",
"Asset",
"XComArg",
]

Expand All @@ -76,7 +76,7 @@
# Things to lazy import in form {local_name: ('target_module', 'target_name', 'deprecated')}
__lazy_imports: dict[str, tuple[str, str, bool]] = {
"DAG": (".models.dag", "DAG", False),
"Dataset": (".datasets", "Dataset", False),
"Asset": (".assets", "Asset", False),
"XComArg": (".models.xcom_arg", "XComArg", False),
"version": (".version", "", False),
# Deprecated lazy imports
Expand All @@ -86,8 +86,8 @@
# These objects are imported by PEP-562, however, static analyzers and IDE's
# have no idea about typing of these objects.
# Add it under TYPE_CHECKING block should help with it.
from airflow.models.asset import Asset
from airflow.models.dag import DAG
from airflow.models.dataset import Dataset
from airflow.models.xcom_arg import XComArg


Expand Down
16 changes: 8 additions & 8 deletions airflow/api_connexion/endpoints/dag_run_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@
format_datetime,
format_parameters,
)
from airflow.api_connexion.schemas.asset_schema import (
AssetEventCollection,
asset_event_collection_schema,
)
from airflow.api_connexion.schemas.dag_run_schema import (
DAGRunCollection,
DAGRunCollectionSchema,
Expand All @@ -50,10 +54,6 @@
set_dagrun_note_form_schema,
set_dagrun_state_form_schema,
)
from airflow.api_connexion.schemas.dataset_schema import (
DatasetEventCollection,
dataset_event_collection_schema,
)
from airflow.api_connexion.schemas.task_instance_schema import (
TaskInstanceReferenceCollection,
task_instance_reference_collection_schema,
Expand Down Expand Up @@ -112,12 +112,12 @@ def get_dag_run(


@security.requires_access_dag("GET", DagAccessEntity.RUN)
@security.requires_access_dataset("GET")
@security.requires_access_asset("GET")
@provide_session
def get_upstream_dataset_events(
*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION
) -> APIResponse:
"""If dag run is dataset-triggered, return the dataset events that triggered it."""
"""If dag run is dataset-triggered, return the asset events that triggered it."""
dag_run: DagRun | None = session.scalar(
select(DagRun).where(
DagRun.dag_id == dag_id,
Expand All @@ -130,8 +130,8 @@ def get_upstream_dataset_events(
detail=f"DAGRun with DAG ID: '{dag_id}' and DagRun ID: '{dag_run_id}' not found",
)
events = dag_run.consumed_dataset_events
return dataset_event_collection_schema.dump(
DatasetEventCollection(dataset_events=events, total_entries=len(events))
return asset_event_collection_schema.dump(
AssetEventCollection(dataset_events=events, total_entries=len(events))
)


Expand Down
Loading