Skip to content

fix PosixPath not working with file create_asset #47880

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mobuchowski merged 1 commit into from
Mar 17, 2025
Merged

fix PosixPath not working with file create_asset #47880

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion providers/amazon/src/airflow/providers/amazon/aws/hooks/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1494,7 +1494,9 @@ def download_file(
get_hook_lineage_collector().add_output_asset(
context=self,
scheme="file",
asset_kwargs={"path": file_path if file_path.is_absolute() else file_path.absolute()},
asset_kwargs={
"path": str(file_path) if file_path.is_absolute() else str(file_path.absolute())
},
)
file = open(file_path, "wb")
else:
Expand Down
12 changes: 11 additions & 1 deletion providers/common/io/src/airflow/providers/common/io/assets/file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from __future__ import annotations

import urllib.parse
from pathlib import PosixPath
from typing import TYPE_CHECKING

from airflow.providers.common.io.version_compat import AIRFLOW_V_3_0_PLUS
Expand All @@ -32,9 +33,18 @@
from airflow.providers.common.compat.openlineage.facet import Dataset as OpenLineageDataset


def create_asset(*, path: str, extra=None) -> Asset:
def create_asset(*, path: str | PosixPath, extra=None) -> Asset:
if isinstance(path, PosixPath):
path = str(path)

if path.startswith("file://"):
path = path[len("file://") :]
elif path.startswith("file:"):
path = path[5:]
while path.startswith("/"):
path = path[1:]
path = "/" + path

return Asset(uri=f"file://{path}", extra=extra)


Expand Down
10 changes: 10 additions & 0 deletions providers/common/io/tests/unit/common/io/assets/test_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations

from pathlib import PosixPath
from urllib.parse import urlsplit, urlunsplit

import pytest
Expand Down Expand Up @@ -54,6 +55,9 @@ def test_sanitize_uri_invalid(uri):
("file:///asdf/fdsa", "file:///asdf/fdsa"),
("file://asdf/fdsa", "file://asdf/fdsa"),
("file://127.0.0.1:8080/dir/file.csv", "file://127.0.0.1:8080/dir/file.csv"),
(PosixPath("file:///tmpdir/some-path.csv"), "file:///tmpdir/some-path.csv"),
("file:/tmpdir/some-path.csv", "file:///tmpdir/some-path.csv"),
("file:tmpdir/some-path.csv", "file:///tmpdir/some-path.csv"),
),
)
def test_file_asset(path, uri):
Expand All @@ -71,6 +75,12 @@ def test_file_asset(path, uri):
("file:///C://dir/file", OpenLineageDataset(namespace="file", name="/C://dir/file")),
("file://asdf.pdf", OpenLineageDataset(namespace="file", name="/asdf.pdf")),
("file:///asdf.pdf", OpenLineageDataset(namespace="file", name="/asdf.pdf")),
(
PosixPath("file:///tmp/pytest/test.log"),
OpenLineageDataset(namespace="file", name="/tmp/pytest/test.log"),
),
("file:/tmp/pytest/test.log", OpenLineageDataset(namespace="file", name="/tmp/pytest/test.log")),
("file:tmp/pytest/test.log", OpenLineageDataset(namespace="file", name="/tmp/pytest/test.log")),
),
)
def test_convert_asset_to_openlineage(path, ol_dataset):
Expand Down