Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add supports for data assets in Registry #28371

Merged
merged 16 commits into from
Feb 8, 2023
Merged

Add supports for data assets in Registry #28371

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
b584e2c
Support list/show and create data asset in a registry
valeriepham04 Jan 13, 2023
50d1c49
Merge branch 'Azure:main' into main
vmpham1012 Jan 17, 2023
e1b3528
Update change log
valeriepham04 Jan 17, 2023
a4ea166
Add unit tests for data operations
valeriepham04 Jan 20, 2023
7d5217e
merge upstream
valeriepham04 Jan 20, 2023
0845a14
Fix lint issue
valeriepham04 Jan 23, 2023
0112df2
Update service client version for data operations
valeriepham04 Jan 23, 2023
e9cb843
Add e2e test for list data in registry
valeriepham04 Jan 25, 2023
cda83b9
Merge remote-tracking branch 'upstream/main'
valeriepham04 Jan 25, 2023
9e995e2
Merge upstream
valeriepham04 Feb 6, 2023
752292c
Add E2E test for view and create data asset in registry
valeriepham04 Feb 7, 2023
d9bd0cc
Add a json file for data e2e test
valeriepham04 Feb 7, 2023
cf0ba36
Add test recordings for data assets in registry
valeriepham04 Feb 7, 2023
6e8ccfd
Update test registry for data asset e2e tests
valeriepham04 Feb 7, 2023
932a45c
Update test registry for data asset e2e tests
valeriepham04 Feb 7, 2023
c34df0a
Removed outdated recording files and marked data registry tests for r…
valeriepham04 Feb 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Removed outdated recording files and marked data registry tests for r… 
…ecording mode only
  • Loading branch information
@valeriepham04
valeriepham04 committed Feb 8, 2023
commit c34df0af82df3ac33ee467653d277a2f785868e4
34 changes: 23 additions & 11 deletions sdk/ml/azure-ai-ml/tests/dataset/e2etests/test_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import pytest
import yaml
from devtools_testutils import AzureRecordedTestCase
from devtools_testutils import AzureRecordedTestCase, is_live
from test_utilities.utils import sleep_if_live

from azure.ai.ml import MLClient, load_data
Expand Down Expand Up @@ -175,8 +175,11 @@ def test_create_mltable(self, client: MLClient, tmp_path: Path, randstr: Callabl
assert data_version.id == generate_data_arm_id(client._operation_scope, name, version)
assert data_version.path.endswith("/tmp_folder/")

def test_create_data_asset_in_registry(self, data_asset_registry_client: MLClient,
randstr: Callable[[], str]) -> None:
@pytest.mark.skipif(condition=not is_live(),
reason="Auth issue in Registry")
def test_create_data_asset_in_registry(
self, data_asset_registry_client: MLClient,
randstr: Callable[[], str]) -> None:
name = randstr("name")
version = "1"
data_asset = load_data(
Expand All @@ -188,10 +191,10 @@ def test_create_data_asset_in_registry(self, data_asset_registry_client: MLClie
}],
)
sleep_if_live(3)
obj = data_asset_registry_client.data.create_or_update(data_asset)
obj = data_asset_registry_client.data.create_or_update(data_asset)
assert obj is not None
assert name == obj.name
data_version = data_asset_registry_client.data.get(name, version)
data_version = data_asset_registry_client.data.get(name, version)

assert data_version.name == obj.name

Expand All @@ -207,25 +210,34 @@ def test_list(self, client: MLClient, data_with_2_versions: str) -> None:
# use a set since ordering of elements returned from list isn't guaranteed
assert {"1", "2"} == {data.version for data in data_list}

def test_list_data_in_registry(self, data_asset_registry_client: MLClient) -> None:
data_iterator = data_asset_registry_client.data.list()
@pytest.mark.skipif(condition=not is_live(),
reason="Auth issue in Registry")
def test_list_data_in_registry(
self, data_asset_registry_client: MLClient) -> None:
data_iterator = data_asset_registry_client.data.list()
assert data_iterator
assert isinstance(data_iterator, ItemPaged)

def test_data_get_latest_label(self, client: MLClient, randstr: Callable[[], str]) -> None:
def test_data_get_latest_label(self, client: MLClient,
randstr: Callable[[], str]) -> None:
name = randstr("name")
versions = ["foo", "bar", "baz", "foobar"]

for version in versions:
client.data.create_or_update(
load_data(
source="./tests/test_configs/dataset/data_file.yaml",
params_override=[{"name": name}, {"version": version}],
)
)
params_override=[{
"name": name
}, {
"version": version
}],
))
sleep_if_live(3)
assert client.data.get(name, label="latest").version == version

@pytest.mark.skipif(condition=not is_live(),
reason="Auth issue in Registry")
def test_data_get_latest_label_in_registry(
self, data_asset_registry_client: MLClient, randstr: Callable[[],
str]) -> None:
Expand Down
Loading