From 9c28447723a295f050f50a20f1955fadff0ec8c5 Mon Sep 17 00:00:00 2001 From: dreamlandliu <205407@qq.com> Date: Fri, 6 Jan 2023 11:17:14 +0800 Subject: [PATCH] feat(client): add filter params for model/dataset/runtime list command (#1672) --- client/starwhale/base/bundle.py | 43 +++++- client/starwhale/base/cloud.py | 17 ++- client/starwhale/core/dataset/cli.py | 26 +++- client/starwhale/core/dataset/model.py | 11 +- client/starwhale/core/dataset/view.py | 13 +- client/starwhale/core/model/cli.py | 26 +++- client/starwhale/core/model/model.py | 11 +- client/starwhale/core/model/view.py | 16 ++- client/starwhale/core/runtime/cli.py | 26 +++- client/starwhale/core/runtime/model.py | 11 +- client/starwhale/core/runtime/view.py | 16 ++- client/tests/base/test_bundle.py | 58 ++++++++ client/tests/base/test_cloud.py | 127 +++++++++++++++++- client/tests/core/test_dataset.py | 43 ++++++ client/tests/core/test_model.py | 37 +++++ client/tests/core/test_runtime.py | 43 ++++++ .../current/reference/cli/dataset.md | 24 +++- .../current/reference/cli/model.md | 26 ++-- .../current/reference/cli/project.md | 16 +-- .../current/reference/cli/runtime.md | 22 ++- 20 files changed, 557 insertions(+), 55 deletions(-) create mode 100644 client/tests/base/test_bundle.py diff --git a/client/starwhale/base/bundle.py b/client/starwhale/base/bundle.py index 29988ef2c1..d57ca43e18 100644 --- a/client/starwhale/base/bundle.py +++ b/client/starwhale/base/bundle.py @@ -29,6 +29,7 @@ from starwhale.utils.config import SWCliConfigMixed from .uri import URI +from .store import BundleField class BaseBundle(metaclass=ABCMeta): @@ -76,9 +77,49 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.Union[t.Dict[str, t.Any], t.List[str]]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filters = filters or {} _cls = cls._get_cls(project_uri) - return _cls.list(project_uri, page, size) # type: ignore + _filter = cls.get_filter_dict(filters, cls.get_filter_fields()) + return _cls.list(project_uri, page, size, _filter) # type: ignore + + @classmethod + def get_filter_dict( + cls, + filters: t.Union[t.Dict[str, t.Any], t.List[str]], + fields: t.Optional[t.List[str]] = None, + ) -> t.Dict[str, t.Any]: + fields = fields or [] + if isinstance(filters, t.Dict): + return {k: v for k, v in filters.items() if k in fields} + + _filter_dict: t.Dict[str, t.Any] = {} + for _f in filters: + _item = _f.split("=", 1) + if _item[0] in fields: + _filter_dict[_item[0]] = _item[1] if len(_item) > 1 else "" + return _filter_dict + + @classmethod + def get_filter_fields(cls) -> t.List[str]: + return ["name", "owner", "latest"] + + @classmethod + def do_bundle_filter( + cls, + bundle_field: BundleField, + filters: t.Union[t.Dict[str, t.Any], t.List[str]], + ) -> bool: + filter_dict = cls.get_filter_dict(filters, cls.get_filter_fields()) + _name = filter_dict.get("name") + if _name and not bundle_field.name.startswith(_name): + return False + _latest = filter_dict.get("latest") is not None + if _latest and "latest" not in bundle_field.tags: + return False + + return True @abstractclassmethod def _get_cls(cls, uri: URI) -> t.Any: diff --git a/client/starwhale/base/cloud.py b/client/starwhale/base/cloud.py index 70be1af871..41dda5e35b 100644 --- a/client/starwhale/base/cloud.py +++ b/client/starwhale/base/cloud.py @@ -235,23 +235,32 @@ def _fetch_bundle_all_list( uri_typ: str, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filter_dict: t.Optional[t.Dict[str, t.Any]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filter_dict = filter_dict or {} + _params = {"pageNum": page, "pageSize": size} + _params.update(filter_dict) r = self.do_http_request( f"/project/{project_uri.project}/{uri_typ}", - params={"pageNum": page, "pageSize": size}, + params=_params, instance_uri=project_uri, ).json() - objects = {} + _page = page + _size = size + if filter_dict.get("latest") is not None: + _page = 1 + _size = 1 + for o in r["data"]["list"]: _name = f"[{o['id']}] {o['name']}" objects[_name] = self._fetch_bundle_history( name=o["id"], project_uri=project_uri, typ=uri_typ, - page=page, - size=size, + page=_page, + size=_size, )[0] return objects, self.parse_pager(r) diff --git a/client/starwhale/core/dataset/cli.py b/client/starwhale/core/dataset/cli.py index 3899e18f69..57e9c9beb0 100644 --- a/client/starwhale/core/dataset/cli.py +++ b/client/starwhale/core/dataset/cli.py @@ -118,7 +118,7 @@ def _diff( view(base_uri).diff(URI(compare_uri, expected_type=URIType.DATASET), show_details) -@dataset_cmd.command("list", aliases=["ls"], help="List dataset") +@dataset_cmd.command("list", aliases=["ls"]) @click.option("-p", "--project", default="", help="Project URI") @click.option("-f", "--fullname", is_flag=True, help="Show fullname of dataset version") @click.option("-sr", "--show-removed", is_flag=True, help="Show removed datasets") @@ -128,6 +128,13 @@ def _diff( @click.option( "--size", type=int, default=DEFAULT_PAGE_SIZE, help="Page size for dataset list" ) +@click.option( + "filters", + "-fl", + "--filter", + multiple=True, + help="Filter output based on conditions provided.", +) @click.pass_obj def _list( view: DatasetTermView, @@ -136,8 +143,23 @@ def _list( show_removed: bool, page: int, size: int, + filters: list, ) -> None: - view.list(project, fullname, show_removed, page, size) + """ + List Dataset + + The filtering flag (-fl or --filter) format is a key=value pair or a flag. + If there is more than one filter, then pass multiple flags.\n + (e.g. --filter name=mnist --filter latest) + + \b + The currently supported filters are: + name\tTEXT\tThe prefix of the dataset name + owner\tTEXT\tThe name or id of the dataset owner + latest\tFLAG\t[Cloud] Only show the latest version + \t \t[Standalone] Only show the version with "latest" tag + """ + view.list(project, fullname, show_removed, page, size, filters) @dataset_cmd.command("info", help="Show dataset details") diff --git a/client/starwhale/core/dataset/model.py b/client/starwhale/core/dataset/model.py index d115cac908..e04b049b14 100644 --- a/client/starwhale/core/dataset/model.py +++ b/client/starwhale/core/dataset/model.py @@ -257,7 +257,9 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.Union[t.Dict[str, t.Any], t.List[str]]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filters = filters or {} rs = defaultdict(list) for _bf in DatasetStorage.iter_all_bundles( @@ -265,6 +267,9 @@ def list( bundle_type=BundleType.DATASET, uri_type=URIType.DATASET, ): + if not cls.do_bundle_filter(_bf, filters): + continue + _mf = _bf.path / DEFAULT_MANIFEST_NAME if not _mf.exists(): continue @@ -521,9 +526,13 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filter_dict: t.Optional[t.Dict[str, t.Any]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filter_dict = filter_dict or {} crm = CloudRequestMixed() - return crm._fetch_bundle_all_list(project_uri, URIType.DATASET, page, size) + return crm._fetch_bundle_all_list( + project_uri, URIType.DATASET, page, size, filter_dict + ) def summary(self) -> t.Optional[DatasetSummary]: resp = self.do_http_request( diff --git a/client/starwhale/core/dataset/view.py b/client/starwhale/core/dataset/view.py index 334ad9624c..f982fed9c7 100644 --- a/client/starwhale/core/dataset/view.py +++ b/client/starwhale/core/dataset/view.py @@ -139,15 +139,16 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: - + filters = filters or [] if isinstance(project_uri, str): _uri = URI(project_uri, expected_type=URIType.PROJECT) else: _uri = project_uri fullname = fullname or (_uri.instance_type == InstanceType.CLOUD) - _datasets, _pager = Dataset.list(_uri, page, size) + _datasets, _pager = Dataset.list(_uri, page, size, filters) _data = BaseTermView.list_data(_datasets, show_removed, fullname) return _data, _pager @@ -235,9 +236,11 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: + filters = filters or [] _datasets, _pager = super().list( - project_uri, fullname, show_removed, page, size + project_uri, fullname, show_removed, page, size, filters ) custom_column: t.Dict[str, t.Callable[[t.Any], str]] = { "tags": lambda x: ",".join(x), @@ -258,9 +261,11 @@ def list( # type: ignore show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> None: + filters = filters or [] _datasets, _pager = super().list( - project_uri, fullname, show_removed, page, size + project_uri, fullname, show_removed, page, size, filters ) cls.pretty_json(_datasets) diff --git a/client/starwhale/core/model/cli.py b/client/starwhale/core/model/cli.py index ce7904561a..a91e2c7c8d 100644 --- a/client/starwhale/core/model/cli.py +++ b/client/starwhale/core/model/cli.py @@ -120,7 +120,7 @@ def _diff( view(base_uri).diff(URI(compare_uri, expected_type=URIType.MODEL), show_details) -@model_cmd.command("list", aliases=["ls"], help="List Model") +@model_cmd.command("list", aliases=["ls"]) @click.option("-p", "--project", default="", help="Project URI") @click.option("-f", "--fullname", is_flag=True, help="Show fullname of model version") @click.option("-sr", "--show-removed", is_flag=True, help="Show removed model") @@ -130,6 +130,13 @@ def _diff( @click.option( "--size", type=int, default=DEFAULT_PAGE_SIZE, help="Page size for model list" ) +@click.option( + "filters", + "-fl", + "--filter", + multiple=True, + help="Filter output based on conditions provided.", +) @click.pass_obj def _list( view: t.Type[ModelTermView], @@ -138,8 +145,23 @@ def _list( show_removed: bool, page: int, size: int, + filters: list, ) -> None: - view.list(project, fullname, show_removed, page, size) + """ + List Model + + The filtering flag (-fl or --filter) format is a key=value pair or a flag. + If there is more than one filter, then pass multiple flags.\n + (e.g. --filter name=mnist --filter latest) + + \b + The currently supported filters are: + name\tTEXT\tThe prefix of the model name + owner\tTEXT\tThe name or id of the model owner + latest\tFLAG\t[Cloud] Only show the latest version + \t \t[Standalone] Only show the version with "latest" tag + """ + view.list(project, fullname, show_removed, page, size, filters) @model_cmd.command("history", help="Show model history") diff --git a/client/starwhale/core/model/model.py b/client/starwhale/core/model/model.py index 79df774996..80cd7d8dee 100644 --- a/client/starwhale/core/model/model.py +++ b/client/starwhale/core/model/model.py @@ -537,13 +537,18 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.Union[t.Dict[str, t.Any], t.List[str]]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filters = filters or {} rs = defaultdict(list) for _bf in ModelStorage.iter_all_bundles( project_uri, bundle_type=BundleType.MODEL, uri_type=URIType.MODEL, ): + if not cls.do_bundle_filter(_bf, filters): + continue + if _bf.path.is_file(): # for origin swmp(tar) _manifest = ModelStorage.get_manifest_by_path( @@ -736,9 +741,13 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filter_dict: t.Optional[t.Dict[str, t.Any]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filter_dict = filter_dict or {} crm = CloudRequestMixed() - return crm._fetch_bundle_all_list(project_uri, URIType.MODEL, page, size) + return crm._fetch_bundle_all_list( + project_uri, URIType.MODEL, page, size, filter_dict + ) def build(self, *args: t.Any, **kwargs: t.Any) -> None: raise NoSupportError("no support build model in the cloud instance") diff --git a/client/starwhale/core/model/view.py b/client/starwhale/core/model/view.py index 970526ec1d..3cf621bfe8 100644 --- a/client/starwhale/core/model/view.py +++ b/client/starwhale/core/model/view.py @@ -201,10 +201,12 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.Union[t.Dict[str, t.Any], t.List[str]]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: + filters = filters or {} _uri = URI(project_uri, expected_type=URIType.PROJECT) fullname = fullname or (_uri.instance_type == InstanceType.CLOUD) - _models, _pager = Model.list(_uri, page, size) + _models, _pager = Model.list(_uri, page, size, filters) _data = BaseTermView.list_data(_models, show_removed, fullname) return _data, _pager @@ -308,8 +310,12 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: - _models, _pager = super().list(project_uri, fullname, show_removed, page, size) + filters = filters or [] + _models, _pager = super().list( + project_uri, fullname, show_removed, page, size, filters + ) custom_column: t.Dict[str, t.Callable[[t.Any], str]] = { "tags": lambda x: ",".join(x), "size": lambda x: pretty_bytes(x), @@ -329,8 +335,12 @@ def list( # type: ignore show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> None: - _models, _pager = super().list(project_uri, fullname, show_removed, page, size) + filters = filters or [] + _models, _pager = super().list( + project_uri, fullname, show_removed, page, size, filters + ) cls.pretty_json(_models) def info(self, fullname: bool = False) -> None: diff --git a/client/starwhale/core/runtime/cli.py b/client/starwhale/core/runtime/cli.py index 778e565152..a7755d950a 100644 --- a/client/starwhale/core/runtime/cli.py +++ b/client/starwhale/core/runtime/cli.py @@ -265,7 +265,7 @@ def _restore(target: str) -> None: RuntimeTermView.restore(target) -@runtime_cmd.command("list", aliases=["ls"], help="List runtime") +@runtime_cmd.command("list", aliases=["ls"]) @click.option("-p", "--project", default="", help="Project URI") @click.option("-f", "--fullname", is_flag=True, help="Show fullname of runtime version") @click.option("-sr", "--show-removed", is_flag=True, help="Show removed runtime") @@ -275,6 +275,13 @@ def _restore(target: str) -> None: @click.option( "--size", type=int, default=DEFAULT_PAGE_SIZE, help="Page size for tasks list" ) +@click.option( + "filters", + "-fl", + "--filter", + multiple=True, + help="Filter output based on conditions provided.", +) @click.pass_obj def _list( view: t.Type[RuntimeTermView], @@ -283,8 +290,23 @@ def _list( show_removed: bool, page: int, size: int, + filters: list, ) -> None: - view.list(project, fullname, show_removed, page, size) + """ + List Runtime + + The filtering flag (-fl or --filter) format is a key=value pair or a flag. + If there is more than one filter, then pass multiple flags.\n + (e.g. --filter name=mnist --filter latest) + + \b + The currently supported filters are: + name\tTEXT\tThe prefix of the runtime name + owner\tTEXT\tThe name or id of the runtime owner + latest\tFLAG\t[Cloud] Only show the latest version + \t \t[Standalone] Only show the version with "latest" tag + """ + view.list(project, fullname, show_removed, page, size, filters) @runtime_cmd.command( diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py index 30bb9fa64d..9c95c49e29 100644 --- a/client/starwhale/core/runtime/model.py +++ b/client/starwhale/core/runtime/model.py @@ -1038,11 +1038,16 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.Union[t.Dict[str, t.Any], t.List[str]]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filters = filters or {} rs = defaultdict(list) for _bf in RuntimeStorage.iter_all_bundles( project_uri, bundle_type=BundleType.RUNTIME, uri_type=URIType.RUNTIME ): + if not cls.do_bundle_filter(_bf, filters): + continue + if not _bf.path.is_file(): continue @@ -1757,9 +1762,13 @@ def list( project_uri: URI, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filter_dict: t.Optional[t.Dict[str, t.Any]] = None, ) -> t.Tuple[t.Dict[str, t.Any], t.Dict[str, t.Any]]: + filter_dict = filter_dict or {} crm = CloudRequestMixed() - return crm._fetch_bundle_all_list(project_uri, URIType.RUNTIME, page, size) + return crm._fetch_bundle_all_list( + project_uri, URIType.RUNTIME, page, size, filter_dict + ) def build(self, *args: t.Any, **kwargs: t.Any) -> None: raise NoSupportError("no support build runtime in the cloud instance") diff --git a/client/starwhale/core/runtime/view.py b/client/starwhale/core/runtime/view.py index dfebeaa879..a8349a3352 100644 --- a/client/starwhale/core/runtime/view.py +++ b/client/starwhale/core/runtime/view.py @@ -206,10 +206,12 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: + filters = filters or [] _uri = URI(project_uri, expected_type=URIType.PROJECT) fullname = fullname or (_uri.instance_type == InstanceType.CLOUD) - _runtimes, _pager = Runtime.list(_uri, page, size) + _runtimes, _pager = Runtime.list(_uri, page, size, filters) _data = BaseTermView.list_data(_runtimes, show_removed, fullname) return _data, _pager @@ -305,8 +307,12 @@ def list( show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> t.Tuple[t.List[t.Dict[str, t.Any]], t.Dict[str, t.Any]]: - _data, _pager = super().list(project_uri, fullname, show_removed, page, size) + filters = filters or [] + _data, _pager = super().list( + project_uri, fullname, show_removed, page, size, filters + ) custom_column: t.Dict[str, t.Callable[[t.Any], str]] = { "tags": lambda x: ",".join(x), @@ -327,8 +333,12 @@ def list( # type: ignore show_removed: bool = False, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE, + filters: t.Optional[t.List[str]] = None, ) -> None: - _data, _pager = super().list(project_uri, fullname, show_removed, page, size) + filters = filters or [] + _data, _pager = super().list( + project_uri, fullname, show_removed, page, size, filters + ) cls.pretty_json(_data) def info(self, fullname: bool = False) -> None: diff --git a/client/tests/base/test_bundle.py b/client/tests/base/test_bundle.py new file mode 100644 index 0000000000..70e2425d3c --- /dev/null +++ b/client/tests/base/test_bundle.py @@ -0,0 +1,58 @@ +from pathlib import Path + +from pyfakefs.fake_filesystem_unittest import TestCase + +from starwhale.base.store import BundleField +from starwhale.base.bundle import BaseBundle + + +class TestBaseBundle(TestCase): + def test_get_filter_dict(self): + _cls = BaseBundle + _fields = ["name", "owner", "latest"] + _filter = ["name=mnist", "owner=starwhale", "latest", "other"] + f_dict = _cls.get_filter_dict(filters=_filter, fields=_fields) + print(f_dict) + assert f_dict.get("name") == "mnist" + assert f_dict.get("owner") == "starwhale" + assert f_dict.get("latest") is not None + assert not f_dict.get("other") + + _filter = {"name": "nmt", "latest": True, "other": True} + f_dict = _cls.get_filter_dict(filters=_filter, fields=_fields) + assert f_dict.get("name") == "nmt" + assert f_dict.get("latest") is not None + assert not f_dict.get("owner") + assert not f_dict.get("other") + + def test_do_bundle_filter(self): + _cls = BaseBundle + _bf = BundleField( + name="mnist", + version="5o66ol6hj3erffoc3wnyzphymgdqjqrnwwezjesk", + tags=["v1", "latest"], + path=Path(), + is_removed=False, + ) + _filter = {"name": "nmt", "latest": True} + assert not _cls.do_bundle_filter(_bf, _filter) + + _filter = {"name": "mnist", "latest": True} + assert _cls.do_bundle_filter(_bf, _filter) + + _filter = {"name": "mn", "latest": True} + assert _cls.do_bundle_filter(_bf, _filter) + + _bf = BundleField( + name="mnist", + version="5o66ol6hj3erffoc3wnyzphymgdqjqrnwwezjesk", + tags=["v1"], + path=Path(), + is_removed=False, + ) + + _filter = {"name": "mnist", "latest": True} + assert not _cls.do_bundle_filter(_bf, _filter) + + _filter = {"name": "mnist"} + assert _cls.do_bundle_filter(_bf, _filter) diff --git a/client/tests/base/test_cloud.py b/client/tests/base/test_cloud.py index ccfc087a47..b983ed346f 100644 --- a/client/tests/base/test_cloud.py +++ b/client/tests/base/test_cloud.py @@ -1,7 +1,14 @@ +from http import HTTPStatus + import yaml +from requests_mock import Mocker from pyfakefs.fake_filesystem_unittest import TestCase -from starwhale.base.cloud import CloudRequestMixed +from starwhale.consts import HTTPMethod +from starwhale.base.uri import URI +from starwhale.base.type import URIType +from starwhale.base.cloud import CloudRequestMixed, CloudBundleModelMixin +from starwhale.utils.config import SWCliConfigMixed class TestCloudRequestMixed(TestCase): @@ -24,3 +31,121 @@ def test_get_bundle_size_from_resp(self): item = {"meta": "no dataset byte size"} size = ins.get_bundle_size_from_resp("dataset", item) assert size == 0 + + @Mocker() + def test_bundle_list(self, rm: Mocker) -> None: + sw = SWCliConfigMixed() + sw.update_instance( + uri="http://1.1.1.1", user_name="test", sw_token="123", alias="test" + ) + + rm.request( + HTTPMethod.GET, + "http://1.1.1.1/api/v1/project/sw/model?pageNum=1&pageSize=20", + json={ + "data": { + "list": [ + {"id": 2, "name": "mnist"}, + {"id": 1, "name": "text_cls"}, + ], + "total": 2, + "size": 2, + } + }, + status_code=HTTPStatus.OK, + ) + + rm.request( + HTTPMethod.GET, + "http://1.1.1.1/api/v1/project/sw/model?name=mnist", + json={ + "data": { + "list": [ + {"id": 2, "name": "mnist"}, + ], + "total": 1, + "size": 1, + } + }, + status_code=HTTPStatus.OK, + ) + + rm.request( + HTTPMethod.GET, + "http://1.1.1.1/api/v1/project/sw/model/1/version", + json={ + "data": { + "list": [ + {"id": 2, "name": "tc_v2", "createdTime": 3000}, + {"id": 1, "name": "tc_v1", "createdTime": 1000}, + ], + "total": 2, + "size": 2, + } + }, + status_code=HTTPStatus.OK, + ) + + rm.request( + HTTPMethod.GET, + "http://1.1.1.1/api/v1/project/sw/model/2/version", + json={ + "data": { + "list": [ + {"id": 5, "name": "mnist_v3", "createdTime": 2000}, + {"id": 4, "name": "mnist_v2", "createdTime": 2000}, + {"id": 3, "name": "mnist_v1", "createdTime": 2000}, + ], + "total": 3, + "size": 3, + } + }, + status_code=HTTPStatus.OK, + ) + + rm.request( + HTTPMethod.GET, + "http://1.1.1.1/api/v1/project/sw/model/2/version?pageNum=1&pageSize=1", + json={ + "data": { + "list": [ + {"id": 5, "name": "mnist_v3", "createdTime": 2000}, + ], + "total": 1, + "size": 1, + } + }, + status_code=HTTPStatus.OK, + ) + + cbm = CloudBundleModelMixin() + _uri = URI("http://1.1.1.1/project/sw", expected_type=URIType.PROJECT) + _models, _pager = cbm._fetch_bundle_all_list(_uri, uri_typ=URIType.MODEL) + + assert len(_models.items()) == 2 + assert len(_models["[2] mnist"]) == 3 + assert len(_models["[1] text_cls"]) == 2 + assert _pager["current"] == 2 + assert _pager["remain"] == 0 + + cbm = CloudBundleModelMixin() + _uri = URI("http://1.1.1.1/project/sw", expected_type=URIType.PROJECT) + _models, _pager = cbm._fetch_bundle_all_list( + _uri, uri_typ=URIType.MODEL, filter_dict={"name": "mnist"} + ) + + assert len(_models.items()) == 1 + assert len(_models["[2] mnist"]) == 3 + assert _pager["current"] == 1 + assert _pager["remain"] == 0 + + cbm = CloudBundleModelMixin() + _uri = URI("http://1.1.1.1/project/sw", expected_type=URIType.PROJECT) + _models, _pager = cbm._fetch_bundle_all_list( + _uri, uri_typ=URIType.MODEL, filter_dict={"name": "mnist", "latest": True} + ) + + assert len(_models.items()) == 1 + assert len(_models["[2] mnist"]) == 1 + assert _pager["current"] == 1 + assert _pager["remain"] == 0 diff --git a/client/tests/core/test_dataset.py b/client/tests/core/test_dataset.py index 07a7426a68..c870bc8df0 100644 --- a/client/tests/core/test_dataset.py +++ b/client/tests/core/test_dataset.py @@ -29,6 +29,7 @@ ObjectStoreType, ) from starwhale.utils.config import SWCliConfigMixed +from starwhale.core.dataset.cli import _list as list_cli from starwhale.core.dataset.cli import _build as build_cli from starwhale.core.dataset.type import ( Line, @@ -518,3 +519,45 @@ def test_to_list(self): ), data_store._get_type(p), ), + + +class CloudDatasetTest(TestCase): + def setUp(self) -> None: + sw_config._config = {} + + def test_cli_list(self) -> None: + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + list_cli, + [ + "--filter", + "name=mnist", + "--filter", + "owner=starwhale", + "--filter", + "latest", + ], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 3 + assert "name=mnist" in call_args[5] + assert "owner=starwhale" in call_args[5] + assert "latest" in call_args[5] + + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + list_cli, + [], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 0 diff --git a/client/tests/core/test_model.py b/client/tests/core/test_model.py index c0ddf3d2e0..652e14f344 100644 --- a/client/tests/core/test_model.py +++ b/client/tests/core/test_model.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest.mock import patch, MagicMock +from click.testing import CliRunner from requests_mock import Mocker from pyfakefs.fake_filesystem_unittest import TestCase @@ -25,6 +26,7 @@ from starwhale.utils.config import SWCliConfigMixed from starwhale.api._impl.job import Context, context_holder from starwhale.core.job.model import Step +from starwhale.core.model.cli import _list as list_cli from starwhale.api._impl.model import PipelineHandler, PPLResultIterator from starwhale.core.model.view import ModelTermView from starwhale.core.model.model import StandaloneModel, resource_to_file_node @@ -396,3 +398,38 @@ def test_serve(self, *args: t.Any): with self.assertRaises(SystemExit): ModelTermView.serve("set", yaml, runtime, "set", host, port) + + +class CloudModelTest(TestCase): + def setUp(self) -> None: + sw_config._config = {} + + def test_cli_list(self) -> None: + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + list_cli, + ["--filter", "name=mn", "--filter", "owner=sw", "--filter", "latest"], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 3 + assert "name=mn" in call_args[5] + assert "owner=sw" in call_args[5] + assert "latest" in call_args[5] + + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + list_cli, + [], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 0 diff --git a/client/tests/core/test_runtime.py b/client/tests/core/test_runtime.py index 0cfb624667..eeb2912b96 100644 --- a/client/tests/core/test_runtime.py +++ b/client/tests/core/test_runtime.py @@ -34,6 +34,7 @@ UnExpectedConfigFieldError, ) from starwhale.utils.config import SWCliConfigMixed +from starwhale.core.runtime.cli import _list as runtime_list_cli from starwhale.core.runtime.cli import _build as runtime_build_cli from starwhale.core.runtime.view import ( get_term_view, @@ -1978,3 +1979,45 @@ def test_create_abnormal_dependencies(self) -> None: with self.assertRaises(FormatError): WheelDependency(["d.d"]) + + +class CloudRuntimeTest(TestCase): + def setUp(self) -> None: + sw_config._config = {} + + def test_cli_list(self) -> None: + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + runtime_list_cli, + [ + "--filter", + "name=pytorch", + "--filter", + "owner=test", + "--filter", + "latest", + ], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 3 + assert "name=pytorch" in call_args[5] + assert "owner=test" in call_args[5] + assert "latest" in call_args[5] + + mock_obj = MagicMock() + runner = CliRunner() + result = runner.invoke( + runtime_list_cli, + [], + obj=mock_obj, + ) + + assert result.exit_code == 0 + assert mock_obj.list.call_count == 1 + call_args = mock_obj.list.call_args[0] + assert len(call_args[5]) == 0 diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/dataset.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/dataset.md index 1da3c3f80f..29965b7ace 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/dataset.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/dataset.md @@ -131,13 +131,20 @@ swcli dataset list [OPTIONS] `dataset list` 命令输出当前选定的instance和project下的所有数据集及相关版本。命令参数如下： -|参数|参数别名|必要性|类型|默认值|说明| -|------|--------|-------|-----------|-----|-----------| +|参数| 参数别名 |必要性|类型|默认值|说明| +|------|-----|----|-----------|-----|-----------| |`--project`|`-p`|❌|String|`swcli project select`命令选定的默认project|Project URI| -|`--fullname`||❌|Boolean|False|显示完整的版本信息，默认只显示版本号的前12位。| -|`--show-removed`||❌|Boolean|False|显示本地已经删除但能恢复的数据集。| -|`--page`||❌|Integer|1|Cloud Instance中分页显示中page序号。| -|`--size`||❌|Integer|20|Cloud Instance中分页显示中每页数量。| +|`--fullname`|`-f` |❌|Boolean|False|显示完整的版本信息，默认只显示版本号的前12位。| +|`--show-removed`|`-sr`| ❌ |Boolean|False|显示本地已经删除但能恢复的数据集。| +|`--page`| | ❌ |Integer|1|Cloud Instance中分页显示中page序号。| +|`--size`| | ❌ |Integer|20|Cloud Instance中分页显示中每页数量。| +| `--filter` |`-fl`| ❌ | String | | 过滤器，使用key=value格式或者flag，可使用多个filter，具体支持的filter如下： | + +| Filter | 类型 | 说明 | 示例| +|----|-----------|-------------------------------------------------------|----| +|`name`| Key-Value | 数据集名称前缀 |--filter name=mnist| +|`owner`| Key-Value | 拥有者名称 |--filter owner=starwhale| +|`latest`|Flag| Cloud Instance: 仅展示最新版本
Standalone Instance: 仅展示带有latest标签的版本 |--filter latest| `dataset list` 的alias命令为 `dataset ls`。 @@ -147,7 +154,10 @@ swcli dataset list [OPTIONS] swcli dataset remove [OPTIONS] DATASET ``` -`dataset remove` 命令可以删除整个数据集或数据集中的某个版本。删除可以分为硬删除和软删除，默认为软删除，指定 `--force` 参数为硬删除。所有的软删除数据集在没有GC之前，都可以通过 `swcli dataset recover` 命令进行恢复。`DATASET` 参数为Dataset URI，当没有指定版本时，会对整个数据集所有版本进行删除，若URI带有版本信息，则只会对该版本进行删除。软删除的数据集，可以通过 `swcli dataset list --show-removed` 命令查看。 +`dataset remove` 命令可以删除整个数据集或数据集中的某个版本。删除可以分为硬删除和软删除，默认为软删除，指定 `--force` +参数为硬删除。所有的软删除数据集在没有GC之前，都可以通过 `swcli dataset recover` 命令进行恢复。`DATASET` 参数为Dataset +URI，当没有指定版本时，会对整个数据集所有版本进行删除，若URI带有版本信息，则只会对该版本进行删除。软删除的数据集，可以通过 `swcli dataset list --show-removed` +命令查看。 |参数|参数别名|必要性|类型|默认值|说明| |------|--------|-------|-----------|-----|-----------| diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/model.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/model.md index 76503ea7a7..0156aaea99 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/model.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/model.md @@ -117,13 +117,20 @@ swcli model list [OPTIONS] `model list` 命令输出当前选定的instance和project下的所有模型包及相关版本。命令参数如下： -|参数|参数别名|必要性|类型|默认值|说明| -|------|--------|-------|-----------|-----|-----------| -|`--project`|`-p`|❌|String|`swcli project select`命令选定的默认project|Project URI| -|`--fullname`||❌|Boolean|False|显示完整的版本信息，默认只显示版本号的前12位。| -|`--show-removed`||❌|Boolean|False|显示本地已经删除但能恢复的模型包。| -|`--page`||❌|Integer|1|Cloud Instance中分页显示中page序号。| -|`--size`||❌|Integer|20|Cloud Instance中分页显示中每页数量。| +|参数|参数别名|必要性|类型|默认值| 说明 | +|---|---|---|---|---|----------------------------------------------------| +|`--project`|`-p`|❌|String|`swcli project select`命令选定的默认project | Project URI| +|`--fullname`|`-f`|❌|Boolean|False| 显示完整的版本信息，默认只显示版本号的前12位。| +|`--show-removed`|`-sr`|❌|Boolean|False| 显示本地已经删除但能恢复的模型包。| +|`--page`| |❌|Integer|1| Cloud Instance中分页显示中page序号。| +|`--size`| |❌|Integer|20| Cloud Instance中分页显示中每页数量。| +|`--filter`|`-fl`|❌|String| | 过滤器，使用key=value格式或者flag，可使用多个filter，具体支持的filter如下： | + +|Filter|类型| 说明 | 示例| +|----|-------|-----------|----| +|`name`| Key-Value | 模型名称前缀 |--filter name=mnist| +|`owner`| Key-Value | 拥有者名称 |--filter owner=starwhale| +|`latest`|Flag| Cloud Instance: 仅展示最新版本
Standalone Instance: 仅展示带有latest标签的版本 |--filter latest| ## 8. 删除模型包 @@ -131,7 +138,10 @@ swcli model list [OPTIONS] swcli model remove [OPTIONS] MODEL ``` -`model remove` 命令可以删除整个模型包或模型包中的某个版本。删除可以分为硬删除和软删除，默认为软删除，指定 `--force` 参数为硬删除。所有的软删除模型包在没有GC之前，都可以通过 `swcli model recover` 命令进行恢复。`MODEL` 参数为Model URI，当没有指定版本时，会对整个模型包所有版本进行删除，若URI带有版本信息，则只会对该版本进行删除。软删除的模型包，可以通过 `swcli model list --show-removed` 命令查看。 +`model remove` 命令可以删除整个模型包或模型包中的某个版本。删除可以分为硬删除和软删除，默认为软删除，指定 `--force` +参数为硬删除。所有的软删除模型包在没有GC之前，都可以通过 `swcli model recover` 命令进行恢复。`MODEL` 参数为Model +URI，当没有指定版本时，会对整个模型包所有版本进行删除，若URI带有版本信息，则只会对该版本进行删除。软删除的模型包，可以通过 `swcli model list --show-removed` +命令查看。 |参数|参数别名|必要性|类型|默认值|说明| |------|--------|-------|-----------|-----|-----------| diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/project.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/project.md index bd406ef182..17e9c1b62c 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/project.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/project.md @@ -14,14 +14,14 @@ project命令提供适用于Standalone Instance和Cloud Instance的Starwhale Pro project包含如下子命令： -|命令|别名|Standalone|Cloud| -|-------|----------|-----| -|create|new,add|✅|✅| -|info||✅|✅| -|list|ls|✅|✅| -|remove|rm|✅|✅| -|recover||✅|✅| -|select|use|✅|✅| +|命令| 别名 |Standalone|Cloud| +|----|---------|----------|-----| +|create| new,add |✅|✅| +|info|| ✅ |✅| +|list| ls |✅|✅| +|remove| rm |✅|✅| +|recover|| ✅ |✅| +|select| use |✅|✅| ## 2. 创建Project diff --git a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/runtime.md b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/runtime.md index 52cc251ef0..3f8954d516 100644 --- a/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/runtime.md +++ b/docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/runtime.md @@ -174,13 +174,20 @@ swcli runtime list [OPTIONS] `runtime list` 命令输出当前选定的instance和project下的所有Runtime及相关版本。命令参数如下： -|参数|参数别名|必要性|类型|默认值|说明| -|------|--------|-------|-----------|-----|-----------| +|参数| 参数别名 |必要性|类型|默认值|说明| +|------|------|-------|-----------|-----|-----------| |`--project`|`-p`|❌|String|`swcli project select`命令选定的默认project|Project URI| -|`--fullname`||❌|Boolean|False|显示完整的版本信息，默认只显示版本号的前12位。| -|`--show-removed`||❌|Boolean|False|显示本地已经删除但能恢复的Runtime。| -|`--page`||❌|Integer|1|Cloud Instance中分页显示中page序号。| -|`--size`||❌|Integer|20|Cloud Instance中分页显示中每页数量。| +|`--fullname`|`-f`|❌|Boolean|False|显示完整的版本信息，默认只显示版本号的前12位。| +|`--show-removed`|`-sr`|❌|Boolean|False|显示本地已经删除但能恢复的Runtime。| +|`--page`| |❌|Integer|1|Cloud Instance中分页显示中page序号。| +|`--size`| |❌|Integer|20|Cloud Instance中分页显示中每页数量。| +|`--filter`|`-fl`|❌| String | | 过滤器，使用key=value格式或者flag，可使用多个filter，具体支持的filter如下： | + +|Filter|类型|说明|示例| +|----|--------|----------|----------| +|`name`|Key-Value|Runtime名称前缀|--filter name=pytorch| +|`owner`| Key-Value|拥有者名称|--filter owner=starwhale| +|`latest`|Flag|Cloud Instance: 仅展示最新版本
Standalone Instance: 仅展示带有latest标签的版本|--filter latest| ## 10. 锁定Python依赖信息 @@ -188,7 +195,8 @@ swcli runtime list [OPTIONS] swcli runtime lock [OPTIONS] [TARGET_DIR] ``` -`runtime lock` 命令能将指定的Python环境中的依赖进行导出，在 `TARGET_DIR` 目录中生成 `requirements-sw-lock.txt` 文件或 `conda-sw-lock.yaml` 文件，典型内容形式如下： +`runtime lock` 命令能将指定的Python环境中的依赖进行导出，在 `TARGET_DIR` 目录中生成 `requirements-sw-lock.txt` +文件或 `conda-sw-lock.yaml` 文件，典型内容形式如下： ```txt # Generated by Starwhale(0.3.0) Runtime Lock