From 10fbe73942da2c0b436c7d607201989dc9106619 Mon Sep 17 00:00:00 2001 From: tianwei Date: Fri, 16 Sep 2022 06:58:24 +0800 Subject: [PATCH] feat(client): support auto-fast-tag for model/dataset/runtime (#1203) support auto-fast-tag for model/dataset/runtime --- client/starwhale/base/bundle.py | 3 ++- client/starwhale/base/bundle_copy.py | 11 ++++++++++ client/starwhale/base/tag.py | 21 ++++++++++++++++++- client/starwhale/core/dataset/model.py | 2 +- client/starwhale/core/model/model.py | 2 +- client/starwhale/core/runtime/model.py | 2 +- client/tests/base/test_copy.py | 27 +++++++++++++++++++----- client/tests/base/test_tag.py | 29 ++++++++++++++++++++++++++ 8 files changed, 87 insertions(+), 10 deletions(-) diff --git a/client/starwhale/base/bundle.py b/client/starwhale/base/bundle.py index 7dad72dcba..62ed76e193 100644 --- a/client/starwhale/base/bundle.py +++ b/client/starwhale/base/bundle.py @@ -137,8 +137,9 @@ def _gen_version(self) -> None: logger.info(f"[step:version]version: {self._version}") console.print(f":new: version {self._version[:SHORT_VERSION_CNT]}") # type: ignore - def _make_latest_tag(self) -> None: + def _make_auto_tags(self) -> None: self.tag.add([LATEST_TAG], quiet=True) # type: ignore + self.tag.add_fast_tag() # type: ignore def _make_tar(self, ftype: str = "") -> None: out = self.store.bundle_dir / f"{self._version}{ftype}" # type: ignore diff --git a/client/starwhale/base/bundle_copy.py b/client/starwhale/base/bundle_copy.py index 7f36cb9bf5..aaa0477caa 100644 --- a/client/starwhale/base/bundle_copy.py +++ b/client/starwhale/base/bundle_copy.py @@ -20,10 +20,12 @@ HTTPMethod, AUTH_ENV_FNAME, VERSION_PREFIX_CNT, + STANDALONE_INSTANCE, DEFAULT_MANIFEST_NAME, DUMPED_SWDS_META_FNAME, ARCHIVED_SWDS_META_FNAME, ) +from starwhale.base.tag import StandaloneTag from starwhale.base.uri import URI from starwhale.utils.fs import ensure_dir from starwhale.base.type import URIType, InstanceType, get_bundle_type_by_uri @@ -212,6 +214,15 @@ def do(self) -> None: else: self._do_download_bundle_tar(progress) + _dest_uri = URI.capsulate_uri( + instance=STANDALONE_INSTANCE, + project=self.dest_uri.project, + obj_type=self.typ, + obj_name=self.bundle_name, + obj_ver=self.bundle_version, + ) + StandaloneTag(_dest_uri).add_fast_tag() + def _do_upload_bundle_dir( self, progress: Progress, add_data_uri_header: bool = False ) -> None: diff --git a/client/starwhale/base/tag.py b/client/starwhale/base/tag.py index 5da99687cc..21f0aed502 100644 --- a/client/starwhale/base/tag.py +++ b/client/starwhale/base/tag.py @@ -38,6 +38,7 @@ def _manifest_path(self) -> Path: def _get_manifest(self) -> t.Dict[str, t.Any]: if not self._manifest_path.exists(): _dft: t.Dict[str, t.Any] = { + "fast_tag_seq": -1, "tags": {}, "versions": {}, } @@ -57,8 +58,26 @@ def _save_manifest(self, _manifest: t.Dict[str, t.Any]) -> None: self._manifest_path, yaml.safe_dump(_manifest, default_flow_style=False) ) - def add(self, tags: t.List[str], quiet: bool = False) -> None: + def add_fast_tag(self) -> None: _manifest = self._get_manifest() + _seq = int(_manifest.get("fast_tag_seq", -1)) + _tag = _seq + 1 + while True: + if f"v{_tag}" in _manifest["tags"]: + _tag = _tag + 1 + else: + break + + _manifest["fast_tag_seq"] = _tag + self.add(tags=[f"v{_tag}"], manifest=_manifest) + + def add( + self, + tags: t.List[str], + quiet: bool = False, + manifest: t.Optional[t.Dict] = None, + ) -> None: + _manifest = manifest or self._get_manifest() _version = self.uri.object.version if not _version and not quiet: diff --git a/client/starwhale/core/dataset/model.py b/client/starwhale/core/dataset/model.py index 6c994789a3..dd891dc8fb 100644 --- a/client/starwhale/core/dataset/model.py +++ b/client/starwhale/core/dataset/model.py @@ -307,7 +307,7 @@ def buildImpl( "render manifest", ), (self._make_swds_meta_tar, 15, "make meta tar"), - (self._make_latest_tag, 5, "make latest tag"), + (self._make_auto_tags, 5, "make auto tags"), ] run_with_progress_bar("swds building...", operations) diff --git a/client/starwhale/core/model/model.py b/client/starwhale/core/model/model.py index edaf177566..5bffc0bd51 100644 --- a/client/starwhale/core/model/model.py +++ b/client/starwhale/core/model/model.py @@ -411,7 +411,7 @@ def buildImpl( "render manifest", ), (self._make_tar, 20, "build model bundle", dict(ftype=BundleType.MODEL)), - (self._make_latest_tag, 5, "make latest tag"), + (self._make_auto_tags, 5, "make auto tags"), ] run_with_progress_bar("model bundle building...", operations) diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py index c17c214d15..488fdf0499 100644 --- a/client/starwhale/core/runtime/model.py +++ b/client/starwhale/core/runtime/model.py @@ -535,7 +535,7 @@ def build( "render manifest", ), (self._make_tar, 20, "make runtime bundle", dict(ftype=BundleType.RUNTIME)), - (self._make_latest_tag, 5, "make latest tag"), + (self._make_auto_tags, 5, "make auto tags"), ] run_with_progress_bar("runtime bundle building...", operations) diff --git a/client/tests/base/test_copy.py b/client/tests/base/test_copy.py index d1ba73e438..131e086928 100644 --- a/client/tests/base/test_copy.py +++ b/client/tests/base/test_copy.py @@ -7,10 +7,13 @@ from starwhale.utils import config as sw_config from starwhale.consts import ( HTTPMethod, + VERSION_PREFIX_CNT, DEFAULT_MANIFEST_NAME, DUMPED_SWDS_META_FNAME, ARCHIVED_SWDS_META_FNAME, ) +from starwhale.base.tag import StandaloneTag +from starwhale.base.uri import URI from starwhale.utils.fs import ensure_dir, ensure_file from starwhale.base.type import URIType from starwhale.utils.config import SWCliConfigMixed, get_swcli_config_path @@ -61,31 +64,45 @@ def test_upload_bundle_file(self, rm: Mocker) -> None: @Mocker() def test_download_bundle_file(self, rm: Mocker) -> None: + version = "112233" rm.request( HTTPMethod.HEAD, - "http://1.1.1.1:8182/api/v1/project/1/model/mnist/version/latest", + f"http://1.1.1.1:8182/api/v1/project/1/model/mnist/version/{version}", json={"message": "existed"}, status_code=HTTPStatus.OK, ) rm.request( HTTPMethod.GET, - "http://1.1.1.1:8182/api/v1/project/1/model/mnist/version/latest/file", + f"http://1.1.1.1:8182/api/v1/project/1/model/mnist/version/{version}/file", content=b"test", ) - dest_dir = self._sw_config.rootdir / "self" / "model" / "mnist" / "la" + dest_dir = ( + self._sw_config.rootdir + / "self" + / "model" + / "mnist" + / f"{version[:VERSION_PREFIX_CNT]}" + ) ensure_dir(dest_dir) bc = BundleCopy( - src_uri="cloud://pre-bare/project/1/model/mnist/version/latest", + src_uri=f"cloud://pre-bare/project/1/model/mnist/version/{version}", dest_uri="self", typ=URIType.MODEL, ) bc.do() - swmp_path = dest_dir / "latest.swmp" + swmp_path = dest_dir / f"{version}.swmp" assert swmp_path.exists() assert swmp_path.read_bytes() == b"test" + st = StandaloneTag( + URI( + f"mnist/version/{version}", + expected_type=URIType.MODEL, + ) + ) + assert st.list() == ["v0"] @Mocker() def test_upload_bundle_dir(self, rm: Mocker) -> None: diff --git a/client/tests/base/test_tag.py b/client/tests/base/test_tag.py index 0141e382e0..5fa814aceb 100644 --- a/client/tests/base/test_tag.py +++ b/client/tests/base/test_tag.py @@ -73,3 +73,32 @@ def test_tag_workflow(self) -> None: assert _manifest["tags"]["test2"] == "gnstmntggi4tinrtmftdgyjzo5wwy2y" assert _manifest["versions"]["me3dmn3gg4ytanrtmftdgyjzpbrgimi"]["me3"] assert _manifest["versions"]["gnstmntggi4tinrtmftdgyjzo5wwy2y"]["test"] + + def test_auto_fast_tag(self) -> None: + version = "me3dmn3gg4ytanrtmftdgyjzpbrgimi" + st = StandaloneTag( + URI( + f"mnist/version/{version}", + expected_type=URIType.MODEL, + ) + ) + assert st._get_manifest()["fast_tag_seq"] == -1 + st.add_fast_tag() + assert st._get_manifest()["fast_tag_seq"] == 0 + st.add_fast_tag() + st.add_fast_tag() + st.add_fast_tag() + assert st._get_manifest()["fast_tag_seq"] == 3 + assert st._get_manifest()["tags"]["v0"] == version + assert st._get_manifest()["tags"]["v3"] == version + st.add(["v4", "v5", "v6"]) + st.add_fast_tag() + assert st._get_manifest()["fast_tag_seq"] == 7 + assert st._get_manifest()["tags"]["v7"] == version + + st.remove(["v7", "v6"], quiet=True) + st.add_fast_tag() + assert st._get_manifest()["fast_tag_seq"] == 8 + assert st._get_manifest()["tags"]["v8"] == version + + assert st.list() == ["v0", "v1", "v2", "v3", "v4", "v5", "v8"]