feat(cli): model eval command support use-docker flag

client/scripts/sw-docker-entrypoint

@@ -134,6 +134,6 @@ case "$1" in
         eval_task_prepare $1 && run
         ;;
     *)
-        exec "$@"
+        eval_task_prepare "starwhale" && exec "$@"
         ;;
 esac

client/starwhale/core/model/cli.py

@@ -207,6 +207,17 @@ def _recover(model: str, force: bool) -> None:
     multiple=True,
     help=f"dataset uri, env is {SWEnv.dataset_uri}",
 )
+@click.option(
+    "--use-docker",
+    is_flag=True,
+    help="[ONLY Standalone]use docker to run evaluation job",
+)
+@click.option("--gencmd", is_flag=True, help="[ONLY Standalone]gen docker run command")
+@click.option(
+    "--image",
+    default="",
+    help="[ONLY Standalone]the image used when use docker",
+)
 def _eval(
     project: str,
     target: str,
@@ -217,6 +228,9 @@ def _eval(
     task_index: int,
     override_task_num: int,
     runtime: str,
+    use_docker: bool,
+    gencmd: bool,
+    image: str,
 ) -> None:
     """
     [ONLY Standalone]Run evaluation processing with root dir of {target}.
@@ -233,6 +247,9 @@ def _eval(
         task_index=task_index,
         task_num=override_task_num,
         dataset_uris=datasets,
+        use_docker=use_docker,
+        gencmd=gencmd,
+        image=image,
     )
 
 

client/starwhale/core/model/view.py

@@ -8,7 +8,14 @@
 from rich.table import Table
 from rich.console import Group
 
-from starwhale.utils import console, load_yaml, pretty_bytes, in_production
+from starwhale.utils import (
+    docker,
+    console,
+    process,
+    load_yaml,
+    pretty_bytes,
+    in_production,
+)
 from starwhale.consts import (
     FileFlag,
     DefaultYAMLName,
@@ -20,7 +27,10 @@
 from starwhale.utils.fs import cmp_file_content
 from starwhale.base.type import URIType, InstanceType
 from starwhale.base.view import BaseTermView
+from starwhale.consts.env import SWEnv
+from starwhale.utils.error import FieldTypeOrValueError
 from starwhale.core.model.store import ModelStorage
+from starwhale.core.runtime.model import StandaloneRuntime
 from starwhale.core.runtime.process import Process as RuntimeProcess
 
 from .model import Model, StandaloneModel
@@ -121,7 +131,38 @@ def eval(
         task_index: int = 0,
         task_num: int = 0,
         runtime_uri: str = "",
+        use_docker: bool = False,
+        gencmd: bool = False,
+        image: str = "",
     ) -> None:
+        if use_docker:
+            if not runtime_uri and not image:
+                raise FieldTypeOrValueError(
+                    "runtime_uri and image both are none when use_docker"
+                )
+            if runtime_uri:
+                runtime = StandaloneRuntime(
+                    URI(runtime_uri, expected_type=URIType.RUNTIME)
+                )
+                image = runtime.store.get_docker_base_image()
+            mnt_paths = (
+                [os.path.abspath(target)]
+                if in_production() or (os.path.exists(target) and os.path.isdir(target))
+                else []
+            )
+            env_vars = {SWEnv.runtime_version: runtime_uri} if runtime_uri else {}
+            cmd = docker.gen_swcli_docker_cmd(
+                image,
+                env_vars=env_vars,
+                mnt_paths=mnt_paths,
+            )
+            console.rule(":elephant: docker cmd", align="left")
+            console.print(f"{cmd}\n")
+            if gencmd:
+                return
+            process.check_call(cmd, shell=True)
+            return
+
         kw = dict(
             project=project,
             version=version,
@@ -148,7 +189,7 @@ def _get_workdir(cls, target: str) -> Path:
         else:
             _uri = URI(target, URIType.MODEL)
             _store = ModelStorage(_uri)
-            workdir = _store.loc
+            workdir = _store.src_dir
         return workdir
 
     @classmethod

client/starwhale/utils/docker.py

@@ -1,10 +1,13 @@
 import os
+import sys
 import typing as t
+import getpass as gt
 import subprocess
+from pwd import getpwnam
 from pathlib import Path
 
-from starwhale.utils import console
-from starwhale.consts import SupportArch
+from starwhale.utils import config, console
+from starwhale.consts import SupportArch, CNTR_DEFAULT_PIP_CACHE_DIR
 from starwhale.utils.error import NoSupportError, MissingFieldError
 from starwhale.utils.process import check_call
 
@@ -149,3 +152,80 @@ def buildx(
     else:
         console.print(":panda_face: start to build image with buildx...")
         check_call(cmd, log=console.print, env=_BUILDX_CMD_ENV)
+
+
+def gen_swcli_docker_cmd(
+    image: str,
+    env_vars: t.Dict[str, str] = {},
+    mnt_paths: t.List[str] = [],
+    name: str = "",
+) -> str:
+
+    if not image:
+        raise ValueError("image should have value")
+    pwd = os.getcwd()
+
+    rootdir = config.load_swcli_config()["storage"]["root"]
+    config_path = config.get_swcli_config_path()
+    cmd = [
+        "docker",
+        "run",
+        "--net=host",
+        "--rm",
+        "-e",
+        "DEBUG=1",
+        "-e",
+        f"SW_USER={gt.getuser()}",
+        "-e",
+        f"SW_USER_ID={getpwnam(gt.getuser()).pw_uid}",
+        "-e",
+        "SW_USER_GROUP_ID=0",
+        "-e",
+        f"SW_LOCAL_STORAGE={rootdir}",
+        "-v",
+        f"{rootdir}:{rootdir}",
+        "-e",
+        f"SW_CLI_CONFIG={config_path}",
+        "-v",
+        f"{config_path}:{config_path}",
+        "-v",
+        f"{pwd}:{pwd}",
+        "-w",
+        f"{pwd}",
+    ]
+
+    if name:
+        cmd += [
+            "--name",
+            f'"{name}"',
+        ]
+
+    if mnt_paths:
+        for cp in mnt_paths:
+            cmd += [
+                "-v",
+                f"{cp}:{cp}",
+            ]
+
+    if env_vars:
+        for _k, _v in env_vars.items():
+            cmd.extend(["-e", f"{_k}={_v}"])
+
+    cntr_cache_dir = os.environ.get("SW_PIP_CACHE_DIR", CNTR_DEFAULT_PIP_CACHE_DIR)
+    host_cache_dir = os.path.expanduser("~/.cache/starwhale-pip")
+    cmd += ["-v", f"{host_cache_dir}:{cntr_cache_dir}"]
+
+    _env = os.environ
+    for _ee in (
+        "SW_PYPI_INDEX_URL",
+        "SW_PYPI_EXTRA_INDEX_URL",
+        "SW_PYPI_TRUSTED_HOST",
+    ):
+        if _ee not in _env:
+            continue
+        cmd.extend(["-e", f"{_ee}={_env[_ee]}"])
+
+    sw_cmd = " ".join([item for item in sys.argv[1:] if "use-docker" not in item])
+
+    cmd.extend([image, f"swcli {sw_cmd}"])
+    return " ".join(cmd)

client/tests/core/test_model.py

@@ -1,5 +1,6 @@
 import os
 import typing as t
+import tempfile
 from pathlib import Path
 from unittest.mock import patch, MagicMock
 
@@ -368,3 +369,15 @@ def some(self):
         )
         context_holder.context = context
         default_handler._invoke(context, "some")
+
+    @patch("starwhale.utils.process.check_call")
+    @patch("starwhale.utils.docker.gen_swcli_docker_cmd")
+    def test_use_docker(self, m_gencmd: MagicMock, m_call: MagicMock):
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            m_gencmd.return_value = "hi"
+            m_call.return_value = 0
+            ModelTermView.eval("", tmpdirname, [], use_docker=True, image="img1")
+            m_gencmd.assert_called_once_with(
+                "img1", env_vars={}, mnt_paths=[tmpdirname]
+            )
+            m_call.assert_called_once_with("hi", shell=True)

client/tests/utils/test_docker.py

@@ -0,0 +1,34 @@
+import os
+import getpass as gt
+import tempfile
+from pwd import getpwnam
+from unittest import TestCase
+
+from starwhale.utils import config
+from starwhale.utils.docker import gen_swcli_docker_cmd
+
+
+class TestDocker(TestCase):
+    def test_gen_cmd(self) -> None:
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            config._config = {}
+            print("created temporary directory", tmpdirname)
+            os.environ["SW_CLI_CONFIG"] = tmpdirname + "/config.yaml"
+            os.environ["SW_LOCAL_STORAGE"] = tmpdirname
+            cmd = gen_swcli_docker_cmd("image1")
+            self.assertTrue(cmd.startswith("docker run"))
+            args = cmd.split()
+            pwd = os.getcwd()
+            self.assertTrue("-w" in args)
+            self.assertTrue(pwd in args)
+            self.assertTrue(f"{pwd}:{pwd}" in args)
+            self.assertTrue("image1" in args)
+            self.assertTrue(f"SW_USER={gt.getuser()}" in args)
+            self.assertTrue(f"SW_USER_ID={getpwnam(gt.getuser()).pw_uid}" in args)
+            self.assertTrue("SW_USER_GROUP_ID=0" in args)
+            rootdir = config.load_swcli_config()["storage"]["root"]
+            self.assertTrue(f"SW_LOCAL_STORAGE={rootdir}" in args)
+            self.assertTrue(f"{rootdir}:{rootdir}" in args)
+            config_path = config.get_swcli_config_path()
+            self.assertTrue(f"SW_CLI_CONFIG={config_path}" in args)
+            self.assertTrue(f"{config_path}:{config_path}" in args)

docs/docs/reference/cli/model.md

@@ -131,7 +131,6 @@ swcli model eval [OPTIONS] MODEL
 
 - This command creates a new job for model evaluation.
 - `MODEL` argument is required. The `Model URI` or model working dir path is ok for the `MODEL` argument.
-- This command does not depend on docker, so you should activate the python environment at first.
 - `model eval` command is beneficial for you to debug ppl/cmp and evaluate models in the standalone instance.
 - Options:
 
@@ -141,7 +140,7 @@ swcli model eval [OPTIONS] MODEL
     |`--name`||❌|String|""|evaluation job name|
     |`--desc`||❌|String|""|evaluation job description|
     |`--project`|`-p`|❌|String|Selected project|Project URI|
-
+    |`--use-docker`||❌|Boolean|False|Only for standalone instance, use docker environment to run model evaluation|
 - Example:
 
     ```bash

docs/i18n/zh/docusaurus-plugin-content-docs/current/reference/cli/model.md

@@ -79,6 +79,9 @@ swcli model eval [OPTIONS] TARGET
 |`--task-index`||❌|Integer|| `--task-index` 参数用来表明 `--step` 中Step的第n个Task会执行，如果不指定 `--task-index`，则该Step中所有Tasks都会执行。Task索引从0开始，会通过 `starwhale.Context` 传入用户程序中，很多场景下Task的索引值用来指导评测数据集如何分割，实现评测的时候同一个Step的不同Task处理不同部分的评测数据集，实现并行提速。另外需要注意的是，`--task-index` 只有在同时设置 `--step` 参数时才生效。|
 |`--runtime`||❌|String||`--runtime`参数为Standalone Instance中的Runtime URI。若设置，则表示模型包构建的时候会使用该Runtime提供的运行时环境；若不设置，则使用当前shell环境作为运行时。设置`--runtime`参数是安全的，只在build运行时才会使用Runtime，不会污染当前shell环境。|
 |`--dataset`||✅||String||Dataset URI，该参数也可以通过 `SW_DATASET_URI` 环境变量来设置。|
+|`--gencmd`||❌|Boolean|False|当选用设置 `--use-docker` 参数后，只输出docker run的命令，不真正运行。该参数只能在Standalone Instance中使用。|
+|`--use-docker`||❌|Boolean|False|选用docker为载体来运行模型评测过程，该参数只能在Standalone Instance中使用。|
+|`--image`||❌|Boolean|False|当选用设置 `--use-docker` 参数后, 该参数生效。此image必须支持swcli命令，你可以先使用`--gencmd`查看具体生成的docker命令。如果`--runtime`被同时指定了，`swcli`会调用runtime的baseimage,本参数不再生效|
 
 ![model-eval.gif](../../img/model-eval.gif)