feat(client): cli supports displaying eval on locally served web pages

.gitignore

@@ -18,7 +18,6 @@ pylint.out
 *.tar.gz
 *.rar
 dist/
-build/
 *.egg
 *.egg/
 *.egg-info/
@@ -62,7 +61,6 @@ node_modules
 console/node/
 console/cypress/example
 coverage
-build
 .DS_Store
 .env.local
 .env.development.local

client/.gitignore

@@ -0,0 +1 @@
+build/

client/setup.py

@@ -35,6 +35,7 @@
     # for system monitor
     "psutil>=5.5.0",
     "GitPython>=3.1.24",
+    "fastapi>=0.71.0",
 ]
 
 extras_require = {

client/starwhale/api/_impl/data_store.py

@@ -657,7 +657,7 @@ def __eq__(self, other: Any) -> bool:
 
 
 def _get_table_path(root_path: str, table_name: str) -> str:
-    return str(pathlib.Path(root_path) / table_name)
+    return str(pathlib.Path(root_path) / table_name.lstrip("/"))
 
 
 def _parse_parquet_name(name: str) -> Tuple[str, int]:

client/starwhale/core/eval/cli.py

@@ -194,11 +194,17 @@ def _cancel(job: str, force: bool) -> None:
     default=DEFAULT_REPORT_COLS,
     help="Max table column size for print",
 )
+@click.option("--web", is_flag=True, help="Open job info page in browser")
 @click.pass_obj
 def _info(
-    view: t.Type[JobTermView], job: str, page: int, size: int, max_report_cols: int
+    view: t.Type[JobTermView],
+    job: str,
+    page: int,
+    size: int,
+    max_report_cols: int,
+    web: bool,
 ) -> None:
-    view(job).info(page, size, max_report_cols)
+    view(job).info(page, size, max_report_cols, web)
 
 
 @eval_job_cmd.command("compare", aliases=["cmp"])

client/starwhale/core/eval/view.py

@@ -119,13 +119,32 @@ def info(
         page: int = DEFAULT_PAGE_IDX,
         size: int = DEFAULT_PAGE_SIZE,
         max_report_cols: int = DEFAULT_REPORT_COLS,
+        web: bool = False,
     ) -> None:
         _rt = self.job.info(page, size)
         if not _rt:
             console.print(":tea: not found info")
             return
 
-        if _rt.get("manifest"):
+        manifest = _rt.get("manifest")
+        if web:
+            if not manifest or not manifest.get("version"):
+                console.print(":tea: not found eval id")
+                sys.exit(1)
+            ver = manifest.get("version")
+            # TODO support changing host and port
+            host = "127.0.0.1"
+            port = 8000
+            url = f"http://{host}:{port}/projects/{self.uri.project}/evaluations/{ver}/results?token=local"
+            console.print(f":tea: open {url} in browser")
+            import uvicorn
+
+            from starwhale.web.server import Server
+
+            uvicorn.run(Server.default(), host=host, port=port, log_level="error")
+            return
+
+        if manifest:
             console.rule(
                 f"[green bold]Inspect {DEFAULT_MANIFEST_NAME} for eval:{self.uri}"
             )
@@ -369,7 +388,7 @@ def _s(x: str) -> str:
             _model = "--"
             if "model" in _m:
                 _model = _s(_m["model"])
-            else:
+            elif "modelName" in _m:
                 _model = f"{_m['modelName']}:{_s(_m['modelVersion'])}"
 
             _name = "--"
@@ -420,6 +439,7 @@ def info(
         page: int = DEFAULT_PAGE_IDX,
         size: int = DEFAULT_PAGE_SIZE,
         max_report_cols: int = DEFAULT_REPORT_COLS,
+        web: bool = False,
     ) -> None:
         _rt = self.job.info(page, size)
         if not _rt:

client/starwhale/web/data_store.py

@@ -0,0 +1,103 @@
+import glob
+import typing as t
+import os.path
+
+from fastapi import APIRouter
+from pydantic import Field, BaseModel
+
+from starwhale.api._impl import wrapper
+from starwhale.utils.config import SWCliConfigMixed
+from starwhale.web.response import success, SuccessResp
+from starwhale.api._impl.data_store import SwType, _get_type, TableDesc, LocalDataStore
+
+router = APIRouter()
+prefix = "datastore"
+
+
+class ListTablesRequest(BaseModel):
+    prefix: str
+
+
+class Filter(BaseModel):
+    operator: str
+    operands: t.List[t.Dict[str, str]]
+
+
+class QueryTableRequest(BaseModel):
+    table_name: str = Field(alias="tableName")
+    filter: t.Optional[Filter]
+    limit: int
+
+
+@router.post("/listTables")
+def list_tables(request: ListTablesRequest) -> SuccessResp:
+    # TODO: use datastore builtin function
+    root = str(SWCliConfigMixed().datastore_dir)
+    path = os.path.join(root, request.prefix)
+    files = glob.glob(f"{path}**", recursive=True)
+    files = [os.path.split(f)[0][len(root) :] for f in files if os.path.isfile(f)]
+    return success({"tables": files})
+
+
+@router.post("/queryTable")
+def query_table(request: QueryTableRequest) -> SuccessResp:
+    eval_id = _is_eval_summary(request)
+    if eval_id:
+        return _eval_summary(eval_id)
+
+    ds = LocalDataStore.get_instance()
+    rows = list(ds.scan_tables([TableDesc(request.table_name)]))
+    col, rows = _rows_to_type_and_records(rows)
+    return success(
+        {
+            "columnTypes": col,
+            "records": rows,
+        }
+    )
+
+
+def _is_eval_summary(request: QueryTableRequest) -> t.Union[str, None]:
+    if not request.table_name.endswith("/summary"):
+        return None
+    if not request.filter:
+        return None
+    op = request.filter.operator
+    operands = request.filter.operands
+    if op != "EQUAL":
+        return None
+    if len(operands) != 2 or {"columnName": "sys/id"} not in operands:
+        return None
+
+    eval_id = (
+        operands[0].get("intValue")
+        if operands[1].get("columnName") == "sys/id"
+        else operands[1].get("intValue")
+    )
+
+    return eval_id
+
+
+def _eval_summary(eval_id: str) -> SuccessResp:
+    evaluation = wrapper.Evaluation(
+        eval_id=eval_id,
+        project="self",
+        instance="local",
+    )
+    summary = evaluation.get_metrics()
+    col, rows = _rows_to_type_and_records(summary)
+    return success(
+        {
+            "columnTypes": col,
+            "records": rows,
+        }
+    )
+
+
+def _rows_to_type_and_records(rows: t.Union[list, dict]) -> t.Tuple[list, list]:
+    if not rows:
+        return [], []
+    if not isinstance(rows, list):
+        rows = [rows]
+    encoders = {k: SwType.encode_schema(_get_type(v)) for k, v in rows[0].items()}
+    column_types = [v.update({"name": k}) or v for k, v in encoders.items()]
+    return column_types, [{k: str(v) for k, v in row.items()} for row in rows]

client/starwhale/web/panel.py

@@ -0,0 +1,32 @@
+import os.path
+from pathlib import Path
+
+from fastapi import APIRouter
+from starlette.requests import Request
+
+from starwhale.consts import SW_AUTO_DIRNAME
+from starwhale.utils.fs import ensure_file
+from starwhale.web.response import success, SuccessResp
+
+router = APIRouter()
+prefix = "panel"
+
+
+def _setting_path() -> str:
+    return os.path.join(os.getcwd(), SW_AUTO_DIRNAME, "panel.json")
+
+
+@router.post("/setting/{project}/{key}")
+async def setting(project: str, key: str, request: Request) -> SuccessResp:
+    content = await request.body()
+    ensure_file(_setting_path(), content.decode("utf-8"), parents=True)
+    return success({})
+
+
+@router.get("/setting/{project}/{key}")
+def get_setting(project: str, key: str) -> SuccessResp:
+    file = Path(_setting_path())
+    if not file.exists() or not file.is_file():
+        return success("")
+    with file.open("r") as f:
+        return success(f.read())

client/starwhale/web/project.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter
+
+from starwhale.web.user import user
+from starwhale.web.response import success, SuccessResp
+
+router = APIRouter()
+prefix = "project"
+
+project = {
+    "id": "1",
+    "name": "self",
+    "privacy": "PRIVATE",
+    "createdTime": 0,
+    "owner": user,
+    "statistics": {},
+}
+
+
+@router.get("/{project_id}")
+def get_project(project_id: str) -> SuccessResp:
+    return success(project)
+
+
+@router.get("/{project_id}/job/{job}")
+def get_job(project_id: str, job: str) -> SuccessResp:
+    return success({"id": "1", "uuid": job, "modelName": "mock-model"})
+
+
+@router.get("/{project_id}/role")
+def get_role(project_id: str) -> SuccessResp:
+    return success(
+        [
+            {
+                "id": "1",
+                "user": user,
+                "project": project,
+                "role": {
+                    "id": "1",
+                    "name": "Owner",
+                    "code": "OWNER",
+                },
+            },
+        ]
+    )

client/starwhale/web/response.py

@@ -0,0 +1,11 @@
+import typing
+
+SuccessResp = typing.Dict[str, typing.Any]
+
+
+def success(data: typing.Any) -> SuccessResp:
+    return {
+        "code": "success",
+        "message": "Success",
+        "data": data,
+    }

client/starwhale/web/server.py

@@ -0,0 +1,63 @@
+import typing as t
+import os.path
+
+import pkg_resources
+from fastapi import FastAPI, APIRouter
+from fastapi.responses import ORJSONResponse
+from typing_extensions import Protocol
+from starlette.requests import Request
+from starlette.responses import FileResponse
+from starlette.exceptions import HTTPException
+from starlette.staticfiles import StaticFiles
+
+from starwhale.web import user, panel, system, project, data_store
+
+STATIC_DIR_DEV = pkg_resources.resource_filename("starwhale", "web/ui")
+
+
+class Component(Protocol):
+    router: APIRouter
+    prefix: str
+
+
+class Server(FastAPI):
+    def __init__(self) -> None:
+        super().__init__(default_response_class=ORJSONResponse)
+
+    def add_component(self, component: Component) -> None:
+        self.include_router(component.router, prefix=f"/{component.prefix.lstrip('/')}")
+
+    @staticmethod
+    def with_components(components: t.List[Component]) -> FastAPI:
+        api = Server()
+        for component in components:
+            api.add_component(component)
+
+        app = FastAPI()
+        app.mount("/api/v1", api)
+
+        @app.get("/")
+        def index() -> FileResponse:
+            return FileResponse(os.path.join(STATIC_DIR_DEV, "index.html"))
+
+        app.mount("/", StaticFiles(directory=STATIC_DIR_DEV), name="assets")
+
+        @app.exception_handler(404)
+        def not_found_exception_handler(
+            request: Request, exc: HTTPException
+        ) -> FileResponse:
+            return FileResponse(os.path.join(STATIC_DIR_DEV, "index.html"))
+
+        return app
+
+    @staticmethod
+    def default() -> FastAPI:
+        return Server.with_components([panel, project, data_store, user, system])
+
+
+# for testing
+if __name__ == "__main__":
+    import uvicorn
+
+    server = Server.default()
+    uvicorn.run("server:server", host="127.0.0.1", port=8000, reload=True)

client/starwhale/web/system.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter
+
+from starwhale.version import STARWHALE_VERSION
+from starwhale.web.response import success, SuccessResp
+
+router = APIRouter()
+prefix = "system"
+
+
+@router.get("/version")
+def version() -> SuccessResp:
+    return success(STARWHALE_VERSION)

client/starwhale/web/ui

@@ -0,0 +1 @@
+../../../console/build