add local server start/stop/status command

client/starwhale/cli/__init__.py

@@ -6,6 +6,7 @@
 from starwhale.version import STARWHALE_VERSION
 from starwhale.cli.deubg import debug_cmd
 from starwhale.utils.cli import AliasedGroup
+from starwhale.cli.server import server_cmd
 from starwhale.utils.debug import init_logger
 from starwhale.core.job.cli import job_cmd
 from starwhale.utils.config import load_swcli_config
@@ -53,6 +54,7 @@ def cli(ctx: click.Context, verbose_cnt: int, output: str) -> None:
     cli.add_command(config_cmd)
     cli.add_command(assistance_cmd)
     cli.add_command(debug_cmd)
+    cli.add_command(server_cmd, aliases=["svr"])  # type: ignore
     add_mngt_command(cli)
 
     return cli

client/starwhale/cli/server/__init__.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import typing as t
+import webbrowser
+from pathlib import Path
+
+import click
+from jinja2 import Environment, FileSystemLoader
+
+from starwhale.version import STARWHALE_VERSION
+from starwhale.utils.fs import ensure_dir, ensure_file
+from starwhale.utils.cli import AliasedGroup
+from starwhale.utils.debug import console
+from starwhale.utils.config import SWCliConfigMixed
+from starwhale.utils.process import check_call
+
+
+@click.group("server", cls=AliasedGroup, help="Manage Starwhale Local Server")
+def server_cmd() -> None:
+    pass
+
+
+@server_cmd.command("start")
+@click.option(
+    "-h",
+    "--host",
+    default="127.0.0.1",
+    show_default=True,
+    help="bind Starwhale Server to the host",
+)
+@click.option(
+    "-p",
+    "--port",
+    default=8082,
+    show_default=True,
+    help="publish Starwhale Server port to the host",
+)
+@click.option(
+    "envs",
+    "-e" "--env",
+    multiple=True,
+    help="set environment variables for Starwhale Server",
+)
+@click.option(
+    "-i",
+    "--server-image",
+    default="",
+    help="set Starwhale Server image. If not set, Starwhale will use the swcli version related server image.",
+)
+@click.option(
+    "--detach/--no-detach",
+    is_flag=True,
+    default=True,
+    show_default=True,
+    help="run Starwhale Server containers in the background",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="render compose yaml file and dry run docker compose",
+)
+@click.option(
+    "--db-image",
+    default="docker-registry.starwhale.cn/bitnami/mysql:8.0.29-debian-10-r2",
+    hidden=True,
+    help="set Starwhale Server database image",
+)
+def start(
+    host: str,
+    port: int,
+    envs: t.List[str],
+    server_image: str,
+    detach: bool,
+    dry_run: bool,
+    db_image: str,
+) -> None:
+    """Start Starwhale Server in the local machine with Docker.
+
+    You need to install Docker(>=19.03) and Docker-Compose(v2) first.
+    When you run this command, a docker compose yaml file will be generated in the ~/.starwhale/.server directory.
+
+    Examples:
+
+    \b
+    # Start Starwhale Server with default settings, then you can visit http://127.0.0.1:8082 to use Starwhale Server.
+    swcli server start
+
+    \b
+    # Start Starwhale Server with custom Server image.
+    swcli server start -i docker-registry.starwhale.cn/star-whale/server:latest
+
+    \b
+    # Start Starwhale Server with custom host and port.
+    swcli server start --port 18082 --host 0.0.0.0
+
+    \b
+    # Start Starwhale Server in the foreground and custom environment variables for pypi.
+    swcli server start --no-detach -e SW_PYPI_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple -e SW_PYPI_EXTRA_INDEX_URL=https://mirrors.aliyun.com/pypi/simple
+    """
+    server_compose_path = SWCliConfigMixed().server_compose
+    ensure_dir(server_compose_path.parent)
+
+    console.print(f":flying_saucer: render compose yaml file: {server_compose_path}")
+    render_compose_yaml(
+        path=server_compose_path,
+        host=host,
+        port=port,
+        envs=envs,
+        server_image=server_image,
+        db_image=db_image,
+    )
+
+    console.print(":ping_pong: start Starwhale Server by docker compose")
+
+    cmd = ["docker", "compose", "-f", str(server_compose_path), "up"]
+    if detach:
+        cmd.append("--detach")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    check_call(cmd, log=console.print)
+
+    if detach:
+        url = f"http://{host}:{port}"
+        console.print(
+            "Starwhale Server is running in the background. \n"
+            "\t :apple: stop: [bold green]swcli server stop[/] \n"
+            "\t :banana: check status: [bold green]swcli server status[/] \n"
+            f"\t :watermelon: more compose command: [bold green]docker compose -f {server_compose_path} sub-command[/] \n"
+            f"\t :carrot: visit web: {url}"
+        )
+        if not dry_run:
+            webbrowser.open(url)
+
+
+@server_cmd.command(
+    "status", aliases=["ps"], help="Show Starwhale Server containers status"
+)
+def status() -> None:
+    # TODO: add more status info
+    check_call(
+        ["docker", "compose", "-f", str(SWCliConfigMixed().server_compose), "ps"],
+        log=console.print,
+    )
+
+
+@server_cmd.command("stop", help="Stop Starwhale Server containers")
+def stop() -> None:
+    check_call(
+        ["docker", "compose", "-f", str(SWCliConfigMixed().server_compose), "stop"],
+        log=console.print,
+    )
+
+
+_TEMPLATE_DIR = Path(__file__).parent
+
+
+def render_compose_yaml(
+    path: Path,
+    host: str,
+    port: int,
+    envs: t.List[str],
+    server_image: str,
+    db_image: str,
+) -> None:
+    jinja2_env = Environment(loader=FileSystemLoader(searchpath=_TEMPLATE_DIR))
+    template = jinja2_env.get_template("compose.tmpl")
+
+    starwhale_version = STARWHALE_VERSION
+    if starwhale_version.startswith("0.0.0"):
+        starwhale_version = ""
+
+    if not server_image:
+        server_image = f"docker-registry.starwhale.cn/star-whale/server:{starwhale_version or 'latest'}"
+
+    content = template.render(
+        host_ip=host,
+        host_port=port,
+        envs=envs,
+        server_image=server_image,
+        db_image=db_image,
+        starwhale_version=starwhale_version,
+    )
+    ensure_file(path, content=content, parents=True)

client/starwhale/cli/server/compose.tmpl

@@ -0,0 +1,79 @@
+# The docker compose yaml file is generated by `swcli server start` command.
+name: starwhale_local
+services:
+  db:
+    image: {{db_image}}
+    restart: always
+    healthcheck:
+      test: ["CMD", '/opt/bitnami/scripts/mysql/healthcheck.sh']
+      interval: 15s
+      timeout: 5s
+      retries: 10
+    networks:
+      - ns
+    volumes:
+      - db:/bitnami/mysql
+    environment:
+      - MYSQL_ROOT_PASSWORD=starwhale
+      - MYSQL_USER=starwhale
+      - MYSQL_PASSWORD=starwhale
+      - MYSQL_DATABASE=starwhale
+    labels:
+      com.starwhale.description: "MySQL for Starwhale Server"
+      com.starwhale.component: "db"
+
+  server:
+    image: {{server_image}}
+    restart: always
+    depends_on:
+      db:
+        condition: service_healthy
+        restart: true
+    environment:
+      - JAR=controller
+      - SW_CONTROLLER_PORT=8082
+      - SW_JWT_TOKEN_EXPIRE_MINUTES=144000
+      - SW_UPLOAD_MAX_FILE_SIZE=20480MB
+      - SW_STORAGE_PREFIX=starwhale
+      - SW_STORAGE_FS_ROOT_DIR=/usr/local/starwhale
+      - SW_DOCKER_REGISTRY_URL=docker-registry.starwhale.cn
+      - SW_SCHEDULER=docker
+      - SW_METADATA_STORAGE_IP=db
+      - SW_METADATA_STORAGE_PORT=3306
+      - SW_METADATA_STORAGE_USER=starwhale
+      - SW_METADATA_STORAGE_PASSWORD=starwhale
+      - SW_DATASTORE_WAL_LOCAL_CACHE_DIR=/opt/starwhale.java/datastore-wal-cache
+      - SW_STORAGE_TYPE=fs
+      - SW_DOCKER_CONTAINER_NETWORK=starwhale_local_ns
+      - SW_INSTANCE_URI=http://server:8082
+      - SW_DATASET_BUILD_PYTHON_VERSION=3.10
+      - SW_DATASET_BUILD_CLI_VERSION={{starwhale_version}}
+      - SW_PYPI_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
+      - SW_PYPI_EXTRA_INDEX_URL=https://mirrors.aliyun.com/pypi/simple
+      {% for env in envs %}
+        - {{env}}
+      {% endfor %}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - wal:/opt/starwhale.java/datastore-wal-cache
+      - fs:/usr/local/starwhale
+    labels:
+      com.starwhale.description: "Starwhale Server"
+      com.starwhale.version: "{{starwhale_version}}"
+      com.starwhale.component: "server"
+    networks:
+      - ns
+    ports:
+      - target: {{host_port}}
+        host_ip: {{host_ip}}
+        published: 8082
+        protocol: tcp
+        mode: host
+
+volumes:
+  db: {}
+  wal: {}
+  fs: {}
+
+networks:
+  ns: {}

client/starwhale/mngt/__init__.py

@@ -160,24 +160,45 @@ def _check_conda() -> str:
         )
         return out.decode().strip().split()[-1]
 
+    def _check_docker_compose() -> str:
+        out = subprocess.check_output(
+            ["docker", "compose", "version", "--short"], stderr=subprocess.STDOUT
+        )
+        return out.decode().strip()
+
     dependencies: t.List[_Dependency] = [
         _Dependency(
             title="Docker",
             min_version="19.03",
             level=_CheckLevel.WARN,
             help=(
                 "Docker is an open platform for developing, shipping, and running applications."
-                "Starwhale uses Docker to run jobs. You can visit https://docs.docker.com/get-docker/ for more details."
+                "Starwhale uses Docker to run jobs. "
+                "In Ubuntu, you can install Docker by `sudo apt-get install docker-ce`."
+                "You can visit https://docs.docker.com/get-docker/ for more details."
             ),
             checker=_check_docker,
         ),
+        _Dependency(
+            title="Docker Compose",
+            min_version="2.0.0",
+            level=_CheckLevel.WARN,
+            help=(
+                "When you run Starwhale Server by `swcli server start`, "
+                "Starwhale will use Docker Compose to manage Starwhale Server."
+                "In Ubuntu, you can install Docker Compose by `sudo apt-get install docker-compose-plugin`."
+                "You can visit https://docs.docker.com/compose/install/ for more details."
+            ),
+            checker=_check_docker_compose,
+        ),
         _Dependency(
             title="Conda",
             min_version="4.0.0",
             level=_CheckLevel.WARN,
             help=(
                 "Conda is an open-source package management system and environment management system."
-                "Starwhale uses Conda to build runtime. You can download it from https://docs.conda.io/en/latest/miniconda.html."
+                "Starwhale uses Conda to build runtime. "
+                "You can download it from https://docs.conda.io/en/latest/miniconda.html."
             ),
             checker=_check_conda,
         ),

client/starwhale/utils/config.py

@@ -80,7 +80,10 @@ def render_default_swcli_config(fpath: str) -> t.Dict[str, t.Any]:
             )
         },
         current_instance=DEFAULT_INSTANCE,
-        storage=dict(root=env_root or str(DEFAULT_SW_LOCAL_STORAGE.resolve())),
+        storage=dict(
+            root=env_root or str(DEFAULT_SW_LOCAL_STORAGE.resolve()),
+            server_compose=f"{env_root or DEFAULT_SW_LOCAL_STORAGE.resolve()}/.server/docker-compose.yaml",
+        ),
         docker=dict(builtin_image_repo=DEFAULT_IMAGE_REPO),
     )
     render_swcli_config(c, fpath)
@@ -114,6 +117,13 @@ def __init__(self, swcli_config: t.Optional[t.Dict[str, t.Any]] = None) -> None:
     def rootdir(self) -> Path:
         return Path(self._config["storage"]["root"])
 
+    @property
+    def server_compose(self) -> Path:
+        if "server_compose" not in self._config["storage"]:
+            return self.rootdir / ".server" / "docker-compose.yaml"
+        else:
+            return Path(self._config["storage"]["server_compose"])
+
     @property
     def datastore_dir(self) -> Path:
         return self.rootdir / DATA_STORE_DIRNAME