diff --git a/client/starwhale/api/_impl/data_store.py b/client/starwhale/api/_impl/data_store.py
index 5666661004..30b2fbf335 100644
--- a/client/starwhale/api/_impl/data_store.py
+++ b/client/starwhale/api/_impl/data_store.py
@@ -42,11 +42,11 @@
import requests
import tenacity
import jsonlines
-from loguru import logger
from filelock import FileLock
from jsonlines import Writer
from typing_extensions import Protocol
+from starwhale.utils import console
from starwhale.consts import STANDALONE_INSTANCE
from starwhale.utils.fs import ensure_dir
from starwhale.consts.env import SWEnv
@@ -1333,7 +1333,7 @@ def update_table(
)
if resp.status_code != HTTPStatus.OK:
- logger.error(
+ console.error(
f"[update-table]Table:{table_name}, resp code:{resp.status_code}, \n resp text: {resp.text}, \n records: {records}"
)
resp.raise_for_status()
@@ -1562,7 +1562,9 @@ def run(self) -> None:
self.table_name, last_schema, to_submit
)
except Exception as e:
- logger.exception(e)
+ from starwhale.utils import console
+
+ console.print_exception()
self._queue_run_exceptions.append(e)
if len(self._queue_run_exceptions) > self._run_exceptions_limits:
break
diff --git a/client/starwhale/api/_impl/dataset/builder/mapping_builder.py b/client/starwhale/api/_impl/dataset/builder/mapping_builder.py
index 5ea8a06fcb..a7ea024ea8 100644
--- a/client/starwhale/api/_impl/dataset/builder/mapping_builder.py
+++ b/client/starwhale/api/_impl/dataset/builder/mapping_builder.py
@@ -12,8 +12,7 @@
from binascii import crc32
from collections import defaultdict
-from loguru import logger
-
+from starwhale.utils import console
from starwhale.consts import STANDALONE_INSTANCE
from starwhale.base.uri import URI
from starwhale.utils.fs import (
@@ -105,7 +104,7 @@ def __exit__(
trace: TracebackType,
) -> None:
if value: # pragma: no cover
- logger.warning(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
self.close()
@@ -254,7 +253,7 @@ def __exit__(
trace: TracebackType,
) -> None:
if value: # pragma: no cover
- logger.warning(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
self.close()
diff --git a/client/starwhale/api/_impl/dataset/loader.py b/client/starwhale/api/_impl/dataset/loader.py
index ac99182e02..3127482fd8 100644
--- a/client/starwhale/api/_impl/dataset/loader.py
+++ b/client/starwhale/api/_impl/dataset/loader.py
@@ -6,9 +6,7 @@
import threading
from functools import total_ordering
-import loguru
-from loguru import logger as _logger
-
+from starwhale.utils import console
from starwhale.consts import HTTPMethod
from starwhale.base.uri import URI
from starwhale.base.type import URIType, InstanceType
@@ -145,7 +143,6 @@ def __init__(
dataset_uri: URI,
start: t.Optional[t.Any] = None,
end: t.Optional[t.Any] = None,
- logger: t.Optional[loguru.Logger] = None,
session_consumption: t.Optional[TabularDatasetSessionConsumption] = None,
cache_size: int = _DEFAULT_LOADER_CACHE_SIZE,
num_workers: int = 2,
@@ -153,7 +150,6 @@ def __init__(
field_transformer: t.Optional[t.Dict] = None,
):
self.dataset_uri = dataset_uri
- self.logger = logger or _logger
self.start = start
self.end = end
self.dataset_scan_revision = dataset_scan_revision
@@ -329,7 +325,7 @@ def __iter__(
else:
yield row
- self.logger.debug(
+ console.debug(
"queue details:"
f"meta fetcher(qsize:{meta_fetched_queue.qsize()}, alive: {meta_fetcher.is_alive()}), "
f"row unpackers(qsize:{row_unpacked_queue.qsize()}, alive: {[t.is_alive() for t in rows_unpackers]})"
@@ -349,13 +345,11 @@ def get_data_loader(
start: t.Optional[t.Any] = None,
end: t.Optional[t.Any] = None,
session_consumption: t.Optional[TabularDatasetSessionConsumption] = None,
- logger: t.Optional[loguru.Logger] = None,
cache_size: int = _DEFAULT_LOADER_CACHE_SIZE,
num_workers: int = 2,
dataset_scan_revision: str = "",
field_transformer: t.Optional[t.Dict] = None,
) -> DataLoader:
-
if session_consumption:
sc_start = session_consumption.session_start # type: ignore
sc_end = session_consumption.session_end # type: ignore
@@ -372,7 +366,6 @@ def get_data_loader(
start=start,
end=end,
session_consumption=session_consumption,
- logger=logger or _logger,
cache_size=cache_size,
num_workers=num_workers,
dataset_scan_revision=dataset_scan_revision,
diff --git a/client/starwhale/api/_impl/dataset/model.py b/client/starwhale/api/_impl/dataset/model.py
index 1b0cae7c99..63221514be 100644
--- a/client/starwhale/api/_impl/dataset/model.py
+++ b/client/starwhale/api/_impl/dataset/model.py
@@ -13,9 +13,8 @@
from itertools import islice
import yaml
-from loguru import logger
-from starwhale.utils import now_str, convert_to_bytes, gen_uniq_version
+from starwhale.utils import console, now_str, convert_to_bytes, gen_uniq_version
from starwhale.consts import (
FileDesc,
HTTPMethod,
@@ -250,7 +249,7 @@ def __exit__(
trace: TracebackType,
) -> None:
if value: # pragma: no cover
- logger.warning(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
self.close()
@@ -919,7 +918,7 @@ def _submit_cloud_version(manifest_path: Path) -> None:
info_revision = _save_info()
manifest_path = _dump_manifest(dataset_revision, info_revision)
- logger.debug(
+ console.debug(
f"dataset commit: revision-{dataset_revision}, info revision-{info_revision}"
)
if self.project_uri.instance == STANDALONE_INSTANCE:
diff --git a/client/starwhale/api/_impl/evaluation.py b/client/starwhale/api/_impl/evaluation.py
index 78b8ceab1e..ac39896c74 100644
--- a/client/starwhale/api/_impl/evaluation.py
+++ b/client/starwhale/api/_impl/evaluation.py
@@ -1,10 +1,7 @@
from __future__ import annotations
-import io
-import sys
import time
import typing as t
-import logging
import threading
from abc import ABCMeta, abstractmethod
from types import TracebackType
@@ -13,13 +10,12 @@
import jsonlines
-from starwhale.utils import now_str
+from starwhale.utils import console, now_str
from starwhale.consts import RunStatus, CURRENT_FNAME, DecoratorInjectAttr
from starwhale.base.uri import URI
from starwhale.utils.fs import ensure_dir, ensure_file
from starwhale.api._impl import wrapper
from starwhale.base.type import URIType, RunSubDirType
-from starwhale.utils.log import StreamWrapper
from starwhale.api.service import Input, Output, Service
from starwhale.utils.error import ParameterError, FieldTypeOrValueError
from starwhale.base.context import Context
@@ -27,15 +23,6 @@
from starwhale.api._impl.dataset import Dataset
from starwhale.core.dataset.tabular import TabularDatasetRow
-if t.TYPE_CHECKING:
- import loguru
-
-
-class _LogType:
- SW = "starwhale"
- USER = "user"
-
-
_jl_writer: t.Callable[[Path], jsonlines.Writer] = lambda p: jsonlines.open(
str((p).resolve()), mode="w"
)
@@ -68,15 +55,8 @@ def __init__(
_logdir / RunSubDirType.RUNLOG / self.context.step / str(self.context.index)
)
self.status_dir = _run_dir / RunSubDirType.STATUS
- self.log_dir = _run_dir / RunSubDirType.LOG
ensure_dir(self.status_dir)
- ensure_dir(self.log_dir)
- self.logger, self._sw_logger = self._init_logger(self.log_dir)
- self._stdout_changed = False
- self._stderr_changed = False
- self._orig_stdout = sys.stdout
- self._orig_stderr = sys.stderr
# TODO: split status/result files
self._timeline_writer = _jl_writer(self.status_dir / "timeline")
@@ -84,47 +64,10 @@ def __init__(
self.evaluation_store = wrapper.Evaluation(
eval_id=self.context.version, project=self.context.project
)
- self._monkey_patch()
self._update_status(RunStatus.START)
- def _init_logger(
- self, log_dir: Path, rotation: str = "500MB"
- ) -> t.Tuple[loguru.Logger, loguru.Logger]:
- # TODO: remove logger first?
- # TODO: add custom log format, include daemonset pod name
- from loguru import logger as _logger
-
- # TODO: configure log rotation size
- _logger.add(
- log_dir / "{time}.log",
- rotation=rotation,
- backtrace=True,
- diagnose=True,
- serialize=True,
- )
- _logger.bind(
- type=_LogType.USER,
- task_id=self.context.index,
- job_id=self.context.version,
- )
- _sw_logger = _logger.bind(type=_LogType.SW)
- return _logger, _sw_logger
-
- def _monkey_patch(self) -> None:
- if not isinstance(sys.stdout, StreamWrapper) and isinstance(
- sys.stdout, io.TextIOWrapper
- ):
- sys.stdout = StreamWrapper(sys.stdout, self.logger, logging.INFO) # type: ignore
- self._stdout_changed = True
-
- if not isinstance(sys.stderr, StreamWrapper) and isinstance(
- sys.stderr, io.TextIOWrapper
- ):
- sys.stderr = StreamWrapper(sys.stderr, self.logger, logging.WARN) # type: ignore
- self._stderr_changed = True
-
def __str__(self) -> str:
- return f"PipelineHandler status@{self.status_dir}, " f"log@{self.log_dir}"
+ return f"PipelineHandler status@{self.status_dir}"
def __enter__(self) -> PipelineHandler:
return self
@@ -135,16 +78,10 @@ def __exit__(
value: t.Optional[BaseException],
trace: TracebackType,
) -> None:
- self._sw_logger.debug(
- f"execute {self.context.step}-{self.context.index} exit func..."
- )
+ console.debug(f"execute {self.context.step}-{self.context.index} exit func...")
if value: # pragma: no cover
- print(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
- if self._stdout_changed:
- sys.stdout = self._orig_stdout
- if self._stderr_changed:
- sys.stderr = self._orig_stderr
self._timeline_writer.close()
@abstractmethod
@@ -160,15 +97,15 @@ def _record_status(func): # type: ignore
@wraps(func) # type: ignore
def _wrapper(*args: t.Any, **kwargs: t.Any) -> None:
self: PipelineHandler = args[0]
- self._sw_logger.info(
+ console.info(
f"start to run {func.__name__} function@{self.context.step}-{self.context.index} ..." # type: ignore
)
self._update_status(RunStatus.RUNNING)
try:
func(*args, **kwargs) # type: ignore
- except Exception as e:
+ except Exception:
self._update_status(RunStatus.FAILED)
- self._sw_logger.exception(f"{func} abort, exception: {e}")
+ console.print_exception()
raise
else:
self._update_status(RunStatus.SUCCESS)
@@ -184,7 +121,7 @@ def _starwhale_internal_run_cmp(self) -> None:
else:
self.cmp()
except Exception as e:
- self._sw_logger.exception(f"cmp exception: {e}")
+ console.exception(f"cmp exception: {e}")
self._timeline_writer.write(
{"time": now, "status": False, "exception": str(e)}
)
@@ -233,7 +170,7 @@ def _starwhale_internal_run_ppl(self) -> None:
]
except Exception as e:
_exception = e
- self._sw_logger.exception(
+ console.exception(
f"[{[r.index for r in rows]}] data handle -> failed"
)
if not self.ignore_error:
@@ -246,7 +183,7 @@ def _starwhale_internal_run_ppl(self) -> None:
cnt += 1
_idx_with_ds = f"{_uri.object}{join_str}{_idx}"
- self._sw_logger.debug(
+ console.debug(
f"[{_idx_with_ds}] use {time.time() - _start:.3f}s, session-id:{self.context.version} @{self.context.step}-{self.context.index}"
)
@@ -278,7 +215,7 @@ def _starwhale_internal_run_ppl(self) -> None:
if self.flush_result and self.ppl_auto_log:
self.evaluation_store.flush_result()
- self._sw_logger.info(
+ console.info(
f"{self.context.step}-{self.context.index} handled {cnt} data items for dataset {self.dataset_uris}"
)
diff --git a/client/starwhale/api/_impl/job.py b/client/starwhale/api/_impl/job.py
index 1fc0c921be..5771372bdd 100644
--- a/client/starwhale/api/_impl/job.py
+++ b/client/starwhale/api/_impl/job.py
@@ -9,7 +9,6 @@
from collections import defaultdict
import yaml
-from loguru import logger
from starwhale.consts import DecoratorInjectAttr
from starwhale.utils.fs import ensure_file
@@ -270,8 +269,6 @@ def generate_jobs_yaml(
package_dir: t.Union[Path, str],
yaml_path: t.Union[Path, str],
) -> None:
- logger.debug(f"ingest run_handlers {search_modules} at {package_dir}")
-
expanded_handlers = Handler.get_registered_handlers_with_expanded_needs(
search_modules, Path(package_dir)
)
diff --git a/client/starwhale/api/_impl/track/collector.py b/client/starwhale/api/_impl/track/collector.py
index c826e5c255..7fe708a72f 100644
--- a/client/starwhale/api/_impl/track/collector.py
+++ b/client/starwhale/api/_impl/track/collector.py
@@ -8,8 +8,8 @@
from collections import defaultdict
import psutil
-from loguru import logger
+from starwhale.utils import console
from starwhale.consts import FMT_DATETIME
from starwhale.utils.venv import guess_current_py_env
from starwhale.utils.error import NotFoundError
@@ -84,8 +84,9 @@ def run(self) -> None:
try:
ret = _action()
self.tracker._log_params(ret, source=_TrackSource.SYSTEM)
- except Exception as e:
- logger.exception(f"failed to inspect {_info}: {e}")
+ except Exception:
+ console.print(":warning: [red]{_info}[/red] inspect failed")
+ console.print_exception()
# TODO: tune the accuracy of inspect and report interval
last_inspect_time = last_report_time = time.monotonic()
@@ -106,7 +107,7 @@ def run(self) -> None:
self._report_metrics()
last_report_time = time.monotonic()
except Exception as e:
- logger.exception(e)
+ console.print_exception()
self._run_exceptions.append(e)
self._raise_run_exceptions()
diff --git a/client/starwhale/api/_impl/track/tracker.py b/client/starwhale/api/_impl/track/tracker.py
index 5344d3a05a..bbb78797ce 100644
--- a/client/starwhale/api/_impl/track/tracker.py
+++ b/client/starwhale/api/_impl/track/tracker.py
@@ -12,9 +12,8 @@
from pathlib import Path
import yaml
-from loguru import logger
-from starwhale.utils import now_str, random_str, gen_uniq_version
+from starwhale.utils import console, now_str, random_str, gen_uniq_version
from starwhale.consts import SW_AUTO_DIRNAME, DEFAULT_MANIFEST_NAME
from starwhale.base.uri import URI
from starwhale.utils.fs import ensure_dir, ensure_file
@@ -133,7 +132,7 @@ def __exit__(
trace: TracebackType,
) -> None:
if value: # pragma: no cover
- logger.warning(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
self.end()
diff --git a/client/starwhale/api/_impl/wrapper.py b/client/starwhale/api/_impl/wrapper.py
index ed1de9caef..f4fcf375ce 100644
--- a/client/starwhale/api/_impl/wrapper.py
+++ b/client/starwhale/api/_impl/wrapper.py
@@ -8,8 +8,8 @@
import dill
import requests
-from loguru import logger
+from starwhale.utils import console
from starwhale.consts import VERSION_PREFIX_CNT, STANDALONE_INSTANCE
from starwhale.consts.env import SWEnv
from starwhale.utils.retry import http_retry
@@ -35,8 +35,8 @@ def close(self) -> None:
try:
writer.close()
except Exception as e:
- logger.exception(f"{writer} exception: {e}")
exceptions.append(e)
+ console.print_exception()
if exceptions:
raise Exception(*exceptions)
diff --git a/client/starwhale/base/bundle.py b/client/starwhale/base/bundle.py
index d7d271ffad..6150f6daa3 100644
--- a/client/starwhale/base/bundle.py
+++ b/client/starwhale/base/bundle.py
@@ -7,7 +7,6 @@
from contextlib import ExitStack
import yaml
-from loguru import logger
from fs.walk import Walker
from starwhale.utils import console, now_str, gen_uniq_version
@@ -168,15 +167,12 @@ def _render_manifest(self) -> None:
# TODO: add signature for import files: model, config
_fpath = self.store.snapshot_workdir / DEFAULT_MANIFEST_NAME # type: ignore
ensure_file(_fpath, yaml.safe_dump(self._manifest, default_flow_style=False))
- logger.info(f"[step:manifest]render manifest: {_fpath}")
def _gen_version(self) -> None:
- logger.info("[step:version]create version...")
if not getattr(self, "_version", ""):
self._version = gen_uniq_version()
self.uri.object.version = self._version # type:ignore
- logger.info(f"[step:version]version: {self._version}")
console.print(f":new: version {self._version[:SHORT_VERSION_CNT]}") # type: ignore
self._manifest["version"] = self._version
self._manifest[CREATED_AT_KEY] = now_str()
@@ -187,13 +183,10 @@ def _make_auto_tags(self) -> None:
def _make_tar(self, ftype: str = "") -> None:
out = self.store.bundle_dir / f"{self._version}{ftype}" # type: ignore
ensure_dir(self.store.bundle_dir) # type: ignore
- logger.info(f"[step:tar]try to tar {out} ...")
-
with tarfile.open(out, "w:") as tar:
tar.add(str(self.store.snapshot_workdir), arcname="") # type: ignore
console.print(f":butterfly: {ftype} bundle:{out}")
- logger.info("[step:tar]finish to make bundle tar")
@classmethod
def _do_validate_yaml(cls, path: Path) -> None:
diff --git a/client/starwhale/base/bundle_copy.py b/client/starwhale/base/bundle_copy.py
index e1bdd4cbd9..78eb505d40 100644
--- a/client/starwhale/base/bundle_copy.py
+++ b/client/starwhale/base/bundle_copy.py
@@ -246,7 +246,7 @@ def do(self) -> None:
TimeElapsedColumn(),
TotalFileSizeColumn(),
TransferSpeedColumn(),
- console=console,
+ console=console.rich_console,
refresh_per_second=0.2,
) as progress:
if self.src_uri.instance_type == InstanceType.STANDALONE:
diff --git a/client/starwhale/base/context.py b/client/starwhale/base/context.py
index 9c29cf181a..e91eb4fbe8 100644
--- a/client/starwhale/base/context.py
+++ b/client/starwhale/base/context.py
@@ -5,8 +5,7 @@
from pathlib import Path
from functools import wraps
-from loguru import logger
-
+from starwhale.utils import console
from starwhale.utils.error import ParameterError
@@ -69,7 +68,7 @@ def set_runtime_context(cls, ctx: Context) -> None:
if val and isinstance(val, Context):
# TODO: _context_holder set only once?
cls._context_holder.value = ctx
- logger.warning(f"runtime context has already be set: {val}")
+ console.warning(f"runtime context has already be set: {val}")
except AttributeError:
cls._context_holder.value = ctx
diff --git a/client/starwhale/base/scheduler/__init__.py b/client/starwhale/base/scheduler/__init__.py
index 9b2aac3575..646b042a6f 100644
--- a/client/starwhale/base/scheduler/__init__.py
+++ b/client/starwhale/base/scheduler/__init__.py
@@ -5,8 +5,7 @@
from pathlib import Path
from concurrent.futures import as_completed, ThreadPoolExecutor
-from loguru import logger
-
+from starwhale.utils import console
from starwhale.consts import RunStatus
from starwhale.base.context import Context
@@ -103,7 +102,7 @@ def _schedule_one_step(self, step_name: str, task_num: int = 0) -> StepResult:
task_num=task_num,
).execute()
- logger.info(
+ console.info(
f"step:{step_name}, result:{result}, run time:{time.time() - start_time}"
)
return result
@@ -132,7 +131,7 @@ def _schedule_one_task(self, step_name: str, task_index: int) -> StepResult:
start_time = time.time()
task_result: TaskResult = _task.execute()
- logger.info(
+ console.info(
f"step:{step_name}, task result:{task_result}, run time:{time.time() - start_time}"
)
return StepResult(name=step_name, task_results=[task_result])
diff --git a/client/starwhale/base/scheduler/step.py b/client/starwhale/base/scheduler/step.py
index f454f71c78..2dac89c02d 100644
--- a/client/starwhale/base/scheduler/step.py
+++ b/client/starwhale/base/scheduler/step.py
@@ -4,9 +4,7 @@
from pathlib import Path
from concurrent.futures import as_completed, ThreadPoolExecutor
-from loguru import logger
-
-from starwhale.utils import load_yaml
+from starwhale.utils import console, load_yaml
from starwhale.consts import RunStatus
from starwhale.base.mixin import ASDictMixin
from starwhale.base.context import Context
@@ -152,7 +150,7 @@ def __repr__(self) -> str:
return f"StepExecutor: step-{self.step}, version-{self.version}, dataset_uris:{self.dataset_uris}"
def execute(self) -> StepResult:
- logger.info(f"start to execute step:{self.step}")
+ console.info(f"start to execute step:{self.step}")
tasks = [
TaskExecutor(
@@ -176,5 +174,5 @@ def execute(self) -> StepResult:
future_tasks = [pool.submit(t.execute) for t in tasks]
task_results = [t.result() for t in as_completed(future_tasks)]
- logger.info(f"finish to execute step:{self.step}")
+ console.info(f"finish to execute step:{self.step}")
return StepResult(name=self.step.name, task_results=task_results)
diff --git a/client/starwhale/base/scheduler/task.py b/client/starwhale/base/scheduler/task.py
index 8ab74f67f0..2051cf884a 100644
--- a/client/starwhale/base/scheduler/task.py
+++ b/client/starwhale/base/scheduler/task.py
@@ -5,8 +5,7 @@
from pathlib import Path
from functools import wraps
-from loguru import logger
-
+from starwhale.utils import console
from starwhale.consts import RunStatus, DecoratorInjectAttr
from starwhale.utils.load import load_module
from starwhale.utils.error import NoSupportError
@@ -143,7 +142,7 @@ def _do_execute(self) -> None:
func()
def execute(self) -> TaskResult:
- logger.info(f"start to execute task with context({self.context}) ...")
+ console.info(f"start to execute task with context({self.context}) ...")
try:
loop = asyncio.get_event_loop()
except RuntimeError:
@@ -154,13 +153,13 @@ def execute(self) -> TaskResult:
Context.set_runtime_context(self.context)
self._do_execute()
except Exception as e:
- logger.exception(e)
+ console.print_exception()
self.exception = e
self.__status = RunStatus.FAILED
else:
self.__status = RunStatus.SUCCESS
finally:
- logger.info(
+ console.info(
f"finish {self.context}, status:{self.status}, error:{self.exception}"
)
loop.close()
diff --git a/client/starwhale/cli/__init__.py b/client/starwhale/cli/__init__.py
index df5ba6c2d0..b04563f9b1 100644
--- a/client/starwhale/cli/__init__.py
+++ b/client/starwhale/cli/__init__.py
@@ -25,6 +25,7 @@ def create_sw_cli() -> click.core.Group:
@click.group(cls=AliasedGroup)
@click.version_option(version=STARWHALE_VERSION, message="%(version)s")
@click.option(
+ "verbose_cnt",
"-v",
"--verbose",
count=True,
@@ -32,9 +33,9 @@ def create_sw_cli() -> click.core.Group:
)
@click.option("-o", "--output", help="Output format", type=click.Choice(["json"]))
@click.pass_context
- def cli(ctx: click.Context, verbose: bool, output: str) -> None:
+ def cli(ctx: click.Context, verbose_cnt: int, output: str) -> None:
load_swcli_config()
- init_logger(verbose)
+ init_logger(verbose_cnt)
ctx.ensure_object(dict)
ctx.obj["output"] = output
diff --git a/client/starwhale/cli/assistance/broker.py b/client/starwhale/cli/assistance/broker.py
index 7416492f79..b503d7a4bf 100644
--- a/client/starwhale/cli/assistance/broker.py
+++ b/client/starwhale/cli/assistance/broker.py
@@ -5,6 +5,8 @@
from fastapi import Body, FastAPI, Request, Response, HTTPException
from pydantic import BaseSettings
+from starwhale.utils import console
+
from .common import (
random_id,
ChunkBuffer,
@@ -105,13 +107,15 @@ def garbage_collect(self) -> None:
new_sessions = {}
for session_id, session in self.sessions.items():
if now - session.last_access_time > settings.gc_timeout_seconds:
- print(f"garbage collection: remove session {session_id}")
+ console.print(f"garbage collection: remove session {session_id}")
else:
new_sessions[session_id] = session
new_commands = {}
for command_id, command in session.commands.items():
if now - command.last_access_time > settings.gc_timeout_seconds:
- print(f"garbage collection: remove command {command_id}")
+ console.print(
+ f"garbage collection: remove command {command_id}"
+ )
else:
new_commands[command_id] = command
session.commands = new_commands
@@ -126,7 +130,7 @@ def start_garbage_collector(self) -> None:
settings.gc_interval_seconds,
)
self.garbage_collector.start()
- print("garbage collector started")
+ console.print("garbage collector started")
def stop_garbage_collector(self) -> None:
with global_lock:
@@ -143,7 +147,7 @@ def start_keep_alive_monitor(self) -> None:
settings.keep_alive_check_interval_seconds,
)
self.keep_alive_monitor.start()
- print("keep alive monitor started")
+ console.print("keep alive monitor started")
def stop_keep_alive_monitor(self) -> None:
with global_lock:
diff --git a/client/starwhale/cli/assistance/cli.py b/client/starwhale/cli/assistance/cli.py
index bcf30fe4de..88ca526b17 100644
--- a/client/starwhale/cli/assistance/cli.py
+++ b/client/starwhale/cli/assistance/cli.py
@@ -2,6 +2,7 @@
import click
+from starwhale.utils import console
from starwhale.utils.cli import AliasedGroup
from .host import CommandRetriever
@@ -23,7 +24,7 @@ def host(broker: str) -> None:
try:
retriever.join()
except KeyboardInterrupt:
- print("stopping...")
+ console.print("stopping...")
retriever.stop()
retriever.join()
@@ -43,6 +44,6 @@ def remote(broker: str, args: Tuple[str]) -> None:
try:
runner.join()
except KeyboardInterrupt:
- print("stopping...")
+ console.print("stopping...")
runner.stop()
runner.join()
diff --git a/client/starwhale/cli/assistance/common.py b/client/starwhale/cli/assistance/common.py
index 44c4e4c3a8..16c1be826a 100644
--- a/client/starwhale/cli/assistance/common.py
+++ b/client/starwhale/cli/assistance/common.py
@@ -9,6 +9,8 @@
import requests
+from starwhale.utils import console
+
def random_id() -> str:
return "".join(
@@ -55,9 +57,9 @@ def run_until_success(
except UnrecoverableError:
raise
except Exception:
- print("===Retry due to exception===")
+ console.print("===Retry due to exception===")
traceback.print_exc()
- print("======")
+ console.print("======")
time.sleep(backoff)
backoff *= 2
if backoff > 1:
@@ -206,9 +208,9 @@ def run(self) -> None:
count = self.chunk_buffer_writer.write(data, 0.5)
data = data[count:]
except Exception as e:
- print("===Stop read from file===")
+ console.print("===Stop read from file===")
traceback.print_exc()
- print("======")
+ console.print("======")
self.chunk_buffer_writer.close(str(e))
finally:
self.file.close()
@@ -239,9 +241,9 @@ def run(self) -> None:
count = os.write(fd, data)
data = data[count:]
except Exception as e:
- print("===Stop write to file===")
+ console.print("===Stop write to file===")
traceback.print_exc()
- print("======")
+ console.print("======")
self.chunk_buffer_reader.close(str(e))
finally:
self.file.close()
@@ -272,9 +274,9 @@ def run(self) -> None:
count = self.chunk_buffer_writer.write(data, 0.5)
data = data[count:]
except Exception as e:
- print("===Stop read from broker===")
+ console.print("===Stop read from broker===")
traceback.print_exc()
- print("======")
+ console.print("======")
self.chunk_buffer_writer.close(str(e))
finally:
self.run_until_success(self._close_read, True)
@@ -316,9 +318,9 @@ def run(self) -> None:
self.offset += count
d = d[count:]
except Exception as e:
- print("===Stop write to broker===")
+ console.print("===Stop write to broker===")
traceback.print_exc()
- print("======")
+ console.print("======")
self.chunk_buffer_reader.close(str(e))
finally:
self.run_until_success(self._close_write, True)
diff --git a/client/starwhale/cli/assistance/host.py b/client/starwhale/cli/assistance/host.py
index 1f35486da4..bc9834b774 100644
--- a/client/starwhale/cli/assistance/host.py
+++ b/client/starwhale/cli/assistance/host.py
@@ -4,6 +4,8 @@
import requests
+from starwhale.utils import console
+
from .common import (
FileReader,
FileWriter,
@@ -135,7 +137,7 @@ def run(self) -> None:
command_id = command["command_id"]
args = command["args"]
if command_id not in self.executions:
- print(f"run command {command_id}:", args)
+ console.print(f"run command {command_id}:", args)
executor = CommandExecutor(
f"command-{command_id}-executor",
f"{self.broker_url}/command/{command_id}",
diff --git a/client/starwhale/consts/__init__.py b/client/starwhale/consts/__init__.py
index 99d90ef8c5..d761f96494 100644
--- a/client/starwhale/consts/__init__.py
+++ b/client/starwhale/consts/__init__.py
@@ -92,6 +92,7 @@ class SupportOS:
FMT_DATETIME = "%Y-%m-%d %H:%M:%S %Z"
+MINI_FMT_DATETIME = "%H:%M:%S"
# TODO: use better DEFAULT words?
DEFAULT_COPY_WORKERS = 4
diff --git a/client/starwhale/core/dataset/copy.py b/client/starwhale/core/dataset/copy.py
index 245dae4e97..baea95056c 100644
--- a/client/starwhale/core/dataset/copy.py
+++ b/client/starwhale/core/dataset/copy.py
@@ -83,7 +83,7 @@ def do(self) -> None:
TimeElapsedColumn(),
TotalFileSizeColumn(),
TransferSpeedColumn(),
- console=console,
+ console=console.rich_console,
refresh_per_second=0.2,
) as progress:
src = TabularDataset(
diff --git a/client/starwhale/core/dataset/store.py b/client/starwhale/core/dataset/store.py
index 1d41fbabbd..389f234ea3 100644
--- a/client/starwhale/core/dataset/store.py
+++ b/client/starwhale/core/dataset/store.py
@@ -14,10 +14,10 @@
import boto3
import requests
-from loguru import logger
from botocore.client import Config as S3Config
from typing_extensions import Protocol
+from starwhale.utils import console
from starwhale.consts import (
HTTPMethod,
SWDSBackendType,
@@ -418,7 +418,7 @@ def get_store(
else:
_store = ObjectStore.from_dataset_uri(owner)
- logger.debug(f"new store backend created for key: {_k}")
+ console.debug(f"new store backend created for key: {_k}")
cls._stores[_k] = _store
return _store
diff --git a/client/starwhale/core/dataset/tabular.py b/client/starwhale/core/dataset/tabular.py
index 35f3434df3..7d893299c6 100644
--- a/client/starwhale/core/dataset/tabular.py
+++ b/client/starwhale/core/dataset/tabular.py
@@ -13,10 +13,9 @@
from collections import UserDict, defaultdict
import requests
-from loguru import logger
from typing_extensions import Protocol
-from starwhale.utils import validate_obj_name
+from starwhale.utils import console, validate_obj_name
from starwhale.consts import ENV_POD_NAME, STANDALONE_INSTANCE
from starwhale.base.uri import URI
from starwhale.base.type import URIType, InstanceType
@@ -113,7 +112,6 @@ def save_to_datastore(self, ds_wrapper: DatastoreWrapperDataset) -> str:
class TabularDatasetRow(ASDictMixin):
-
_FEATURES_PREFIX = "features/"
def __init__(
@@ -308,7 +306,7 @@ def __exit__(
trace: TracebackType,
) -> None:
if value: # pragma: no cover
- logger.warning(f"type:{type}, exception:{value}, traceback:{trace}")
+ console.warning(f"type:{type}, exception:{value}, traceback:{trace}")
self.close()
@@ -501,7 +499,7 @@ def get_scan_range(
if not self._todo_queue.empty():
task: StandaloneTDSC._BatchTask = self._todo_queue.get()
self._doing_consumption[consumer_id][f"{task.start}-{task.end}"] = task
- logger.info(
+ console.info(
f"{consumer_id} handle scan-range: ({task.start}, {task.end})"
)
return task.start, task.end
diff --git a/client/starwhale/core/job/view.py b/client/starwhale/core/job/view.py
index acfc19e4f1..73f586a8a3 100644
--- a/client/starwhale/core/job/view.py
+++ b/client/starwhale/core/job/view.py
@@ -2,7 +2,6 @@
import typing as t
from rich import box
-from loguru import logger
from rich.panel import Panel
from rich.table import Table
from rich.pretty import Pretty
@@ -31,7 +30,6 @@ def __init__(self, job_uri: str) -> None:
super().__init__()
self.raw_uri = job_uri
self.uri = URI(job_uri, expected_type=URIType.JOB)
- logger.debug(f"eval job:{self.raw_uri}")
self.job = Job.get_job(self.uri)
self._action_run_map = {
JobOperationType.CANCEL: self.job.cancel,
diff --git a/client/starwhale/core/model/model.py b/client/starwhale/core/model/model.py
index 39a7796004..269a2ec524 100644
--- a/client/starwhale/core/model/model.py
+++ b/client/starwhale/core/model/model.py
@@ -14,7 +14,6 @@
import yaml
from fs import open_fs
-from loguru import logger
from fs.walk import Walker
from starwhale.utils import (
@@ -354,16 +353,9 @@ def run(
exceptions.append(_tr.exception)
if exceptions:
raise Exception(*exceptions)
-
- logger.debug(
- f"-->[Finished] run[{version[:SHORT_VERSION_CNT]}] execute finished, results info:{results}"
- )
- except Exception as e:
+ except Exception:
scheduler_status = RunStatus.FAILED
- error_message = str(e)
- logger.error(
- f"-->[Failed] run[{version[:SHORT_VERSION_CNT]}] execute failed, error info:{e}"
- )
+ console.print_exception()
raise
finally:
_manifest: t.Dict[str, t.Any] = {
@@ -557,7 +549,6 @@ def list(
def buildImpl(self, workdir: Path, **kw: t.Any) -> None: # type: ignore[override]
model_config: ModelConfig = kw["model_config"]
- logger.debug(f"build workdir:{workdir}")
operations = [
(self._gen_version, 5, "gen version"),
@@ -693,7 +684,7 @@ def _make_meta_tar(
total_size += size
self._manifest["size"] = total_size
- console.print(f":basket: resource files size: {pretty_bytes(total_size)}")
+ console.debug(f":basket: resource files size: {pretty_bytes(total_size)}")
ensure_file(
self.store.resource_files_path,
@@ -738,8 +729,6 @@ def load_model_config(cls, yaml_path: Path, workdir: Path) -> ModelConfig:
return _config
def _prepare_snapshot(self) -> None:
- logger.info("[step:prepare-snapshot]prepare model snapshot dirs...")
-
ensure_dir(self.store.snapshot_workdir)
ensure_dir(self.store.src_dir)
@@ -760,6 +749,9 @@ def _copy_src(self, workdir: Path, model_config: ModelConfig) -> None:
if ignore.exists():
with open(ignore, "r") as f:
excludes = [line.strip() for line in f.readlines()]
+ console.debug(
+ f"copy dir: {workdir} -> {self.store.src_dir}, excludes: {excludes}"
+ )
self._object_store.copy_dir(
str(workdir.resolve()), str(self.store.src_dir.resolve()), excludes=excludes
)
@@ -774,8 +766,6 @@ def _copy_src(self, workdir: Path, model_config: ModelConfig) -> None:
self.store.src_dir / DefaultYAMLName.MODEL, model_yaml, parents=True
)
- logger.info("[step:copy]finish copy files")
-
@classmethod
def _load_config_envs(cls, _config: ModelConfig) -> None:
for _env in _config.run.envs:
diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py
index 07f80a2ab3..0e05912634 100644
--- a/client/starwhale/core/runtime/model.py
+++ b/client/starwhale/core/runtime/model.py
@@ -13,7 +13,6 @@
import yaml
import jinja2
from fs import open_fs
-from loguru import logger
from fs.copy import copy_fs, copy_file
from fs.tarfs import TarFS
from typing_extensions import Protocol
@@ -300,12 +299,12 @@ def _get_wheels(self, src_dir: Path) -> t.Generator[Path, None, None]:
def conda_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
for path in self._get_wheels(src_dir):
- logger.debug(f"conda run pip install: {path}")
+ console.debug(f"conda run pip install: {path}")
conda_install_req(req=path, prefix_path=env_dir, configs=configs)
def venv_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
for path in self._get_wheels(src_dir):
- logger.debug(f"venv pip install: {path}")
+ console.debug(f"venv pip install: {path}")
venv_install_req(
env_dir, path, pip_config=configs.get("pip")
) # type:ignore
@@ -332,7 +331,7 @@ def conda_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
_conda_pkgs = " ".join([repr(_p) for _p in self.deps if _p])
_conda_pkgs = _conda_pkgs.strip()
if _conda_pkgs:
- logger.debug(f"conda install: {_conda_pkgs}")
+ console.debug(f"conda install: {_conda_pkgs}")
conda_install_req(
req=_conda_pkgs,
prefix_path=env_dir,
@@ -341,7 +340,7 @@ def conda_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
)
def venv_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
- logger.warning("no support install conda pkg in the venv environment")
+ console.warning("no support install conda pkg in the venv environment")
class CondaEnvFileDependency(ASDictMixin, BaseDependency):
@@ -371,7 +370,7 @@ def conda_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
conda_env_update(env_fpath=env_fpath, target_env=env_dir)
def venv_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
- logger.warning(
+ console.warning(
"no support install/update conda environment file in the venv environment"
)
@@ -400,12 +399,12 @@ def _get_pkgs(self) -> t.Generator[str, None, None]:
def conda_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
# TODO: merge deps
for pkg in self._get_pkgs():
- logger.debug(f"conda run pip install: {pkg}")
+ console.debug(f"conda run pip install: {pkg}")
conda_install_req(req=pkg, prefix_path=env_dir, configs=configs)
def venv_install(self, src_dir: Path, env_dir: Path, configs: t.Dict) -> None:
for pkg in self._get_pkgs():
- logger.debug(f"venv pip install: {pkg}")
+ console.debug(f"venv pip install: {pkg}")
venv_install_req(env_dir, pkg, pip_config=configs.get("pip")) # type:ignore
@@ -496,7 +495,7 @@ def __init__(self, deps: t.Optional[t.List[str]] = None) -> None:
self._unparsed.append(d)
if self._unparsed:
- logger.warning(f"unparsed dependencies:{self._unparsed}")
+ console.warning(f"unparsed dependencies:{self._unparsed}")
def __str__(self) -> str:
return f"Starwhale Runtime Dependencies: {len(self.deps)}, unparsed: {len(self._unparsed)}"
@@ -1041,12 +1040,12 @@ def _copy_src(
RuntimeArtifactType.FILES: [],
}
- logger.info("[step:copy-wheels]start to copy wheels...")
+ console.info("[step:copy-wheels]start to copy wheels...")
ensure_dir(self.store.snapshot_workdir / RuntimeArtifactType.WHEELS)
for _fname in config.dependencies._wheels:
_fpath = workdir / _fname
if not _fpath.exists():
- logger.warning(f"not found wheel: {_fpath}")
+ console.warning(f"not found wheel: {_fpath}")
continue
_dest = f"{RuntimeArtifactType.WHEELS}/{_fname.lstrip('/')}"
@@ -1058,13 +1057,13 @@ def _copy_src(
_dest,
)
- logger.info("[step:copy-files]start to copy files...")
+ console.info("[step:copy-files]start to copy files...")
ensure_dir(self.store.snapshot_workdir / RuntimeArtifactType.FILES)
for _f in config.dependencies._files:
_src = workdir / _f["src"]
_dest = f"{RuntimeArtifactType.FILES}/{_f['src'].lstrip('/')}"
if not _src.exists():
- logger.warning(f"not found src-file: {_src}")
+ console.warning(f"not found src-file: {_src}")
continue
self._manifest["artifacts"][RuntimeArtifactType.FILES].append(_f)
@@ -1076,13 +1075,13 @@ def _copy_src(
ensure_dir((self.store.snapshot_workdir / _dest).parent)
copy_file(workdir_fs, _f["src"], snapshot_fs, _dest)
- logger.info("[step:copy-deps]start to copy pip/conda requirement files")
+ console.info("[step:copy-deps]start to copy pip/conda requirement files")
ensure_dir(self.store.snapshot_workdir / RuntimeArtifactType.DEPEND)
for _fname in config.dependencies._conda_files + config.dependencies._pip_files:
_fpath = workdir / _fname
if not _fpath.exists():
- logger.warning(f"not found dependencies: {_fpath}")
+ console.warning(f"not found dependencies: {_fpath}")
continue
_dest = f"{RuntimeArtifactType.DEPEND}/{_fname.lstrip('/')}"
@@ -1200,7 +1199,7 @@ def _dump_dependencies(
"conda_files": deps._conda_files,
}
- logger.info("[step:dep]finish dump dep")
+ console.info("[step:dep]finish dump dep")
if download_all_deps:
packaged = package_python_env(
@@ -1213,7 +1212,7 @@ def _dump_dependencies(
self._manifest["dependencies"]["local_packaged_env"] = packaged
def _prepare_snapshot(self) -> None:
- logger.info("[step:prepare-snapshot]prepare runtime snapshot dirs...")
+ console.info("[step:prepare-snapshot]prepare runtime snapshot dirs...")
# TODO: graceful clear?
if self.store.snapshot_workdir.exists():
@@ -1319,7 +1318,7 @@ def _copy_file(src: Path, dest: Path) -> None:
raise NotFoundError(src)
if dest.exists() and not force:
- logger.warning(f"{dest} existed, skip copy")
+ console.warning(f"{dest} existed, skip copy")
ensure_dir(dest.parent)
shutil.copy(str(src), str(dest))
@@ -1826,7 +1825,7 @@ def _install_dependencies_with_runtime_yaml(
deps_config = Dependencies(runtime_yaml.get("dependencies", []))
for dep in deps_config.deps:
if dep.kind in skip_deps:
- logger.debug(f"skip {dep} to install")
+ console.debug(f"skip {dep} to install")
continue
_func = (
diff --git a/client/starwhale/core/runtime/process.py b/client/starwhale/core/runtime/process.py
index ad9b991e64..e75d012614 100644
--- a/client/starwhale/core/runtime/process.py
+++ b/client/starwhale/core/runtime/process.py
@@ -5,6 +5,7 @@
import copy
import typing as t
from pathlib import Path
+from functools import partial
from starwhale.utils import console
from starwhale.consts import PythonRunEnv, DEFAULT_MANIFEST_NAME
@@ -91,7 +92,7 @@ def run(self) -> None:
self.EnvInActivatedProcess: "1",
self.ActivatedRuntimeURI: str(self._uri),
},
- log=print,
+ log=partial(console.print, without_timestamp=True),
)
def _restore_runtime(
diff --git a/client/starwhale/utils/__init__.py b/client/starwhale/utils/__init__.py
index 19a1cb06d4..84c7098ac5 100644
--- a/client/starwhale/utils/__init__.py
+++ b/client/starwhale/utils/__init__.py
@@ -9,24 +9,23 @@
import platform
from pathlib import Path
from datetime import datetime
-from functools import cmp_to_key
+from functools import partial, cmp_to_key
from contextlib import contextmanager
import yaml
-from rich.console import Console
from starwhale.consts import (
FMT_DATETIME,
+ MINI_FMT_DATETIME,
SW_DEV_DUMMY_VERSION,
ENV_DISABLE_PROGRESS_BAR,
)
from starwhale.version import STARWHALE_VERSION
from starwhale.utils.error import NoSupportError
-console = Console(soft_wrap=True)
-now_str: t.Callable[[], str] = (
- lambda: datetime.now().astimezone().strftime(FMT_DATETIME)
-)
+now: t.Callable[[str], str] = lambda x: datetime.now().astimezone().strftime(x)
+now_str: t.Callable[[], str] = partial(now, FMT_DATETIME)
+now_mini_str: t.Callable[[], str] = partial(now, MINI_FMT_DATETIME)
def timestamp_to_datatimestr(timestamp: float) -> str:
diff --git a/client/starwhale/utils/console.py b/client/starwhale/utils/console.py
new file mode 100644
index 0000000000..97c003a8b0
--- /dev/null
+++ b/client/starwhale/utils/console.py
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import typing as t
+import logging
+import threading
+
+from rich.console import Console
+
+from . import now
+
+rich_console = Console(soft_wrap=True)
+
+_min_level = logging.ERROR
+_min_level_lock = threading.Lock()
+
+
+def set_level(level: int) -> None:
+ global _min_level
+ with _min_level_lock:
+ if level not in _levels:
+ raise ValueError(f"invalid log level: {level}")
+ _min_level = level
+
+
+class _LevelInfo(t.NamedTuple):
+ number: int
+ name: str
+ color: str
+ emoji: str
+ style: str
+
+
+_levels: t.Dict[int, _LevelInfo] = {
+ logging.DEBUG: _LevelInfo(logging.DEBUG, "DEBUG", "blue", ":speaker:", "default"),
+ logging.INFO: _LevelInfo(logging.INFO, "INFO", "green", ":bulb:", "default"),
+ logging.WARN: _LevelInfo(
+ logging.WARN, "WARN", "yellow", ":question:", "bold magenta"
+ ),
+ logging.ERROR: _LevelInfo(logging.ERROR, "ERROR", "red", ":x:", "bold red"),
+ logging.FATAL: _LevelInfo(
+ logging.FATAL,
+ "FATAL",
+ "bold red",
+ ":x:",
+ "red on white",
+ ),
+}
+
+
+def print(*args: t.Any, **kwargs: t.Any) -> None:
+ extend_args = list(args or [])
+ global _min_level
+ without_timestamp = kwargs.pop("without_timestamp", False)
+ if _min_level <= logging.DEBUG and not without_timestamp:
+ extend_args.insert(0, f"[{_get_datetime_str()}]")
+
+ rich_console.print(*extend_args, **kwargs)
+
+
+def print_exception(*args: t.Any, **kwargs: t.Any) -> None:
+ rich_console.print_exception(*args, **kwargs)
+
+
+def debug(*args: t.Any) -> None:
+ _log(logging.DEBUG, *args)
+
+
+def info(*args: t.Any) -> None:
+ _log(logging.INFO, *args)
+
+
+def warn(*args: t.Any) -> None:
+ _log(logging.WARN, *args)
+
+
+warning = warn
+
+
+def error(*args: t.Any) -> None:
+ _log(logging.ERROR, *args)
+
+
+def exception(*args: t.Any) -> None:
+ _log(logging.ERROR, *args)
+ print_exception()
+
+
+def fatal(*args: t.Any) -> None:
+ _log(logging.FATAL, *args)
+
+
+def log(*args: t.Any) -> None:
+ _log(logging.INFO, *args)
+
+
+def _log(level: int, *args: t.Any) -> None:
+ level_info = _levels.get(level) or _levels[logging.INFO]
+
+ global _min_level
+ if level_info.number < _min_level:
+ return
+
+ datetime_str = _get_datetime_str()
+ rich_console.print(
+ f"[{datetime_str}] {level_info.emoji} [{level_info.color}]|{level_info.name}|[/]",
+ *args,
+ style=level_info.style,
+ )
+
+
+def rule(*args: t.Any, **kwargs: t.Any) -> None:
+ rich_console.rule(*args, **kwargs)
+
+
+def _get_datetime_str() -> str:
+ global _min_level
+ if _min_level <= logging.DEBUG:
+ return now("%Y-%m-%d %H:%M:%S.%f")
+ else:
+ return now("%H:%M:%S")
diff --git a/client/starwhale/utils/debug.py b/client/starwhale/utils/debug.py
index 9c5e7ded9e..d03c02378f 100644
--- a/client/starwhale/utils/debug.py
+++ b/client/starwhale/utils/debug.py
@@ -1,37 +1,34 @@
import os
-import sys
import logging
from rich import traceback
-from loguru import logger
-from starwhale.consts import ENV_LOG_LEVEL, ENV_LOG_VERBOSE_COUNT
+from starwhale.utils import console
+from starwhale.consts import (
+ ENV_LOG_LEVEL,
+ ENV_LOG_VERBOSE_COUNT,
+ ENV_DISABLE_PROGRESS_BAR,
+)
def init_logger(verbose: int) -> None:
- fmt = "{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | {message}"
- if verbose <= 0:
- lvl = logging.WARNING
+ if verbose == 0:
+ lvl = logging.ERROR
elif verbose == 1:
+ lvl = logging.WARNING
+ elif verbose == 2:
lvl = logging.INFO
else:
lvl = logging.DEBUG
+ console.set_level(lvl)
lvl_name = logging.getLevelName(lvl)
os.environ[ENV_LOG_LEVEL] = lvl_name
os.environ[ENV_LOG_VERBOSE_COUNT] = str(verbose)
+ if verbose > 0:
+ os.environ[ENV_DISABLE_PROGRESS_BAR] = "1"
+ console.print(f":space_invader: verbosity: {verbose}, log level: {lvl_name}")
+
# TODO: custom debug for tb install
traceback.install(show_locals=True, max_frames=1, width=200)
-
- logger.remove()
- logger.add(
- sys.stderr,
- level=lvl_name,
- colorize=True,
- backtrace=True,
- diagnose=True,
- catch=True,
- format=fmt,
- )
- logger.debug(f"verbosity: {verbose}, log level: {lvl_name}")
diff --git a/client/starwhale/utils/http.py b/client/starwhale/utils/http.py
index 96a596bfad..8811ac6a2d 100644
--- a/client/starwhale/utils/http.py
+++ b/client/starwhale/utils/http.py
@@ -4,10 +4,10 @@
from functools import wraps
import requests
-from rich import print as rprint
-from loguru import logger
from rich.panel import Panel
+from starwhale.utils import console
+
def wrap_sw_error_resp(
r: requests.Response,
@@ -17,14 +17,13 @@ def wrap_sw_error_resp(
silent: bool = False,
ignore_status_codes: t.List[int] = [],
) -> None:
-
if silent:
- _rprint: t.Callable = lambda x: x
+ _print: t.Callable = lambda x: x
else:
- _rprint = rprint
+ _print = console.print
if r.status_code == HTTPStatus.OK:
- _rprint(f":clap: {header} success")
+ _print(f":clap: {header} success")
return
msg = f":disappointed_face: url:{r.url}\n:dog: http status code: {r.status_code} \n"
@@ -40,7 +39,7 @@ def wrap_sw_error_resp(
if r.status_code in ignore_status_codes:
return
- rprint(Panel.fit(msg, title=":space_invader: error details")) # type: ignore
+ _print(Panel.fit(msg, title=":space_invader: error details")) # type: ignore
if exit:
sys.exit(1)
@@ -55,7 +54,7 @@ def _wrapper(*args: t.Any, **kwargs: t.Any) -> t.Any:
try:
return func(*args, **kwargs)
except Exception as e:
- logger.warning(f"{func} error: {e}")
+ console.warning(f"{func} error: {e}")
return default_ret
return _wrapper
diff --git a/client/starwhale/utils/load.py b/client/starwhale/utils/load.py
index 97f5836b01..d4f4ed1622 100644
--- a/client/starwhale/utils/load.py
+++ b/client/starwhale/utils/load.py
@@ -4,7 +4,6 @@
from pathlib import Path
import pkg_resources
-from loguru import logger
from starwhale.utils import console
from starwhale.utils.venv import (
@@ -32,21 +31,22 @@ def import_object(
for _path in external_paths[::-1]:
if _path not in sys.path:
- logger.debug(f"insert sys.path: '{_path}'")
sys.path.insert(0, _path)
pkg_resources.working_set.add_entry(_path)
sys_changed = True
try:
module_name, handler_name = handler_path.split(":", 1)
- logger.debug(f"import module:{module_name}, handler:{handler_name}")
+ console.print(
+ f":speaking_head: [green]import module:{module_name}, handler:{handler_name}[/]"
+ )
_module = importlib.import_module(module_name, package=workdir_path)
_obj = getattr(_module, handler_name, None)
if not _obj:
raise ModuleNotFoundError(f"{handler_path}")
- except Exception as e:
- logger.exception(e)
+ except Exception:
+ console.print_exception()
if sys_changed:
sys.path[:] = prev_paths
raise
@@ -60,7 +60,6 @@ def load_module(module: str, path: Path) -> t.Any:
external_paths = [workdir_path]
for _path in external_paths[::-1]:
if _path not in sys.path:
- logger.debug(f"insert sys.path: '{_path}'")
sys.path.insert(0, _path)
pkg_resources.working_set.add_entry(_path)
diff --git a/client/starwhale/utils/process.py b/client/starwhale/utils/process.py
index e29733e1ee..912d62a0f2 100644
--- a/client/starwhale/utils/process.py
+++ b/client/starwhale/utils/process.py
@@ -2,11 +2,11 @@
import typing as t
from subprocess import PIPE, Popen, STDOUT, CalledProcessError
-from loguru import logger
+from starwhale.utils import console
def log_check_call(*args: t.Any, **kwargs: t.Any) -> int:
- log = kwargs.pop("log", logger.debug)
+ log = kwargs.pop("log", console.debug)
kwargs["bufsize"] = 1
kwargs["stdout"] = PIPE
kwargs["stderr"] = STDOUT
@@ -18,7 +18,7 @@ def log_check_call(*args: t.Any, **kwargs: t.Any) -> int:
output = []
p = Popen(*args, **kwargs)
- logger.debug(f"cmd: {p.args!r}")
+ console.debug(f"cmd: {p.args!r}")
while True:
line = p.stdout.readline() # type: ignore
diff --git a/client/starwhale/utils/progress.py b/client/starwhale/utils/progress.py
index 0e52946540..cdc98af269 100644
--- a/client/starwhale/utils/progress.py
+++ b/client/starwhale/utils/progress.py
@@ -3,7 +3,6 @@
from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
-from starwhale.utils import console
from starwhale.consts import ENV_DISABLE_PROGRESS_BAR
@@ -12,7 +11,7 @@ def run_with_progress_bar(
operations: t.Sequence[t.Tuple[t.Any, ...]],
**kw: t.Any,
) -> None:
- if os.environ.get(ENV_DISABLE_PROGRESS_BAR) == "0":
+ if os.environ.get(ENV_DISABLE_PROGRESS_BAR) == "1":
for op in operations:
if len(op) == 4:
op[0](**op[3])
@@ -23,7 +22,6 @@ def run_with_progress_bar(
SpinnerColumn(),
*Progress.get_default_columns(),
TimeElapsedColumn(),
- console=console,
refresh_per_second=1,
) as progress:
task = progress.add_task(
diff --git a/client/starwhale/utils/venv.py b/client/starwhale/utils/venv.py
index 44d879445e..90b55e550a 100644
--- a/client/starwhale/utils/venv.py
+++ b/client/starwhale/utils/venv.py
@@ -8,7 +8,6 @@
import conda_pack
import virtualenv
-from loguru import logger
from starwhale.utils import console, is_linux, venv_pack, get_downloadable_sw_version
from starwhale.consts import (
@@ -257,7 +256,7 @@ def pip_freeze_by_pybin(
include_local_wheel: bool = False,
) -> None:
lock_fpath = Path(lock_fpath)
- logger.info(f"{py_bin}: pip freeze...")
+ console.info(f"{py_bin}: pip freeze...")
content = [f"# Generated by Starwhale({STARWHALE_VERSION}) Runtime Lock"]
@@ -329,7 +328,7 @@ def check_python_interpreter_consistency(mode: str) -> t.Tuple[bool, str, str]:
or os.environ.get(ENV_CONDA_PREFIX)
or sys.base_prefix
)
- logger.debug(
+ console.debug(
f"current python interpreter base_prefix:{sys.base_prefix}, expected env base_prefix:{ep_base_prefix}"
)
_ok = ep_base_prefix == sys.base_prefix
@@ -337,7 +336,7 @@ def check_python_interpreter_consistency(mode: str) -> t.Tuple[bool, str, str]:
cur_version = get_python_version()
user_version = get_user_python_version(mode)
if not user_version.startswith(cur_version):
- logger.error(
+ console.error(
f"swcli use python:{cur_version}, but runtime venv/conda python:{user_version}"
)
raise PythonEnvironmentError(
@@ -357,9 +356,9 @@ def guess_current_py_env() -> str:
def get_user_python_sys_paths(py_env: str) -> t.List[str]:
- logger.debug(f"get env({py_env}) sys path")
+ console.debug(f"get env({py_env}) sys path")
_py_bin = get_user_runtime_python_bin(py_env)
- logger.info(f"{_py_bin}: sys.path")
+ console.info(f"{_py_bin}: sys.path")
output = subprocess.check_output(
[
_py_bin,
@@ -530,7 +529,7 @@ def package_python_env(
sys_name = platform.system()
if not is_linux():
- logger.warning(
+ console.warning(
f"[info:dep]{sys_name} will skip conda/venv to generate local all bundles env"
)
return False
@@ -636,10 +635,10 @@ def create_python_env(
if isolated_env_dir.exists() and not force:
raise ExistedError(str(isolated_env_dir))
- logger.info(f"create venv @ {isolated_env_dir}...")
+ console.info(f"create venv @ {isolated_env_dir}...")
venv_setup(isolated_env_dir, python_version=python_version, prompt=name)
elif mode == PythonRunEnv.CONDA:
- logger.info(
+ console.info(
f"create conda {name}:{isolated_env_dir}, use python {python_version}..."
)
conda_setup(python_version, prefix=isolated_env_dir)
@@ -652,7 +651,7 @@ def get_python_version() -> str:
def get_python_version_by_bin(py_bin: str) -> str:
- logger.info(f"{py_bin}: python version")
+ console.info(f"{py_bin}: python version")
output = subprocess.check_output(
[
py_bin,
@@ -795,7 +794,7 @@ def install_starwhale(
str(prefix_path / "bin" / "python3"), SW_PYPI_PKG_NAME
)
if _existed and not force:
- logger.info(f"{SW_PYPI_PKG_NAME} has already be installed at {prefix_path}")
+ console.info(f"{SW_PYPI_PKG_NAME} has already be installed at {prefix_path}")
return
configs = configs or {}
diff --git a/client/tests/utils/test_common.py b/client/tests/utils/test_common.py
index 40817e40f8..87cb9a1222 100644
--- a/client/tests/utils/test_common.py
+++ b/client/tests/utils/test_common.py
@@ -19,7 +19,7 @@
pretty_merge_list,
validate_obj_name,
)
-from starwhale.consts import HTTPMethod, ENV_LOG_LEVEL
+from starwhale.consts import HTTPMethod, ENV_LOG_LEVEL, ENV_DISABLE_PROGRESS_BAR
from starwhale.utils.debug import init_logger
from starwhale.utils.retry import http_retry
@@ -37,9 +37,15 @@ def test_valid_object_name() -> None:
assert validate_obj_name("v1-alpha1")[0]
+@patch("os.environ", {})
def test_logger() -> None:
init_logger(0)
+ assert os.environ[ENV_LOG_LEVEL] == "ERROR"
+ assert os.environ.get(ENV_DISABLE_PROGRESS_BAR, "0") == "0"
+
+ init_logger(1)
assert os.environ[ENV_LOG_LEVEL] == "WARNING"
+ assert os.environ.get(ENV_DISABLE_PROGRESS_BAR, "0") == "1"
init_logger(3)
assert os.environ[ENV_LOG_LEVEL] == "DEBUG"
diff --git a/example/mnist/mnist/custom_evaluator.py b/example/mnist/mnist/custom_evaluator.py
index f3a09a9fca..5b677a8ea1 100644
--- a/example/mnist/mnist/custom_evaluator.py
+++ b/example/mnist/mnist/custom_evaluator.py
@@ -8,7 +8,6 @@
import torch
import gradio
from PIL import Image as PILImage
-from loguru import logger
from torchvision import transforms
from starwhale import (
@@ -45,29 +44,23 @@ def run_ppl(self, context: Context) -> None:
ds = dataset(_uri)
ds.make_distributed_consumption(session_id=context.version)
for rows in ds.batch_iter(self.batch_size):
- try:
- pred_values, probability_matrixs = self.batch_ppl(
- [r[1] for r in rows]
+ pred_values, probability_matrixs = self.batch_ppl([r[1] for r in rows])
+ for (
+ (_idx, _data),
+ pred_value,
+ probability_matrix,
+ ) in zip(rows, pred_values, probability_matrixs):
+ _unique_id = f"{_uri.object}_{_idx}"
+
+ evaluation.log(
+ category="results",
+ id=_unique_id,
+ metrics=dict(
+ pred_value=dill.dumps(pred_value),
+ probability_matrix=dill.dumps(probability_matrix),
+ label=_data["label"],
+ ),
)
- for (
- (_idx, _data),
- pred_value,
- probability_matrix,
- ) in zip(rows, pred_values, probability_matrixs):
- _unique_id = f"{_uri.object}_{_idx}"
-
- evaluation.log(
- category="results",
- id=_unique_id,
- metrics=dict(
- pred_value=dill.dumps(pred_value),
- probability_matrix=dill.dumps(probability_matrix),
- label=_data["label"],
- ),
- )
- except Exception:
- logger.error(f"[{[r[0] for r in rows]}] data handle -> failed")
- raise
@handler(needs=[run_ppl], name="cmp")
@multi_classification(
diff --git a/example/ucf101/ucf101/custom_evaluator.py b/example/ucf101/ucf101/custom_evaluator.py
index 1c5d0f6d1b..12d983560a 100644
--- a/example/ucf101/ucf101/custom_evaluator.py
+++ b/example/ucf101/ucf101/custom_evaluator.py
@@ -9,7 +9,6 @@
import dill
import numpy as np
import torch
-from loguru import logger
from starwhale import (
URI,
@@ -145,19 +144,15 @@ def run_ppl(context: Context) -> None:
rows, pred_values, probability_matrixs
):
_unique_id = f"{_uri.object}_{_idx}"
- try:
- evaluation.log(
- category="results",
- id=_unique_id,
- metrics=dict(
- pred_value=dill.dumps(pred_value),
- probability_matrix=dill.dumps(probability_matrix),
- annotations=_annotations,
- ),
- )
- except Exception:
- logger.error(f"[{_unique_id}] data handle -> failed")
- raise
+ evaluation.log(
+ category="results",
+ id=_unique_id,
+ metrics=dict(
+ pred_value=dill.dumps(pred_value),
+ probability_matrix=dill.dumps(probability_matrix),
+ annotations=_annotations,
+ ),
+ )
@handler(needs=[run_ppl])
diff --git a/scripts/client_test/cmds/base/invoke.py b/scripts/client_test/cmds/base/invoke.py
index 03bab13b62..75718d4989 100644
--- a/scripts/client_test/cmds/base/invoke.py
+++ b/scripts/client_test/cmds/base/invoke.py
@@ -2,7 +2,7 @@
import subprocess
from typing import Dict, List, Tuple, Optional
-from loguru import logger
+from starwhale.utils import console
def invoke_with_react(args: List[str], input_content: str = "yes") -> Tuple[int, str]:
@@ -15,7 +15,7 @@ def invoke_with_react(args: List[str], input_content: str = "yes") -> Tuple[int,
)
_stdout, _err = p.communicate(input=input_content)
if _err:
- logger.warning(f"args:{args}, error is:{_err}")
+ console.warning(f"args:{args}, error is:{_err}")
return p.returncode, _stdout
@@ -36,14 +36,14 @@ def invoke(
universal_newlines=True,
)
- logger.debug(f"cmd: {p.args!r}, env: {external_env}")
+ console.debug(f"cmd: {p.args!r}, env: {external_env}")
output = []
while True:
line = p.stdout.readline() # type: ignore
if line:
if log:
- logger.debug(line)
+ console.debug(line)
output.append(line)
if p.poll() is not None:
@@ -53,13 +53,13 @@ def invoke(
for line in p.stdout.readlines(): # type: ignore
if line:
if log:
- logger.debug(line)
+ console.debug(line)
output.append(line)
try:
p.stdout.close() # type: ignore
except Exception as ex:
- logger.error(f"failed to close stdout:{ex}")
+ console.error(f"failed to close stdout:{ex}")
if raise_err and p.returncode != 0:
cmd = args[0]