diff --git a/client/starwhale/base/type.py b/client/starwhale/base/type.py index cd0de039b5..c647c38f60 100644 --- a/client/starwhale/base/type.py +++ b/client/starwhale/base/type.py @@ -40,6 +40,7 @@ class RuntimeArtifactType: DEPEND = "dependencies" WHEELS = "wheels" FILES = "files" + CONFIGS = "configs" class RuntimeLockFileType: diff --git a/client/starwhale/core/runtime/cli.py b/client/starwhale/core/runtime/cli.py index 58afb5826d..31ed959cb1 100644 --- a/client/starwhale/core/runtime/cli.py +++ b/client/starwhale/core/runtime/cli.py @@ -248,6 +248,18 @@ def _quickstart( help="Include editable packages", hidden=True, ) +@optgroup.option( # type: ignore[no-untyped-call] + "-epo", + "--emit-pip-options", + is_flag=True, + help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}", +) +@optgroup.option( # type: ignore[no-untyped-call] + "-ecc", + "--emit-condarc", + is_flag=True, + help="Emit to dump ~/.condarc from the local machine, default is False", +) @optgroup.option( # type: ignore[no-untyped-call] "-ilw", "--include-local-wheel", @@ -280,6 +292,8 @@ def _build( shell: bool, yaml: str, docker: str, + emit_pip_options: bool, + emit_condarc: bool, ) -> None: """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs. Runtime build only works in the Standalone instance. @@ -346,6 +360,8 @@ def _build( download_all_deps=download_all_deps, include_editable=include_editable, include_local_wheel=include_local_wheel, + emit_condarc=emit_condarc, + emit_pip_options=emit_pip_options, ) else: RuntimeTermView.build_from_runtime_yaml( @@ -358,6 +374,8 @@ def _build( include_local_wheel=include_local_wheel, no_cache=no_cache, disable_env_lock=disable_env_lock, + emit_condarc=emit_condarc, + emit_pip_options=emit_pip_options, ) diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py index 743f763666..b722532aff 100644 --- a/client/starwhale/core/runtime/model.py +++ b/client/starwhale/core/runtime/model.py @@ -575,8 +575,14 @@ def __init__( class CondaConfig(ASDictMixin): - def __init__(self, channels: t.Optional[t.List[str]] = None, **kw: t.Any) -> None: + def __init__( + self, + channels: t.Optional[t.List[str]] = None, + condarc: t.Optional[t.Dict] = None, + **kw: t.Any, + ) -> None: self.channels = channels or [DEFAULT_CONDA_CHANNEL] + self.condarc = condarc or {} class Configs(ASDictMixin): @@ -746,6 +752,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, + emit_pip_options: bool = False, + emit_condarc: bool = False, ) -> None: raise NotImplementedError @@ -761,6 +769,8 @@ def build_from_runtime_yaml( env_prefix_path: str = "", env_name: str = "", env_use_shell: bool = False, + emit_pip_options: bool = False, + emit_condarc: bool = False, ) -> None: raise NotImplementedError @@ -897,6 +907,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, + emit_pip_options: bool = False, + emit_condarc: bool = False, ) -> None: if conda_name or conda_prefix: prefix_path = ( @@ -951,6 +963,8 @@ def build_from_python_env( env_name=conda_name, env_prefix_path=conda_prefix or venv_prefix, env_use_shell=env_use_shell, + emit_pip_options=emit_pip_options, + emit_condarc=emit_condarc, ) finally: empty_dir(workdir) @@ -967,6 +981,8 @@ def build_from_runtime_yaml( env_prefix_path: str = "", env_name: str = "", env_use_shell: bool = False, + emit_pip_options: bool = False, + emit_condarc: bool = False, ) -> None: workdir = Path(workdir) yaml_path = Path(yaml_path) @@ -987,6 +1003,7 @@ def build_from_runtime_yaml( env_name=env_name, env_prefix_path=env_prefix_path, env_use_shell=env_use_shell, + emit_pip_options=emit_pip_options, ) # try getting starwhale version for line in content.splitlines(): @@ -1001,10 +1018,10 @@ def build_from_runtime_yaml( (self._gen_version, 5, "gen version"), (self._prepare_snapshot, 5, "prepare snapshot"), ( - self._dump_context, + self._dump_configs, 5, - "dump environment and configs", - dict(config=swrt_config), + "dump configs", + dict(config=swrt_config, emit_condarc=emit_condarc), ), ( self._lock_environment, @@ -1057,10 +1074,35 @@ def build_from_runtime_yaml( ] run_with_progress_bar("runtime bundle building...", operations) - def _dump_context(self, config: RuntimeConfig) -> None: - # TODO: refactor docker image in environment + @staticmethod + def _prepare_configs(workdir: Path, mode: str) -> None: + if mode == PythonRunEnv.CONDA: + condarc_path = workdir / RuntimeArtifactType.CONFIGS / "condarc" + if condarc_path.exists(): + # CONDARC env only works in the current process and subprocess by the starwhale.utils.process.check_call function + os.environ["CONDARC"] = str(condarc_path) + + def _dump_configs(self, config: RuntimeConfig, emit_condarc: bool = False) -> None: self._manifest["configs"] = config.configs.asdict() + condarc_path = Path.home() / ".condarc" + if ( + config.mode == PythonRunEnv.CONDA + and not emit_condarc + and condarc_path.exists() + ): + local_condarc = load_yaml(condarc_path) + local_condarc.update(config.configs.conda.condarc) + self._manifest["configs"]["conda"]["condarc"] = local_condarc + + condarc = self._manifest["configs"]["conda"]["condarc"] + if condarc: + ensure_file( + self.store.snapshot_workdir / RuntimeArtifactType.CONFIGS / "condarc", + yaml.safe_dump(condarc, default_flow_style=False), + parents=True, + ) + def _copy_src( self, config: RuntimeConfig, @@ -1795,6 +1837,12 @@ def restore( else: operations.extend( [ + ( + cls._prepare_configs, + 5, + "prepare configs", + dict(workdir=workdir, mode=_env["mode"]), + ), ( cls._setup_python_env, 20, diff --git a/client/starwhale/core/runtime/view.py b/client/starwhale/core/runtime/view.py index e34b93d300..3ae06010d5 100644 --- a/client/starwhale/core/runtime/view.py +++ b/client/starwhale/core/runtime/view.py @@ -171,6 +171,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, + emit_condarc: bool = False, + emit_pip_options: bool = False, ) -> Resource: set_args = list(filter(bool, (conda_name, conda_prefix, venv_prefix))) if len(set_args) >= 2: @@ -212,6 +214,8 @@ def build_from_python_env( download_all_deps=download_all_deps, include_editable=include_editable, include_local_wheel=include_local_wheel, + emit_condarc=emit_condarc, + emit_pip_options=emit_pip_options, ) return runtime_uri @@ -228,6 +232,8 @@ def build_from_runtime_yaml( include_local_wheel: bool = False, no_cache: bool = False, disable_env_lock: bool = False, + emit_pip_options: bool = False, + emit_condarc: bool = False, ) -> Resource: workdir = Path(workdir) yaml_path = Path(yaml_path) @@ -258,6 +264,8 @@ def build_from_runtime_yaml( include_local_wheel=include_local_wheel, no_cache=no_cache, disable_env_lock=disable_env_lock, + emit_condarc=emit_condarc, + emit_pip_options=emit_pip_options, ) return _runtime_uri diff --git a/client/tests/core/test_runtime.py b/client/tests/core/test_runtime.py index 3dabc2a997..2c4cb549de 100644 --- a/client/tests/core/test_runtime.py +++ b/client/tests/core/test_runtime.py @@ -820,7 +820,7 @@ def test_build_from_runtime_yaml_in_venv_mode( _manifest = load_yaml(os.path.join(runtime_workdir, DEFAULT_MANIFEST_NAME)) assert _manifest["configs"] == { - "conda": {"channels": ["conda-forge"]}, + "conda": {"channels": ["conda-forge"], "condarc": {}}, "docker": {"image": ""}, "pip": {"extra_index_url": [""], "index_url": "", "trusted_host": [""]}, } @@ -1263,7 +1263,7 @@ def test_lock_ret_val(self, m_mkstemp: MagicMock, m_check: MagicMock, *args: t.A @patch("starwhale.base.bundle.LocalStorageBundleMixin._gen_version") @patch("starwhale.core.runtime.model.StandaloneRuntime._prepare_snapshot") - @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_context") + @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_configs") @patch("starwhale.utils.venv.check_call") @patch("starwhale.utils.venv.subprocess.check_output") @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_dependencies") diff --git a/example/LLM/belle-bloom/runtime_conda.yaml b/example/LLM/belle-bloom/runtime_conda.yaml index 230bb26e84..37edf97dda 100644 --- a/example/LLM/belle-bloom/runtime_conda.yaml +++ b/example/LLM/belle-bloom/runtime_conda.yaml @@ -10,6 +10,22 @@ environment: configs: pip: index_url: https://mirrors.aliyun.com/pypi/simple + conda: + condarc: # custom condarc config file + channels: + - defaults + show_channel_urls: true + default_channels: + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r + - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2 + custom_channels: + conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud + pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud + pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud + nvidia: https://mirrors.aliyun.com/anaconda/cloud + ssl_verify: false + default_threads: 10 dependencies: - pip: # pip dependencies according to the following requirements: diff --git a/scripts/example/runtime_conda.yaml b/scripts/example/runtime_conda.yaml index 3412eb1f7e..dba235d388 100644 --- a/scripts/example/runtime_conda.yaml +++ b/scripts/example/runtime_conda.yaml @@ -11,3 +11,8 @@ environment: os: ubuntu:20.04 mode: conda name: simple-test-conda +config: + conda: + condarc: + ssl_verify: false + default_threads: 4