support condarc for runtime yaml

client/starwhale/base/type.py

@@ -40,6 +40,7 @@ class RuntimeArtifactType:
     DEPEND = "dependencies"
     WHEELS = "wheels"
     FILES = "files"
+    CONFIGS = "configs"
 
 
 class RuntimeLockFileType:

client/starwhale/core/runtime/cli.py

@@ -248,6 +248,18 @@ def _quickstart(
     help="Include editable packages",
     hidden=True,
 )
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "-epo",
+    "--emit-pip-options",
+    is_flag=True,
+    help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}",
+)
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "-ecc",
+    "--emit-condarc",
+    is_flag=True,
+    help="Emit to dump ~/.condarc from the local machine, default is False",
+)
 @optgroup.option(  # type: ignore[no-untyped-call]
     "-ilw",
     "--include-local-wheel",
@@ -280,6 +292,8 @@ def _build(
     shell: bool,
     yaml: str,
     docker: str,
+    emit_pip_options: bool,
+    emit_condarc: bool,
 ) -> None:
     """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs.
     Runtime build only works in the Standalone instance.
@@ -346,6 +360,8 @@ def _build(
             download_all_deps=download_all_deps,
             include_editable=include_editable,
             include_local_wheel=include_local_wheel,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
     else:
         RuntimeTermView.build_from_runtime_yaml(
@@ -358,6 +374,8 @@ def _build(
             include_local_wheel=include_local_wheel,
             no_cache=no_cache,
             disable_env_lock=disable_env_lock,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
 
 

client/starwhale/core/runtime/model.py

@@ -575,8 +575,14 @@ def __init__(
 
 
 class CondaConfig(ASDictMixin):
-    def __init__(self, channels: t.Optional[t.List[str]] = None, **kw: t.Any) -> None:
+    def __init__(
+        self,
+        channels: t.Optional[t.List[str]] = None,
+        condarc: t.Optional[t.Dict] = None,
+        **kw: t.Any,
+    ) -> None:
         self.channels = channels or [DEFAULT_CONDA_CHANNEL]
+        self.condarc = condarc or {}
 
 
 class Configs(ASDictMixin):
@@ -746,6 +752,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         raise NotImplementedError
 
@@ -761,6 +769,8 @@ def build_from_runtime_yaml(
         env_prefix_path: str = "",
         env_name: str = "",
         env_use_shell: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         raise NotImplementedError
 
@@ -897,6 +907,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         if conda_name or conda_prefix:
             prefix_path = (
@@ -951,6 +963,8 @@ def build_from_python_env(
                 env_name=conda_name,
                 env_prefix_path=conda_prefix or venv_prefix,
                 env_use_shell=env_use_shell,
+                emit_pip_options=emit_pip_options,
+                emit_condarc=emit_condarc,
             )
         finally:
             empty_dir(workdir)
@@ -967,6 +981,8 @@ def build_from_runtime_yaml(
         env_prefix_path: str = "",
         env_name: str = "",
         env_use_shell: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         workdir = Path(workdir)
         yaml_path = Path(yaml_path)
@@ -987,6 +1003,7 @@ def build_from_runtime_yaml(
                 env_name=env_name,
                 env_prefix_path=env_prefix_path,
                 env_use_shell=env_use_shell,
+                emit_pip_options=emit_pip_options,
             )
             # try getting starwhale version
             for line in content.splitlines():
@@ -1001,10 +1018,10 @@ def build_from_runtime_yaml(
             (self._gen_version, 5, "gen version"),
             (self._prepare_snapshot, 5, "prepare snapshot"),
             (
-                self._dump_context,
+                self._dump_configs,
                 5,
-                "dump environment and configs",
-                dict(config=swrt_config),
+                "dump configs",
+                dict(config=swrt_config, emit_condarc=emit_condarc),
             ),
             (
                 self._lock_environment,
@@ -1057,10 +1074,35 @@ def build_from_runtime_yaml(
         ]
         run_with_progress_bar("runtime bundle building...", operations)
 
-    def _dump_context(self, config: RuntimeConfig) -> None:
-        # TODO: refactor docker image in environment
+    @staticmethod
+    def _prepare_configs(workdir: Path, mode: str) -> None:
+        if mode == PythonRunEnv.CONDA:
+            condarc_path = workdir / RuntimeArtifactType.CONFIGS / "condarc"
+            if condarc_path.exists():
+                # CONDARC env only works in the current process and subprocess by the starwhale.utils.process.check_call function
+                os.environ["CONDARC"] = str(condarc_path)
+
+    def _dump_configs(self, config: RuntimeConfig, emit_condarc: bool = False) -> None:
         self._manifest["configs"] = config.configs.asdict()
 
+        condarc_path = Path.home() / ".condarc"
+        if (
+            config.mode == PythonRunEnv.CONDA
+            and not emit_condarc
+            and condarc_path.exists()
+        ):
+            local_condarc = load_yaml(condarc_path)
+            local_condarc.update(config.configs.conda.condarc)
+            self._manifest["configs"]["conda"]["condarc"] = local_condarc
+
+        condarc = self._manifest["configs"]["conda"]["condarc"]
+        if condarc:
+            ensure_file(
+                self.store.snapshot_workdir / RuntimeArtifactType.CONFIGS / "condarc",
+                yaml.safe_dump(condarc, default_flow_style=False),
+                parents=True,
+            )
+
     def _copy_src(
         self,
         config: RuntimeConfig,
@@ -1795,6 +1837,12 @@ def restore(
         else:
             operations.extend(
                 [
+                    (
+                        cls._prepare_configs,
+                        5,
+                        "prepare configs",
+                        dict(workdir=workdir, mode=_env["mode"]),
+                    ),
                     (
                         cls._setup_python_env,
                         20,

client/starwhale/core/runtime/view.py

@@ -171,6 +171,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_condarc: bool = False,
+        emit_pip_options: bool = False,
     ) -> Resource:
         set_args = list(filter(bool, (conda_name, conda_prefix, venv_prefix)))
         if len(set_args) >= 2:
@@ -212,6 +214,8 @@ def build_from_python_env(
             download_all_deps=download_all_deps,
             include_editable=include_editable,
             include_local_wheel=include_local_wheel,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
         return runtime_uri
 
@@ -228,6 +232,8 @@ def build_from_runtime_yaml(
         include_local_wheel: bool = False,
         no_cache: bool = False,
         disable_env_lock: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> Resource:
         workdir = Path(workdir)
         yaml_path = Path(yaml_path)
@@ -258,6 +264,8 @@ def build_from_runtime_yaml(
             include_local_wheel=include_local_wheel,
             no_cache=no_cache,
             disable_env_lock=disable_env_lock,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
         return _runtime_uri
 

client/tests/core/test_runtime.py

@@ -820,7 +820,7 @@ def test_build_from_runtime_yaml_in_venv_mode(
         _manifest = load_yaml(os.path.join(runtime_workdir, DEFAULT_MANIFEST_NAME))
 
         assert _manifest["configs"] == {
-            "conda": {"channels": ["conda-forge"]},
+            "conda": {"channels": ["conda-forge"], "condarc": {}},
             "docker": {"image": ""},
             "pip": {"extra_index_url": [""], "index_url": "", "trusted_host": [""]},
         }
@@ -1263,7 +1263,7 @@ def test_lock_ret_val(self, m_mkstemp: MagicMock, m_check: MagicMock, *args: t.A
 
     @patch("starwhale.base.bundle.LocalStorageBundleMixin._gen_version")
     @patch("starwhale.core.runtime.model.StandaloneRuntime._prepare_snapshot")
-    @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_context")
+    @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_configs")
     @patch("starwhale.utils.venv.check_call")
     @patch("starwhale.utils.venv.subprocess.check_output")
     @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_dependencies")

example/LLM/belle-bloom/runtime_conda.yaml

@@ -10,6 +10,22 @@ environment:
 configs:
   pip:
     index_url: https://mirrors.aliyun.com/pypi/simple
+  conda:
+    condarc: # custom condarc config file
+      channels:
+        - defaults
+      show_channel_urls: true
+      default_channels:
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
+      custom_channels:
+        conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        nvidia: https://mirrors.aliyun.com/anaconda/cloud
+      ssl_verify: false
+      default_threads: 10
 dependencies:
   - pip:
       # pip dependencies according to the following requirements:

scripts/example/runtime_conda.yaml

@@ -11,3 +11,8 @@ environment:
   os: ubuntu:20.04
 mode: conda
 name: simple-test-conda
+config:
+  conda:
+    condarc:
+      ssl_verify: false
+      default_threads: 4