feat(client): support condarc for runtime yaml

client/starwhale/base/type.py

@@ -40,6 +40,7 @@ class RuntimeArtifactType:
     DEPEND = "dependencies"
     WHEELS = "wheels"
     FILES = "files"
+    CONFIGS = "configs"
 
 
 class RuntimeLockFileType:

client/starwhale/core/runtime/cli.py

@@ -248,6 +248,18 @@ def _quickstart(
     help="Include editable packages",
     hidden=True,
 )
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "-epo",
+    "--emit-pip-options",
+    is_flag=True,
+    help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}",
+)
+@optgroup.option(  # type: ignore[no-untyped-call]
+    "-ecc",
+    "--emit-condarc",
+    is_flag=True,
+    help="Emit to dump ~/.condarc from the local machine, default is False",
+)
 @optgroup.option(  # type: ignore[no-untyped-call]
     "-ilw",
     "--include-local-wheel",
@@ -280,6 +292,8 @@ def _build(
     shell: bool,
     yaml: str,
     docker: str,
+    emit_pip_options: bool,
+    emit_condarc: bool,
 ) -> None:
     """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs.
     Runtime build only works in the Standalone instance.
@@ -346,6 +360,8 @@ def _build(
             download_all_deps=download_all_deps,
             include_editable=include_editable,
             include_local_wheel=include_local_wheel,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
     else:
         RuntimeTermView.build_from_runtime_yaml(
@@ -358,6 +374,8 @@ def _build(
             include_local_wheel=include_local_wheel,
             no_cache=no_cache,
             disable_env_lock=disable_env_lock,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
 
 

client/starwhale/core/runtime/model.py

@@ -575,8 +575,14 @@ def __init__(
 
 
 class CondaConfig(ASDictMixin):
-    def __init__(self, channels: t.Optional[t.List[str]] = None, **kw: t.Any) -> None:
+    def __init__(
+        self,
+        channels: t.Optional[t.List[str]] = None,
+        condarc: t.Optional[t.Dict] = None,
+        **kw: t.Any,
+    ) -> None:
         self.channels = channels or [DEFAULT_CONDA_CHANNEL]
+        self.condarc = condarc or {}
 
 
 class Configs(ASDictMixin):
@@ -746,6 +752,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         raise NotImplementedError
 
@@ -761,6 +769,8 @@ def build_from_runtime_yaml(
         env_prefix_path: str = "",
         env_name: str = "",
         env_use_shell: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         raise NotImplementedError
 
@@ -897,6 +907,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         if conda_name or conda_prefix:
             prefix_path = (
@@ -951,6 +963,8 @@ def build_from_python_env(
                 env_name=conda_name,
                 env_prefix_path=conda_prefix or venv_prefix,
                 env_use_shell=env_use_shell,
+                emit_pip_options=emit_pip_options,
+                emit_condarc=emit_condarc,
             )
         finally:
             empty_dir(workdir)
@@ -967,6 +981,8 @@ def build_from_runtime_yaml(
         env_prefix_path: str = "",
         env_name: str = "",
         env_use_shell: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> None:
         workdir = Path(workdir)
         yaml_path = Path(yaml_path)
@@ -987,6 +1003,7 @@ def build_from_runtime_yaml(
                 env_name=env_name,
                 env_prefix_path=env_prefix_path,
                 env_use_shell=env_use_shell,
+                emit_pip_options=emit_pip_options,
             )
             # try getting starwhale version
             for line in content.splitlines():
@@ -1001,10 +1018,10 @@ def build_from_runtime_yaml(
             (self._gen_version, 5, "gen version"),
             (self._prepare_snapshot, 5, "prepare snapshot"),
             (
-                self._dump_context,
+                self._dump_configs,
                 5,
-                "dump environment and configs",
-                dict(config=swrt_config),
+                "dump configs",
+                dict(config=swrt_config, emit_condarc=emit_condarc),
             ),
             (
                 self._lock_environment,
@@ -1057,10 +1074,35 @@ def build_from_runtime_yaml(
         ]
         run_with_progress_bar("runtime bundle building...", operations)
 
-    def _dump_context(self, config: RuntimeConfig) -> None:
-        # TODO: refactor docker image in environment
+    @staticmethod
+    def _prepare_configs(workdir: Path, mode: str) -> None:
+        if mode == PythonRunEnv.CONDA:
+            condarc_path = workdir / RuntimeArtifactType.CONFIGS / "condarc"
+            if condarc_path.exists():
+                # CONDARC env only works in the current process and subprocess by the starwhale.utils.process.check_call function
+                os.environ["CONDARC"] = str(condarc_path)
+
+    def _dump_configs(self, config: RuntimeConfig, emit_condarc: bool = False) -> None:
         self._manifest["configs"] = config.configs.asdict()
 
+        condarc_path = Path.home() / ".condarc"
+        if (
+            config.mode == PythonRunEnv.CONDA
+            and not emit_condarc
+            and condarc_path.exists()
+        ):
+            local_condarc = load_yaml(condarc_path)
+            local_condarc.update(config.configs.conda.condarc)
+            self._manifest["configs"]["conda"]["condarc"] = local_condarc
+
+        condarc = self._manifest["configs"]["conda"]["condarc"]
+        if condarc:
+            ensure_file(
+                self.store.snapshot_workdir / RuntimeArtifactType.CONFIGS / "condarc",
+                yaml.safe_dump(condarc, default_flow_style=False),
+                parents=True,
+            )
+
     def _copy_src(
         self,
         config: RuntimeConfig,
@@ -1795,6 +1837,12 @@ def restore(
         else:
             operations.extend(
                 [
+                    (
+                        cls._prepare_configs,
+                        5,
+                        "prepare configs",
+                        dict(workdir=workdir, mode=_env["mode"]),
+                    ),
                     (
                         cls._setup_python_env,
                         20,

client/starwhale/core/runtime/view.py

@@ -171,6 +171,8 @@ def build_from_python_env(
         download_all_deps: bool = False,
         include_editable: bool = False,
         include_local_wheel: bool = False,
+        emit_condarc: bool = False,
+        emit_pip_options: bool = False,
     ) -> Resource:
         set_args = list(filter(bool, (conda_name, conda_prefix, venv_prefix)))
         if len(set_args) >= 2:
@@ -212,6 +214,8 @@ def build_from_python_env(
             download_all_deps=download_all_deps,
             include_editable=include_editable,
             include_local_wheel=include_local_wheel,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
         return runtime_uri
 
@@ -228,6 +232,8 @@ def build_from_runtime_yaml(
         include_local_wheel: bool = False,
         no_cache: bool = False,
         disable_env_lock: bool = False,
+        emit_pip_options: bool = False,
+        emit_condarc: bool = False,
     ) -> Resource:
         workdir = Path(workdir)
         yaml_path = Path(yaml_path)
@@ -258,6 +264,8 @@ def build_from_runtime_yaml(
             include_local_wheel=include_local_wheel,
             no_cache=no_cache,
             disable_env_lock=disable_env_lock,
+            emit_condarc=emit_condarc,
+            emit_pip_options=emit_pip_options,
         )
         return _runtime_uri
 

client/tests/core/test_runtime.py

@@ -70,6 +70,7 @@ def tearDown(self) -> None:
     def _clear_cache(self) -> None:
         sw_config._config = {}
         get_conda_bin.cache_clear()
+        os.environ.pop("CONDARC", None)
 
     @patch("starwhale.utils.venv.check_call")
     @patch("starwhale.utils.venv.virtualenv.cli_run")
@@ -405,6 +406,11 @@ def test_build_from_conda_name(
         ensure_dir(os.path.join(conda_prefix, "conda-meta"))
         conda_name = "starwhale"
 
+        self.fs.create_file(
+            Path.home() / ".condarc",
+            contents=yaml.safe_dump({"ssl_verify": False}, default_flow_style=False),
+        )
+
         lock_fpath = Path("/tmp/conda-sw-lock.yaml")
         lock_content = "numpy==1.19.5\nPillow==8.3.1"
         ensure_file(lock_fpath, content=lock_content, parents=True)
@@ -479,6 +485,11 @@ def test_build_from_conda_name(
             "mode": "conda",
             "python": "3.8",
         }
+        assert _manifest["configs"] == {
+            "conda": {"channels": ["conda-forge"], "condarc": {"ssl_verify": False}},
+            "docker": {"image": ""},
+            "pip": {"extra_index_url": [""], "index_url": "", "trusted_host": [""]},
+        }
         assert _manifest["version"] == uri.version
 
         _runtime_yaml = load_yaml(os.path.join(runtime_workdir, "runtime.yaml"))
@@ -495,6 +506,8 @@ def test_build_from_conda_name(
             "mode": "conda",
             "name": "test",
         }
+        _condarc = load_yaml(os.path.join(runtime_workdir, "configs", "condarc"))
+        assert _condarc == {"ssl_verify": False}
 
     @patch("starwhale.base.uri.resource.Resource._refine_local_rc_info")
     @patch("starwhale.utils.venv.check_call")
@@ -820,7 +833,7 @@ def test_build_from_runtime_yaml_in_venv_mode(
         _manifest = load_yaml(os.path.join(runtime_workdir, DEFAULT_MANIFEST_NAME))
 
         assert _manifest["configs"] == {
-            "conda": {"channels": ["conda-forge"]},
+            "conda": {"channels": ["conda-forge"], "condarc": {}},
             "docker": {"image": ""},
             "pip": {"extra_index_url": [""], "index_url": "", "trusted_host": [""]},
         }
@@ -1083,6 +1096,11 @@ def test_build_from_conda_shell(
             "pip_pkgs": ["starwhale"],
             "raw_deps": [{"deps": ["starwhale"], "kind": "pip_pkg"}],
         }
+        assert _manifest["configs"] == {
+            "conda": {"channels": ["conda-forge"], "condarc": {}},
+            "docker": {"image": ""},
+            "pip": {"extra_index_url": [""], "index_url": "", "trusted_host": [""]},
+        }
 
         _env = _manifest["environment"]
         assert _env["auto_lock_dependencies"]
@@ -1095,6 +1113,8 @@ def test_build_from_conda_shell(
         for p in _manifest["artifacts"]["dependencies"]:
             assert os.path.exists(os.path.join(runtime_workdir, p))
 
+        assert not os.path.exists(os.path.join(runtime_workdir, "configs", "condarc"))
+
     @patch("starwhale.core.runtime.model.conda_export")
     @patch("starwhale.utils.venv.check_call")
     @patch("starwhale.utils.venv.subprocess.check_output")
@@ -1263,7 +1283,7 @@ def test_lock_ret_val(self, m_mkstemp: MagicMock, m_check: MagicMock, *args: t.A
 
     @patch("starwhale.base.bundle.LocalStorageBundleMixin._gen_version")
     @patch("starwhale.core.runtime.model.StandaloneRuntime._prepare_snapshot")
-    @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_context")
+    @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_configs")
     @patch("starwhale.utils.venv.check_call")
     @patch("starwhale.utils.venv.subprocess.check_output")
     @patch("starwhale.core.runtime.model.StandaloneRuntime._dump_dependencies")
@@ -1649,6 +1669,13 @@ def test_restore_conda_with_auto_lock(
         )
         ensure_dir(workdir)
 
+        self.fs.create_dir(os.path.join(workdir, "configs"))
+        condarc_path = os.path.join(workdir, "configs", "condarc")
+        self.fs.create_file(
+            condarc_path,
+            contents=yaml.safe_dump({"verbosity": 3}, default_flow_style=False),
+        )
+
         self.fs.create_file(
             os.path.join(workdir, DEFAULT_MANIFEST_NAME),
             contents=yaml.safe_dump(
@@ -1701,9 +1728,11 @@ def test_restore_conda_with_auto_lock(
         self.fs.create_file(req_lock_fpath, contents="fake content")
         self.fs.create_file(wheel_fpath, contents="")
 
+        assert "CONDARC" not in os.environ
         m_machine.return_value = "arm64"
         Runtime.restore(Path(workdir))
 
+        assert os.environ["CONDARC"] == condarc_path
         assert m_call.call_count == 4
         conda_cmds = [cm[0][0] for cm in m_call.call_args_list]
         conda_prefix_dir = os.path.join(export_dir, "conda")
@@ -1831,6 +1860,7 @@ def test_restore_conda(
         m_machine.return_value = "arm64"
         Runtime.restore(Path(workdir))
 
+        assert "CONDARC" not in os.environ
         assert m_call.call_count == 8
         conda_cmds = [cm[0][0] for cm in m_call.call_args_list]
         conda_prefix_dir = os.path.join(export_dir, "conda")

example/LLM/belle-bloom/runtime_conda.yaml

@@ -10,6 +10,22 @@ environment:
 configs:
   pip:
     index_url: https://mirrors.aliyun.com/pypi/simple
+  conda:
+    condarc: # custom condarc config file
+      channels:
+        - defaults
+      show_channel_urls: true
+      default_channels:
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
+        - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
+      custom_channels:
+        conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
+        nvidia: https://mirrors.aliyun.com/anaconda/cloud
+      ssl_verify: false
+      default_threads: 10
 dependencies:
   - pip:
       # pip dependencies according to the following requirements:

scripts/example/runtime_conda.yaml

@@ -11,3 +11,8 @@ environment:
   os: ubuntu:20.04
 mode: conda
 name: simple-test-conda
+config:
+  conda:
+    condarc:
+      ssl_verify: false
+      default_threads: 4