feat: implement mirror and oci settings

Cargo.toml

@@ -27,12 +27,12 @@ slow_integration_tests = []
 [dependencies]
 assert_matches = "1.5.0"
 async-once-cell = "0.5.3"
-async-recursion = "1.0.5"
+async-recursion = "1.1.0"
 async-scoped = { version = "0.9.0", features = ["use-tokio"] }
-blake3 = "1.5.0"
+blake3 = "1.5.1"
 cfg-if = "0.1"
-chrono = "0.4.34"
-clap = { version = "4.5.1", default-features = false, features = [
+chrono = "0.4.35"
+clap = { version = "4.5.3", default-features = false, features = [
     "derive",
     "usage",
     "wrap_help",
@@ -65,61 +65,61 @@ install-wheel-rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 is_executable = "1.0.1"
 itertools = "0.12.1"
 lazy_static = "1.4.0"
-miette = { version = "7.1.0", features = [
+miette = { version = "7.2.0", features = [
     "fancy",
     "supports-color",
     "supports-hyperlinks",
     "supports-unicode",
     "terminal_size",
     "textwrap",
 ] }
-minijinja = { version = "1.0.12", features = ["builtins"] }
+minijinja = { version = "1.0.13", features = ["builtins"] }
 once_cell = "1.19.0"
 pep440_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 pep508_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 platform-host = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 platform-tags = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 pypi-types = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
-rattler = { version = "0.19.2", default-features = false, features = ["cli-tools"] }
-rattler_conda_types = { version = "0.20.1", default-features = false }
-rattler_digest = { version = "0.19.1", default-features = false }
-rattler_lock = { version = "0.20.1", default-features = false }
-rattler_networking = { version = "0.19.1", default-features = false }
-rattler_repodata_gateway = { version = "0.19.2", default-features = false, features = [
+rattler = { version = "0.19.3", default-features = false, features = ["cli-tools"] }
+rattler_conda_types = { version = "0.20.2", default-features = false }
+rattler_digest = { version = "0.19.2", default-features = false }
+rattler_lock = { version = "0.20.2", default-features = false }
+rattler_networking = { version = "0.19.2", default-features = false }
+rattler_repodata_gateway = { version = "0.19.3", default-features = false, features = [
     "sparse",
 ] }
-rattler_shell = { version = "0.19.2", default-features = false, features = [
+rattler_shell = { version = "0.19.3", default-features = false, features = [
     "sysinfo",
 ] }
-rattler_solve = { version = "0.20.1", default-features = false, features = [
+rattler_solve = { version = "0.20.2", default-features = false, features = [
     "resolvo",
 ] }
-rattler_virtual_packages = { version = "0.19.2", default-features = false }
+rattler_virtual_packages = { version = "0.19.3", default-features = false }
 regex = "1.10.3"
-reqwest = { version = "0.11.24", default-features = false }
-reqwest-middleware = "0.2.4"
+reqwest = { version = "0.11.26", default-features = false }
+reqwest-middleware = "0.2.5"
 reqwest-retry = "0.4.0"
 self-replace = "1.3.7"
 serde = "1.0.197"
 serde-untagged = "0.1.5"
 serde_json = "1.0.114"
 serde_spanned = "0.6.5"
-serde_with = { version = "3.6.1", features = ["indexmap"] }
-serde_yaml = "0.9.32"
+serde_with = { version = "3.7.0", features = ["indexmap"] }
+serde_yaml = "0.9.33"
 shlex = "1.3.0"
 spdx = "0.10.4"
 strsim = "0.11.0"
 tabwriter = { version = "1.4.0", features = ["ansi_formatting"] }
 tar = "0.4.40"
 tempfile = "3.10.1"
-thiserror = "1.0.57"
+thiserror = "1.0.58"
 tokio = { version = "1.36.0", features = [
     "macros",
     "rt-multi-thread",
     "signal",
 ] }
 tokio-util = "0.7.10"
-toml_edit = { version = "0.22.6", features = ["serde"] }
+toml_edit = { version = "0.22.7", features = ["serde"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 url = "2.5.0"
@@ -150,29 +150,31 @@ signal-hook = "0.3.17"
 
 [dev-dependencies]
 insta = { version = "1.36.1", features = ["yaml", "glob"] }
-rattler_digest = "0.19.1"
+rattler_digest = "0.19.2"
 rstest = "0.18.2"
 serde_json = "1.0.114"
 serial_test = "3.0.0"
 tokio = { version = "1.36.0", features = ["rt"] }
-toml = "0.8.10"
+toml = "0.8.11"
 
 [patch.crates-io]
 # For pyproject-toml
 # If you change this also change the versions in the the patch section for astral
 pep440_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 pep508_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
+#deno_task_shell = { path = "../deno_task_shell" }
 
-#rattler = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_conda_types = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_digest = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_lock = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_networking = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_repodata_gateway = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_shell = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_solve = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#rattler_virtual_packages = { git = "https://github.com/mamba-org/rattler", rev = "7e0a130f0603fc2ff204649df9adba78422de2c8" }
-#deno_task_shell = { path = "../deno_task_shell" }#rattler = { path = "../rattler/crates/rattler" }
+# rattler = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_conda_types = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_digest = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_lock = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_networking = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_repodata_gateway = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_shell = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_solve = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+# rattler_virtual_packages = { git = "https://github.com/mamba-org/rattler", rev = "e3784e0bc65b98f3cf3dcb7e2b8924113f3ec463" }
+
+#rattler = { path = "../rattler/crates/rattler" }
 #rattler_conda_types = { path = "../rattler/crates/rattler_conda_types" }
 #rattler_digest = { path = "../rattler/crates/rattler_digest" }
 #rattler_networking = { path = "../rattler/crates/rattler_networking" }
@@ -182,7 +184,6 @@ pep508_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }
 #rattler_virtual_packages = { path = "../rattler/crates/rattler_virtual_packages" }
 #rattler_lock = { path = "../rattler/crates/rattler_lock" }
 
-
 # Change these lines if you want a patched version of astral
 # [patch.'https://github.com/astral-sh/uv']
 # pep440_rs = { git = "https://github.com/astral-sh/uv", tag = "0.1.16" }

docs/advanced/global_configuration.md

@@ -1,15 +1,19 @@
 # Global configuration in pixi
 
-Pixi supports some global configuration options, as well as project-scoped configuration (that does not belong into the project file).
-The configuration is loaded in the following order:
+Pixi supports some global configuration options, as well as project-scoped
+configuration (that does not belong into the project file). The configuration is
+loaded in the following order:
 
-1. Global configuration folder (e.g. `~/.config/pixi/config.toml` on Linux, dependent on XDG_CONFIG_HOME)
-2. Global .pixi folder: `~/.pixi/config.toml` (or `$PIXI_HOME/config.toml` if the `PIXI_HOME` environment variable is set)
+1. Global configuration folder (e.g. `~/.config/pixi/config.toml` on Linux,
+   dependent on XDG_CONFIG_HOME)
+2. Global .pixi folder: `~/.pixi/config.toml` (or `$PIXI_HOME/config.toml` if
+   the `PIXI_HOME` environment variable is set)
 3. Project-local .pixi folder: `$PIXI_PROJECT/.pixi/config.toml`
 4. Command line arguments (`--tls-no-verify`, `--change-ps1=false` etc.)
 
 !!! note
-    To find the locations where `pixi` looks for configuration files, run `pixi` with `-v` or `--verbose`.
+    To find the locations where `pixi` looks for configuration files, run
+    `pixi` with `-v` or `--verbose`.
 
 ## Reference
 
@@ -35,4 +39,62 @@ tls_no_verify = false
 # file as fallback. This option allows you to force the use of a JSON file.
 # Read more in the authentication section.
 authentication_override_file = "/path/to/your/override.json"
+
+# configuration for conda channel-mirrors
+[mirrors]
+# redirect all requests for conda-forge to the prefix.dev mirror
+"https://conda.anaconda.org/conda-forge" = [
+    "https://prefix.dev/conda-forge"
+]
+
+# redirect all requests for bioconda to one of the three listed mirrors
+# Note: for repodata we try the first mirror first.
+"https://conda.anaconda.org/bioconda" = [
+    "https://conda.anaconda.org/bioconda",
+    # OCI registries are also supported
+    "oci://ghcr.io/channel-mirrors/bioconda",
+    "https://prefix.dev/bioconda",
+]
 ```
+
+## Mirror configuration
+
+You can configure mirrors for conda channels. We expect that mirrors are exact
+copies of the original channel. The implementation will look for the mirror key
+(a URL) in the `mirrors` section of the configuration file and replace the
+original URL with the mirror URL.
+
+To also include the original URL, you have to repeat it in the list of mirrors.
+
+The mirrors are prioritized based on the order of the list. We attempt to fetch
+the repodata (the most important file) from the first mirror in the list. The
+repodata contains all the SHA256 hashes of the individual packages, so it is
+important to get this file from a trusted source.
+
+You can also specify mirrors for an entire "host", e.g.
+
+```toml
+[mirrors]
+"https://conda.anaconda.org" = [
+    "https://prefix.dev/"
+]
+```
+
+This will forward all request to channels on anaconda.org to prefix.dev.
+Channels that are not currently mirrored on prefix.dev will fail in the above example.
+
+### OCI Mirrors
+
+You can also specify mirrors on the OCI registry. There is a public mirror on
+the Github container registry (ghcr.io) that is maintained by the conda-forge
+team. You can use it like this:
+
+```toml
+[mirrors]
+"https://conda.anaconda.org/conda-forge" = [
+    "oci://ghcr.io/channel-mirrors/conda-forge"
+]
+```
+
+The GHCR mirror also contains `bioconda` packages. You can search the [available
+packages on Github](https://github.com/orgs/channel-mirrors/packages).

src/config.rs

@@ -2,9 +2,11 @@ use clap::{ArgAction, Parser};
 use miette::{Context, IntoDiagnostic};
 use rattler_conda_types::{Channel, ChannelConfig, ParseChannelError};
 use serde::Deserialize;
+use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
+use url::Url;
 
 use crate::consts;
 
@@ -103,6 +105,9 @@ pub struct Config {
     #[serde(default)]
     tls_no_verify: Option<bool>,
 
+    #[serde(default)]
+    mirrors: HashMap<Url, Vec<Url>>,
+
     #[serde(skip)]
     pub loaded_from: Vec<PathBuf>,
 
@@ -210,6 +215,8 @@ impl Config {
             self.authentication_override_file = other.authentication_override_file.clone();
         }
 
+        self.mirrors.extend(other.mirrors.clone());
+
         self.loaded_from.extend(other.loaded_from.iter().cloned());
     }
 
@@ -259,6 +266,10 @@ impl Config {
             .map(|c| Channel::from_str(c, &self.channel_config))
             .collect::<Result<Vec<Channel>, _>>()
     }
+
+    pub fn mirror_map(&self) -> &std::collections::HashMap<Url, Vec<Url>> {
+        &self.mirrors
+    }
 }
 
 #[cfg(test)]

src/install.rs

@@ -1,8 +1,8 @@
-use crate::default_retry_policy;
 use crate::progress::{
     default_progress_style, finished_progress_style, global_multi_progress,
     ProgressBarMessageFormatter,
 };
+use crate::utils::reqwest::default_retry_policy;
 use futures::future::ready;
 use futures::{stream, FutureExt, StreamExt, TryFutureExt, TryStreamExt};
 use indicatif::ProgressBar;

src/lib.rs

@@ -34,11 +34,3 @@ pub use task::{
     CmdArgs, ExecutableTask, FindTaskError, FindTaskSource, RunOutput, SearchEnvironments, Task,
     TaskDisambiguation, TaskExecutionError, TaskGraph, TaskGraphError,
 };
-
-use rattler_networking::retry_policies::ExponentialBackoff;
-
-/// The default retry policy employed by pixi.
-/// TODO: At some point we might want to make this configurable.
-pub fn default_retry_policy() -> ExponentialBackoff {
-    ExponentialBackoff::builder().build_with_max_retries(3)
-}

src/project/manifest/mod.rs

@@ -42,7 +42,7 @@ use std::{
 pub use system_requirements::{LibCSystemRequirement, SystemRequirements};
 pub use target::{Target, TargetSelector, Targets};
 use thiserror::Error;
-use toml_edit::{value, Array, Document, Item, Table, TomlError, Value};
+use toml_edit::{value, Array, DocumentMut, Item, Table, TomlError, Value};
 
 /// Errors that can occur when getting a feature.
 #[derive(Debug, Clone, Error, Diagnostic)]
@@ -66,7 +66,7 @@ pub struct Manifest {
     pub contents: String,
 
     /// Editable toml document
-    pub document: toml_edit::Document,
+    pub document: toml_edit::DocumentMut,
 
     /// The parsed manifest
     pub parsed: ProjectManifest,
@@ -87,7 +87,7 @@ impl Manifest {
     pub fn from_str(root: &Path, contents: impl Into<String>) -> miette::Result<Self> {
         let contents = contents.into();
         let (manifest, document) = match ProjectManifest::from_toml_str(&contents)
-            .and_then(|manifest| contents.parse::<Document>().map(|doc| (manifest, doc)))
+            .and_then(|manifest| contents.parse::<DocumentMut>().map(|doc| (manifest, doc)))
         {
             Ok(result) => result,
             Err(e) => {
@@ -779,7 +779,7 @@ fn get_nested_toml_table_name(
 /// for a specific platform.
 /// If table not found, its inserted into the document.
 fn get_or_insert_toml_table<'a>(
-    doc: &'a mut Document,
+    doc: &'a mut DocumentMut,
     platform: Option<Platform>,
     feature: &FeatureName,
     table_name: &str,

src/project/manifest/snapshots/pixi__project__manifest__system_requirements__tests__fam_misspelled.snap

@@ -5,8 +5,5 @@ expression: "\n        [system-requirements.libc]\n        version = \"2.12\"\n
 TOML parse error at line 2, column 9
   |
 2 |         [system-requirements.libc]
-  |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  |         ^^^^^^^^^^^^^^^^^^^^^^^^^^
 unknown field `fam`, expected `family` or `version`
-
-
-

src/snapshots/pixi__config__tests__config_merge.snap

@@ -15,6 +15,7 @@ Config {
     tls_no_verify: Some(
         false,
     ),
+    mirrors: {},
     loaded_from: [
         "path/config_1.toml",
         "path/config_2.toml",