Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: use the gateway to fetch repodata #1307

Merged
merged 14 commits into from
May 14, 2024
Prev Previous commit
Next Next commit
feat: update to latest
  • Loading branch information
baszalmstra committed May 2, 2024
commit 92cb8543070a99be2300db8cab2c1d376821a94e
22 changes: 11 additions & 11 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 9 additions & 3 deletions src/cli/add.rs
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,12 @@ async fn determine_latest_versions(
// Get the records for the package
let records = project
.repodata_gateway()
.load_records(project.channels(), platforms, [name.clone()])
.query(
project.channels().into_iter().cloned(),
platforms,
[name.clone()],
)
.recursive(false)
.await
.into_diagnostic()?;

Expand Down Expand Up @@ -431,11 +436,12 @@ pub async fn determine_best_version<'p>(
// Extract the package names from all the dependencies
let fetch_repodata_start = Instant::now();
let available_packages = repodata_gateway
.load_records_recursive(
channels,
.query(
channels.into_iter().cloned(),
[platform, Platform::NoArch],
dependencies.clone().into_match_specs(),
)
.recursive(true)
.await
.into_diagnostic()?;
let total_records = available_packages.iter().map(RepoData::len).sum::<usize>();
Expand Down
3 changes: 2 additions & 1 deletion src/lock_file/update.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1176,11 +1176,12 @@ async fn spawn_solve_conda_environment_task(
pb.set_message("loading repodata");
let fetch_repodata_start = Instant::now();
let available_packages = repodata_gateway
.load_records_recursive(
.query(
channels,
[platform, Platform::NoArch],
dependencies.clone().into_match_specs(),
)
.recursive(true)
.await
.into_diagnostic()?;
let total_records = available_packages.iter().map(RepoData::len).sum::<usize>();
Expand Down
21 changes: 16 additions & 5 deletions src/repodata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@ use indicatif::ProgressBar;
use itertools::Itertools;
use miette::{IntoDiagnostic, WrapErr};
use rattler_conda_types::{Channel, Platform};
use rattler_repodata_gateway::fetch;
use rattler_repodata_gateway::fetch::FetchRepoDataOptions;
use rattler_repodata_gateway::sparse::SparseRepoData;
use rattler_repodata_gateway::{fetch, Reporter};
use reqwest_middleware::ClientWithMiddleware;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use url::Url;

pub async fn fetch_sparse_repodata(
channels: impl IntoIterator<Item = &'_ Channel>,
Expand Down Expand Up @@ -121,6 +123,17 @@ pub async fn fetch_sparse_repodata_targets(
repo_data.wrap_err("failed to fetch repodata from channels")
}

struct DownloadProgressReporter {
progress_bar: ProgressBar,
}

impl Reporter for DownloadProgressReporter {
fn on_download_progress(&self, _url: &Url, _index: usize, bytes: usize, total: Option<usize>) {
self.progress_bar.set_length(total.unwrap_or(bytes) as u64);
self.progress_bar.set_position(bytes as u64);
}
}

/// Given a channel and platform, download and cache the `repodata.json` for it. This function
/// reports its progress via a CLI progressbar.
async fn fetch_repo_data_records_with_progress(
Expand All @@ -133,15 +146,13 @@ async fn fetch_repo_data_records_with_progress(
fetch_options: FetchRepoDataOptions,
) -> miette::Result<Option<SparseRepoData>> {
// Download the repodata.json
let download_progress_progress_bar = progress_bar.clone();
let result = fetch::fetch_repo_data(
channel.platform_url(platform),
client,
repodata_cache.to_path_buf(),
fetch_options,
Some(Box::new(move |fetch::DownloadProgress { total, bytes }| {
download_progress_progress_bar.set_length(total.unwrap_or(bytes));
download_progress_progress_bar.set_position(bytes);
Some(Arc::new(DownloadProgressReporter {
progress_bar: progress_bar.clone(),
})),
)
.await;
Expand Down