Download crates in parallel

Author: ishitatsuyuki

Cargo.toml

@@ -21,7 +21,7 @@ atty = "0.2"
 crates-io = { path = "src/crates-io", version = "0.16" }
 crossbeam = "0.3"
 crypto-hash = "0.3"
-curl = "0.4.6"
+curl = "0.4.11"
 docopt = "0.8.1"
 env_logger = "0.5"
 failure = "0.1.1"
@@ -91,3 +91,9 @@ doc = false
 [[test]]
 name = "testsuite"
 path = "tests/testsuite/lib.rs"
+
+[patch.crates-io]
+curl = { git = "https://github.com/alexcrichton/curl-rust" }
+curl-sys = { git = "https://github.com/alexcrichton/curl-rust" }
+openssl = { git = "https://github.com/sfackler/rust-openssl", branch = "more-sync" }
+openssl-sys = { git = "https://github.com/sfackler/rust-openssl", branch = "more-sync" }

src/cargo/core/package.rs

@@ -197,22 +197,32 @@ impl<'cfg> PackageSet<'cfg> {
         Box::new(self.packages.keys())
     }
 
-    pub fn get(&self, id: &PackageId) -> CargoResult<&Package> {
-        let slot = self.packages.get(id).ok_or_else(|| {
-            internal(format!("couldn't find `{}` in package set", id))
-        })?;
-        if let Some(pkg) = slot.borrow() {
-            return Ok(pkg)
+    pub fn get(&self, ids: &[&PackageId]) -> CargoResult<Vec<&Package>> {
+        let mut pending = BTreeMap::new();
+        for &id in ids {
+            let slot = self.packages.get(id).ok_or_else(|| {
+                internal(format!("couldn't find `{}` in package set", id))
+            })?;
+            if slot.borrow().is_none() {
+                let &mut (ref mut pending_ids, ref mut slots) = pending.entry(id.source_id()).or_insert_with(|| (Vec::new(), Vec::new()));
+                pending_ids.push(id);
+                slots.push(slot);
+            }
         }
         let mut sources = self.sources.borrow_mut();
-        let source = sources.get_mut(id.source_id()).ok_or_else(|| {
-            internal(format!("couldn't find source for `{}`", id))
-        })?;
-        let pkg = source.download(id).chain_err(|| {
-            format_err!("unable to get packages from source")
-        })?;
-        assert!(slot.fill(pkg).is_ok());
-        Ok(slot.borrow().unwrap())
+        for (source_id, &(ref pending_ids, ref slots)) in &pending {
+            let source = sources.get_mut(source_id).ok_or_else(|| {
+                internal(format!("couldn't find source `{}`", source_id))
+            })?;
+            let pkgs = source.download(&*pending_ids).chain_err(|| {
+                format_err!("unable to get packages from source")
+            })?;
+            for (pkg, slot) in pkgs.into_iter().zip(slots) {
+                assert!(slot.fill(pkg).is_ok());
+            }
+        }
+
+        Ok(ids.iter().map(|id| self.packages.get(id).unwrap().borrow().unwrap()).collect())
     }
 
     pub fn sources(&self) -> Ref<SourceMap<'cfg>> {

src/cargo/core/source/mod.rs

@@ -20,7 +20,7 @@ pub trait Source: Registry {
 
     /// The download method fetches the full package for each name and
     /// version specified.
-    fn download(&mut self, package: &PackageId) -> CargoResult<Package>;
+    fn download(&mut self, ids: &[&PackageId]) -> CargoResult<Vec<Package>>;
 
     /// Generates a unique string which represents the fingerprint of the
     /// current state of the source.
@@ -57,8 +57,8 @@ impl<'a, T: Source + ?Sized + 'a> Source for Box<T> {
     }
 
     /// Forwards to `Source::download`
-    fn download(&mut self, id: &PackageId) -> CargoResult<Package> {
-        (**self).download(id)
+    fn download(&mut self, ids: &[&PackageId]) -> CargoResult<Vec<Package>> {
+        (**self).download(ids)
     }
 
     /// Forwards to `Source::fingerprint`

src/cargo/ops/cargo_clean.rs

@@ -45,11 +45,10 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> {
                                    profiles)?;
     let mut units = Vec::new();
 
-    for spec in opts.spec {
-        // Translate the spec to a Package
-        let pkgid = resolve.query(spec)?;
-        let pkg = packages.get(pkgid)?;
+    let pkg_ids: CargoResult<Vec<_>> = opts.spec.iter().map(|spec| resolve.query(spec)).collect();
+    let pkgs = packages.get(&*pkg_ids?)?;
 
+    for pkg in pkgs {
         // Generate all relevant `Unit` targets for this package
         for target in pkg.targets() {
             for kind in [Kind::Host, Kind::Target].iter() {

src/cargo/ops/cargo_compile.rs

@@ -247,12 +247,11 @@ pub fn compile_ws<'a>(ws: &Workspace<'a>,
                                               )?;
     let (packages, resolve_with_overrides) = resolve;
 
-    let to_builds = specs.iter().map(|p| {
-        let pkgid = p.query(resolve_with_overrides.iter())?;
-        let p = packages.get(pkgid)?;
-        p.manifest().print_teapot(ws.config());
-        Ok(p)
-    }).collect::<CargoResult<Vec<_>>>()?;
+    let pkg_ids: CargoResult<Vec<_>> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect();
+    let to_builds = packages.get(&*pkg_ids?)?;
+    for pkg in &to_builds {
+        pkg.manifest().print_teapot(ws.config());
+    }
 
     let mut general_targets = Vec::new();
     let mut package_targets = Vec::new();

src/cargo/ops/cargo_doc.rs

@@ -22,10 +22,8 @@ pub fn doc(ws: &Workspace, options: &DocOptions) -> CargoResult<()> {
                                             &specs)?;
     let (packages, resolve_with_overrides) = resolve;
 
-    let pkgs = specs.iter().map(|p| {
-        let pkgid = p.query(resolve_with_overrides.iter())?;
-        packages.get(pkgid)
-    }).collect::<CargoResult<Vec<_>>>()?;
+    let pkg_ids: CargoResult<Vec<_>> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect();
+    let pkgs = packages.get(&*pkg_ids?)?;
 
     let mut lib_names = HashMap::new();
     let mut bin_names = HashMap::new();

src/cargo/ops/cargo_fetch.rs

@@ -5,8 +5,9 @@ use util::CargoResult;
 /// Executes `cargo fetch`.
 pub fn fetch<'a>(ws: &Workspace<'a>) -> CargoResult<(Resolve, PackageSet<'a>)> {
     let (packages, resolve) = ops::resolve_ws(ws)?;
-    for id in resolve.iter() {
-        packages.get(id)?;
+    {
+        let pkg_ids: Vec<_> = resolve.iter().collect();
+        packages.get(&*pkg_ids)?;
     }
     Ok((resolve, packages))
 }

src/cargo/ops/cargo_install.rs

@@ -397,8 +397,8 @@ fn select_pkg<'a, T>(mut source: T,
             let deps = source.query_vec(&dep)?;
             match deps.iter().map(|p| p.package_id()).max() {
                 Some(pkgid) => {
-                    let pkg = source.download(pkgid)?;
-                    Ok((pkg, Box::new(source)))
+                    let pkg = source.download(&[pkgid])?;
+                    Ok((pkg.into_iter().next().unwrap(), Box::new(source)))
                 }
                 None => {
                     let vers_info = vers.map(|v| format!(" with version `{}`", v))

src/cargo/ops/cargo_output_metadata.rs

@@ -54,9 +54,8 @@ fn metadata_full(ws: &Workspace,
                                          &specs)?;
     let (packages, resolve) = deps;
 
-    let packages = packages.package_ids()
-                           .map(|i| packages.get(i).map(|p| p.clone()))
-                           .collect::<CargoResult<Vec<_>>>()?;
+    let package_ids: Vec<_> = packages.package_ids().collect();
+    let packages: Vec<_> = packages.get(&*package_ids)?.into_iter().cloned().collect();
 
     Ok(ExportInfo {
         packages,

src/cargo/ops/cargo_rustc/context.rs

@@ -780,7 +780,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
 
         let id = unit.pkg.package_id();
         let deps = self.resolve.deps(id);
-        let mut ret = deps.filter(|dep| {
+        let pkg_ids: Vec<_> = deps.filter(|dep| {
             unit.pkg.dependencies().iter().filter(|d| {
                 d.name() == dep.name() && d.version_req().matches(dep.version())
             }).any(|d| {
@@ -814,22 +814,19 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
                 // actually used!
                 true
             })
-        }).filter_map(|id| {
-            match self.get_package(id) {
-                Ok(pkg) => {
-                    pkg.targets().iter().find(|t| t.is_lib()).map(|t| {
-                        let unit = Unit {
-                            pkg,
-                            target: t,
-                            profile: self.lib_or_check_profile(unit, t),
-                            kind: unit.kind.for_target(t),
-                        };
-                        Ok(unit)
-                    })
+        }).collect();
+
+        let pkgs = self.get_packages(&*pkg_ids)?;
+        let mut ret: Vec<_> = pkgs.into_iter().filter_map(|pkg| {
+            pkg.targets().iter().find(|t| t.is_lib()).map(|t| {
+                Unit {
+                    pkg,
+                    target: t,
+                    profile: self.lib_or_check_profile(unit, t),
+                    kind: unit.kind.for_target(t),
                 }
-                Err(e) => Some(Err(e))
-            }
-        }).collect::<CargoResult<Vec<_>>>()?;
+            })
+        }).collect();
 
         // If this target is a build script, then what we've collected so far is
         // all we need. If this isn't a build script, then it depends on the
@@ -913,7 +910,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
 
     /// Returns the dependencies necessary to document a package
     fn doc_deps(&self, unit: &Unit<'a>) -> CargoResult<Vec<Unit<'a>>> {
-        let deps = self.resolve.deps(unit.pkg.package_id()).filter(|dep| {
+        let dep_ids: Vec<_> = self.resolve.deps(unit.pkg.package_id()).filter(|dep| {
             unit.pkg.dependencies().iter().filter(|d| {
                 d.name() == dep.name()
             }).any(|dep| {
@@ -923,16 +920,15 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
                     _ => false,
                 }
             })
-        }).map(|dep| {
-            self.get_package(dep)
-        });
+        }).collect();
+
+        let deps = self.get_packages(&*dep_ids)?;
 
         // To document a library, we depend on dependencies actually being
         // built. If we're documenting *all* libraries, then we also depend on
         // the documentation of the library being built.
         let mut ret = Vec::new();
         for dep in deps {
-            let dep = dep?;
             let lib = match dep.targets().iter().find(|t| t.is_lib()) {
                 Some(lib) => lib,
                 None => continue,
@@ -1007,7 +1003,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
     }
 
     /// Gets a package for the given package id.
-    pub fn get_package(&self, id: &PackageId) -> CargoResult<&'a Package> {
+    pub fn get_packages(&self, id: &[&PackageId]) -> CargoResult<Vec<&'a Package>> {
         self.packages.get(id)
     }