Throttle futures

[micprog]

Due to an update to tokio and futures in #94, the throttling for futures was removed. If this causes instability when running bender, the throttling will have to be refactored as well. Please report any instability you may notice (possibly too many simultaneous git requests).

Relevant code sections:
src/future_throttle.rs

bender/src/sess.rs

Lines 500 to 519 in f82ef0e

	// TODO MICHAERO: May need throttle
	future::lazy(|_| {
	stageln!("Cloning", "{} ({})", name2, url2);
	Ok(())
	})
	.and_then(|_| git.spawn_with(|c| c.arg("init").arg("--bare")))
	.and_then(|_| git.spawn_with(|c| c.arg("remote").arg("add").arg("origin").arg(url)))
	.and_then(|_| git.fetch("origin"))
	.await
	.map_err(move |cause| {
	if url3.contains("git@") {
	warnln!("Please ensure your public ssh key is added to the git server.");
	}
	warnln!("Please ensure the url is correct and you have access to the repository.");
	Error::chain(
	format!("Failed to initialize git database in {:?}.", db_dir),
	cause,
	)
	})
	.map(move |_| git)

bender/src/sess.rs

Lines 528 to 545 in f82ef0e

	// TODO MICHAERO: May need throttle
	future::lazy(|_| {
	stageln!("Fetching", "{} ({})", name2, url2);
	Ok(())
	})
	.and_then(|_| git.fetch("origin"))
	.await
	.map_err(move |cause| {
	if url3.contains("git@") {
	warnln!("Please ensure your public ssh key is added to the git server.");
	}
	warnln!("Please ensure the url is correct and you have access to the repository.");
	Error::chain(
	format!("Failed to update git database in {:?}.", db_dir),
	cause,
	)
	})
	.map(move |_| git)

bender/src/sess.rs

Lines 809 to 834 in f82ef0e

	// TODO MICHAERO: May need proper chaining to previous future using and_then
	if !path.exists() {
	stageln!("Checkout", "{} ({})", name, url);
	// First generate a tag to be cloned in the database. This is
	// necessary since `git clone` does not accept commits, but only
	// branches or tags for shallow clones.
	let tag_name_0 = format!("bender-tmp-{}", revision);
	let tag_name_1 = tag_name_0.clone();
	let git = self.git_database(name, url, false).await?;
	// .and_then(move |git| {
	git.clone()
	.spawn_with(move |c| c.arg("tag").arg(tag_name_0).arg(revision).arg("--force"))
	.await?;
	git.clone()
	.spawn_with(move |c| {
	c.arg("clone")
	.arg(git.path)
	.arg(path)
	.arg("--recursive")
	.arg("--branch")
	.arg(tag_name_1)
	})
	.await?;
	}
	Ok(path)

bender/src/sess.rs

Lines 892 to 961 in f82ef0e

	// TODO MICHAERO: May need proper chaining using and_then
	let db = self.git_database(&dep.name, url, false).await?;
	let entries = db.list_files(rev, Some("Bender.yml")).await?;
	let data = match entries.into_iter().next() {
	None => Ok(None),
	Some(entry) => db.cat_file(entry.hash).await.map(|f| Some(f)),
	}?;
	let manifest: Result<_> = match data {
	Some(data) => {
	let partial: config::PartialManifest = serde_yaml::from_str(&data)
	.map_err(|cause| {
	Error::chain(
	format!(
	"Syntax error in manifest of dependency `{}` at \
	revision `{}`.",
	dep_name, rev
	),
	cause,
	)
	})?;
	let mut full = partial.validate().map_err(|cause| {
	Error::chain(
	format!(
	"Error in manifest of dependency `{}` at revision \
	`{}`.",
	dep_name, rev
	),
	cause,
	)
	})?;
	// Add base path to path dependencies within git repositories
	for dep in full.dependencies.iter_mut() {
	match dep {
	(_, config::Dependency::Path(ref path)) => {
	if !path.starts_with("/") {
	if !self.get_package_path(dep_id).exists() {
	warnln!("Please note that dependencies for {:?} may not be available unless {:?} is properly checked out.\n (to checkout run `bender sources` and then `bender update` again).", dep.0, full.package.name);
	}
	*dep.1 = config::Dependency::Path(
	self.get_package_path(dep_id).join(path).clone(),
	);
	}
	}
	(_, _) => {}
	}
	}
	Ok(Some(self.sess.intern_manifest(full)))
	}
	None => Ok(None),
	};
	let manifest = manifest?;
	self.sess
	.cache
	.dependency_manifest_version
	.lock()
	.unwrap()
	.insert(cache_key, manifest);
	if dep.name
	!= match manifest {
	Some(x) => &x.package.name,
	None => "dead",
	}
	{
	warnln!("Dependency name and package name do not match for {:?} / {:?}, this can cause unwanted behavior",
	dep.name, match manifest {
	Some(x) => &x.package.name,
	None => "dead"
	}); // TODO (micprog): This should be an error
	}
	Ok(manifest)