Switch to rustwide for sandboxing

Cargo.toml

@@ -39,6 +39,7 @@ tokio = "0.1"
 systemstat = "0.1.4"
 prometheus = { version = "0.7.0", default-features = false }
 lazy_static = "1.0.0"
+rustwide = { git =  "https://github.com/rust-lang/rustwide" }
 
 # iron dependencies
 iron = "0.5"

src/bin/cratesfyi.rs

@@ -12,7 +12,7 @@ use std::path::PathBuf;
 
 use clap::{Arg, App, SubCommand};
 use cratesfyi::{DocBuilder, DocBuilderOptions, db};
-use cratesfyi::utils::{build_doc, add_crate_to_queue};
+use cratesfyi::utils::{build_doc, build_doc_rustwide, add_crate_to_queue};
 use cratesfyi::start_web_server;
 use cratesfyi::db::{add_path_into_database, connect_db};
 
@@ -37,6 +37,20 @@ pub fn main() {
                 .index(3)
                 .required(false)
                 .help("The target platform to compile for")))
+        .subcommand(SubCommand::with_name("doc_rustwide")
+            .about("Builds documentation of a crate with rustwide")
+            .arg(Arg::with_name("CRATE_NAME")
+                .index(1)
+                .required(true)
+                .help("Crate name"))
+            .arg(Arg::with_name("CRATE_VERSION")
+                .index(2)
+                .required(true)
+                .help("Crate version"))
+            .arg(Arg::with_name("TARGET")
+                .index(3)
+                .required(false)
+                .help("The target platform to compile for")))
         .subcommand(SubCommand::with_name("build")
             .about("Builds documentation in a chroot environment")
             .arg(Arg::with_name("PREFIX")
@@ -160,6 +174,13 @@ pub fn main() {
         if let Err(e) = build_doc(name, version, target) {
             panic!("{:#?}", e);
         }
+    } else if let Some(matches) = matches.subcommand_matches("doc_rustwide") {
+        let name = matches.value_of("CRATE_NAME").unwrap();
+        let version = matches.value_of("CRATE_VERSION").unwrap();
+        let target = matches.value_of("TARGET");
+        if let Err(e) = build_doc_rustwide(name, version, target) {
+            panic!("{:#?}", e);
+        }
     } else if let Some(matches) = matches.subcommand_matches("build") {
         let docbuilder_opts = {
             let mut docbuilder_opts = if let Some(prefix) = matches.value_of("PREFIX") {

src/lib.rs

@@ -40,6 +40,7 @@ extern crate rusoto_credential;
 extern crate futures;
 extern crate tokio;
 extern crate systemstat;
+extern crate rustwide;
 
 pub use self::docbuilder::DocBuilder;
 pub use self::docbuilder::ChrootBuilderResult;

src/utils/build_doc.rs

@@ -127,7 +127,7 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> Result
     Ok(try!(ws.current()).clone())
 }
 
-fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box<dyn Source + 'cfg>)
+pub fn resolve_deps<'cfg>(pkg: &Package, config: &'cfg Config, src: Box<dyn Source + 'cfg>)
     -> CargoResult<Vec<(String, Package)>>
 {
     let mut registry = try!(PackageRegistry::new(config));

src/utils/build_doc_rustwide.rs

@@ -0,0 +1,122 @@
+use cargo::core::{enable_nightly_features, Package, SourceId, Workspace as CargoWorkspace};
+use cargo::sources::SourceConfigMap;
+use cargo::util::{internal, Config};
+use error::Result;
+use rustwide::{
+    cmd::{Command, SandboxBuilder},
+    Crate, Toolchain, Workspace, WorkspaceBuilder,
+};
+use std::collections::HashSet;
+use std::path::Path;
+use utils::{parse_rustc_version, resolve_deps};
+use Metadata;
+
+// TODO: 1GB might not be enough
+const SANDBOX_MEMORY_LIMIT: usize = 1024 * 1025 * 1024; // 1GB
+const SANDBOX_NETWORKING: bool = false;
+
+pub fn build_doc_rustwide(name: &str, version: &str, target: Option<&str>) -> Result<Package> {
+    // TODO: Handle workspace path correctly
+    let rustwide_workspace =
+        WorkspaceBuilder::new(Path::new("/tmp/docs-builder"), "docsrs").init()?;
+
+    // TODO: Instead of using just nightly, we can pin a version.
+    //       Docs.rs can only use nightly (due to unstable docs.rs features in rustdoc)
+    let toolchain = Toolchain::Dist {
+        name: "nightly".into(),
+    };
+    toolchain.install(&rustwide_workspace)?;
+
+    let krate = Crate::crates_io(name, version);
+    krate.fetch(&rustwide_workspace)?;
+
+    let sandbox = SandboxBuilder::new()
+        .memory_limit(Some(SANDBOX_MEMORY_LIMIT))
+        .enable_networking(SANDBOX_NETWORKING);
+
+    let mut build_dir = rustwide_workspace.build_dir(&format!("{}-{}", name, version));
+    let pkg = build_dir.build(&toolchain, &krate, sandbox, |build| {
+        enable_nightly_features();
+        let config = Config::default()?;
+        let source_id = try!(SourceId::crates_io(&config));
+        let source_cfg_map = try!(SourceConfigMap::new(&config));
+        let manifest_path = build.host_source_dir().join("Cargo.toml");
+        let ws = CargoWorkspace::new(&manifest_path, &config)?;
+        let pkg = ws.load(&manifest_path)?;
+
+        let metadata = Metadata::from_package(&pkg).map_err(|e| internal(e.to_string()))?;
+
+        let mut rustdoc_flags: Vec<String> = vec![
+            "-Z".to_string(),
+            "unstable-options".to_string(),
+            "--resource-suffix".to_string(),
+            format!(
+                "-{}",
+                parse_rustc_version(rustc_version(&rustwide_workspace, &toolchain)?)?
+            ),
+            "--static-root-path".to_string(),
+            "/".to_string(),
+            "--disable-per-crate-search".to_string(),
+        ];
+
+        let source = try!(source_cfg_map.load(source_id, &HashSet::new()));
+        let _lock = try!(config.acquire_package_cache_lock());
+
+        for (name, dep) in try!(resolve_deps(&pkg, &config, source)) {
+            rustdoc_flags.push("--extern-html-root-url".to_string());
+            rustdoc_flags.push(format!(
+                "{}=https://docs.rs/{}/{}",
+                name.replace("-", "_"),
+                dep.name(),
+                dep.version()
+            ));
+        }
+
+        let mut cargo_args = vec!["doc".to_owned(), "--lib".to_owned(), "--no-deps".to_owned()];
+        if let Some(features) = &metadata.features {
+            cargo_args.push("--features".to_owned());
+            cargo_args.push(features.join(" "));
+        }
+        if metadata.all_features {
+            cargo_args.push("--all-features".to_owned());
+        }
+        if metadata.no_default_features {
+            cargo_args.push("--no-default-features".to_owned());
+        }
+
+        // TODO: We need to use build result here
+        // FIXME: We also need build log (basically stderr message)
+        let result = build
+            .cargo()
+            .env(
+                "RUSTFLAGS",
+                metadata
+                    .rustc_args
+                    .map(|args| args.join(""))
+                    .unwrap_or("".to_owned()),
+            )
+            .env("RUSTDOCFLAGS", rustdoc_flags.join(" "))
+            .args(&cargo_args)
+            .run();
+
+        // TODO: We need to return build result as well
+        Ok(pkg)
+    })?;
+
+    Ok(pkg)
+}
+
+fn rustc_version(workspace: &Workspace, toolchain: &Toolchain) -> Result<String> {
+    let res = Command::new(workspace, toolchain.rustc())
+        .args(&["--version"])
+        .log_output(false)
+        .run_capture()?;
+
+    if let Some(line) = res.stdout_lines().iter().next() {
+        Ok(line.clone())
+    } else {
+        Err(::failure::err_msg(
+            "invalid output returned by `rustc --version`",
+        ))
+    }
+}

src/utils/mod.rs

@@ -1,7 +1,8 @@
 //! Various utilities for cratesfyi
 
 
-pub use self::build_doc::{build_doc, get_package, source_path, update_sources};
+pub use self::build_doc::{build_doc, get_package, source_path, update_sources, resolve_deps};
+pub use self::build_doc_rustwide::build_doc_rustwide;
 pub use self::copy::{copy_dir, copy_doc_dir};
 pub use self::github_updater::github_updater;
 pub use self::release_activity_updater::update_release_activity;
@@ -12,6 +13,7 @@ pub use self::queue::add_crate_to_queue;
 
 mod github_updater;
 mod build_doc;
+mod build_doc_rustwide;
 mod copy;
 mod release_activity_updater;
 mod daemon;