Merge pull request #182 from onur/rustdoc-resource-suffix

rustdoc resource suffix changes

Author: onur

.travis.yml

@@ -1,11 +1,6 @@
-dist: precise
 language: rust
-addons:
-  apt:
-    sources:
-      - kalakris-cmake
-    packages:
-      - cmake
+cache: cargo
 before_install:
 - sudo apt-get update -qq
 - sudo apt-get install -qq libmagic1 libmagic-dev
+script: cargo build --verbose && cargo test --verbose -- --test-threads=1

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "cratesfyi"
-version = "0.4.2"
+version = "0.5.0"
 authors = ["Onur Aslan <onuraslan@gmail.com>"]
 readme = "README.md"
 license = "MIT"

Cargo.toml

@@ -1,16 +1,16 @@
 
 use super::DocBuilder;
 use super::crates::crates_from_path;
-use utils::{get_package, source_path, copy_dir, copy_doc_dir, update_sources};
+use utils::{get_package, source_path, copy_dir, copy_doc_dir,
+            update_sources, parse_rustc_version, command_result};
 use db::{connect_db, add_package_into_database, add_build_into_database, add_path_into_database};
 use cargo::core::Package;
-use std::process::{Command, Output};
+use std::process::Command;
 use std::path::PathBuf;
 use std::fs::remove_dir_all;
 use postgres::Connection;
 use rustc_serialize::json::Json;
 use error::Result;
-use regex::Regex;
 
 
 /// List of targets supported by docs.rs
@@ -208,7 +208,7 @@ impl DocBuilder {
             .join(target.unwrap_or(""));
         copy_doc_dir(crate_doc_path,
                      destination,
-                     parse_rustc_version(rustc_version).trim(),
+                     parse_rustc_version(rustc_version)?.trim(),
                      target.is_some())
     }
 
@@ -353,7 +353,7 @@ impl DocBuilder {
         // acme-client-0.0.0 is an empty library crate and it will always build
         let pkg = try!(get_package("acme-client", Some("=0.0.0")));
         let res = self.build_package_in_chroot(&pkg);
-        let rustc_version = parse_rustc_version(&res.rustc_version);
+        let rustc_version = parse_rustc_version(&res.rustc_version)?;
 
         if !res.build_success {
             return Err(format!("Failed to build empty crate for: {}", res.rustc_version).into());
@@ -362,7 +362,14 @@ impl DocBuilder {
         info!("Copying essential files for: {}", res.rustc_version);
 
         let files = (// files require rustc version subfix
-                     ["rustdoc.css", "main.css", "main.js", "normalize.css"],
+                     ["brush.svg",
+                      "dark.css",
+                      "main.css",
+                      "main.js",
+                      "normalize.css",
+                      "rustdoc.css",
+                      "storage.js",
+                      "theme.js"],
                      // files doesn't require rustc version subfix
                      ["FiraSans-Medium.woff",
                       "FiraSans-Regular.woff",
@@ -385,11 +392,10 @@ impl DocBuilder {
         try!(create_dir_all(&destination));
 
         for file in files.0.iter() {
-            let source_path = source.join(file);
-            let destination_path = {
-                let spl: Vec<&str> = file.split('.').collect();
-                destination.join(format!("{}-{}.{}", spl[0], rustc_version, spl[1]))
-            };
+            let spl: Vec<&str> = file.split('.').collect();
+            let file_name = format!("{}-{}.{}", spl[0], rustc_version, spl[1]);
+            let source_path = source.join(&file_name);
+            let destination_path = destination.join(&file_name);
             try!(copy(source_path, destination_path));
         }
 
@@ -409,18 +415,6 @@ impl DocBuilder {
 }
 
 
-/// Simple function to capture command output
-fn command_result(output: Output) -> Result<String> {
-    let mut command_out = String::from_utf8_lossy(&output.stdout).into_owned();
-    command_out.push_str(&String::from_utf8_lossy(&output.stderr).into_owned()[..]);
-    match output.status.success() {
-        true => Ok(command_out),
-        false => Err(command_out.into()),
-    }
-}
-
-
-
 /// Returns canonical name of a package.
 ///
 /// It's just package-version. All directory structure used in cratesfyi is
@@ -432,20 +426,6 @@ fn canonical_name(package: &Package) -> String {
 }
 
 
-/// Parses rustc commit hash from rustc version string
-fn parse_rustc_version<S: AsRef<str>>(version: S) -> String {
-    let version_regex = Regex::new(r" ([\w-.]+) \((\w+) (\d+)-(\d+)-(\d+)\)").unwrap();
-    let captures = version_regex.captures(version.as_ref()).expect("Failed to parse rustc version");
-
-    format!("{}{}{}-{}-{}",
-            captures.get(3).unwrap().as_str(),
-            captures.get(4).unwrap().as_str(),
-            captures.get(5).unwrap().as_str(),
-            captures.get(1).unwrap().as_str(),
-            captures.get(2).unwrap().as_str())
-}
-
-
 /// Runs `func` with the all crates from crates-io.index repository path.
 ///
 /// First argument of func is the name of crate and
@@ -460,7 +440,6 @@ fn crates<F>(path: PathBuf, mut func: F) -> Result<()>
 #[cfg(test)]
 mod test {
     extern crate env_logger;
-    use super::parse_rustc_version;
     use std::path::PathBuf;
     use {DocBuilder, DocBuilderOptions};
 
@@ -484,14 +463,6 @@ mod test {
         assert!(res.is_ok());
     }
 
-    #[test]
-    fn test_parse_rustc_version() {
-        assert_eq!(parse_rustc_version("rustc 1.10.0-nightly (57ef01513 2016-05-23)"),
-                   "20160523-1.10.0-nightly-57ef01513");
-        assert_eq!(parse_rustc_version("cratesfyi 0.2.0 (ba9ae23 2016-05-26)"),
-                   "20160526-0.2.0-ba9ae23");
-    }
-
     #[test]
     #[ignore]
     fn test_add_essential_files() {

src/docbuilder/chroot_builder.rs

@@ -7,6 +7,7 @@ use cargo;
 use reqwest;
 use magic::MagicError;
 use git2;
+use regex;
 
 
 error_chain! {
@@ -19,5 +20,6 @@ error_chain! {
         Git2Error(git2::Error);
         MagicError(MagicError);
         CargoError(Box<cargo::CargoError>);
+        RegexError(regex::Error);
     }
 }

src/error.rs

@@ -12,6 +12,9 @@ use cargo::util::{CargoResult, Config, human, Filesystem};
 use cargo::sources::SourceConfigMap;
 use cargo::ops::{self, Packages, DefaultExecutor};
 
+use utils::{get_current_versions, parse_rustc_version};
+use error::Result;
+
 use Metadata;
 
 
@@ -23,7 +26,7 @@ use Metadata;
 // idea is to make cargo to download
 // and build a crate and its documentation
 // instead of doing it manually like in the previous version of cratesfyi
-pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoResult<Package> {
+pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> Result<Package> {
     let config = try!(Config::default());
     let source_id = try!(SourceId::crates_io(&config));
 
@@ -55,6 +58,16 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoR
         env::set_var("RUSTFLAGS", rustc_args.join(" "));
     }
 
+    // since https://github.com/rust-lang/rust/pull/48511 we can pass --resource-suffix to
+    // add correct version numbers to css and javascript files
+    let mut rustdoc_args: Vec<String> =
+        vec!["-Z".to_string(), "unstable-options".to_string(),
+             "--resource-suffix".to_string(),
+             format!("-{}", parse_rustc_version(get_current_versions()?.0)?)];
+    if let Some(package_rustdoc_args) = metadata.rustdoc_args {
+        rustdoc_args.append(&mut package_rustdoc_args.iter().map(|s| s.to_owned()).collect());
+    }
+
     let opts = ops::CompileOptions {
         config: &config,
         jobs: None,
@@ -72,7 +85,7 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoR
                                         &[], false,
                                         &[], false),
         target_rustc_args: None,
-        target_rustdoc_args: metadata.rustdoc_args.as_ref().map(Vec::as_slice),
+        target_rustdoc_args: Some(rustdoc_args.as_slice()),
     };
 
     let ws = try!(Workspace::ephemeral(pkg, &config, Some(Filesystem::new(target_dir)), false));
@@ -92,6 +105,8 @@ pub fn get_package(name: &str, vers: Option<&str>) -> CargoResult<Package> {
     let source_map = try!(SourceConfigMap::new(&config));
     let mut source = try!(source_map.load(&source_id));
 
+    try!(source.update());
+
     let dep = try!(Dependency::parse_no_deprecated(name, vers, &source_id));
     let deps = try!(source.query(&dep));
     let pkg = try!(deps.iter().map(|p| p.package_id()).max()
@@ -128,23 +143,8 @@ pub fn source_path(pkg: &Package) -> Option<&Path> {
 #[cfg(test)]
 mod test {
     use std::path::Path;
-    use std::fs::remove_dir_all;
     use super::*;
 
-    #[test]
-    fn test_build_doc() {
-        let doc = build_doc("rand", None, None);
-        assert!(doc.is_ok());
-
-        let root_path = Path::new("cratesfyi");
-        assert!(root_path.join("doc").join("rand").exists());
-
-        remove_dir_all(root_path).unwrap();
-
-        let doc = build_doc("SOMECRATEWICHWILLBENVEREXISTS", None, None);
-        assert!(doc.is_err());
-    }
-
     #[test]
     fn test_get_package() {
         let pkg = get_package("rand", None);

src/utils/build_doc.rs

@@ -2,8 +2,6 @@
 // FIXME: There is so many PathBuf's in this module
 //        Conver them to Path
 
-use std::io::prelude::*;
-use std::io;
 use std::path::{Path, PathBuf};
 use std::fs;
 use error::Result;
@@ -48,13 +46,17 @@ fn copy_files_and_handle_html(source: PathBuf,
                               target: bool)
                               -> Result<()> {
 
+    // FIXME: handle_html is useless since we started using --resource-suffix
+    //        argument with rustdoc
+
     // Make sure destination directory is exists
     if !destination.exists() {
         try!(fs::create_dir_all(&destination));
     }
 
-    // Avoid copying duplicated files
-    let dup_regex = Regex::new(r"(\.lock|\.txt|\.woff|jquery\.js|playpen\.js|main\.js|\.css)$")
+    // Avoid copying common files
+    let dup_regex = Regex::new(
+        r"(\.lock|\.txt|\.woff|\.svg|\.css|main-.*\.css|main-.*\.js|normalize-.*\.js|rustdoc-.*\.css|storage-.*\.js|theme-.*\.js)$")
         .unwrap();
 
     for file in try!(source.read_dir()) {
@@ -72,8 +74,6 @@ fn copy_files_and_handle_html(source: PathBuf,
                                             handle_html,
                                             &rustc_version,
                                             target));
-        } else if handle_html && file.file_name().into_string().unwrap().ends_with(".html") {
-            try!(copy_html(&file.path(), &destination_full_path, rustc_version, target));
         } else if handle_html && dup_regex.is_match(&file.file_name().into_string().unwrap()[..]) {
             continue;
         } else {
@@ -85,45 +85,6 @@ fn copy_files_and_handle_html(source: PathBuf,
 }
 
 
-fn copy_html(source: &PathBuf,
-             destination: &PathBuf,
-             rustc_version: &str,
-             target: bool)
-             -> Result<()> {
-
-    let source_file = try!(fs::File::open(source));
-    let mut destination_file = try!(fs::OpenOptions::new()
-        .write(true)
-        .create(true)
-        .open(destination));
-
-    let reader = io::BufReader::new(source_file);
-
-    // FIXME: We don't need to store common libraries (jquery and normalize) for the each version
-    //        of rustc. I believe storing only one version of this files should work in every
-    //        documentation page.
-    let replace_regex =
-        Regex::new(r#"(href|src)="(.*)(main|jquery|rustdoc|playpen|normalize)\.(css|js)""#)
-            .unwrap();
-    let replace_str = format!("$1=\"{}../../$2$3-{}.$4\"",
-                              if target { "../" } else { "" },
-                              rustc_version);
-
-    for line in reader.lines() {
-        let mut line = try!(line);
-
-        // replace css links
-        line = replace_regex.replace_all(&line[..], &replace_str[..]).into_owned();
-
-        try!(destination_file.write(line.as_bytes()));
-        // need to write consumed newline
-        try!(destination_file.write(&['\n' as u8]));
-    }
-
-    Ok(())
-}
-
-
 
 #[cfg(test)]
 mod test {

src/utils/copy.rs

@@ -6,10 +6,12 @@ pub use self::copy::{copy_dir, copy_doc_dir};
 pub use self::github_updater::github_updater;
 pub use self::release_activity_updater::update_release_activity;
 pub use self::daemon::start_daemon;
+pub use self::rustc_version::{parse_rustc_version, get_current_versions, command_result};
 
 mod github_updater;
 mod build_doc;
 mod copy;
 mod release_activity_updater;
 mod daemon;
 mod pubsubhubbub;
+mod rustc_version;