Add index preprocessor (#685)

weihanglo · Michael-F-Bryan · commit 69599646e729 · 2018-05-04T19:41:28.000+08:00
* Add index preprocessor

README.md is a de facto index file in markdown-based documentation.
Hence, we respect to README.md and convert it into index.html.

* Fix warning for unused variables

* Update tests for config

* Match file stem case-insensitively for IndexPreprocessor

* Add tests for IndexPreprocessor

* Update book example to fit index preprocessor
diff --git a/book-example/src/SUMMARY.md b/book-example/src/SUMMARY.md
@@ -1,24 +1,26 @@
 # Summary
 
 - [mdBook](README.md)
-- [Command Line Tool](cli/cli-tool.md)
+- [Command Line Tool](cli/README.md)
     - [init](cli/init.md)
     - [build](cli/build.md)
     - [watch](cli/watch.md)
     - [serve](cli/serve.md)
     - [test](cli/test.md)
     - [clean](cli/clean.md)
-- [Format](format/format.md)
+- [Format](format/README.md)
     - [SUMMARY.md](format/summary.md)
     - [Configuration](format/config.md)
-    - [Theme](format/theme/theme.md)
+    - [Theme](format/theme/README.md)
         - [index.hbs](format/theme/index-hbs.md)
         - [Syntax highlighting](format/theme/syntax-highlighting.md)
         - [Editor](format/theme/editor.md)
     - [MathJax Support](format/mathjax.md)
     - [mdBook specific features](format/mdbook.md)
-- [For Developers](for_developers/index.md)
+- [For Developers](for_developers/README.md)
     - [Preprocessors](for_developers/preprocessors.md)
     - [Alternate Backends](for_developers/backends.md)
+
 -----------
+
 [Contributors](misc/contributors.md)
diff --git a/book-example/src/cli/README.md b/book-example/src/cli/README.md
diff --git a/book-example/src/for_developers/README.md b/book-example/src/for_developers/README.md
diff --git a/book-example/src/format/README.md b/book-example/src/format/README.md
diff --git a/book-example/src/format/theme/README.md b/book-example/src/format/theme/README.md
diff --git a/book-example/src/misc/contributors.md b/book-example/src/misc/contributors.md
@@ -16,3 +16,4 @@ If you have contributed to mdBook and I forgot to add you, don't hesitate to add
 - [projektir](https://github.com/projektir)
 - [Phaiax](https://github.com/Phaiax)
 - [Matt Ickstadt](https://github.com/mattico)
+- Weihang Lo ([@weihanglo](https://github.com/weihanglo))
diff --git a/src/book/mod.rs b/src/book/mod.rs
@@ -21,7 +21,12 @@ use toml::Value;
 
 use utils;
 use renderer::{CmdRenderer, HtmlHandlebars, RenderContext, Renderer};
-use preprocess::{LinkPreprocessor, Preprocessor, PreprocessorContext};
+use preprocess::{
+    LinkPreprocessor,
+    IndexPreprocessor,
+    Preprocessor,
+    PreprocessorContext
+};
 use errors::*;
 
 use config::Config;
@@ -218,6 +223,7 @@ impl MDBook {
         let preprocess_context = PreprocessorContext::new(self.root.clone(), self.config.clone());
 
         LinkPreprocessor::new().run(&preprocess_context, &mut self.book)?;
+        IndexPreprocessor::new().run(&preprocess_context, &mut self.book)?;
 
         for item in self.iter() {
             if let BookItem::Chapter(ref ch) = *item {
@@ -322,15 +328,19 @@ fn determine_renderers(config: &Config) -> Vec<Box<Renderer>> {
 }
 
 fn default_preprocessors() -> Vec<Box<Preprocessor>> {
-    vec![Box::new(LinkPreprocessor::new())]
+    vec![
+        Box::new(LinkPreprocessor::new()),
+        Box::new(IndexPreprocessor::new()),
+    ]
 }
 
 /// Look at the `MDBook` and try to figure out what preprocessors to run.
 fn determine_preprocessors(config: &Config) -> Result<Vec<Box<Preprocessor>>> {
     let preprocess_list = match config.build.preprocess {
         Some(ref p) => p,
-        // If no preprocessor field is set, default to the LinkPreprocessor. This allows you
-        // to disable the LinkPreprocessor by setting "preprocess" to an empty list.
+        // If no preprocessor field is set, default to the LinkPreprocessor and
+        // IndexPreprocessor. This allows you to disable default preprocessors
+        // by setting "preprocess" to an empty list.
         None => return Ok(default_preprocessors()),
     };
 
@@ -339,6 +349,7 @@ fn determine_preprocessors(config: &Config) -> Result<Vec<Box<Preprocessor>>> {
     for key in preprocess_list {
         match key.as_ref() {
             "links" => preprocessors.push(Box::new(LinkPreprocessor::new())),
+            "index" => preprocessors.push(Box::new(IndexPreprocessor::new())),
             _ => bail!("{:?} is not a recognised preprocessor", key),
         }
     }
@@ -403,7 +414,7 @@ mod tests {
     }
 
     #[test]
-    fn config_defaults_to_link_preprocessor_if_not_set() {
+    fn config_defaults_to_link_and_index_preprocessor_if_not_set() {
         let cfg = Config::default();
 
         // make sure we haven't got anything in the `output` table
@@ -412,8 +423,9 @@ mod tests {
         let got = determine_preprocessors(&cfg);
 
         assert!(got.is_ok());
-        assert_eq!(got.as_ref().unwrap().len(), 1);
+        assert_eq!(got.as_ref().unwrap().len(), 2);
         assert_eq!(got.as_ref().unwrap()[0].name(), "links");
+        assert_eq!(got.as_ref().unwrap()[1].name(), "index");
     }
 
     #[test]
diff --git a/src/preprocess/index.rs b/src/preprocess/index.rs
@@ -0,0 +1,91 @@
+use std::path::Path;
+use regex::Regex;
+
+use errors::*;
+
+use super::{Preprocessor, PreprocessorContext};
+use book::{Book, BookItem};
+
+/// A preprocessor for converting file name `README.md` to `index.md` since
+/// `README.md` is the de facto index file in a markdown-based documentation.
+pub struct IndexPreprocessor;
+
+impl IndexPreprocessor {
+    /// Create a new `IndexPreprocessor`.
+    pub fn new() -> Self {
+        IndexPreprocessor
+    }
+}
+
+impl Preprocessor for IndexPreprocessor {
+    fn name(&self) -> &str {
+        "index"
+    }
+
+    fn run(&self, ctx: &PreprocessorContext, book: &mut Book) -> Result<()> {
+        let source_dir = ctx.root.join(&ctx.config.book.src);
+        book.for_each_mut(|section: &mut BookItem| {
+            if let BookItem::Chapter(ref mut ch) = *section {
+                if is_readme_file(&ch.path) {
+                    let index_md = source_dir
+                        .join(ch.path.with_file_name("index.md"));
+                    if index_md.exists() {
+                        warn_readme_name_conflict(&ch.path, &index_md);
+                    }
+
+                    ch.path.set_file_name("index.md");
+                }
+            }
+        });
+
+        Ok(())
+    }
+}
+
+fn warn_readme_name_conflict<P: AsRef<Path>>(readme_path: P, index_path: P) {
+    let file_name = readme_path.as_ref().file_name().unwrap_or_default();
+    let parent_dir = index_path.as_ref().parent().unwrap_or(index_path.as_ref());
+    warn!("It seems that there are both {:?} and index.md under \"{}\".", file_name, parent_dir.display());
+    warn!("mdbook converts {:?} into index.html by default. It may cause", file_name);
+    warn!("unexpected behavior if putting both files under the same directory.");
+    warn!("To solve the warning, try to rearrange the book structure or disable");
+    warn!("\"index\" preprocessor to stop the conversion.");
+}
+
+fn is_readme_file<P: AsRef<Path>>(path: P) -> bool {
+    lazy_static! {
+        static ref RE: Regex = Regex::new(r"(?i)^readme$").unwrap();
+    }
+    RE.is_match(
+        path.as_ref()
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .unwrap_or_default()
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn file_stem_exactly_matches_readme_case_insensitively() {
+        let path = "path/to/Readme.md";
+        assert!(is_readme_file(path));
+
+        let path = "path/to/README.md";
+        assert!(is_readme_file(path));
+
+        let path = "path/to/rEaDmE.md";
+        assert!(is_readme_file(path));
+
+        let path = "path/to/README.markdown";
+        assert!(is_readme_file(path));
+
+        let path = "path/to/README";
+        assert!(is_readme_file(path));
+
+        let path = "path/to/README-README.md";
+        assert!(!is_readme_file(path));
+    }
+}
diff --git a/src/preprocess/mod.rs b/src/preprocess/mod.rs
@@ -1,16 +1,18 @@
 //! Book preprocessing.
 
 pub use self::links::LinkPreprocessor;
+pub use self::index::IndexPreprocessor;
 
 mod links;
+mod index;
 
 use book::Book;
 use config::Config;
 use errors::*;
 
 use std::path::PathBuf;
 
-/// Extra information for a `Preprocessor` to give them more context when 
+/// Extra information for a `Preprocessor` to give them more context when
 /// processing a book.
 pub struct PreprocessorContext {
     /// The location of the book directory on disk.
@@ -26,7 +28,7 @@ impl PreprocessorContext {
     }
 }
 
-/// An operation which is run immediately after loading a book into memory and 
+/// An operation which is run immediately after loading a book into memory and
 /// before it gets rendered.
 pub trait Preprocessor {
     /// Get the `Preprocessor`'s name.
@@ -35,4 +37,4 @@ pub trait Preprocessor {
     /// Run this `Preprocessor`, allowing it to update the book before it is
     /// given to a renderer.
     fn run(&self, ctx: &PreprocessorContext, book: &mut Book) -> Result<()>;
-}
+}
diff --git a/tests/dummy_book/src2/README.md b/tests/dummy_book/src2/README.md
@@ -0,0 +1 @@
+# Root README
diff --git a/tests/dummy_book/src2/SUMMARY.md b/tests/dummy_book/src2/SUMMARY.md
@@ -0,0 +1,7 @@
+# This dummy book is for testing the conversion of README.md to index.html by IndexPreprocessor
+
+[Root README](README.md)
+
+- [1st README](first/README.md)
+- [2nd README](second/README.md)
+    - [2nd index](second/index.md)
diff --git a/tests/dummy_book/src2/first/README.md b/tests/dummy_book/src2/first/README.md
@@ -0,0 +1 @@
+# First README
diff --git a/tests/dummy_book/src2/second/README.md b/tests/dummy_book/src2/second/README.md
@@ -0,0 +1 @@
+# Second README
diff --git a/tests/dummy_book/src2/second/index.md b/tests/dummy_book/src2/second/index.md
@@ -0,0 +1 @@
+# Second index
diff --git a/tests/rendered_output.rs b/tests/rendered_output.rs
@@ -340,6 +340,36 @@ fn book_with_a_reserved_filename_does_not_build() {
     assert!(got.is_err());
 }
 
+#[test]
+fn by_default_mdbook_use_index_preprocessor_to_convert_readme_to_index() {
+    let temp = DummyBook::new().build().unwrap();
+    let mut cfg = Config::default();
+    cfg.set("book.src", "src2").expect("Couldn't set config.book.src to \"src2\".");
+    let md = MDBook::load_with_config(temp.path(), cfg).unwrap();
+    md.build().unwrap();
+
+    let first_index = temp.path()
+        .join("book")
+        .join("first")
+        .join("index.html");
+    let expected_strings = vec![
+        r#"href="first/index.html""#,
+        r#"href="second/index.html""#,
+        "First README",
+    ];
+    assert_contains_strings(&first_index, &expected_strings);
+    assert_doesnt_contain_strings(&first_index, &vec!["README.html"]);
+
+    let second_index = temp.path()
+        .join("book")
+        .join("second")
+        .join("index.html");
+    let unexpected_strings = vec![
+        "Second README",
+    ];
+    assert_doesnt_contain_strings(&second_index, &unexpected_strings);
+}
+
 #[cfg(feature = "search")]
 mod search {
     extern crate serde_json;
