Rollup merge of #124731 - dalance:add_translation_support, r=ehuss

matthiaskrgr · web-flow · commit 3b6eefe8bf64 · 2024-06-06T04:17:25.000+02:00
Add translation support by mdbook-i18n-helpers to bootstrap This PR add translation support by [mdbook-i18n-helpers](https://github.com/google/mdbook-i18n-helpers) to bootstrap. This is draft PR because there is the dependency to my forked mdbook-i18n-helpers. If this PR is acceptable, I'll send a PR to the original mdbook-i18n-helpers and remove draft after changing the dependency to the original. Closes #124641
diff --git a/Cargo.lock b/Cargo.lock
@@ -372,7 +372,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
 dependencies = [
  "memchr",
- "regex-automata 0.3.9",
+ "regex-automata 0.3.7",
  "serde",
 ]
 
@@ -2206,6 +2206,21 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "line-wrap"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e"
+
+[[package]]
+name = "linereader"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d921fea6860357575519aca014c6e22470585accdd543b370c404a8a72d0dd1d"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "linkchecker"
 version = "0.1.0"
@@ -2214,6 +2229,12 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "lint-docs"
 version = "0.1.0"
@@ -2364,6 +2385,25 @@ dependencies = [
  "topological-sort",
 ]
 
+[[package]]
+name = "mdbook-i18n-helpers"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1f71f5961d6f3376e1ff3e5989c2e3ecccc3e8a00f3a3acde446847f84852e4"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "mdbook",
+ "polib",
+ "pulldown-cmark 0.10.3",
+ "pulldown-cmark-to-cmark",
+ "regex",
+ "semver",
+ "serde_json",
+ "syntect",
+ "textwrap",
+]
+
 [[package]]
 name = "mdbook-trpl-listing"
 version = "0.1.0"
@@ -2650,6 +2690,28 @@ version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
 
+[[package]]
+name = "onig"
+version = "6.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "opener"
 version = "0.6.1"
@@ -2968,6 +3030,29 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "plist"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9d34169e64b3c7a80c8621a48adaf44e0cf62c78a9b25dd9dd35f1881a17cf9"
+dependencies = [
+ "base64",
+ "indexmap",
+ "line-wrap",
+ "quick-xml",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "polib"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b393b155cf9be86249cba1b56cc81be0e6212c66d94ac0d76d37a1761f3bb1b"
+dependencies = [
+ "linereader",
+]
+
 [[package]]
 name = "polonius-engine"
 version = "0.13.0"
@@ -3126,6 +3211,15 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe"
 
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quine-mc_cluskey"
 version = "0.2.4"
@@ -3261,12 +3355,13 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29"
 dependencies = [
  "aho-corasick",
  "memchr",
+ "regex-automata 0.3.7",
  "regex-syntax 0.7.5",
 ]
 
@@ -3290,9 +3385,14 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.9"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9"
+checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax 0.7.5",
+]
 
 [[package]]
 name = "regex-lite"
@@ -3367,6 +3467,7 @@ dependencies = [
  "clap",
  "env_logger",
  "mdbook",
+ "mdbook-i18n-helpers",
  "mdbook-trpl-listing",
  "mdbook-trpl-note",
 ]
@@ -5372,6 +5473,28 @@ dependencies = [
  "syn 2.0.64",
 ]
 
+[[package]]
+name = "syntect"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
+dependencies = [
+ "bincode",
+ "bitflags 1.3.2",
+ "flate2",
+ "fnv",
+ "once_cell",
+ "onig",
+ "plist",
+ "regex-syntax 0.8.3",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "thiserror",
+ "walkdir",
+ "yaml-rust",
+]
+
 [[package]]
 name = "sysinfo"
 version = "0.30.12"
@@ -5497,6 +5620,12 @@ dependencies = [
  "std",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
+
 [[package]]
 name = "thin-vec"
 version = "0.2.13"
@@ -6476,6 +6605,15 @@ dependencies = [
  "lzma-sys",
 ]
 
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
 [[package]]
 name = "yansi-term"
 version = "0.1.2"
diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
@@ -29,7 +29,7 @@ macro_rules! submodule_helper {
 }
 
 macro_rules! book {
-    ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
+    ($($name:ident, $path:expr, $book_name:expr, $lang:expr $(, submodule $(= $submodule:literal)? )? ;)+) => {
         $(
             #[derive(Debug, Clone, Hash, PartialEq, Eq)]
         pub struct $name {
@@ -61,6 +61,7 @@ macro_rules! book {
                     name: $book_name.to_owned(),
                     src: builder.src.join($path),
                     parent: Some(self),
+                    languages: $lang.into(),
                 })
             }
         }
@@ -74,15 +75,15 @@ macro_rules! book {
 // FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules
 // and checking against it?).
 book!(
-    CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo";
-    ClippyBook, "src/tools/clippy/book", "clippy";
-    EditionGuide, "src/doc/edition-guide", "edition-guide", submodule;
-    EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule;
-    Nomicon, "src/doc/nomicon", "nomicon", submodule;
-    Reference, "src/doc/reference", "reference", submodule;
-    RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule;
-    RustdocBook, "src/doc/rustdoc", "rustdoc";
-    StyleGuide, "src/doc/style-guide", "style-guide";
+    CargoBook, "src/tools/cargo/src/doc", "cargo", &[], submodule = "src/tools/cargo";
+    ClippyBook, "src/tools/clippy/book", "clippy", &[];
+    EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule;
+    EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule;
+    Nomicon, "src/doc/nomicon", "nomicon", &[], submodule;
+    Reference, "src/doc/reference", "reference", &[], submodule;
+    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja"], submodule;
+    RustdocBook, "src/doc/rustdoc", "rustdoc", &[];
+    StyleGuide, "src/doc/style-guide", "style-guide", &[];
 );
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
@@ -110,6 +111,7 @@ impl Step for UnstableBook {
             name: "unstable-book".to_owned(),
             src: builder.md_doc_out(self.target).join("unstable-book"),
             parent: Some(self),
+            languages: vec![],
         })
     }
 }
@@ -120,6 +122,7 @@ struct RustbookSrc<P: Step> {
     name: String,
     src: PathBuf,
     parent: Option<P>,
+    languages: Vec<&'static str>,
 }
 
 impl<P: Step> Step for RustbookSrc<P> {
@@ -151,7 +154,19 @@ impl<P: Step> Step for RustbookSrc<P> {
             builder.info(&format!("Rustbook ({target}) - {name}"));
             let _ = fs::remove_dir_all(&out);
 
-            builder.run(rustbook_cmd.arg("build").arg(src).arg("-d").arg(out));
+            builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out));
+
+            for lang in &self.languages {
+                let out = out.join(lang);
+
+                builder.info(&format!("Rustbook ({target}) - {name} - {lang}"));
+                let _ = fs::remove_dir_all(&out);
+
+                let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);
+                builder.run(
+                    rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).arg("-l").arg(lang),
+                );
+            }
         }
 
         if self.parent.is_some() {
@@ -214,6 +229,7 @@ impl Step for TheBook {
             name: "book".to_owned(),
             src: absolute_path.clone(),
             parent: Some(self),
+            languages: vec![],
         });
 
         // building older edition redirects
@@ -225,6 +241,7 @@ impl Step for TheBook {
                 // There should only be one book that is marked as the parent for each target, so
                 // treat the other editions as not having a parent.
                 parent: Option::<Self>::None,
+                languages: vec![],
             });
         }
 
@@ -1208,6 +1225,7 @@ impl Step for RustcBook {
             name: "rustc".to_owned(),
             src: out_base,
             parent: Some(self),
+            languages: vec![],
         });
     }
 }
diff --git a/src/tools/rustbook/Cargo.toml b/src/tools/rustbook/Cargo.toml
@@ -9,6 +9,7 @@ clap = "4.0.32"
 env_logger = "0.11"
 mdbook-trpl-listing = { path = "../../doc/book/packages/mdbook-trpl-listing" }
 mdbook-trpl-note = { path = "../../doc/book/packages/mdbook-trpl-note" }
+mdbook-i18n-helpers = "0.3.3"
 
 [dependencies.mdbook]
 version = "0.4.37"
diff --git a/src/tools/rustbook/src/main.rs b/src/tools/rustbook/src/main.rs
@@ -7,6 +7,7 @@ use clap::{arg, ArgMatches, Command};
 
 use mdbook::errors::Result as Result3;
 use mdbook::MDBook;
+use mdbook_i18n_helpers::preprocessors::Gettext;
 
 use mdbook_trpl_listing::TrplListing;
 use mdbook_trpl_note::TrplNote;
@@ -19,6 +20,11 @@ fn main() {
     .required(false)
     .value_parser(clap::value_parser!(PathBuf));
 
+    let l_arg = arg!(-l --"lang" <LANGUAGE>
+"The output language")
+    .required(false)
+    .value_parser(clap::value_parser!(String));
+
     let dir_arg = arg!([dir] "Root directory for the book\n\
                               (Defaults to the current directory when omitted)")
     .value_parser(clap::value_parser!(PathBuf));
@@ -33,6 +39,7 @@ fn main() {
             Command::new("build")
                 .about("Build the book from the markdown files")
                 .arg(d_arg)
+                .arg(l_arg)
                 .arg(&dir_arg),
         )
         .subcommand(
@@ -63,6 +70,12 @@ pub fn build(args: &ArgMatches) -> Result3<()> {
     let book_dir = get_book_dir(args);
     let mut book = load_book(&book_dir)?;
 
+    if let Some(lang) = args.get_one::<String>("lang") {
+        let gettext = Gettext;
+        book.with_preprocessor(gettext);
+        book.config.set("book.language", lang).unwrap();
+    }
+
     // Set this to allow us to catch bugs in advance.
     book.config.build.create_missing = false;
 
