generate-copyright: use rinja to format the output

jonathanpallant · jonathanpallant · commit f7e6bf61a9c3 · 2024-08-06T11:04:55.000+01:00
I can't find a way to derive rinja::Template for Node - I think because it is a recursive type. So I rendered it manually using html_escape.
diff --git a/src/tools/generate-copyright/Cargo.toml b/src/tools/generate-copyright/Cargo.toml
@@ -8,8 +8,10 @@ description = "Produces a manifest of all the copyrighted materials in the Rust
 
 [dependencies]
 anyhow = "1.0.65"
+cargo_metadata = "0.18.1"
+html-escape = "0.2.13"
+rinja = "0.2.0"
 serde = { version = "1.0.147", features = ["derive"] }
 serde_json = "1.0.85"
-thiserror = "1"
 tempfile = "3"
-cargo_metadata = "0.18.1"
+thiserror = "1"
diff --git a/src/tools/generate-copyright/src/cargo_metadata.rs b/src/tools/generate-copyright/src/cargo_metadata.rs
@@ -1,7 +1,7 @@
 //! Gets metadata about a workspace from Cargo
 
 use std::collections::BTreeMap;
-use std::ffi::{OsStr, OsString};
+use std::ffi::OsStr;
 use std::path::Path;
 
 /// Describes how this module can fail
@@ -36,7 +36,9 @@ pub struct PackageMetadata {
     /// A list of important files from the package, with their contents.
     ///
     /// This includes *COPYRIGHT*, *NOTICE*, *AUTHOR*, *LICENSE*, and *LICENCE* files, case-insensitive.
-    pub notices: BTreeMap<OsString, String>,
+    pub notices: BTreeMap<String, String>,
+    /// If this is true, this dep is in the Rust Standard Library
+    pub is_in_libstd: Option<bool>,
 }
 
 /// Use `cargo metadata` and `cargo vendor` to get a list of dependencies and their license data.
@@ -101,6 +103,7 @@ pub fn get_metadata(
                     license: package.license.unwrap_or_else(|| String::from("Unspecified")),
                     authors: package.authors,
                     notices: BTreeMap::new(),
+                    is_in_libstd: None,
                 },
             );
         }
@@ -161,8 +164,9 @@ fn load_important_files(
                 if metadata.is_dir() {
                     // scoop up whole directory
                 } else if metadata.is_file() {
-                    println!("Scraping {}", filename.to_string_lossy());
-                    dep.notices.insert(filename.to_owned(), std::fs::read_to_string(path)?);
+                    let filename = filename.to_string_lossy();
+                    println!("Scraping {}", filename);
+                    dep.notices.insert(filename.to_string(), std::fs::read_to_string(path)?);
                 }
             }
         }
diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs
@@ -1,37 +1,17 @@
 use std::collections::BTreeMap;
-use std::io::Write;
 use std::path::{Path, PathBuf};
 
 use anyhow::Error;
+use rinja::Template;
 
 mod cargo_metadata;
 
-static TOP_BOILERPLATE: &str = r##"
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="UTF-8">
-    <title>Copyright notices for The Rust Toolchain</title>
-</head>
-<body>
-
-<h1>Copyright notices for The Rust Toolchain</h1>
-
-<p>This file describes the copyright and licensing information for the source
-code within The Rust Project git tree, and the third-party dependencies used
-when building the Rust toolchain (including the Rust Standard Library).</p>
-
-<h2>Table of Contents</h2>
-<ul>
-    <li><a href="#in-tree-files">In-tree files</a></li>
-    <li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
-</ul>
-"##;
-
-static BOTTOM_BOILERPLATE: &str = r#"
-</body>
-</html>
-"#;
+#[derive(Template)]
+#[template(path = "COPYRIGHT.html")]
+struct CopyrightTemplate {
+    in_tree: Node,
+    dependencies: BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
+}
 
 /// The entry point to the binary.
 ///
@@ -53,150 +33,114 @@ fn main() -> Result<(), Error> {
         Path::new("./src/tools/cargo/Cargo.toml"),
         Path::new("./library/std/Cargo.toml"),
     ];
-    let collected_cargo_metadata =
+    let mut collected_cargo_metadata =
         cargo_metadata::get_metadata_and_notices(&cargo, &out_dir, &root_path, &workspace_paths)?;
 
     let stdlib_set =
         cargo_metadata::get_metadata(&cargo, &root_path, &[Path::new("./library/std/Cargo.toml")])?;
 
-    let mut buffer = Vec::new();
+    for (key, value) in collected_cargo_metadata.iter_mut() {
+        value.is_in_libstd = Some(stdlib_set.contains_key(key));
+    }
 
-    writeln!(buffer, "{}", TOP_BOILERPLATE)?;
+    let template = CopyrightTemplate {
+        in_tree: collected_tree_metadata.files,
+        dependencies: collected_cargo_metadata,
+    };
 
-    writeln!(
-        buffer,
-        r#"<h2 id="in-tree-files">In-tree files</h2><p>The following licenses cover the in-tree source files that were used in this release:</p>"#
-    )?;
-    render_tree_recursive(&collected_tree_metadata.files, &mut buffer)?;
+    let output = template.render()?;
 
-    writeln!(
-        buffer,
-        r#"<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2><p>The following licenses cover the out-of-tree crates that were used in this release:</p>"#
-    )?;
-    render_deps(&collected_cargo_metadata, &stdlib_set, &mut buffer)?;
+    std::fs::write(&dest_file, output)?;
 
-    writeln!(buffer, "{}", BOTTOM_BOILERPLATE)?;
+    Ok(())
+}
 
-    std::fs::write(&dest_file, &buffer)?;
+/// Describes a tree of metadata for our filesystem tree
+#[derive(serde::Deserialize)]
+struct Metadata {
+    files: Node,
+}
 
+/// Describes one node in our metadata tree
+#[derive(serde::Deserialize)]
+#[serde(rename_all = "kebab-case", tag = "type")]
+pub(crate) enum Node {
+    Root { children: Vec<Node> },
+    Directory { name: String, children: Vec<Node>, license: Option<License> },
+    File { name: String, license: License },
+    Group { files: Vec<String>, directories: Vec<String>, license: License },
+}
+
+fn with_box<F>(fmt: &mut std::fmt::Formatter<'_>, inner: F) -> std::fmt::Result
+where
+    F: FnOnce(&mut std::fmt::Formatter<'_>) -> std::fmt::Result,
+{
+    writeln!(fmt, r#"<div style="border:1px solid black; padding: 5px;">"#)?;
+    inner(fmt)?;
+    writeln!(fmt, "</div>")?;
     Ok(())
 }
 
-/// Recursively draw the tree of files/folders we found on disk and their licenses, as
-/// markdown, into the given Vec.
-fn render_tree_recursive(node: &Node, buffer: &mut Vec<u8>) -> Result<(), Error> {
-    writeln!(buffer, r#"<div style="border:1px solid black; padding: 5px;">"#)?;
-    match node {
-        Node::Root { children } => {
-            for child in children {
-                render_tree_recursive(child, buffer)?;
+impl std::fmt::Display for Node {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Node::Root { children } => {
+                if children.len() > 1 {
+                    with_box(fmt, |f| {
+                        for child in children {
+                            writeln!(f, "{child}")?;
+                        }
+                        Ok(())
+                    })
+                } else {
+                    for child in children {
+                        writeln!(fmt, "{child}")?;
+                    }
+                    Ok(())
+                }
             }
-        }
-        Node::Directory { name, children, license } => {
-            render_tree_license(std::iter::once(name), license.as_ref(), buffer)?;
-            if !children.is_empty() {
-                writeln!(buffer, "<p><b>Exceptions:</b></p>")?;
-                for child in children {
-                    render_tree_recursive(child, buffer)?;
+            Node::Directory { name, children, license } => with_box(fmt, |f| {
+                render_tree_license(std::iter::once(name), license.as_ref(), f)?;
+                if !children.is_empty() {
+                    writeln!(f, "<p><b>Exceptions:</b></p>")?;
+                    for child in children {
+                        writeln!(f, "{child}")?;
+                    }
                 }
+                Ok(())
+            }),
+            Node::Group { files, directories, license } => with_box(fmt, |f| {
+                render_tree_license(directories.iter().chain(files.iter()), Some(license), f)
+            }),
+            Node::File { name, license } => {
+                with_box(fmt, |f| render_tree_license(std::iter::once(name), Some(license), f))
             }
         }
-        Node::Group { files, directories, license } => {
-            render_tree_license(directories.iter().chain(files.iter()), Some(license), buffer)?;
-        }
-        Node::File { name, license } => {
-            render_tree_license(std::iter::once(name), Some(license), buffer)?;
-        }
     }
-    writeln!(buffer, "</div>")?;
-
-    Ok(())
 }
 
-/// Draw a series of sibling files/folders, as markdown, into the given Vec.
+/// Draw a series of sibling files/folders, as HTML, into the given formatter.
 fn render_tree_license<'a>(
     names: impl Iterator<Item = &'a String>,
     license: Option<&License>,
-    buffer: &mut Vec<u8>,
-) -> Result<(), Error> {
-    writeln!(buffer, "<p><b>File/Directory:</b> ")?;
+    f: &mut std::fmt::Formatter<'_>,
+) -> std::fmt::Result {
+    writeln!(f, "<p><b>File/Directory:</b> ")?;
     for name in names {
-        writeln!(buffer, "<code>{name}</code>")?;
+        writeln!(f, "<code>{}</code>", html_escape::encode_text(&name))?;
     }
-    writeln!(buffer, "</p>")?;
+    writeln!(f, "</p>")?;
 
     if let Some(license) = license {
-        writeln!(buffer, "<p><b>License:</b> {}</p>", license.spdx)?;
+        writeln!(f, "<p><b>License:</b> {}</p>", html_escape::encode_text(&license.spdx))?;
         for copyright in license.copyright.iter() {
-            writeln!(buffer, "<p><b>Copyright:</b> {copyright}</p>")?;
+            writeln!(f, "<p><b>Copyright:</b> {}</p>", html_escape::encode_text(&copyright))?;
         }
     }
 
     Ok(())
 }
 
-/// Render a list of out-of-tree dependencies as markdown into the given Vec.
-fn render_deps(
-    all_deps: &BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
-    stdlib_set: &BTreeMap<cargo_metadata::Package, cargo_metadata::PackageMetadata>,
-    buffer: &mut Vec<u8>,
-) -> Result<(), Error> {
-    for (package, metadata) in all_deps {
-        let authors_list = if metadata.authors.is_empty() {
-            "None Specified".to_owned()
-        } else {
-            metadata.authors.join(", ")
-        };
-        let url = format!("https://crates.io/crates/{}/{}", package.name, package.version);
-        writeln!(buffer)?;
-        writeln!(
-            buffer,
-            r#"<h3>📦 {name}-{version}</h3>"#,
-            name = package.name,
-            version = package.version,
-        )?;
-        writeln!(buffer, r#"<p><b>URL:</b> <a href="{url}">{url}</a></p>"#,)?;
-        writeln!(
-            buffer,
-            "<p><b>In libstd:</b> {}</p>",
-            if stdlib_set.contains_key(package) { "Yes" } else { "No" }
-        )?;
-        writeln!(buffer, "<p><b>Authors:</b> {}</p>", escape_html(&authors_list))?;
-        writeln!(buffer, "<p><b>License:</b> {}</p>", escape_html(&metadata.license))?;
-        writeln!(buffer, "<p><b>Notices:</b> ")?;
-        if metadata.notices.is_empty() {
-            writeln!(buffer, "None")?;
-        } else {
-            for (name, contents) in &metadata.notices {
-                writeln!(
-                    buffer,
-                    "<details><summary><code>{}</code></summary>",
-                    name.to_string_lossy()
-                )?;
-                writeln!(buffer, "<pre>\n{}\n</pre>", contents)?;
-                writeln!(buffer, "</details>")?;
-            }
-        }
-        writeln!(buffer, "</p>")?;
-    }
-    Ok(())
-}
-/// Describes a tree of metadata for our filesystem tree
-#[derive(serde::Deserialize)]
-struct Metadata {
-    files: Node,
-}
-
-/// Describes one node in our metadata tree
-#[derive(serde::Deserialize)]
-#[serde(rename_all = "kebab-case", tag = "type")]
-pub(crate) enum Node {
-    Root { children: Vec<Node> },
-    Directory { name: String, children: Vec<Node>, license: Option<License> },
-    File { name: String, license: License },
-    Group { files: Vec<String>, directories: Vec<String>, license: License },
-}
-
 /// A License has an SPDX license name and a list of copyright holders.
 #[derive(serde::Deserialize)]
 struct License {
@@ -212,13 +156,3 @@ fn env_path(var: &str) -> Result<PathBuf, Error> {
         anyhow::bail!("missing environment variable {var}")
     }
 }
-
-/// Escapes any invalid HTML characters
-fn escape_html(input: &str) -> String {
-    static MAPPING: [(char, &str); 3] = [('&', "&amp;"), ('<', "&lt;"), ('>', "&gt;")];
-    let mut output = input.to_owned();
-    for (ch, s) in &MAPPING {
-        output = output.replace(*ch, s);
-    }
-    output
-}
diff --git a/src/tools/generate-copyright/templates/COPYRIGHT.html b/src/tools/generate-copyright/templates/COPYRIGHT.html
@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>Copyright notices for The Rust Toolchain</title>
+</head>
+<body>
+
+<h1>Copyright notices for The Rust Toolchain</h1>
+
+<p>This file describes the copyright and licensing information for the source
+code within The Rust Project git tree, and the third-party dependencies used
+when building the Rust toolchain (including the Rust Standard Library).</p>
+
+<h2>Table of Contents</h2>
+<ul>
+    <li><a href="#in-tree-files">In-tree files</a></li>
+    <li><a href="#out-of-tree-dependencies">Out-of-tree dependencies</a></li>
+</ul>
+
+<h2 id="in-tree-files">In-tree files</h2>
+
+<p>The following licenses cover the in-tree source files that were used in this
+release:</p>
+
+{{ in_tree|safe }}
+
+<h2 id="out-of-tree-dependencies">Out-of-tree dependencies</h2>
+
+<p>The following licenses cover the out-of-tree crates that were used in this
+release:</p>
+
+{% for (key, value) in dependencies %}
+    <h3>📦 {{key.name}}-{{key.version}}</h3>
+    <p><b>URL:</b> <a href="https://crates.io/crates/{{ key.name }}/{{ key.version }}">https://crates.io/crates/{{ key.name }}/{{ key.version }}</a></p>
+    <p><b>In libstd:</b> {% if value.is_in_libstd.unwrap() %} Yes {% else %} No {% endif %}</p>
+    <p><b>Authors:</b> {{ value.authors|join(", ") }}</p>
+    <p><b>License:</b> {{ value.license }}</p>
+    {% let len = value.notices.len() %}
+    {% if len > 0 %}
+        <p><b>Notices:</b>
+        {% for (notice_name, notice_text) in value.notices %}
+            <details>
+                <summary><code>{{ notice_name }}</code></summary>
+                <pre>
+{{ notice_text }}
+                </pre>
+            </details>
+        {% endfor %}
+        </p>
+    {% endif %}
+{% endfor %}
+</body>
+</html>
