From b3fc3f49cf3dd0c6a2ca5be8ef2fa749a13a82b6 Mon Sep 17 00:00:00 2001
From: Martin Vassor <martin.vassor@alumni.epfl.ch>
Date: Thu, 25 Jul 2024 10:02:33 +0100
Subject: [PATCH] Add fields to disable sitemap.xml and robots.txt generation
 (#2579)

* Add feature to disable robots.txt and sitemap.xml from the config file.

Addresses feature request #2248

* Add documentation for no_sitemap & no_robots

* Rename no_robots and no_sitemap into generate_robots_txt and generate_sitemap (default to true)

* fix rustfmt issues

* Change documentation to show defaults

* Update documentation for the fields generate_sitemaps (resp. robots_txt), now that the default is true and false is needed to disable
---
 components/config/src/config/mod.rs           | 74 +++++++++++++++++++
 components/site/src/lib.rs                    | 12 ++-
 .../getting-started/configuration.md          |  6 ++
 3 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/components/config/src/config/mod.rs b/components/config/src/config/mod.rs
index 57ff1a805..88d949fc2 100644
--- a/components/config/src/config/mod.rs
+++ b/components/config/src/config/mod.rs
@@ -98,6 +98,10 @@ pub struct Config {
     pub markdown: markup::Markdown,
     /// All user params set in `[extra]` in the config
     pub extra: HashMap<String, Toml>,
+    /// Enables the generation of Sitemap.xml
+    pub generate_sitemap: bool,
+    /// Enables the generation of robots.txt
+    pub generate_robots_txt: bool,
 }
 
 #[derive(Serialize)]
@@ -117,6 +121,8 @@ pub struct SerializedConfig<'a> {
     extra: &'a HashMap<String, Toml>,
     markdown: &'a markup::Markdown,
     search: search::SerializedSearch<'a>,
+    generate_sitemap: bool,
+    generate_robots_txt: bool,
 }
 
 impl Config {
@@ -332,6 +338,8 @@ impl Config {
             extra: &self.extra,
             markdown: &self.markdown,
             search: self.search.serialize(),
+            generate_sitemap: self.generate_sitemap,
+            generate_robots_txt: self.generate_robots_txt,
         }
     }
 }
@@ -395,6 +403,8 @@ impl Default for Config {
             search: search::Search::default(),
             markdown: markup::Markdown::default(),
             extra: HashMap::new(),
+            generate_sitemap: true,
+            generate_robots_txt: true,
         }
     }
 }
@@ -992,4 +1002,68 @@ feed_filename = "test.xml"
 
         Config::parse(config).unwrap();
     }
+
+    #[test]
+    fn parse_generate_sitemap_true() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+generate_sitemap = true
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(config.generate_sitemap);
+    }
+
+    #[test]
+    fn parse_generate_sitemap_false() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+generate_sitemap = false
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(!config.generate_sitemap);
+    }
+
+    #[test]
+    fn default_no_sitemap_true() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(config.generate_sitemap);
+    }
+
+    #[test]
+    fn parse_generate_robots_true() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+generate_robots_txt = true
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(config.generate_robots_txt);
+    }
+
+    #[test]
+    fn parse_generate_robots_false() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+generate_robots_txt = false
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(!config.generate_robots_txt);
+    }
+
+    #[test]
+    fn default_no_robots_true() {
+        let config = r#"
+title = "My Site"
+base_url = "example.com"
+"#;
+        let config = Config::parse(config).unwrap();
+        assert!(config.generate_robots_txt);
+    }
 }
diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs
index 46c62c959..cd48c219e 100644
--- a/components/site/src/lib.rs
+++ b/components/site/src/lib.rs
@@ -742,8 +742,10 @@ impl Site {
         start = log_time(start, "Rendered sections");
         self.render_orphan_pages()?;
         start = log_time(start, "Rendered orphan pages");
-        self.render_sitemap()?;
-        start = log_time(start, "Rendered sitemap");
+        if self.config.generate_sitemap {
+            self.render_sitemap()?;
+            start = log_time(start, "Rendered sitemap");
+        }
 
         let library = self.library.read().unwrap();
         if self.config.generate_feeds {
@@ -769,8 +771,10 @@ impl Site {
         start = log_time(start, "Rendered themes css");
         self.render_404()?;
         start = log_time(start, "Rendered 404");
-        self.render_robots()?;
-        start = log_time(start, "Rendered robots.txt");
+        if self.config.generate_robots_txt {
+            self.render_robots()?;
+            start = log_time(start, "Rendered robots.txt");
+        }
         self.render_taxonomies()?;
         start = log_time(start, "Rendered taxonomies");
         // We process images at the end as we might have picked up images to process from markdown
diff --git a/docs/content/documentation/getting-started/configuration.md b/docs/content/documentation/getting-started/configuration.md
index 103ae0730..2565e0913 100644
--- a/docs/content/documentation/getting-started/configuration.md
+++ b/docs/content/documentation/getting-started/configuration.md
@@ -100,6 +100,12 @@ taxonomies = []
 # content for `default_language`.
 build_search_index = false
 
+# When set to "false", Sitemap.xml is not generated
+generate_sitemap = true
+
+# When set to "false", robots.txt is not generated
+generate_robots_txt = true
+
 # Configuration of the Markdown rendering
 [markdown]
 # When set to "true", all code blocks are highlighted.