feat: add robots.txt

Author: zeldan

src/main.rs

@@ -5,7 +5,7 @@ async fn main() {
     use blog::app::{self, shell};
     use blog::ssr::app_state::AppState;
     use blog::ssr::redirect::redirect_www;
-    use blog::ssr::server_utils::{connect, rss_handler, sitemap_handler};
+    use blog::ssr::server_utils::{connect, robots_handler, rss_handler, sitemap_handler};
     use dotenvy::dotenv;
     use leptos::logging;
     use leptos::prelude::*;
@@ -58,6 +58,7 @@ async fn main() {
         )
         .route("/rss.xml", get(rss_handler))
         .route("/sitemap.xml", get(sitemap_handler))
+        .route("/robots.txt", get(robots_handler))
         .layer(
             tower::ServiceBuilder::new()
                 .layer(TraceLayer::new_for_http())

src/ssr/server_utils.rs

@@ -332,3 +332,12 @@ pub async fn sitemap_handler(State(state): State<AppState>) -> Response<String>
         .body(sitemap)
         .unwrap()
 }
+
+pub async fn robots_handler() -> Response<String> {
+    let mut robots = String::new();
+    robots.push_str("User-agent: *\nDisallow:\n\nAllow: /\n\nSitemap: https://rust-dd.com/sitemap.xml\n");
+    Response::builder()
+        .header("Content-Type", "text/plain")
+        .body(robots)
+        .unwrap()
+}