Update robots.txt

raselshikdar · web-flow · commit 11e4c67141c0 · 2024-10-05T23:00:42.000+06:00
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,2 +1,16 @@
 User-agent: *
 Allow: /
+
+Disallow: /wp-admin/
+Disallow: /cgi-bin/
+Disallow: /search
+
+# Disallow access to sensitive files
+Disallow: /private/
+Disallow: /temp/
+
+# Allow all crawlers to index sitemap
+Sitemap: https://info.rasel.us.kg/sitemap.xml
+
+# Optional: Specify crawl delay to prevent server overload
+Crawl-delay: 10
