feat: Create sitemap.xml dynamically

Author: linkdotnet

docs/SEO/Readme.md

@@ -34,7 +34,7 @@ This blog also offers an RSS feed ([RSS 2.0 specification](https://validator.w3.
 
 ### Sitemap
 
-This blog offers to generate a [sitemap](https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap) that lists all blog posts, the archive and pages of the blog. A sitemap can be generated in the Admin tab of the navigation bar under "Sitemap". This allows, especially new sites that don't have many inbound links, to be indexed easier by search engines.
+This blog automatically generates a [sitemap](https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap) that lists all blog posts, the archive and pages of the blog.
 
 ## JSON LD
 This blog supports a JSON-LD for structured data. The current support is limited / rudimentary. Information like `Headline` (the title of the blog post), `Author`, `PublishDated` and `PreviewImage` are present.

src/LinkDotNet.Blog.Web/Controller/SitemapController.cs

@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using System.Xml;
+using System.Xml.Serialization;
+using LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LinkDotNet.Blog.Web.Controller;
+
+[EnableRateLimiting("ip")]
+[Route("sitemap.xml")]
+public sealed class SitemapController : ControllerBase
+{
+    private readonly ISitemapService sitemapService;
+    private readonly IXmlWriter xmlWriter;
+    private readonly IMemoryCache memoryCache;
+
+    public SitemapController(
+        ISitemapService sitemapService,
+        IXmlWriter xmlWriter,
+        IMemoryCache memoryCache)
+    {
+        this.sitemapService = sitemapService;
+        this.xmlWriter = xmlWriter;
+        this.memoryCache = memoryCache;
+    }
+
+    [ResponseCache(Duration = 3600)]
+    [HttpGet]
+    public async Task<IActionResult> GetSitemap()
+    {
+        var buffer = await memoryCache.GetOrCreateAsync("sitemap.xml", async e =>
+                     {
+                         e.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1);
+                         return await GetSitemapBuffer();
+                     })
+                     ?? throw new InvalidOperationException("Buffer is null");
+
+        return File(buffer, "application/xml");
+    }
+
+    private async Task<byte[]> GetSitemapBuffer()
+    {
+        var baseUri = $"{Request.Scheme}://{Request.Host}{Request.PathBase}";
+        var sitemap = await sitemapService.CreateSitemapAsync(baseUri);
+        var buffer = await xmlWriter.WriteToBuffer(sitemap);
+        return buffer;
+    }
+}

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/Services/ISitemapService.cs

@@ -4,7 +4,5 @@ namespace LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
 
 public interface ISitemapService
 {
-    Task<SitemapUrlSet> CreateSitemapAsync();
-
-    Task SaveSitemapToFileAsync(SitemapUrlSet sitemap);
-}
+    Task<SitemapUrlSet> CreateSitemapAsync(string baseUri);
+}

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/Services/IXmlFileWriter.cs

@@ -2,7 +2,7 @@
 
 namespace LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
 
-public interface IXmlFileWriter
+public interface IXmlWriter
 {
-    Task WriteObjectToXmlFileAsync<T>(T objectToSave, string fileName);
-}
+    Task<byte[]> WriteToBuffer<T>(T objectToSave);
+}

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/Services/SitemapService.cs

@@ -2,66 +2,63 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using LinkDotNet.Blog.Domain;
 using LinkDotNet.Blog.Infrastructure.Persistence;
 using Microsoft.AspNetCore.Components;
+using Microsoft.Extensions.Caching.Memory;
 
 namespace LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
 
 public sealed class SitemapService : ISitemapService
 {
     private readonly IRepository<BlogPost> repository;
-    private readonly NavigationManager navigationManager;
-    private readonly IXmlFileWriter xmlFileWriter;
 
-    public SitemapService(
-        IRepository<BlogPost> repository,
-        NavigationManager navigationManager,
-        IXmlFileWriter xmlFileWriter)
+    public SitemapService(IRepository<BlogPost> repository)
     {
         this.repository = repository;
-        this.navigationManager = navigationManager;
-        this.xmlFileWriter = xmlFileWriter;
     }
 
-    public async Task<SitemapUrlSet> CreateSitemapAsync()
+    public async Task<SitemapUrlSet> CreateSitemapAsync(string baseUri)
     {
+        ArgumentException.ThrowIfNullOrEmpty(baseUri);
+
         var urlSet = new SitemapUrlSet();
 
+        if (!baseUri.EndsWith('/'))
+        {
+            baseUri += "/";
+        }
+
         var blogPosts = await repository.GetAllAsync(f => f.IsPublished, b => b.UpdatedDate);
 
-        urlSet.Urls.Add(new SitemapUrl { Location = navigationManager.BaseUri });
-        urlSet.Urls.Add(new SitemapUrl { Location = $"{navigationManager.BaseUri}archive" });
-        urlSet.Urls.AddRange(CreateUrlsForBlogPosts(blogPosts));
-        urlSet.Urls.AddRange(CreateUrlsForTags(blogPosts));
+        urlSet.Urls.Add(new SitemapUrl { Location = baseUri });
+        urlSet.Urls.Add(new SitemapUrl { Location = $"{baseUri}archive" });
+        urlSet.Urls.AddRange(CreateUrlsForBlogPosts(blogPosts, baseUri));
+        urlSet.Urls.AddRange(CreateUrlsForTags(blogPosts, baseUri));
 
         return urlSet;
     }
 
-    public async Task SaveSitemapToFileAsync(SitemapUrlSet sitemap)
-    {
-        await xmlFileWriter.WriteObjectToXmlFileAsync(sitemap, "wwwroot/sitemap.xml");
-    }
-
-    private ImmutableArray<SitemapUrl> CreateUrlsForBlogPosts(IEnumerable<BlogPost> blogPosts)
+    private static ImmutableArray<SitemapUrl> CreateUrlsForBlogPosts(IEnumerable<BlogPost> blogPosts, string baseUri)
     {
         return blogPosts.Select(b => new SitemapUrl
         {
-            Location = $"{navigationManager.BaseUri}blogPost/{b.Id}",
+            Location = $"{baseUri}blogPost/{b.Id}",
             LastModified = b.UpdatedDate.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
         }).ToImmutableArray();
     }
 
-    private IEnumerable<SitemapUrl> CreateUrlsForTags(IEnumerable<BlogPost> blogPosts)
+    private static IEnumerable<SitemapUrl> CreateUrlsForTags(IEnumerable<BlogPost> blogPosts, string baseUri)
     {
         return blogPosts
             .SelectMany(b => b.Tags)
             .Distinct()
             .Select(t => new SitemapUrl
             {
-                Location = $"{navigationManager.BaseUri}searchByTag/{Uri.EscapeDataString(t)}",
+                Location = $"{baseUri}searchByTag/{Uri.EscapeDataString(t)}",
             });
     }
 }

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/Services/XmlFileWriter.cs

@@ -0,0 +1,19 @@
+using System.IO;
+using System.Threading.Tasks;
+using System.Xml;
+using System.Xml.Serialization;
+
+namespace LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
+
+public sealed class XmlWriter : IXmlWriter
+{
+    public async Task<byte[]> WriteToBuffer<T>(T objectToSave)
+    {
+        await using var memoryStream = new MemoryStream();
+        await using var xmlWriter = System.Xml.XmlWriter.Create(memoryStream, new XmlWriterSettings { Indent = true, Async = true });
+        var serializer = new XmlSerializer(typeof(T));
+        serializer.Serialize(xmlWriter, objectToSave);
+        xmlWriter.Close();
+        return memoryStream.ToArray();
+    }
+}

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/Services/XmlWriter.cs

@@ -16,7 +16,6 @@
                 <li><hr class="dropdown-divider"></li>
                 <li><h6 class="dropdown-header">Others</h6></li>
                 <li><a class="dropdown-item" href="short-codes">Shortcodes</a></li>
-                <li><a class="dropdown-item" href="Sitemap">Sitemap</a></li>
                 <li><hr class="dropdown-divider"></li>
 	            <li><a class="dropdown-item" target="_blank" href="https://github.com/linkdotnet/Blog/releases" rel="noreferrer">Releases</a></li>
             </ul>

src/LinkDotNet.Blog.Web/Features/Admin/Sitemap/SitemapPage.razor

@@ -20,7 +20,7 @@ public static IServiceCollection AddApplicationServices(this IServiceCollection
         services.AddScoped<ISortOrderCalculator, SortOrderCalculator>();
         services.AddScoped<IUserRecordService, UserRecordService>();
         services.AddScoped<ISitemapService, SitemapService>();
-        services.AddScoped<IXmlFileWriter, XmlFileWriter>();
+        services.AddScoped<IXmlWriter, XmlWriter>();
         services.AddScoped<IFileProcessor, FileProcessor>();
 
         services.AddSingleton<CacheService>();

src/LinkDotNet.Blog.Web/Features/Home/Components/AccessControl.razor

@@ -1,59 +1,41 @@
-using System;
+using System;
 using System.IO;
 using System.Threading.Tasks;
 using LinkDotNet.Blog.Domain;
 using LinkDotNet.Blog.Infrastructure.Persistence;
+using LinkDotNet.Blog.TestUtilities;
 using LinkDotNet.Blog.Web.Features.Admin.Sitemap.Services;
 using Microsoft.AspNetCore.Components;
 using TestContext = Xunit.TestContext;
 
 namespace LinkDotNet.Blog.IntegrationTests.Web.Shared.Services;
 
-public sealed class SitemapServiceTests : IDisposable
+public sealed class SitemapServiceTests : SqlDatabaseTestBase<BlogPost>
 {
-    private const string OutputDirectory = "wwwroot";
-    private const string OutputFilename = $"{OutputDirectory}/sitemap.xml";
     private readonly SitemapService sut;
 
     public SitemapServiceTests()
-    {
-        var repositoryMock = Substitute.For<IRepository<BlogPost>>();
-        sut = new SitemapService(repositoryMock, Substitute.For<NavigationManager>(), new XmlFileWriter());
-        Directory.CreateDirectory("wwwroot");
-    }
+        => sut = new SitemapService(Repository);
 
     [Fact]
     public async Task ShouldSaveSitemapUrlInCorrectFormat()
     {
-        var urlSet = new SitemapUrlSet
-        {
-            Urls =
-            [
-                new SitemapUrl { Location = "here", }
-            ],
-        };
-        await sut.SaveSitemapToFileAsync(urlSet);
-
-        var lines = await File.ReadAllTextAsync(OutputFilename, TestContext.Current.CancellationToken);
-        lines.ShouldBe(
-            @"<?xml version=""1.0"" encoding=""utf-8""?>
-<urlset xmlns:xsi=""http://www.w3.org/2001/XMLSchema-instance"" xmlns:xsd=""http://www.w3.org/2001/XMLSchema"" xmlns=""http://www.sitemaps.org/schemas/sitemap/0.9"">
-  <url>
-    <loc>here</loc>
-  </url>
-</urlset>");
-    }
-
-    public void Dispose()
-    {
-        if (File.Exists(OutputFilename))
-        {
-            File.Delete(OutputFilename);
-        }
-
-        if (Directory.Exists(OutputDirectory))
-        {
-            Directory.Delete(OutputDirectory, true);
-        }
+        var publishedBlogPost = new BlogPostBuilder()
+            .WithTitle("Title 1")
+            .WithUpdatedDate(new DateTime(2024, 12, 24))
+            .IsPublished()
+            .Build();
+        var unpublishedBlogPost = new BlogPostBuilder()
+            .IsPublished(false)
+            .Build();
+        await Repository.StoreAsync(publishedBlogPost);
+        await Repository.StoreAsync(unpublishedBlogPost);
+        
+        var sitemap = await sut.CreateSitemapAsync("https://www.linkdotnet.blog");
+        
+        sitemap.Urls.Count.ShouldBe(3);
+        sitemap.Urls.ShouldContain(u => u.Location == "https://www.linkdotnet.blog/");
+        sitemap.Urls.ShouldContain(u => u.Location == "https://www.linkdotnet.blog/archive");
+        sitemap.Urls.ShouldContain(u => u.Location == "https://www.linkdotnet.blog/blogPost/" + publishedBlogPost.Id);
     }
 }