Refactor LinkIndexProvider into own project

docs-builder.sln

@@ -99,6 +99,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate",
 		actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.LinkIndex", "src\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj", "{FD1AC230-798B-4AB9-8CE6-A06264885DBC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -160,6 +162,10 @@ Global
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -184,5 +190,6 @@ Global
 		{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
 		{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 	EndGlobalSection
 EndGlobal

src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="AWSSDK.S3" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
+    </ItemGroup>
+
+</Project>

src/Elastic.Documentation.LinkIndex/ILinkIndexReader.cs

@@ -0,0 +1,14 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Links;
+
+namespace Elastic.Documentation.LinkIndex;
+
+public interface ILinkIndexReader
+{
+	Task<LinkRegistry> GetRegistry(Cancel cancellationToken = default);
+	Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancellationToken = default);
+	string RegistryUrl { get; }
+}

src/Elastic.Documentation.LinkIndex/ILinkIndexReaderWriter.cs

@@ -0,0 +1,7 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Documentation.LinkIndex;
+
+public interface ILinkIndexReaderWriter : ILinkIndexReader, ILinkIndexWriter;

src/Elastic.Documentation.LinkIndex/ILinkIndexWriter.cs

@@ -0,0 +1,12 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Links;
+
+namespace Elastic.Documentation.LinkIndex;
+
+public interface ILinkIndexWriter
+{
+	Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default);
+}

src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs

@@ -0,0 +1,56 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Net;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Elastic.Documentation.Links;
+
+namespace Elastic.Documentation.LinkIndex;
+
+public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader
+{
+
+	// <summary>
+	// Using <see cref="AnonymousAWSCredentials"/> to access the link index
+	// allows to read from the link index without the need to provide AWS credentials.
+	// </summary>
+	public static Aws3LinkIndexReader CreateAnonymous()
+	{
+		var credentials = new AnonymousAWSCredentials();
+		var config = new AmazonS3Config
+		{
+			RegionEndpoint = Amazon.RegionEndpoint.USEast2
+		};
+		var s3Client = new AmazonS3Client(credentials, config);
+		return new AwsS3LinkIndexReaderWriter(s3Client);
+	}
+
+	public async Task<LinkRegistry> GetRegistry(Cancel cancellationToken = default)
+	{
+		var getObjectRequest = new GetObjectRequest
+		{
+			BucketName = bucketName,
+			Key = registryKey
+		};
+		var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
+		await using var stream = getObjectResponse.ResponseStream;
+		var linkIndex = LinkRegistry.Deserialize(stream);
+		return linkIndex with { ETag = getObjectResponse.ETag };
+	}
+	public async Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancellationToken)
+	{
+		var getObjectRequest = new GetObjectRequest
+		{
+			BucketName = bucketName,
+			Key = key
+		};
+		var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
+		await using var stream = getObjectResponse.ResponseStream;
+		return RepositoryLinks.Deserialize(stream);
+	}
+
+	public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
+}

src/Elastic.Documentation.LinkIndex/LinkIndexReaderWriter.cs

@@ -0,0 +1,41 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Net;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Elastic.Documentation.Links;
+
+namespace Elastic.Documentation.LinkIndex;
+
+public class AwsS3LinkIndexReaderWriter(
+	IAmazonS3 s3Client,
+	string bucketName = "elastic-docs-link-index",
+	string registryKey = "link-index.json"
+) : Aws3LinkIndexReader(s3Client, bucketName, registryKey), ILinkIndexReaderWriter
+{
+	private readonly IAmazonS3 _s3Client = s3Client;
+	private readonly string _bucketName = bucketName;
+	private readonly string _registryKey = registryKey;
+
+	public async Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default)
+	{
+		if (registry.ETag == null)
+			// The ETag should not be null if the LinkReferenceRegistry was retrieved from GetLinkIndex()
+			throw new InvalidOperationException($"{nameof(LinkRegistry)}.{nameof(registry.ETag)} cannot be null");
+		var json = LinkRegistry.Serialize(registry);
+		var putObjectRequest = new PutObjectRequest
+		{
+			BucketName = _bucketName,
+			Key = _registryKey,
+			ContentBody = json,
+			ContentType = "application/json",
+			IfMatch = registry.ETag // Only update if the ETag matches. Meaning the object has not been changed in the meantime.
+		};
+		var putResponse = await _s3Client.PutObjectAsync(putObjectRequest, cancellationToken);
+		if (putResponse.HttpStatusCode != HttpStatusCode.OK)
+			throw new Exception($"Unable to save {nameof(LinkRegistry)} to s3://{_bucketName}/{_registryKey}");
+	}
+}

src/Elastic.Documentation/Links/LinkReferenceRegistry.cs → src/Elastic.Documentation/Links/LinkRegistry.cs

@@ -8,20 +8,54 @@
 
 namespace Elastic.Documentation.Links;
 
-public record LinkReferenceRegistry
+public record LinkRegistry
 {
 	/// Map of branch to <see cref="LinkRegistryEntry"/>
 	[JsonPropertyName("repositories")]
 	public required Dictionary<string, Dictionary<string, LinkRegistryEntry>> Repositories { get; init; }
 
-	public static LinkReferenceRegistry Deserialize(Stream json) =>
-		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
+	[JsonIgnore]
+	public string? ETag { get; init; }
 
-	public static LinkReferenceRegistry Deserialize(string json) =>
-		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
+	public LinkRegistry WithLinkRegistryEntry(LinkRegistryEntry entry)
+	{
+		var copiedRepositories = new Dictionary<string, Dictionary<string, LinkRegistryEntry>>(Repositories);
+		var repository = entry.Repository;
+		var branch = entry.Branch;
+		// repository already exists in links.json
+		if (copiedRepositories.TryGetValue(repository, out var existingRepositoryEntry))
+		{
+			// The branch already exists in the repository entry
+			if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry))
+			{
+				if (entry.UpdatedAt > existingBranchEntry.UpdatedAt)
+					existingRepositoryEntry[branch] = entry;
+			}
+			// branch does not exist in the repository entry
+			else
+			{
+				existingRepositoryEntry[branch] = entry;
+			}
+		}
+		// onboarding new repository
+		else
+		{
+			copiedRepositories.Add(repository, new Dictionary<string, LinkRegistryEntry>
+			{
+				{ branch, entry }
+			});
+		}
+		return this with { Repositories = copiedRepositories };
+	}
 
-	public static string Serialize(LinkReferenceRegistry referenceRegistry) =>
-		JsonSerializer.Serialize(referenceRegistry, SourceGenerationContext.Default.LinkReferenceRegistry);
+	public static LinkRegistry Deserialize(Stream json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!;
+
+	public static LinkRegistry Deserialize(string json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!;
+
+	public static string Serialize(LinkRegistry registry) =>
+		JsonSerializer.Serialize(registry, SourceGenerationContext.Default.LinkRegistry);
 }
 
 public record LinkRegistryEntry
@@ -46,4 +80,3 @@ public record LinkRegistryEntry
 	[JsonPropertyName("updated_at")]
 	public DateTime UpdatedAt { get; init; } = DateTime.MinValue;
 }
-

src/Elastic.Documentation/Links/LinkReference.cs → src/Elastic.Documentation/Links/RepositoryLinks.cs

@@ -39,7 +39,7 @@ public record LinkRedirect : LinkSingleRedirect
 	public LinkSingleRedirect[]? Many { get; init; }
 }
 
-public record LinkReference
+public record RepositoryLinks
 {
 	[JsonPropertyName("origin")]
 	public required GitCheckoutInformation Origin { get; init; }
@@ -61,12 +61,12 @@ public record LinkReference
 	public static string SerializeRedirects(Dictionary<string, LinkRedirect>? redirects) =>
 		JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect);
 
-	public static LinkReference Deserialize(Stream json) =>
-		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
+	public static RepositoryLinks Deserialize(Stream json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!;
 
-	public static LinkReference Deserialize(string json) =>
-		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
+	public static RepositoryLinks Deserialize(string json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!;
 
-	public static string Serialize(LinkReference reference) =>
-		JsonSerializer.Serialize(reference, SourceGenerationContext.Default.LinkReference);
+	public static string Serialize(RepositoryLinks reference) =>
+		JsonSerializer.Serialize(reference, SourceGenerationContext.Default.RepositoryLinks);
 }

src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

@@ -12,8 +12,8 @@ namespace Elastic.Documentation.Serialization;
 
 [JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)]
 [JsonSerializable(typeof(GenerationState))]
-[JsonSerializable(typeof(LinkReference))]
+[JsonSerializable(typeof(RepositoryLinks))]
 [JsonSerializable(typeof(GitCheckoutInformation))]
-[JsonSerializable(typeof(LinkReferenceRegistry))]
+[JsonSerializable(typeof(LinkRegistry))]
 [JsonSerializable(typeof(LinkRegistryEntry))]
 public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -276,11 +276,11 @@ private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<
 		return false;
 	}
 
-	private async Task<LinkReference> GenerateLinkReference(Cancel ctx)
+	private async Task<RepositoryLinks> GenerateLinkReference(Cancel ctx)
 	{
 		var file = DocumentationSet.LinkReferenceFile;
 		var state = DocumentationSet.CreateLinkReference();
-		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference);
+		var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.RepositoryLinks);
 		await DocumentationSet.OutputDirectory.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx);
 		return state;
 	}

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -64,6 +64,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj" />
     <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
     <ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj" />
   </ItemGroup>

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -8,8 +8,8 @@
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Configuration.TableOfContents;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
-using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Extensions;
 using Elastic.Markdown.Extensions.DetectionRules;
 using Elastic.Markdown.IO.Navigation;
@@ -126,7 +126,7 @@ public DocumentationSet(
 		SourceDirectory = context.DocumentationSourceDirectory;
 		OutputDirectory = context.DocumentationOutputDirectory;
 		LinkResolver =
-			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger));
+			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, Aws3LinkIndexReader.CreateAnonymous(), logger));
 		Configuration = context.Configuration;
 		EnabledExtensions = InstantiateExtensions();
 		treeCollector ??= new TableOfContentsTreeCollector();
@@ -361,7 +361,7 @@ MarkdownFile ExtensionOrDefaultMarkdown()
 		}
 	}
 
-	public LinkReference CreateLinkReference()
+	public RepositoryLinks CreateLinkReference()
 	{
 		var redirects = Configuration.Redirects;
 		var crossLinks = Context.Collector.CrossLinks.ToHashSet().ToArray();
@@ -375,7 +375,7 @@ public LinkReference CreateLinkReference()
 				return new LinkMetadata { Anchors = anchors, Hidden = v.File.Hidden };
 			});
 
-		return new LinkReference
+		return new RepositoryLinks
 		{
 			Redirects = redirects,
 			UrlPathPrefix = Context.UrlPathPrefix,

src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -5,16 +5,17 @@
 using System.Collections.Frozen;
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration.Builder;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.Links.CrossLinks;
 
-public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger)
+public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILinkIndexReader linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{
-		var linkReferences = new Dictionary<string, LinkReference>();
+		var linkReferences = new Dictionary<string, RepositoryLinks>();
 		var linkIndexEntries = new Dictionary<string, LinkRegistryEntry>();
 		var declaredRepositories = new HashSet<string>();
 		foreach (var repository in configuration.CrossLinkRepositories)