Command for generation of EOL annotation data file

src/Microsoft.DotNet.ImageBuilder/src/Commands/AnnotateEolDigestsCommand.cs

@@ -43,7 +43,7 @@ public AnnotateEolDigestsCommand(
 
         public override async Task ExecuteAsync()
         {
-            EolAnnotationsData eolAnnotations = LoadEolAnnotationsData(Options.EolDigestsListPath);
+            EolAnnotationsData eolAnnotations = LoadEolAnnotationsData(Options.EolDigestsListOutputPath);
             DateOnly? globalEolDate = eolAnnotations?.EolDate;
 
             await _registryCredentialsProvider.ExecuteWithCredentialsAsync(
@@ -62,7 +62,7 @@ await _registryCredentialsProvider.ExecuteWithCredentialsAsync(
                                 {
                                     // We will capture all failures and log the json data at the end.
                                     // Json data can be used to rerun the failed annotations.
-                                    _failedAnnotations.Add(new EolDigestData { Digest = a.Digest, EolDate = eolDate });
+                                    _failedAnnotations.Add(new EolDigestData { Digest = a.Digest, EolDate = eolDate, Tag = a.Tag });
                                 }
                             }
                             else

src/Microsoft.DotNet.ImageBuilder/src/Commands/AnnotateEolDigestsOptions.cs

@@ -15,7 +15,7 @@ public class AnnotateEolDigestsOptions : Options
     {
         public RegistryCredentialsOptions CredentialsOptions { get; set; } = new();
 
-        public string EolDigestsListPath { get; set; } = string.Empty;
+        public string EolDigestsListOutputPath { get; set; } = string.Empty;
         public string AcrName { get; set; } = string.Empty;
         public bool Force { get; set; } = false;
     }
@@ -41,7 +41,7 @@ public override IEnumerable<Argument> GetCliArguments() =>
                 .Concat(
                     new Argument[]
                     {
-                        new Argument<string>(nameof(AnnotateEolDigestsOptions.EolDigestsListPath),
+                        new Argument<string>(nameof(AnnotateEolDigestsOptions.EolDigestsListOutputPath),
                             "EOL annotations digests list path"),
                         new Argument<string>(nameof(AnnotateEolDigestsOptions.AcrName),
                             "Azure registry name")

src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateEolAnnotationDataCommand.cs

@@ -0,0 +1,195 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Azure.Containers.ContainerRegistry;
+using Kusto.Cloud.Platform.Utils;
+using Microsoft.DotNet.ImageBuilder.Models.Annotations;
+using Microsoft.DotNet.ImageBuilder.Models.Image;
+using Newtonsoft.Json;
+
+#nullable enable
+namespace Microsoft.DotNet.ImageBuilder.Commands;
+
+[Export(typeof(ICommand))]
+public class GenerateEolAnnotationDataCommand : Command<GenerateEolAnnotationDataOptions, GenerateEolAnnotationDataOptionsBuilder>
+{
+    private readonly IDotNetReleasesService _dotNetReleasesService;
+    private readonly ILoggerService _loggerService;
+    private readonly IContainerRegistryClientFactory _acrClientFactory;
+    private readonly IAzureTokenCredentialProvider _tokenCredentialProvider;
+    private readonly IOrasService _orasService;
+    private readonly DateOnly _eolDate;
+
+    [ImportingConstructor]
+    public GenerateEolAnnotationDataCommand(
+        IDotNetReleasesService dotNetReleasesService,
+        ILoggerService loggerService,
+        IContainerRegistryClientFactory acrClientFactory,
+        IAzureTokenCredentialProvider tokenCredentialProvider,
+        IOrasService orasService)
+    {
+        _dotNetReleasesService = dotNetReleasesService ?? throw new ArgumentNullException(nameof(dotNetReleasesService));
+        _loggerService = loggerService ?? throw new ArgumentNullException(nameof(loggerService));
+        _acrClientFactory = acrClientFactory ?? throw new ArgumentNullException(nameof(acrClientFactory));
+        _tokenCredentialProvider = tokenCredentialProvider ?? throw new ArgumentNullException(nameof(tokenCredentialProvider));
+        _orasService = orasService ?? throw new ArgumentNullException(nameof(orasService));
+
+        _eolDate = DateOnly.FromDateTime(DateTime.UtcNow); // default EOL date
+    }
+
+    protected override string Description => "Generate EOL annotation data";
+
+    public override async Task ExecuteAsync()
+    {
+        List<EolDigestData> digestsToAnnotate = await GetDigestsToAnnotate();
+        WriteDigestDataJson(digestsToAnnotate);
+    }
+
+    private void WriteDigestDataJson(List<EolDigestData> digestsToAnnotate)
+    {
+        EolAnnotationsData eolAnnotations = new(digestsToAnnotate, _eolDate);
+
+        string annotationsJson = JsonConvert.SerializeObject(
+            eolAnnotations, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
+        File.WriteAllText(Options.EolDigestsListPath, annotationsJson);
+    }
+
+    private async Task<List<EolDigestData>> GetDigestsToAnnotate()
+    {
+        Dictionary<string, DateOnly> productEolDates = await _dotNetReleasesService.GetProductEolDatesFromReleasesJson();
+        ImageArtifactDetails oldImageArtifactDetails = LoadImageInfoData(Options.OldImageInfoPath);
+        ImageArtifactDetails newImageArtifactDetails = LoadImageInfoData(Options.NewImageInfoPath);
+
+        List<EolDigestData> digestDataList = [];
+
+        try
+        {
+            // Find all the digests that need to be annotated for EOL by querying the registry for all the digests, scoped to those repos associated with
+            // the image info. The repo scoping is done because there may be some cases where multiple image info files are used for different repositories.
+            // The intent is to annotate all of the digests that do not exist in the image info file. So this scoping ensures we don't annotate digests that
+            // are associated with another image info file. However, we also need to account for the deletion of an entire repository. In that case, we want
+            // all the digests in that repo to be annotated. But since the repo is deleted, it doesn't show up in the newly generated image info file. So we
+            // need the previous version of the image info file to know that the repo had previously existed and so that repo is included in the scope for
+            // the query of the digests.
+            IEnumerable<string> repoNames = newImageArtifactDetails.Repos.Select(repo => repo.Repo)
+                .Union(oldImageArtifactDetails.Repos.Select(repo => repo.Repo));
+            IEnumerable<(string Digest, string? Tag)> registryDigests = await GetAllDigestsFromRegistry(repoNames);
+
+            IEnumerable<string> supportedDigests = GetSupportedDigests(newImageArtifactDetails);
+            IEnumerable<EolDigestData> unsupportedDigests = GetUnsupportedDigests(registryDigests, supportedDigests);
+
+            // Annotate digests that are not already annotated for EOL
+            ConcurrentBag<EolDigestData> digetsToAnnotate = [];
+            Parallel.ForEach(unsupportedDigests, digest =>
+            {
+                if (!_orasService.IsDigestAnnotatedForEol(digest.Digest, _loggerService, Options.IsDryRun))
+                {
+                    digetsToAnnotate.Add(digest);
+                }
+            });
+
+            digestDataList.AddRange(digetsToAnnotate);
+
+            if (Options.AnnotateEolProducts)
+            {
+                // Annotate images for eol products in new image info
+                foreach (ImageData image in newImageArtifactDetails.Repos.SelectMany(repo => repo.Images))
+                {
+                    digestDataList.AddRange(GetProductEolDigests(image, productEolDates));
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            _loggerService.WriteError($"Error occurred while generating EOL annotation data: {e}");
+            throw;
+        }
+
+        digestDataList = digestDataList.OrderBy(item => item.Digest).ToList();
+
+        return digestDataList;
+    }
+
+    /// <summary>
+    /// Finds all the digests that are in the registry but not in the supported digests list.
+    /// </summary>
+    private static IEnumerable<EolDigestData> GetUnsupportedDigests(IEnumerable<(string Digest, string? Tag)> registryDigests, IEnumerable<string> supportedDigests) =>
+        registryDigests
+            .Where(registryDigest => !supportedDigests.Contains(registryDigest.Digest))
+            .Select(registryDigest => new EolDigestData(registryDigest.Digest) { Tag = registryDigest.Tag });
+
+    private static IEnumerable<string> GetSupportedDigests(ImageArtifactDetails newImageArtifactDetails) =>
+        newImageArtifactDetails.Repos
+            .SelectMany(repo => repo.Images)
+            .SelectMany(GetImageDigests)
+            .Select(digest => digest.Digest);
+
+    private static IEnumerable<(string Digest, string? Tag)> GetImageDigests(ImageData image)
+    {
+        if (image.Manifest is not null)
+        {
+            yield return (image.Manifest.Digest, GetLongestTag(image.Manifest.SharedTags));
+        }
+
+        foreach (PlatformData platform in image.Platforms)
+        {
+            yield return (platform.Digest, GetLongestTag(platform.SimpleTags));
+        }
+    }
+
+    private static string? GetLongestTag(IEnumerable<string> tags) =>
+        tags.OrderByDescending(tag => tag.Length).FirstOrDefault();
+
+    private async Task<IEnumerable<(string Digest, string? Tag)>> GetAllDigestsFromRegistry(IEnumerable<string> repoNames)
+    {
+        IContainerRegistryClient acrClient = _acrClientFactory.Create(Options.RegistryName, _tokenCredentialProvider.GetCredential());
+        IAsyncEnumerable<string> repositoryNames = acrClient.GetRepositoryNamesAsync();
+
+        ConcurrentBag<(string Digest, string? Tag)> digests = [];
+        await foreach (string repositoryName in repositoryNames.Where(name => repoNames.Contains(name)))
+        {
+            ContainerRepository repo = acrClient.GetRepository(repositoryName);
+            IAsyncEnumerable<ArtifactManifestProperties> manifests = repo.GetAllManifestPropertiesAsync();
+            await foreach (ArtifactManifestProperties manifestProps in manifests)
+            {
+                string imageName = DockerHelper.GetImageName(Options.RegistryName, repositoryName, digest: manifestProps.Digest);
+                digests.Add((imageName, GetLongestTag(manifestProps.Tags)));
+            }
+        }
+
+        return digests;
+    }
+
+    private static IEnumerable<EolDigestData> GetProductEolDigests(ImageData image, Dictionary<string, DateOnly> productEolDates)
+    {
+        if (image.ProductVersion == null)
+        {
+            return [];
+        }
+
+        string dotnetVersion = Version.Parse(image.ProductVersion).ToString(2);
+        if (!productEolDates.TryGetValue(dotnetVersion, out DateOnly date))
+        {
+            return [];
+        }
+
+        return GetImageDigests(image).Select(val => new EolDigestData(val.Digest) { Tag = val.Tag, EolDate = date });
+    }
+
+    private static ImageArtifactDetails LoadImageInfoData(string imageInfoPath)
+    {
+        string imageInfoJson = File.ReadAllText(imageInfoPath);
+        ImageArtifactDetails? imageArtifactDetails = JsonConvert.DeserializeObject<ImageArtifactDetails>(imageInfoJson);
+        return imageArtifactDetails is null
+            ? throw new JsonException($"Unable to correctly deserialize path '{imageInfoJson}'.")
+            : imageArtifactDetails;
+    }
+}

src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateEolAnnotationDataOptions.cs

@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.CommandLine;
+using System.Linq;
+using static Microsoft.DotNet.ImageBuilder.Commands.CliHelper;
+
+#nullable enable
+namespace Microsoft.DotNet.ImageBuilder.Commands;
+
+public class GenerateEolAnnotationDataOptions : Options
+{
+    public string EolDigestsListPath { get; set; } = string.Empty;
+    public string OldImageInfoPath { get; set; } = string.Empty;
+    public string NewImageInfoPath { get; set; } = string.Empty;
+    public bool AnnotateEolProducts { get; set; }
+    public string RepoPrefix { get; set; } = string.Empty;
+    public string RegistryName { get; set; } = string.Empty;
+}
+
+public class GenerateEolAnnotationDataOptionsBuilder : CliOptionsBuilder
+{
+    public override IEnumerable<Option> GetCliOptions() =>
+        base.GetCliOptions()
+            .Concat(
+                [
+                    CreateOption<bool>("annotate-eol-products", nameof(GenerateEolAnnotationDataOptions.AnnotateEolProducts),
+                        "Annotate images of EOL products"),
+                ]
+            );
+
+    public override IEnumerable<Argument> GetCliArguments() =>
+        base.GetCliArguments()
+            .Concat(
+                [
+                    new Argument<string>(nameof(GenerateEolAnnotationDataOptions.EolDigestsListPath),
+                        "EOL annotations digests list output path"),
+                    new Argument<string>(nameof(GenerateEolAnnotationDataOptions.OldImageInfoPath),
+                        "Old image-info file"),
+                    new Argument<string>(nameof(GenerateEolAnnotationDataOptions.NewImageInfoPath),
+                        "New image-info file"),
+                    new Argument<string>(nameof(GenerateEolAnnotationDataOptions.RepoPrefix),
+                        "Prefix to add to the repo names specified in the manifest"),
+                    new Argument<string>(nameof(GenerateEolAnnotationDataOptions.RegistryName),
+                        "Name of the registry"),
+                ]
+            );
+}

src/Microsoft.DotNet.ImageBuilder/src/DotNetReleasesService.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.Composition;
+using System.Threading.Tasks;
+using Microsoft.Deployment.DotNet.Releases;
+
+#nullable enable
+namespace Microsoft.DotNet.ImageBuilder
+{
+    [Export(typeof(IDotNetReleasesService))]
+    public class DotNetReleasesService : IDotNetReleasesService
+    {
+        public async Task<Dictionary<string, DateOnly>> GetProductEolDatesFromReleasesJson()
+        {
+            Dictionary<string, DateOnly> productEolDates = [];
+
+            ProductCollection dotnetProducts = await ProductCollection.GetAsync();
+
+            foreach (Product product in dotnetProducts)
+            {
+                if (product.EndOfLifeDate != null &&
+                    product.EndOfLifeDate <= DateTime.Today)
+                {
+                    productEolDates.Add(product.ProductVersion, DateOnly.FromDateTime((DateTime)product.EndOfLifeDate));
+                }
+            }
+
+            return productEolDates;
+        }
+    }
+}

src/Microsoft.DotNet.ImageBuilder/src/IDotNetReleasesService.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+#nullable enable
+namespace Microsoft.DotNet.ImageBuilder
+{
+    public interface IDotNetReleasesService
+    {
+        Task<Dictionary<string, DateOnly>> GetProductEolDatesFromReleasesJson();
+    }
+}
+#nullable disable

src/Microsoft.DotNet.ImageBuilder/src/Microsoft.DotNet.ImageBuilder.csproj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <!-- Upgrade explicitly referenced package version referenced by Microsoft.Data.SqlClient.5.1.1 and Microsoft.Azure.Kusto.Ingest.11.3.4 -->
     <PackageReference Include="Azure.Containers.ContainerRegistry" Version="1.1.1" />
+    <PackageReference Include="Azure.Monitor.Query" Version="1.4.0" />
     <PackageReference Include="Azure.Identity" Version="1.11.4" />
     <PackageReference Include="Azure.ResourceManager.ContainerRegistry" Version="1.2.1" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.20.0" />
@@ -19,6 +20,7 @@
     <PackageReference Include="LibGit2Sharp" Version="0.29.0" />
     <PackageReference Include="Microsoft.Azure.Kusto.Ingest" Version="12.2.1" />
     <PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.0" />
+    <PackageReference Include="Microsoft.Deployment.DotNet.Releases" Version="1.0.0" />
     <PackageReference Include="Microsoft.DotNet.Git.IssueManager" Version="9.0.0-beta.24123.3" />
     <PackageReference Include="Microsoft.DotNet.VersionTools" Version="9.0.0-beta.24151.5" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.225.1" />

src/Microsoft.DotNet.ImageBuilder/src/Models/Annotations/AcrEventEntry.cs

@@ -0,0 +1,10 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+#nullable enable
+namespace Microsoft.DotNet.ImageBuilder.Models.Annotations;
+
+public record AcrEventEntry(DateTime TimeGenerated, string Digest);

src/Microsoft.DotNet.ImageBuilder/src/Models/Annotations/EolAnnotationsData.cs

@@ -23,6 +23,6 @@ public EolAnnotationsData(List<EolDigestData> eolDigests, DateOnly? eolDate = nu
 
         public DateOnly? EolDate { get; set; }
 
-        public List<EolDigestData>? EolDigests { get; set; }
+        public List<EolDigestData> EolDigests { get; set; } = [];
     }
 }