Remove force annotate option

src/Microsoft.DotNet.ImageBuilder/src/Commands/AnnotateEolDigestsCommand.cs

@@ -4,37 +4,42 @@
 
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.ComponentModel.Composition;
 using System.IO;
+using System.Linq;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.DotNet.ImageBuilder.Models.Annotations;
-using Newtonsoft.Json;
+using Microsoft.DotNet.ImageBuilder.Models.MarBulkDeletion;
 
 #nullable enable
 namespace Microsoft.DotNet.ImageBuilder.Commands
 {
     [Export(typeof(ICommand))]
     public class AnnotateEolDigestsCommand : Command<AnnotateEolDigestsOptions, AnnotateEolDigestsOptionsBuilder>
     {
-        private readonly IDockerService _dockerService;
         private readonly ILoggerService _loggerService;
-        private readonly IProcessService _processService;
         private readonly IOrasService _orasService;
         private readonly IRegistryCredentialsProvider _registryCredentialsProvider;
+        private readonly ConcurrentBag<EolDigestData> _failedAnnotationImageDigests = [];
+        private readonly ConcurrentBag<EolDigestData> _skippedAnnotationImageDigests = [];
+        private readonly ConcurrentBag<EolDigestData> _existingAnnotationImageDigests = [];
+        private readonly ConcurrentBag<string> _existingAnnotationDigests = [];
 
-        private ConcurrentBag<EolDigestData> _failedAnnotations = new ();
+        private static readonly JsonSerializerOptions s_jsonSerializerOptions = new()
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+            WriteIndented = true
+        };
 
         [ImportingConstructor]
         public AnnotateEolDigestsCommand(
-            IDockerService dockerService,
             ILoggerService loggerService,
-            IProcessService processService,
             IOrasService orasService,
             IRegistryCredentialsProvider registryCredentialsProvider)
         {
-            _dockerService = new DockerServiceCache(dockerService ?? throw new ArgumentNullException(nameof(dockerService)));
             _loggerService = loggerService ?? throw new ArgumentNullException(nameof(loggerService));
-            _processService = processService ?? throw new ArgumentNullException(nameof(processService));
             _orasService = orasService ?? throw new ArgumentNullException(nameof(orasService));
             _registryCredentialsProvider = registryCredentialsProvider ?? throw new ArgumentNullException(nameof(registryCredentialsProvider));
         }
@@ -44,57 +49,106 @@ public AnnotateEolDigestsCommand(
         public override async Task ExecuteAsync()
         {
             EolAnnotationsData eolAnnotations = LoadEolAnnotationsData(Options.EolDigestsListOutputPath);
-            DateOnly? globalEolDate = eolAnnotations?.EolDate;
+            DateOnly? globalEolDate = eolAnnotations.EolDate;
 
             await _registryCredentialsProvider.ExecuteWithCredentialsAsync(
                 Options.IsDryRun,
-                async () =>
+                () =>
                 {
-                    Parallel.ForEach(eolAnnotations.EolDigests, (a) =>
-                    {
-                        if (Options.Force || !_orasService.IsDigestAnnotatedForEol(a.Digest, _loggerService, Options.IsDryRun))
-                        {
-                            DateOnly? eolDate = a.EolDate ?? globalEolDate;
-                            if (eolDate != null)
-                            {
-                                _loggerService.WriteMessage($"Annotating EOL for digest '{a.Digest}', date '{eolDate}'");
-                                if (!_orasService.AnnotateEolDigest(a.Digest, eolDate.Value, _loggerService, Options.IsDryRun))
-                                {
-                                    // We will capture all failures and log the json data at the end.
-                                    // Json data can be used to rerun the failed annotations.
-                                    _failedAnnotations.Add(new EolDigestData { Digest = a.Digest, EolDate = eolDate, Tag = a.Tag });
-                                }
-                            }
-                            else
-                            {
-                                _loggerService.WriteError($"EOL date is not specified for digest '{a.Digest}'.");
-                            }
-                        }
-                        else
-                        {
-                            _loggerService.WriteMessage($"Digest '{a.Digest}' is already annotated for EOL.");
-                        }
-                    });
-
+                    Parallel.ForEach(eolAnnotations.EolDigests, digestData => AnnotateDigest(digestData, globalEolDate));
+                    return Task.CompletedTask;
                 },
                 Options.CredentialsOptions,
                 registryName: Options.AcrName,
                 ownedAcr: Options.AcrName);
 
-            if (_failedAnnotations.Count > 0)
+            WriteNonEmptySummaryForImageDigests(_skippedAnnotationImageDigests,
+                "The following image digests were skipped because they have existing annotations with matching EOL dates.");
+
+            WriteNonEmptySummaryForImageDigests(_existingAnnotationImageDigests,
+                "The following image digests were skipped because they have existing annotations with non-matching EOL dates. These need to be deleted from MAR before they can be re-annotated.");
+
+            WriteNonEmptySummaryForAnnotationDigests(_existingAnnotationDigests,
+                "These are the digests of the annotations with the non-matching EOL dates. This JSON can be used as input for the bulk deletion in MAR.");
+
+            WriteNonEmptySummaryForImageDigests(_failedAnnotationImageDigests,
+                "The following digests had annotation failures:");
+
+            if (!_existingAnnotationImageDigests.IsEmpty || !_failedAnnotationImageDigests.IsEmpty)
             {
-                _loggerService.WriteMessage("JSON file for rerunning failed annotations:");
-                _loggerService.WriteMessage("");
-                _loggerService.WriteMessage(JsonConvert.SerializeObject(new EolAnnotationsData(eolDigests: [.. _failedAnnotations])));
-                _loggerService.WriteMessage("");
-                throw new InvalidOperationException($"Failed to annotate {_failedAnnotations.Count} digests for EOL.");
+                throw new InvalidOperationException(
+                    $"Some digest annotations failed or were skipped due to existing non-matching EOL date annotations (failed: {_failedAnnotationImageDigests.Count}, skipped: {_existingAnnotationImageDigests.Count}).");
+            }
+        }
+
+        private void WriteNonEmptySummaryForAnnotationDigests(IEnumerable<string> annotationDigests, string message)
+        {
+            if (annotationDigests.Any())
+            {
+                WriteNonEmptySummary(new BulkDeletionDescription { Digests = [.. annotationDigests] }, message);
+            }
+        }
+
+        private void WriteNonEmptySummaryForImageDigests(IEnumerable<EolDigestData> eolDigests, string message)
+        {
+            if (eolDigests.Any())
+            {
+                WriteNonEmptySummary(new EolAnnotationsData(eolDigests: [.. eolDigests]), message);
+            }
+        }
+
+        private void WriteNonEmptySummary(object value, string message)
+        {
+            _loggerService.WriteMessage(message);
+            _loggerService.WriteMessage();
+            _loggerService.WriteMessage(JsonSerializer.Serialize(value, s_jsonSerializerOptions));
+            _loggerService.WriteMessage();
+        }
+
+        private void AnnotateDigest(EolDigestData digestData, DateOnly? globalEolDate)
+        {
+            DateOnly? eolDate = digestData.EolDate ?? globalEolDate;
+            if (eolDate is null)
+            {
+                _failedAnnotationImageDigests.Add(new EolDigestData { Digest = digestData.Digest, EolDate = eolDate });
+                _loggerService.WriteError($"EOL date is not specified for digest '{digestData.Digest}'.");
+                return;
+            }
+
+            if (!_orasService.IsDigestAnnotatedForEol(digestData.Digest, _loggerService, Options.IsDryRun, out OciManifest? lifecycleArtifactManifest))
+            {
+                _loggerService.WriteMessage($"Annotating EOL for digest '{digestData.Digest}', date '{eolDate}'");
+                if (!_orasService.AnnotateEolDigest(digestData.Digest, eolDate.Value, _loggerService, Options.IsDryRun))
+                {
+                    // We will capture all failures and log the json data at the end.
+                    // Json data can be used to rerun the failed annotations.
+                    _failedAnnotationImageDigests.Add(new EolDigestData { Digest = digestData.Digest, EolDate = eolDate, Tag = digestData.Tag });
+                }
+            }
+            else
+            {
+                if (lifecycleArtifactManifest.Annotations[OrasService.EndOfLifeAnnotation] == eolDate?.ToString(OrasService.EolDateFormat))
+                {
+                    _loggerService.WriteMessage($"Skipping digest '{digestData.Digest}' because it is already annotated with a matching EOL date.");
+                    _skippedAnnotationImageDigests.Add(digestData);
+                }
+                else
+                {
+                    _loggerService.WriteError($"Could not annotate digest '{digestData.Digest}' because it has an existing non-matching EOL date: {eolDate}.");
+                    _existingAnnotationImageDigests.Add(new EolDigestData { Digest = digestData.Digest, EolDate = eolDate });
+
+                    // Reference is a fully-qualified digest name. We want to remove the registry and repo prefix from the name to reflect the repo-qualified
+                    // name that exists in MAR.
+                    string refDigest = lifecycleArtifactManifest.Reference.TrimStart($"{Options.AcrName}/{Options.RepoPrefix}");
+                    _existingAnnotationDigests.Add(refDigest);
+                }
             }
         }
 
         private static EolAnnotationsData LoadEolAnnotationsData(string eolDigestsListPath)
         {
             string eolAnnotationsJson = File.ReadAllText(eolDigestsListPath);
-            EolAnnotationsData? eolAnnotations = JsonConvert.DeserializeObject<EolAnnotationsData>(eolAnnotationsJson);
+            EolAnnotationsData? eolAnnotations = JsonSerializer.Deserialize<EolAnnotationsData>(eolAnnotationsJson);
             return eolAnnotations is null
                 ? throw new JsonException($"Unable to correctly deserialize path '{eolAnnotationsJson}'.")
                 : eolAnnotations;

src/Microsoft.DotNet.ImageBuilder/src/Commands/AnnotateEolDigestsOptions.cs

@@ -2,11 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections.Generic;
 using System.CommandLine;
 using System.Linq;
-using static Microsoft.DotNet.ImageBuilder.Commands.CliHelper;
 
 #nullable enable
 namespace Microsoft.DotNet.ImageBuilder.Commands
@@ -17,7 +15,7 @@ public class AnnotateEolDigestsOptions : Options
 
         public string EolDigestsListOutputPath { get; set; } = string.Empty;
         public string AcrName { get; set; } = string.Empty;
-        public bool Force { get; set; } = false;
+        public string RepoPrefix { get; set; } = string.Empty;
     }
 
     public class AnnotateEolDigestsOptionsBuilder : CliOptionsBuilder
@@ -26,14 +24,7 @@ public class AnnotateEolDigestsOptionsBuilder : CliOptionsBuilder
 
         public override IEnumerable<Option> GetCliOptions() =>
             base.GetCliOptions()
-                .Concat(_registryCredentialsOptionsBuilder.GetCliOptions())
-                .Concat(
-                    new Option[]
-                    {
-                        CreateOption<bool>("force", nameof(AnnotateEolDigestsOptions.Force),
-                            "Annotate always, without checking if digest is already annotated for EOL"),
-                    }
-                );
+                .Concat(_registryCredentialsOptionsBuilder.GetCliOptions());
 
         public override IEnumerable<Argument> GetCliArguments() =>
             base.GetCliArguments()
@@ -44,7 +35,9 @@ public override IEnumerable<Argument> GetCliArguments() =>
                         new Argument<string>(nameof(AnnotateEolDigestsOptions.EolDigestsListOutputPath),
                             "EOL annotations digests list path"),
                         new Argument<string>(nameof(AnnotateEolDigestsOptions.AcrName),
-                            "Azure registry name")
+                            "Azure registry name"),
+                        new Argument<string>(nameof(GenerateEolAnnotationDataOptions.RepoPrefix),
+                            "Publish prefix of the repo names"),
                     }
                 );
     }

src/Microsoft.DotNet.ImageBuilder/src/Commands/GenerateEolAnnotationDataCommand.cs

@@ -90,7 +90,7 @@ private async Task<List<EolDigestData>> GetDigestsToAnnotate()
             ConcurrentBag<EolDigestData> digetsToAnnotate = [];
             Parallel.ForEach(unsupportedDigests, digest =>
             {
-                if (!_orasService.IsDigestAnnotatedForEol(digest.Digest, _loggerService, Options.IsDryRun))
+                if (!_orasService.IsDigestAnnotatedForEol(digest.Digest, _loggerService, Options.IsDryRun, out _))
                 {
                     digetsToAnnotate.Add(digest);
                 }

src/Microsoft.DotNet.ImageBuilder/src/IOrasService.cs

@@ -3,13 +3,15 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.DotNet.ImageBuilder.Models.Annotations;
 
 #nullable enable
 namespace Microsoft.DotNet.ImageBuilder
 {
     public interface IOrasService
     {
-        bool IsDigestAnnotatedForEol(string digest, ILoggerService loggerService, bool isDryRun);
+        bool IsDigestAnnotatedForEol(string digest, ILoggerService loggerService, bool isDryRun, [MaybeNullWhen(false)] out OciManifest lifecycleArtifactManifest);
 
         bool AnnotateEolDigest(string digest, DateOnly date, ILoggerService loggerService, bool isDryRun);
     }

src/Microsoft.DotNet.ImageBuilder/src/Models/Annotations/EolAnnotationsData.cs

@@ -4,7 +4,6 @@
 
 using System;
 using System.Collections.Generic;
-using Newtonsoft.Json;
 
 #nullable enable
 namespace Microsoft.DotNet.ImageBuilder.Models.Annotations

src/Microsoft.DotNet.ImageBuilder/src/Models/Annotations/OciManifest.cs

@@ -9,10 +9,10 @@ namespace Microsoft.DotNet.ImageBuilder.Models.Annotations
 {
     public class OciManifest
     {
-        public string? ArtifactType { get; set; }
+        public string ArtifactType { get; set; } = string.Empty;
 
-        public OciManifest()
-        {
-        }
+        public string Reference { get; set; } = string.Empty;
+
+        public Dictionary<string, string> Annotations { get; set; } = [];
     }
 }

src/Microsoft.DotNet.ImageBuilder/src/Models/MarBulkDeletion/BulkDeletionDescription.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+
+namespace Microsoft.DotNet.ImageBuilder.Models.MarBulkDeletion;
+
+#nullable enable
+internal record BulkDeletionDescription
+{
+    public string RegistryType { get; init; } = "public";
+    public List<string> Digests { get; init;} = [];
+}

src/Microsoft.DotNet.ImageBuilder/src/OrasService.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.ComponentModel.Composition;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Microsoft.DotNet.ImageBuilder.Models.Annotations;
 using Newtonsoft.Json;
@@ -15,19 +16,22 @@ namespace Microsoft.DotNet.ImageBuilder
     public class OrasService : IOrasService
     {
         private const string LifecycleArtifactType = "application/vnd.microsoft.artifact.lifecycle";
+        public const string EndOfLifeAnnotation = "vnd.microsoft.artifact.lifecycle.end-of-life.date";
+        public const string EolDateFormat = "yyyy-MM-dd";
 
-        public bool IsDigestAnnotatedForEol(string digest, ILoggerService loggerService, bool isDryRun)
+        public bool IsDigestAnnotatedForEol(string digest, ILoggerService loggerService, bool isDryRun, [MaybeNullWhen(false)] out OciManifest lifecycleArtifactManifest)
         {
             string? stdOut = ExecuteHelper.ExecuteWithRetry(
                 "oras",
                 $"discover --artifact-type {LifecycleArtifactType} {digest} --format json",
                 isDryRun);
 
-            if (!string.IsNullOrEmpty(stdOut) && LifecycleAnnotationExists(stdOut, loggerService))
+            if (!string.IsNullOrEmpty(stdOut) && LifecycleAnnotationExists(stdOut, loggerService, out lifecycleArtifactManifest))
             {
                 return true;
             }
 
+            lifecycleArtifactManifest = null;
             return false;
         }
 
@@ -37,7 +41,7 @@ public bool AnnotateEolDigest(string digest, DateOnly date, ILoggerService logge
             {
                 ExecuteHelper.ExecuteWithRetry(
                     "oras",
-                    $"attach --artifact-type {LifecycleArtifactType} --annotation \"vnd.microsoft.artifact.lifecycle.end-of-life.date={date:yyyy-MM-dd}\" {digest}",
+                    $"attach --artifact-type {LifecycleArtifactType} --annotation \"{EndOfLifeAnnotation}={date.ToString(EolDateFormat)}\" {digest}",
                     isDryRun);
             }
             catch (InvalidOperationException ex)
@@ -49,21 +53,23 @@ public bool AnnotateEolDigest(string digest, DateOnly date, ILoggerService logge
             return true;
         }
 
-        private static bool LifecycleAnnotationExists(string json, ILoggerService loggerService)
+        private static bool LifecycleAnnotationExists(string json, ILoggerService loggerService, [MaybeNullWhen(false)] out OciManifest lifecycleArtifactManifest)
         {
             try
             {
                 OrasDiscoverData? orasDiscoverData = JsonConvert.DeserializeObject<OrasDiscoverData>(json);
                 if (orasDiscoverData?.Manifests != null)
                 {
-                    return orasDiscoverData.Manifests.Where(m => m.ArtifactType == LifecycleArtifactType).Any();
+                    lifecycleArtifactManifest = orasDiscoverData.Manifests.FirstOrDefault(m => m.ArtifactType == LifecycleArtifactType);
+                    return lifecycleArtifactManifest is not null;
                 }
             }
             catch (JsonException ex)
             {
                 loggerService.WriteError($"Failed to deserialize 'oras discover' json: {ex.Message}");
             }
 
+            lifecycleArtifactManifest = null;
             return false;
         }
     }