Support custom ado fields that mark work items as duplicate (#3467)

* Add field to ado config for checking duplicate work items

* Make duplicate fields nullable and add it to python models

* Update broken tests

* Update docs to include new ado_duplicate_fields property

Author: kananb

contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json

@@ -13,6 +13,10 @@
       "System.AreaPath": "OneFuzz-Ado-Integration",
       "System.Title": "{{report.task_id}}"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+    },
     "on_duplicate": {
       "increment": [],
       "comment": "DUP {{report.input_sha256}} <br> Repro Command: <br> <pre> {{ repro_cmd }} </pre> ",

docs/notifications/ado.md

@@ -51,6 +51,13 @@ clickable, make it a link.
       "System.Title": "{{ report.crash_site }} - {{ report.executable }}",
       "Microsoft.VSTS.TCM.ReproSteps": "This is my call stack: <ul> {{ for item in report.call_stack }} <li> {{ item }} </li> {{ end }} </ul>"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+      // note: the fields and values below are checked by default and don't need to be specified
+      // "System.Reason": "Duplicate"
+      // "Microsoft.VSTS.Common.ResolvedReason": "Duplicate"
+    },
     "comment": "This is my comment. {{ report.input_sha256 }} {{ input_url }} <br> <pre>{{ repro_cmd }}</pre>",
     "unique_fields": ["System.Title", "System.AreaPath"],
     "on_duplicate": {

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -689,6 +689,7 @@ public record AdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
     ) : NotificationTemplate {
     public async Task<OneFuzzResultVoid> Validate() {
@@ -704,8 +705,9 @@ public record RenderedAdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
-    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);
+    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, AdoDuplicateFields, Comment);
 
 public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
     public Task<OneFuzzResultVoid> Validate() {

src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs

@@ -239,7 +239,7 @@ private static async Async.Task ProcessNotification(IOnefuzzContext context, Con
 
         var renderedConfig = RenderAdoTemplate(logTracer, renderer, config, instanceUrl);
         var ado = new AdoConnector(renderedConfig, project!, client, instanceUrl, logTracer, await GetValidFields(client, project));
-        await ado.Process(notificationInfo);
+        await ado.Process(notificationInfo, config.AdoDuplicateFields);
     }
 
     public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer renderer, AdoTemplate original, Uri instanceUrl) {
@@ -291,6 +291,7 @@ public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer
             original.UniqueFields,
             adoFields,
             onDuplicate,
+            original.AdoDuplicateFields,
             original.Comment != null ? Render(renderer, original.Comment, instanceUrl, logTracer) : null
         );
     }
@@ -525,7 +526,7 @@ private async Async.Task<WorkItem> CreateNew() {
             return (taskType, document);
         }
 
-        public async Async.Task Process(IList<(string, string)> notificationInfo) {
+        public async Async.Task Process(IList<(string, string)> notificationInfo, Dictionary<string, string>? duplicateFields) {
             var updated = false;
             WorkItem? oldestWorkItem = null;
             await foreach (var workItem in ExistingWorkItems(notificationInfo)) {
@@ -535,7 +536,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) {
                     _logTracer.AddTags(new List<(string, string)> { ("MatchingWorkItemIds", $"{workItem.Id}") });
                     _logTracer.LogInformation("Found matching work item");
                 }
-                if (IsADODuplicateWorkItem(workItem)) {
+                if (IsADODuplicateWorkItem(workItem, duplicateFields)) {
                     continue;
                 }
 
@@ -575,13 +576,17 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) {
             }
         }
 
-        private static bool IsADODuplicateWorkItem(WorkItem wi) {
+        private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary<string, string>? duplicateFields) {
             // A work item could have System.State == Resolve && System.Reason == Duplicate
             // OR it could have System.State == Closed && System.Reason == Duplicate
             // I haven't found any other combinations where System.Reason could be duplicate but just to be safe
             // we're explicitly _not_ checking the state of the work item to determine if it's duplicate
             return wi.Fields.ContainsKey("System.Reason") && string.Equals(wi.Fields["System.Reason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase)
             || wi.Fields.ContainsKey("Microsoft.VSTS.Common.ResolvedReason") && string.Equals(wi.Fields["Microsoft.VSTS.Common.ResolvedReason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase)
+            || duplicateFields?.Any(fieldPair => {
+                var (field, value) = fieldPair;
+                return wi.Fields.ContainsKey(field) && string.Equals(wi.Fields[field].ToString(), value, StringComparison.OrdinalIgnoreCase);
+            }) == true
             // Alternatively, the work item can also specify a 'relation' to another work item.
             // This is typically used to create parent/child relationships between work items but can also
             // Be used to mark duplicates so we should check this as well.

src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs

@@ -111,6 +111,7 @@ public async Async.Task OptionalFieldsAreSupported() {
                 },
                 "{{ if org }} blah {{ end }}"
             ),
+            null,
             "{{ if org }} blah {{ end }}"
         );
 
@@ -137,6 +138,7 @@ public async Async.Task All_ADO_Fields_Are_Migrated() {
                 },
                 "{% if org %} comment {% endif %}"
             ),
+            null,
             "{% if org %} comment {% endif %}"
         );
 

src/ApiService/Tests/OrmModelsTest.cs

@@ -232,6 +232,7 @@ from authToken in Arb.Generate<SecretData<string>>()
                         from str in Arb.Generate<NonEmptyString>()
                         from fields in Arb.Generate<List<string>>()
                         from adoFields in Arb.Generate<Dictionary<string, string>>()
+                        from adoDuplicateFields in Arb.Generate<Dictionary<string, string>>()
                         from dupeTemplate in Arb.Generate<ADODuplicateTemplate>()
                         select new AdoTemplate(
                             baseUrl,
@@ -241,6 +242,7 @@ from dupeTemplate in Arb.Generate<ADODuplicateTemplate>()
                             fields,
                             adoFields,
                             dupeTemplate,
+                            adoDuplicateFields,
                             str.Get));
 
         public static Arbitrary<TeamsTemplate> ArbTeamsTemplate()

src/pytypes/onefuzztypes/models.py

@@ -273,6 +273,7 @@ class ADOTemplate(BaseModel):
     unique_fields: List[str]
     comment: Optional[str]
     ado_fields: Dict[str, str]
+    ado_duplicate_fields: Optional[Dict[str, str]]
     on_duplicate: ADODuplicateTemplate
 
     # validator needed to convert auth_token to SecretData