Fix exclude consistency fields 13131

CHANGELOG.md

@@ -57,6 +57,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added "Hanging Indent" as the default selected bibliography body format for CSL styles that specify it (e.g. APA). [#13074](https://github.com/JabRef/jabref/pull/13074)
 - We fixed an issue where bibliography entries generated from CSL styles had leading spaces. [#13074](https://github.com/JabRef/jabref/pull/13074)
 - We fixed an issue where the preview area in the "Select Style" dialog of the LibreOffice integration was too small to display full content. [#13051](https://github.com/JabRef/jabref/issues/13051)
+- We excluded specific fields (e.g., `comment`, `pdf`, `sortkey`) from the consistency check to reduce false positives [#13131](https://github.com/JabRef/jabref/issues/13131)
 - We fixed an issue where the tab showing the fulltext search results was not displayed. [#12865](https://github.com/JabRef/jabref/issues/12865)
 - We fixed an issue showing an empty tooltip in maintable. [#11681](https://github.com/JabRef/jabref/issues/11681)
 

jablib/src/main/java/org/jabref/logic/quality/consistency/BibliographyConsistencyCheck.java

@@ -8,17 +8,62 @@
 import java.util.Map;
 import java.util.SequencedCollection;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.function.BiConsumer;
 
+
 import org.jabref.logic.bibtex.comparator.BibEntryByCitationKeyComparator;
 import org.jabref.logic.bibtex.comparator.BibEntryByFieldsComparator;
 import org.jabref.logic.bibtex.comparator.FieldComparatorStack;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.field.Field;
+import org.jabref.model.entry.field.SpecialField;
+import org.jabref.model.entry.field.StandardField;
+import org.jabref.model.entry.field.UserSpecificCommentField;
 import org.jabref.model.entry.types.EntryType;
 
 public class BibliographyConsistencyCheck {
 
+    // ----------------------------------------------------------------
+    // Fields that we do *not* want to check for consistency:
+    //   – every field in StandardField.AUTOMATIC_FIELDS
+    //   – COMMENT, CROSSREF, CITES, PDF, REVIEW, SORTKEY, SORTNAME, TYPE, XREF, GROUPS
+    //   – plus any SpecialField or UserSpecificCommentField (filtered at runtime)
+    // ----------------------------------------------------------------
+    private static final Set<Field> FILTERED_FIELDS = Stream
+            // start with JabRef’s built‐in AUTOMATIC_FIELDS set
+            .concat(
+                    StandardField.AUTOMATIC_FIELDS.stream(),
+                    // then add the extra fields to ignore
+                    Stream.of(
+                            StandardField.COMMENT,
+                            StandardField.CROSSREF,
+                            StandardField.CITES,
+                            StandardField.PDF,
+                            StandardField.REVIEW,
+                            StandardField.SORTKEY,
+                            StandardField.SORTNAME,
+                            StandardField.TYPE,
+                            StandardField.XREF,
+                            StandardField.GROUPS
+                    )
+            )
+            .collect(Collectors.toSet());
+
+    /**
+     * Drop any field we don’t want to check.
+     */
+    private static Set<Field> filterFields(Collection<Field> fields) {
+        return fields.stream()
+                     .filter(f -> !FILTERED_FIELDS.contains(f))
+                     .filter(f -> !(f instanceof SpecialField))
+                     .filter(f -> !(f instanceof UserSpecificCommentField))
+                     .collect(Collectors.toSet());
+    }
+
+
+
     public record Result(Map<EntryType, EntryTypeResult> entryTypeToResultMap) {
     }
 
@@ -69,7 +114,10 @@ public Result check(List<BibEntry> entries, BiConsumer<Integer, Integer> entries
 
             List<BibEntry> differingEntries = entryTypeToEntriesMap
                     .get(entryType).stream()
-                    .filter(entry -> !entry.getFields().equals(commonFields))
+                    .filter(entry -> {
+                        Set<Field> entryFiltered = filterFields(entry.getFields());
+                        return !entryFiltered.equals(commonFields);
+                    })
                     .sorted(comparatorStack)
                     .toList();
 
@@ -80,17 +128,25 @@ public Result check(List<BibEntry> entries, BiConsumer<Integer, Integer> entries
     }
 
     private static void collectEntriesIntoMaps(List<BibEntry> entries, Map<EntryType, Set<Field>> entryTypeToFieldsInAnyEntryMap, Map<EntryType, Set<Field>> entryTypeToFieldsInAllEntriesMap, Map<EntryType, Set<BibEntry>> entryTypeToEntriesMap) {
-        for (BibEntry entry : entries) {
-            EntryType entryType = entry.getType();
-
-            Set<Field> fieldsInAnyEntry = entryTypeToFieldsInAnyEntryMap.computeIfAbsent(entryType, _ -> new HashSet<>());
-            fieldsInAnyEntry.addAll(entry.getFields());
-
-            Set<Field> fieldsInAllEntries = entryTypeToFieldsInAllEntriesMap.computeIfAbsent(entryType, _ -> new HashSet<>(entry.getFields()));
-            fieldsInAllEntries.retainAll(entry.getFields());
-
-            Set<BibEntry> entriesOfType = entryTypeToEntriesMap.computeIfAbsent(entryType, _ -> new HashSet<>());
-            entriesOfType.add(entry);
-        }
+        entries.forEach(entry -> {
+                      EntryType entryType = entry.getType();
+
+                      // 1) filter out AUTOMATIC_FIELDS + COMMENT, CROSSREF, CITES, PDF, REVIEW,
+                      //    SORTKEY, SORTNAME, TYPE, XREF, GROUPS
+                      // 2) remove any SpecialField or UserSpecificCommentField
+                      Set<Field> filteredFields = filterFields(entry.getFields());
+
+                      Set<Field> fieldsInAnyEntry = entryTypeToFieldsInAnyEntryMap
+                                       .computeIfAbsent(entryType, k -> new HashSet<>());
+                      fieldsInAnyEntry.addAll(filteredFields);
+
+                      Set<Field> fieldsInAllEntries = entryTypeToFieldsInAllEntriesMap
+                                        .computeIfAbsent(entryType, k -> new HashSet<>(filteredFields));
+                      fieldsInAllEntries.retainAll(filteredFields);
+
+                      Set<BibEntry> entriesOfType = entryTypeToEntriesMap
+                                        .computeIfAbsent(entryType, k -> new HashSet<>());
+                      entriesOfType.add(entry);
+                    });
     }
 }

jablib/src/test/java/org/jabref/logic/quality/consistency/BibliographyConsistencyCheckTest.java

@@ -8,8 +8,10 @@
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.logic.importer.fileformat.BibtexImporter;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.SpecialField;
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.field.UnknownField;
+import org.jabref.model.entry.field.UserSpecificCommentField;
 import org.jabref.model.entry.types.StandardEntryType;
 import org.jabref.model.util.DummyFileUpdateMonitor;
 
@@ -18,6 +20,8 @@
 import org.mockito.Answers;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 
 class BibliographyConsistencyCheckTest {
@@ -102,4 +106,43 @@ void checkLibraryWithoutIssues(@TempDir Path tempDir) {
         BibliographyConsistencyCheck.Result expected = new BibliographyConsistencyCheck.Result(Map.of());
         assertEquals(expected, result);
     }
+
+    /**
+     * Entries that differ only by automatic, comment, PDF, special or user-specific comment fields
+     * should produce no consistency results.
+     */
+    @Test
+    void filteredFieldsAreIgnored() {
+        // MISC entries differing only in filtered fields:
+        BibEntry a = new BibEntry(StandardEntryType.Misc, "a")
+                .withField(StandardField.COMMENT, "note")
+                .withField(StandardField.PDF, "file.pdf")
+                .withField(new UserSpecificCommentField("XYZ"), "foo")
+                .withField(SpecialField.PRIORITY, "high");
+        BibEntry b = new BibEntry(StandardEntryType.Misc, "b")
+                .withField(StandardField.COMMENT, "another note")
+                .withField(StandardField.PDF, "other.pdf");
+
+        var result = new BibliographyConsistencyCheck().check(List.of(a, b));
+
+        // Since all differences are in filtered fields, there should be no entries reported:
+        assertTrue(result.entryTypeToResultMap().isEmpty(),
+                "Differences only in filtered fields must be ignored");
+    }
+
+    @SuppressWarnings("checkstyle:RegexpMultiline")
+    @Test
+    void nonFilteredFieldDifferenceIsReported() {
+        // Two MISC entries differing in AUTHOR (not filtered)
+        BibEntry withAuthor = new BibEntry(StandardEntryType.Misc, "1")
+                .withField(StandardField.AUTHOR, "Knuth");
+        BibEntry withoutAuthor = new BibEntry(StandardEntryType.Misc, "2");
+
+        var result = new BibliographyConsistencyCheck().check(List.of(withAuthor, withoutAuthor));
+
+        // AUTHOR should be reported as a unique field
+        var typeResult = result.entryTypeToResultMap().get(StandardEntryType.Misc);
+        assertNotNull(typeResult);
+        assertTrue(typeResult.fields().contains(StandardField.AUTHOR));
+    }
 }