Move to extended enums for fields and entry types (#5148)

* Introduce interface for fields

* Continue using fields interface instead of strings

* Introduce OrFields class

* Move FieldProperty

* Convert entry types to extended enum

* Start fixing build

* Fix build

* Start fixing tests

* Everything compiles again

* Make model tests pass

* Fix most tests

* Fix build

* Fix a few more tests

* Fix a few more tests

* Fix a few more tests

* Fix a few more tests

* Fix checkstyle issues

* Fix more tests

* Fix more tests

* Disable test that fails on CI

* Disable test that fails on CI

Author: tobiasdiez

CHANGELOG.md

@@ -11,6 +11,8 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 ## [Unreleased]
 
 ### Changed
+- All fields are now properly sorted alphabetically (in the subgroups of required/optional fields) when the entry is written to the bib file.
+- We fixed an issue where some importers used the field `pubstatus` instead of the standard BibTeX field `pubstate`.
 - We changed the latex command removal for docbook exporter. [#3838](https://github.com/JabRef/jabref/issues/3838)
 - We changed the location of some fields in the entry editor (you might need to reset your preferences for these changes to come into effect)
   - Journal/Year/Month in biblatex mode -> Deprecated (if filled)

src/jmh/java/org/jabref/benchmarks/Benchmarks.java

@@ -22,6 +22,9 @@
 import org.jabref.model.database.BibDatabaseMode;
 import org.jabref.model.database.BibDatabaseModeDetection;
 import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibEntryTypesManager;
+import org.jabref.model.entry.field.StandardField;
+import org.jabref.model.entry.field.UnknownField;
 import org.jabref.model.groups.GroupHierarchyType;
 import org.jabref.model.groups.KeywordGroup;
 import org.jabref.model.groups.WordKeywordGroup;
@@ -54,12 +57,12 @@ public void init() throws Exception {
         for (int i = 0; i < 1000; i++) {
             BibEntry entry = new BibEntry();
             entry.setCiteKey("id" + i);
-            entry.setField("title", "This is my title " + i);
-            entry.setField("author", "Firstname Lastname and FirstnameA LastnameA and FirstnameB LastnameB" + i);
-            entry.setField("journal", "Journal Title " + i);
-            entry.setField("keyword", "testkeyword");
-            entry.setField("year", "1" + i);
-            entry.setField("rnd", "2" + randomizer.nextInt());
+            entry.setField(StandardField.TITLE, "This is my title " + i);
+            entry.setField(StandardField.AUTHOR, "Firstname Lastname and FirstnameA LastnameA and FirstnameB LastnameB" + i);
+            entry.setField(StandardField.JOURNAL, "Journal Title " + i);
+            entry.setField(StandardField.KEYWORDS, "testkeyword");
+            entry.setField(StandardField.YEAR, "1" + i);
+            entry.setField(new UnknownField("rnd"), "2" + randomizer.nextInt());
             database.insertEntry(entry);
         }
 
@@ -72,7 +75,7 @@ public void init() throws Exception {
 
     private StringWriter getOutputWriter() throws IOException {
         StringWriter outputWriter = new StringWriter();
-        BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(outputWriter, mock(SavePreferences.class));
+        BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(outputWriter, mock(SavePreferences.class), new BibEntryTypesManager());
         databaseWriter.savePartOfDatabase(
                 new BibDatabaseContext(database, new MetaData(), new Defaults()), database.getEntries());
         return outputWriter;
@@ -128,7 +131,7 @@ public String htmlToLatexConversion() {
 
     @Benchmark
     public boolean keywordGroupContains() {
-        KeywordGroup group = new WordKeywordGroup("testGroup", GroupHierarchyType.INDEPENDENT, "keyword", "testkeyword", false, ',', false);
+        KeywordGroup group = new WordKeywordGroup("testGroup", GroupHierarchyType.INDEPENDENT, StandardField.KEYWORDS, "testkeyword", false, ',', false);
         return group.containsAll(database.getEntries());
     }
 

src/main/java/org/jabref/Globals.java

@@ -21,6 +21,7 @@
 import org.jabref.logic.protectedterms.ProtectedTermsLoader;
 import org.jabref.logic.remote.server.RemoteListenerServerLifecycle;
 import org.jabref.logic.util.BuildInfo;
+import org.jabref.model.entry.BibEntryTypesManager;
 import org.jabref.model.util.FileUpdateMonitor;
 import org.jabref.preferences.JabRefPreferences;
 
@@ -61,6 +62,7 @@ public class Globals {
     public static StateManager stateManager = new StateManager();
     public static ExporterFactory exportFactory;
     public static CountingUndoManager undoManager = new CountingUndoManager();
+    public static BibEntryTypesManager entryTypesManager = new BibEntryTypesManager();
     // Key binding preferences
     private static KeyBindingRepository keyBindingRepository;
     private static DefaultFileUpdateMonitor fileUpdateMonitor;

src/main/java/org/jabref/JabRefMain.java

@@ -22,9 +22,7 @@
 import org.jabref.logic.util.JavaVersion;
 import org.jabref.logic.util.OS;
 import org.jabref.migrations.PreferencesMigrations;
-import org.jabref.model.EntryTypes;
 import org.jabref.model.database.BibDatabaseMode;
-import org.jabref.model.entry.InternalBibtexFields;
 import org.jabref.preferences.JabRefPreferences;
 
 import org.slf4j.Logger;
@@ -153,21 +151,14 @@ private static boolean handleMultipleAppInstances(String[] args) {
     }
 
     private static void applyPreferences(JabRefPreferences preferences) {
-        // Update handling of special fields based on preferences
-        InternalBibtexFields.updateSpecialFields(Globals.prefs.getBoolean(JabRefPreferences.SERIALIZESPECIALFIELDS));
-        // Update name of the time stamp field based on preferences
-        InternalBibtexFields.updateTimeStampField(Globals.prefs.getTimestampPreferences().getTimestampField());
-        // Update which fields should be treated as numeric, based on preferences:
-        InternalBibtexFields.setNumericFields(Globals.prefs.getStringList(JabRefPreferences.NUMERIC_FIELDS));
-
         // Read list(s) of journal names and abbreviations
         Globals.journalAbbreviationLoader = new JournalAbbreviationLoader();
 
         // Build list of Import and Export formats
         Globals.IMPORT_FORMAT_READER.resetImportFormats(Globals.prefs.getImportFormatPreferences(),
                                                         Globals.prefs.getXMPPreferences(), Globals.getFileUpdateMonitor());
-        EntryTypes.loadCustomEntryTypes(preferences.loadCustomEntryTypes(BibDatabaseMode.BIBTEX),
-                                        preferences.loadCustomEntryTypes(BibDatabaseMode.BIBLATEX));
+        Globals.entryTypesManager.addCustomizedEntryTypes(preferences.loadBibEntryTypes(BibDatabaseMode.BIBTEX),
+                preferences.loadBibEntryTypes(BibDatabaseMode.BIBLATEX));
         Globals.exportFactory = Globals.prefs.getExporterFactory(Globals.journalAbbreviationLoader);
 
         // Initialize protected terms loader

src/main/java/org/jabref/cli/ArgumentProcessor.java

@@ -44,7 +44,6 @@
 import org.jabref.logic.util.OS;
 import org.jabref.logic.xmp.XmpPreferences;
 import org.jabref.model.Defaults;
-import org.jabref.model.EntryTypes;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.BibDatabaseMode;
@@ -391,7 +390,7 @@ private void saveDatabase(BibDatabase newBase, String subName) {
             System.out.println(Localization.lang("Saving") + ": " + subName);
             SavePreferences prefs = Globals.prefs.loadForSaveFromPreferences();
             AtomicFileWriter fileWriter = new AtomicFileWriter(Paths.get(subName), prefs.getEncoding());
-            BibDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, prefs);
+            BibDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, prefs, Globals.entryTypesManager);
             Defaults defaults = new Defaults(Globals.prefs.getDefaultBibDatabaseMode());
             databaseWriter.saveDatabase(new BibDatabaseContext(newBase, defaults));
 
@@ -458,8 +457,8 @@ private void exportFile(List<ParserResult> loaded, String[] data) {
     private void importPreferences() {
         try {
             Globals.prefs.importPreferences(cli.getPreferencesImport());
-            EntryTypes.loadCustomEntryTypes(Globals.prefs.loadCustomEntryTypes(BibDatabaseMode.BIBTEX),
-                    Globals.prefs.loadCustomEntryTypes(BibDatabaseMode.BIBLATEX));
+            Globals.entryTypesManager.addCustomizedEntryTypes(Globals.prefs.loadBibEntryTypes(BibDatabaseMode.BIBTEX),
+                    Globals.prefs.loadBibEntryTypes(BibDatabaseMode.BIBLATEX));
             List<TemplateExporter> customExporters = Globals.prefs.getCustomExportFormats(Globals.journalAbbreviationLoader);
             LayoutFormatterPreferences layoutPreferences = Globals.prefs
                     .getLayoutFormatterPreferences(Globals.journalAbbreviationLoader);

src/main/java/org/jabref/cli/CrossrefFetcherEvaluator.java

@@ -16,7 +16,7 @@
 import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.entry.BibEntry;
-import org.jabref.model.entry.FieldName;
+import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.entry.identifier.DOI;
 import org.jabref.preferences.JabRefPreferences;
 
@@ -53,7 +53,7 @@ public static void main(String[] args) throws IOException, InterruptedException
 
                     @Override
                     public void run() {
-                        Optional<DOI> origDOI = entry.getField(FieldName.DOI).flatMap(DOI::parse);
+                        Optional<DOI> origDOI = entry.getField(StandardField.DOI).flatMap(DOI::parse);
                         if (origDOI.isPresent()) {
                             dois.incrementAndGet();
                             try {

src/main/java/org/jabref/gui/BasePanel.java

@@ -92,13 +92,14 @@
 import org.jabref.model.database.shared.DatabaseLocation;
 import org.jabref.model.database.shared.DatabaseSynchronizer;
 import org.jabref.model.entry.BibEntry;
-import org.jabref.model.entry.FieldName;
-import org.jabref.model.entry.InternalBibtexFields;
 import org.jabref.model.entry.LinkedFile;
 import org.jabref.model.entry.event.EntryChangedEvent;
 import org.jabref.model.entry.event.EntryEventSource;
-import org.jabref.model.entry.specialfields.SpecialField;
-import org.jabref.model.entry.specialfields.SpecialFieldValue;
+import org.jabref.model.entry.field.Field;
+import org.jabref.model.entry.field.FieldFactory;
+import org.jabref.model.entry.field.SpecialField;
+import org.jabref.model.entry.field.SpecialFieldValue;
+import org.jabref.model.entry.field.StandardField;
 import org.jabref.preferences.JabRefPreferences;
 import org.jabref.preferences.PreviewPreferences;
 
@@ -573,13 +574,13 @@ private void openExternalFile() {
         }
         JabRefExecutorService.INSTANCE.execute(() -> {
             final BibEntry entry = selectedEntries.get(0);
-            if (!entry.hasField(FieldName.FILE)) {
+            if (!entry.hasField(StandardField.FILE)) {
                 // no bibtex field
                 new SearchAndOpenFile(entry, BasePanel.this).searchAndOpen();
                 return;
             }
             FileListTableModel fileListTableModel = new FileListTableModel();
-            entry.getField(FieldName.FILE).ifPresent(fileListTableModel::setContent);
+            entry.getField(StandardField.FILE).ifPresent(fileListTableModel::setContent);
             if (fileListTableModel.getRowCount() == 0) {
                 // content in BibTeX field is not readable
                 new SearchAndOpenFile(entry, BasePanel.this).searchAndOpen();
@@ -637,7 +638,7 @@ public void insertEntry(final BibEntry bibEntry) {
 
                 // Create an UndoableInsertEntry object.
                 getUndoManager().addEdit(new UndoableInsertEntry(bibDatabaseContext.getDatabase(), bibEntry));
-                output(Localization.lang("Added new '%0' entry.", bibEntry.getType()));
+                output(Localization.lang("Added new '%0' entry.", bibEntry.getType().getDisplayName()));
 
                 markBaseChanged(); // The database just changed.
                 if (Globals.prefs.getBoolean(JabRefPreferences.AUTO_OPEN_FORM)) {
@@ -650,11 +651,11 @@ public void insertEntry(final BibEntry bibEntry) {
         }
     }
 
-    public void editEntryAndFocusField(BibEntry entry, String fieldName) {
+    public void editEntryAndFocusField(BibEntry entry, Field field) {
         showAndEdit(entry);
         Platform.runLater(() -> {
             // Focus field and entry in main table (async to give entry editor time to load)
-            entryEditor.setFocusToField(fieldName);
+            entryEditor.setFocusToField(field);
             clearAndSelect(entry);
         });
     }
@@ -808,7 +809,7 @@ public void updateSearchManager() {
     }
 
     private void instantiateSearchAutoCompleter() {
-        searchAutoCompleter = new PersonNameSuggestionProvider(InternalBibtexFields.getPersonNameFields());
+        searchAutoCompleter = new PersonNameSuggestionProvider(FieldFactory.getPersonNameFields());
         for (BibEntry entry : bibDatabaseContext.getDatabase().getEntries()) {
             searchAutoCompleter.indexEntry(entry);
         }
@@ -1300,11 +1301,11 @@ private class OpenURLAction implements BaseAction {
         public void action() {
             final List<BibEntry> bes = mainTable.getSelectedEntries();
             if (bes.size() == 1) {
-                String field = FieldName.DOI;
-                Optional<String> link = bes.get(0).getField(FieldName.DOI);
-                if (bes.get(0).hasField(FieldName.URL)) {
-                    link = bes.get(0).getField(FieldName.URL);
-                    field = FieldName.URL;
+                Field field = StandardField.DOI;
+                Optional<String> link = bes.get(0).getField(StandardField.DOI);
+                if (bes.get(0).hasField(StandardField.URL)) {
+                    link = bes.get(0).getField(StandardField.URL);
+                    field = StandardField.URL;
                 }
                 if (link.isPresent()) {
                     try {
@@ -1320,9 +1321,9 @@ public void action() {
                     List<LinkedFile> files = bes.get(0).getFiles();
 
                     Optional<LinkedFile> linkedFile = files.stream()
-                                                           .filter(file -> (FieldName.URL.equalsIgnoreCase(file.getFileType())
-                                                                            || FieldName.PS.equalsIgnoreCase(file.getFileType())
-                                                                            || FieldName.PDF.equalsIgnoreCase(file.getFileType())))
+                                                           .filter(file -> (StandardField.URL.getName().equalsIgnoreCase(file.getFileType())
+                                                                   || StandardField.PS.getName().equalsIgnoreCase(file.getFileType())
+                                                                   || StandardField.PDF.getName().equalsIgnoreCase(file.getFileType())))
                                                            .findFirst();
 
                     if (linkedFile.isPresent()) {

src/main/java/org/jabref/gui/ClipBoardManager.java

@@ -81,7 +81,7 @@ public void setContent(String string) {
 
     public void setContent(List<BibEntry> entries) throws IOException {
         final ClipboardContent content = new ClipboardContent();
-        BibEntryWriter writer = new BibEntryWriter(new LatexFieldFormatter(Globals.prefs.getLatexFieldFormatterPreferences()), false);
+        BibEntryWriter writer = new BibEntryWriter(new LatexFieldFormatter(Globals.prefs.getLatexFieldFormatterPreferences()), Globals.entryTypesManager);
         String serializedEntries = writer.serializeAll(entries, BibDatabaseMode.BIBTEX);
         content.put(DragAndDropDataFormats.ENTRIES, serializedEntries);
         content.putString(serializedEntries);

src/main/java/org/jabref/gui/EntryTypeView.java

@@ -13,13 +13,14 @@
 import javafx.scene.control.TitledPane;
 import javafx.scene.layout.FlowPane;
 
+import org.jabref.Globals;
 import org.jabref.gui.util.BaseDialog;
 import org.jabref.gui.util.ControlHelper;
 import org.jabref.gui.util.ViewModelListCellFactory;
 import org.jabref.logic.importer.IdBasedFetcher;
 import org.jabref.logic.l10n.Localization;
-import org.jabref.model.EntryTypes;
 import org.jabref.model.database.BibDatabaseMode;
+import org.jabref.model.entry.BibEntryType;
 import org.jabref.model.entry.BiblatexEntryTypes;
 import org.jabref.model.entry.BibtexEntryTypes;
 import org.jabref.model.entry.EntryType;
@@ -84,10 +85,9 @@ public EntryTypeView(BasePanel basePanel, DialogService dialogService, JabRefPre
 
     }
 
-    private void addEntriesToPane(FlowPane pane, Collection<? extends EntryType> entries) {
-
-        for (EntryType entryType : entries) {
-            Button entryButton = new Button(entryType.getName());
+    private void addEntriesToPane(FlowPane pane, Collection<? extends BibEntryType> entries) {
+        for (BibEntryType entryType : entries) {
+            Button entryButton = new Button(entryType.getType().getDisplayName());
             entryButton.setUserData(entryType);
             entryButton.setOnAction(event -> setEntryTypeForReturnAndClose(entryType));
             pane.getChildren().add(entryButton);
@@ -124,7 +124,7 @@ public void initialize() {
             bibTexTitlePane.setVisible(false);
             ieeeTranTitlePane.setVisible(false);
 
-            List<EntryType> customTypes = EntryTypes.getAllCustomTypes(BibDatabaseMode.BIBLATEX);
+            List<BibEntryType> customTypes = Globals.entryTypesManager.getAllCustomTypes(BibDatabaseMode.BIBLATEX);
             if (customTypes.isEmpty()) {
                 customTitlePane.setVisible(false);
             } else {
@@ -136,7 +136,7 @@ public void initialize() {
             addEntriesToPane(bibTexPane, BibtexEntryTypes.ALL);
             addEntriesToPane(ieeetranPane, IEEETranEntryTypes.ALL);
 
-            List<EntryType> customTypes = EntryTypes.getAllCustomTypes(BibDatabaseMode.BIBTEX);
+            List<BibEntryType> customTypes = Globals.entryTypesManager.getAllCustomTypes(BibDatabaseMode.BIBTEX);
             if (customTypes.isEmpty()) {
                 customTitlePane.setVisible(false);
             } else {
@@ -162,8 +162,8 @@ private void focusTextField(Event event) {
         idTextField.selectAll();
     }
 
-    private void setEntryTypeForReturnAndClose(EntryType entryType) {
-        type = entryType;
+    private void setEntryTypeForReturnAndClose(BibEntryType entryType) {
+        type = entryType.getType();
         viewModel.stopFetching();
         this.close();
     }

src/main/java/org/jabref/gui/EntryTypeViewModel.java

@@ -14,6 +14,7 @@
 import javafx.concurrent.Task;
 import javafx.concurrent.Worker;
 
+import org.jabref.Globals;
 import org.jabref.gui.duplicationFinder.DuplicateResolverDialog;
 import org.jabref.logic.bibtex.DuplicateCheck;
 import org.jabref.logic.bibtexkeypattern.BibtexKeyGenerator;
@@ -138,7 +139,7 @@ public void runFetcherWorker() {
             Optional<BibEntry> result = fetcherWorker.getValue();
             if (result.isPresent()) {
                 final BibEntry entry = result.get();
-                Optional<BibEntry> duplicate = DuplicateCheck.containsDuplicate(basePanel.getDatabase(), entry, basePanel.getBibDatabaseContext().getMode());
+                Optional<BibEntry> duplicate = new DuplicateCheck(Globals.entryTypesManager).containsDuplicate(basePanel.getDatabase(), entry, basePanel.getBibDatabaseContext().getMode());
                 if ((duplicate.isPresent())) {
                     DuplicateResolverDialog dialog = new DuplicateResolverDialog(entry, duplicate.get(), DuplicateResolverDialog.DuplicateResolverType.IMPORT_CHECK, basePanel.getBibDatabaseContext());
                     switch (dialog.showAndWait().orElse(DuplicateResolverDialog.DuplicateResolverResult.BREAK)) {