Add ability to run arbitrary formatters as cleanup actions

CHANGELOG.md

@@ -14,6 +14,7 @@ to [sourceforge feature requests](https://sourceforge.net/p/jabref/features/) by
 
 ### Changed
 - Implemented [#455](https://github.com/JabRef/jabref/issues/455): Add button in preference dialog to reset preferences
+- Add ability to run arbitrary formatters as cleanup actions (some old cleanup jobs are replaced by this functionality)
 - Implemented [#756](https://github.com/JabRef/jabref/issues/756): Add possibility to reformat all entries on save (under Preferences, File)
 - Comments and preamble are serialized with capitalized first letter, i.e. `@Comment` instead of `@comment` and `@Preamble` instead of `@PREAMBLE`.
 - Global sorting options and preferences are removed. Databases can still be sorted on save, but this is configured locally and stored in the file

src/main/java/net/sf/jabref/JabRefPreferences.java

@@ -36,13 +36,18 @@
 
 import javax.swing.*;
 
+import net.sf.jabref.exporter.FieldFormatterCleanups;
 import net.sf.jabref.gui.*;
 import net.sf.jabref.gui.entryeditor.EntryEditorTabList;
 import net.sf.jabref.gui.maintable.PersistenceTableColumnListener;
 import net.sf.jabref.gui.preftabs.ImportSettingsTab;
 import net.sf.jabref.importer.fileformat.ImportFormat;
 import net.sf.jabref.logic.autocompleter.AutoCompletePreferences;
 import net.sf.jabref.logic.cleanup.CleanupPreset;
+import net.sf.jabref.logic.cleanup.FieldFormatterCleanup;
+import net.sf.jabref.logic.formatter.BibtexFieldFormatters;
+import net.sf.jabref.logic.formatter.bibtexfields.*;
+import net.sf.jabref.logic.formatter.casechanger.CaseKeeper;
 import net.sf.jabref.logic.l10n.Localization;
 import net.sf.jabref.logic.labelpattern.GlobalLabelPattern;
 import net.sf.jabref.logic.util.OS;
@@ -62,6 +67,7 @@
 public class JabRefPreferences {
     private static final Log LOGGER = LogFactory.getLog(JabRefPreferences.class);
     public static final String EXTERNAL_FILE_TYPES = "externalFileTypes";
+
     /**
      * HashMap that contains all preferences which are set by default
      */
@@ -313,28 +319,32 @@ public class JabRefPreferences {
 
     public static final String AKS_AUTO_NAMING_PDFS_AGAIN = "AskAutoNamingPDFsAgain";
     public static final String CLEANUP_DOI = "CleanUpDOI";
-    public static final String CLEANUP_MONTH = "CleanUpMonth";
-    public static final String CLEANUP_PAGE_NUMBERS = "CleanUpPageNumbers";
-    public static final String CLEANUP_DATE = "CleanUpDate";
     public static final String CLEANUP_MAKE_PATHS_RELATIVE = "CleanUpMakePathsRelative";
     public static final String CLEANUP_RENAME_PDF = "CleanUpRenamePDF";
     public static final String CLEANUP_RENAME_PDF_ONLY_RELATIVE_PATHS = "CleanUpRenamePDFonlyRelativePaths";
     public static final String CLEANUP_UPGRADE_EXTERNAL_LINKS = "CleanUpUpgradeExternalLinks";
     public static final String CLEANUP_SUPERSCRIPTS = "CleanUpSuperscripts";
-    public static final String CLEANUP_HTML = "CleanUpHTML";
-    public static final String CLEANUP_CASE = "CleanUpCase";
-    public static final String CLEANUP_LATEX = "CleanUpLaTeX";
-    public static final String CLEANUP_UNITS = "CleanUpUnits";
     public static final String CLEANUP_UNICODE = "CleanUpUnicode";
     public static final String CLEANUP_CONVERT_TO_BIBLATEX = "CleanUpConvertToBiblatex";
     public static final String CLEANUP_FIX_FILE_LINKS = "CleanUpFixFileLinks";
+    public static final String CLEANUP_FORMATTERS = "CleanUpFormatters";
     public static final CleanupPreset CLEANUP_DEFAULT_PRESET;
     static {
         EnumSet<CleanupPreset.CleanupStep> deactivedJobs = EnumSet.of(
                 CleanupPreset.CleanupStep.CLEAN_UP_UPGRADE_EXTERNAL_LINKS,
                 CleanupPreset.CleanupStep.RENAME_PDF_ONLY_RELATIVE_PATHS,
                 CleanupPreset.CleanupStep.CONVERT_TO_BIBLATEX);
-        CLEANUP_DEFAULT_PRESET = new CleanupPreset(EnumSet.complementOf(deactivedJobs));
+
+        List<FieldFormatterCleanup> activeFormatterCleanups = new ArrayList<>();
+        activeFormatterCleanups.add(new FieldFormatterCleanup("pages", BibtexFieldFormatters.PAGE_NUMBERS));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("date", BibtexFieldFormatters.DATE));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("month", new MonthFormatter()));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("title", new CaseKeeper()));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("title", new UnitFormatter()));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("title", new LatexFormatter()));
+        activeFormatterCleanups.add(new FieldFormatterCleanup("title", new HTMLToLatexFormatter()));
+        FieldFormatterCleanups formatterCleanups = new FieldFormatterCleanups(true, activeFormatterCleanups);
+        CLEANUP_DEFAULT_PRESET = new CleanupPreset(EnumSet.complementOf(deactivedJobs), formatterCleanups);
     }
 
 
@@ -942,9 +952,12 @@ public void putStringList(String key, List<String> value) {
             return;
         }
 
-        put(key, value.stream().map(val -> StringUtil.quote(val, ";", '\\')).collect(Collectors.joining(";")));
+        put(key, convertListToString(value));
     }
 
+    private static String convertListToString(List<String> value) {
+        return value.stream().map(val -> StringUtil.quote(val, ";", '\\')).collect(Collectors.joining(";"));
+    }
 
     /**
      * Returns a List of Strings containing the chosen columns.
@@ -1306,19 +1319,13 @@ private static void insertCleanupPreset(Map<String, Object> storage, CleanupPres
 
         storage.put(CLEANUP_SUPERSCRIPTS, preset.isCleanUpSuperscripts());
         storage.put(CLEANUP_DOI, preset.isCleanUpDOI());
-        storage.put(CLEANUP_MONTH, preset.isCleanUpMonth());
-        storage.put(CLEANUP_PAGE_NUMBERS, preset.isCleanUpPageNumbers());
-        storage.put(CLEANUP_DATE, preset.isCleanUpDate());
         storage.put(CLEANUP_MAKE_PATHS_RELATIVE, preset.isMakePathsRelative());
         storage.put(CLEANUP_RENAME_PDF, preset.isRenamePDF());
         storage.put(CLEANUP_RENAME_PDF_ONLY_RELATIVE_PATHS, preset.isRenamePdfOnlyRelativePaths());
         storage.put(CLEANUP_UPGRADE_EXTERNAL_LINKS, preset.isCleanUpUpgradeExternalLinks());
-        storage.put(CLEANUP_HTML, preset.isConvertHTMLToLatex());
-        storage.put(CLEANUP_CASE, preset.isConvertCase());
-        storage.put(CLEANUP_LATEX, preset.isConvertLaTeX());
-        storage.put(CLEANUP_UNITS, preset.isConvertUnits());
         storage.put(CLEANUP_UNICODE, preset.isConvertUnicodeToLatex());
         storage.put(CLEANUP_CONVERT_TO_BIBLATEX, preset.isConvertToBiblatex());
         storage.put(CLEANUP_FIX_FILE_LINKS, preset.isFixFileLinks());
+        storage.put(CLEANUP_FORMATTERS, convertListToString(preset.getFormatterCleanups().convertToString()));
     }
 }

src/main/java/net/sf/jabref/MetaData.java

@@ -18,7 +18,7 @@
 import java.io.*;
 import java.util.*;
 
-import net.sf.jabref.exporter.SaveActions;
+import net.sf.jabref.exporter.FieldFormatterCleanups;
 import net.sf.jabref.groups.GroupTreeNode;
 import net.sf.jabref.logic.config.SaveOrderConfig;
 import net.sf.jabref.logic.labelpattern.AbstractLabelPattern;
@@ -33,9 +33,10 @@ public class MetaData implements Iterable<String> {
 
     public static final String META_FLAG = "jabref-meta: ";
     public static final String SAVE_ORDER_CONFIG = "saveOrderConfig";
+    public static final String SAVE_ACTIONS = "saveActions";
     private static final String PREFIX_KEYPATTERN = "keypattern_";
     private static final String KEYPATTERNDEFAULT = "keypatterndefault";
-    static final String DATABASE_TYPE = "DATABASE_TYPE";
+    public static final String DATABASE_TYPE = "DATABASE_TYPE";
     public static final String GROUPSVERSION = "groupsversion";
     public static final String GROUPSTREE = "groupstree";
     public static final String GROUPS = "groups";
@@ -386,13 +387,13 @@ public void setLabelPattern(AbstractLabelPattern labelPattern) {
         this.labelPattern = labelPattern;
     }
 
-    public SaveActions getSaveActions() {
-        if (this.getData(SaveActions.META_KEY) == null) {
-            return new SaveActions(false, "");
+    public FieldFormatterCleanups getSaveActions() {
+        if (this.getData(SAVE_ACTIONS) == null) {
+            return new FieldFormatterCleanups(false, new ArrayList<>());
         } else {
-            boolean enablementStatus = this.getData(SaveActions.META_KEY).get(0).equals("enabled");
-            String formatterString = this.getData(SaveActions.META_KEY).get(1);
-            return new SaveActions(enablementStatus, formatterString);
+            boolean enablementStatus = this.getData(SAVE_ACTIONS).get(0).equals("enabled");
+            String formatterString = this.getData(SAVE_ACTIONS).get(1);
+            return new FieldFormatterCleanups(enablementStatus, formatterString);
         }
     }
 
@@ -450,7 +451,7 @@ public Map<String, String> serialize() throws IOException {
                 stringBuilder.append(StringUtil.quote(dataItem, ";", '\\')).append(";");
 
                 //in case of save actions, add an additional newline after the enabled flag
-                if (metaItem.getKey().equals(SaveActions.META_KEY) && "enabled".equals(dataItem)) {
+                if (metaItem.getKey().equals(SAVE_ACTIONS) && "enabled".equals(dataItem)) {
                     stringBuilder.append(Globals.NEWLINE);
                 }
             }
@@ -485,19 +486,9 @@ public Map<String, String> serialize() throws IOException {
         return serializedMetaData;
     }
 
-    public void setSaveActions(SaveActions saveActions) {
-        List<String> actionsSerialized = new ArrayList<>();
-
-        if (saveActions.isEnabled()) {
-            actionsSerialized.add("enabled");
-        } else {
-            actionsSerialized.add("disabled");
-        }
-
-        String formatterString = SaveActions.getMetaDataString(saveActions.getConfiguredActions());
-        actionsSerialized.add(formatterString);
-
-        putData(SaveActions.META_KEY, actionsSerialized);
+    public void setSaveActions(FieldFormatterCleanups saveActions) {
+        List<String> actionsSerialized = saveActions.convertToString();
+        putData(SAVE_ACTIONS, actionsSerialized);
     }
 
     public void setSaveOrderConfig(SaveOrderConfig saveOrderConfig) {

src/main/java/net/sf/jabref/exporter/BibDatabaseWriter.java

@@ -228,9 +228,9 @@ public SaveSession savePartOfDatabase(BibDatabaseContext bibDatabaseContext, Sav
     private static List<FieldChange> applySaveActions(List<BibEntry> toChange, MetaData metaData) {
         List<FieldChange> changes = new ArrayList<>();
 
-        if (metaData.getData(SaveActions.META_KEY) != null) {
+        if (metaData.getData(MetaData.SAVE_ACTIONS) != null) {
             // save actions defined -> apply for every entry
-            SaveActions saveActions = metaData.getSaveActions();
+            FieldFormatterCleanups saveActions = metaData.getSaveActions();
 
             for (BibEntry entry : toChange) {
                 changes.addAll(saveActions.applySaveActions(entry));

src/main/java/net/sf/jabref/exporter/SaveActions.java → src/main/java/net/sf/jabref/exporter/FieldFormatterCleanups.java

@@ -12,36 +12,32 @@
 
 import java.util.*;
 
-public class SaveActions {
+public class FieldFormatterCleanups {
 
     private final List<FieldFormatterCleanup> actions;
 
-    private List<Formatter> availableFormatters;
-
-    public static final String META_KEY = "saveActions";
+    private static List<Formatter> availableFormatters;
 
     private boolean enabled;
 
-    public static final SaveActions DEFAULT_ACTIONS;
+    public static final FieldFormatterCleanups DEFAULT_SAVE_ACTIONS;
 
     static {
+        availableFormatters = new ArrayList<>();
+        availableFormatters.addAll(BibtexFieldFormatters.ALL);
+        availableFormatters.addAll(CaseChangers.ALL);
+
         String defaultFormatterString = "pages[PageNumbersFormatter]month[MonthFormatter]booktitle[SuperscriptFormatter]";
-        DEFAULT_ACTIONS = new SaveActions(false, defaultFormatterString);
+        DEFAULT_SAVE_ACTIONS = new FieldFormatterCleanups(false, defaultFormatterString);
     }
 
-    public SaveActions(boolean enabled, String formatterString) {
+    public FieldFormatterCleanups(boolean enabled, String formatterString) {
+        this(enabled, parse(formatterString));
+    }
 
-        actions = new ArrayList<>();
-        setAvailableFormatters();
+    public FieldFormatterCleanups(boolean enabled, List<FieldFormatterCleanup> actions) {
         this.enabled = enabled;
-
-        if ((formatterString == null) || "".equals(formatterString)) {
-            // no save actions defined in the meta data
-            return;
-        } else {
-            parseSaveActions(formatterString);
-        }
-
+        this.actions = Objects.requireNonNull(actions);
     }
 
     public boolean isEnabled() {
@@ -65,7 +61,7 @@ public boolean equals(Object o) {
             return false;
         }
 
-        SaveActions that = (SaveActions) o;
+        FieldFormatterCleanups that = (FieldFormatterCleanups) o;
 
         if (enabled != that.enabled) {
             return false;
@@ -79,7 +75,15 @@ public int hashCode() {
         return Objects.hash(actions, enabled);
     }
 
-    private void parseSaveActions(String formatterString) {
+    private static List<FieldFormatterCleanup> parse(String formatterString) {
+
+        if ((formatterString == null) || formatterString.isEmpty()) {
+            // no save actions defined in the meta data
+            return new ArrayList<>();
+        }
+
+        List<FieldFormatterCleanup> actions = new ArrayList<>();
+
         //read concrete actions
         int startIndex = 0;
 
@@ -121,15 +125,9 @@ private void parseSaveActions(String formatterString) {
         } catch (StringIndexOutOfBoundsException ignore) {
             // if this exception occurs, the remaining part of the save actions string is invalid.
             // Thus we stop parsing and take what we have parsed until now
-            return;
+            return actions;
         }
-    }
-
-    private void setAvailableFormatters() {
-        availableFormatters = new ArrayList<>();
-
-        availableFormatters.addAll(BibtexFieldFormatters.ALL);
-        availableFormatters.addAll(CaseChangers.ALL);
+        return actions;
     }
 
     public List<FieldChange> applySaveActions(BibEntry entry) {
@@ -150,7 +148,7 @@ private List<FieldChange> applyAllActions(BibEntry entry) {
         return result;
     }
 
-    private Formatter getFormatterFromString(String formatterName) {
+    private static Formatter getFormatterFromString(String formatterName) {
         for (Formatter formatter : availableFormatters) {
             if (formatterName.equals(formatter.getKey())) {
                 return formatter;
@@ -159,7 +157,7 @@ private Formatter getFormatterFromString(String formatterName) {
         return new IdentityFormatter();
     }
 
-    public static String getMetaDataString(List<FieldFormatterCleanup> actionList) {
+    private static String getMetaDataString(List<FieldFormatterCleanup> actionList) {
         //first, group all formatters by the field for which they apply
         Map<String, List<String>> groupedByField = new HashMap<>();
         for (FieldFormatterCleanup cleanup : actionList) {
@@ -190,5 +188,30 @@ public static String getMetaDataString(List<FieldFormatterCleanup> actionList) {
         return result.toString();
     }
 
+    public List<String> convertToString() {
+        List<String> stringRepresentation = new ArrayList<>();
 
+        if (enabled) {
+            stringRepresentation.add("enabled");
+        } else {
+            stringRepresentation.add("disabled");
+        }
+
+        String formatterString = FieldFormatterCleanups.getMetaDataString(actions);
+        stringRepresentation.add(formatterString);
+        return stringRepresentation;
+    }
+
+    public static FieldFormatterCleanups parseFromString(List<String> formatterMetaList) {
+
+        if (formatterMetaList != null && formatterMetaList.size() >= 2) {
+            boolean enablementStatus = "enabled".equals(formatterMetaList.get(0));
+            String formatterString = formatterMetaList.get(1);
+            return new FieldFormatterCleanups(enablementStatus, formatterString);
+        } else {
+            // return default actions
+            return FieldFormatterCleanups.DEFAULT_SAVE_ACTIONS;
+        }
+
+    }
 }