Implement 1356

CHANGELOG.md

@@ -12,7 +12,8 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 ## [Unreleased]
 
 ### Changed
-- Implemented [#661](https://github.com/JabRef/jabref/issues/661): Introducing a "check for updates" mechanism (manually/automatic at startup)
+- Implemented [#1356](https://github.com/JabRef/jabref/issues/1356): Added a formatter for converting HTML to Unicode
+- Implemented [#661](https://github.com/JabRef/jabref/issues/661): Introducing a "check for updates" mechnism (manually/automatic at startup)
 - [#1485](https://github.com/JabRef/jabref/issues/1485): Biblatex field shorttitle is now exported/imported as standard field ShortTitle to Word bibliography
 - [#1431](https://github.com/JabRef/jabref/issues/1431): Import dialog shows file extensions and filters the view
 - Updated German translation

build.gradle

@@ -103,6 +103,8 @@ dependencies {
 
     compile 'commons-logging:commons-logging:1.2'
 
+    compile 'org.apache.commons:commons-lang3:3.4'
+
     compile 'org.jsoup:jsoup:1.9.2'
     compile 'com.mashape.unirest:unirest-java:1.4.9'
     compile 'info.debatty:java-string-similarity:0.13'

src/main/java/net/sf/jabref/logic/formatter/Formatters.java

@@ -23,6 +23,7 @@
 
 import net.sf.jabref.logic.formatter.bibtexfields.ClearFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.HtmlToLatexFormatter;
+import net.sf.jabref.logic.formatter.bibtexfields.HtmlToUnicodeFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.LatexCleanupFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.NormalizeDateFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.NormalizeMonthFormatter;
@@ -45,6 +46,7 @@ public class Formatters {
 
     public static final List<Formatter> CONVERTERS = Arrays.asList(
             new HtmlToLatexFormatter(),
+            new HtmlToUnicodeFormatter(),
             new LatexToUnicodeFormatter(),
             new UnicodeToLatexFormatter()
     );

src/main/java/net/sf/jabref/logic/formatter/bibtexfields/HtmlToUnicodeFormatter.java

@@ -0,0 +1,36 @@
+package net.sf.jabref.logic.formatter.bibtexfields;
+
+import net.sf.jabref.logic.formatter.Formatter;
+import net.sf.jabref.logic.l10n.Localization;
+import net.sf.jabref.logic.layout.LayoutFormatter;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+
+public class HtmlToUnicodeFormatter implements LayoutFormatter, Formatter {
+
+    @Override
+    public String getName() {
+        return "HTML to Unicode";
+    }
+
+    @Override
+    public String getKey() {
+        return "html_to_unicode";
+    }
+
+    @Override
+    public String getDescription() {
+        return Localization.lang("Converts HTML code to Unicode.");
+    }
+
+    @Override
+    public String getExampleInput() {
+        return "<b>bread</b> &amp; butter";
+    }
+
+    @Override
+    public String format(String fieldText) {
+        // StringEscapeUtils converts characters and regex kills tags
+        return StringEscapeUtils.unescapeHtml4(fieldText).replaceAll("\\<[^>]*>","");
+    }
+}

src/main/resources/l10n/JabRef_da.properties

@@ -1705,3 +1705,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_de.properties

@@ -2279,6 +2279,7 @@ Changes_the_first_letter_of_all_words_to_capital_case_and_the_remaining_letters_
 Cleans_up_LaTeX_code.=Räumt_LaTeX-Code_auf.
 
 Converts_HTML_code_to_LaTeX_code.=Konvertiere_HTML-Code_in_LaTeX-Code.
+Converts_HTML_code_to_Unicode.=Konvertiere_HTML-Code_in_Unicode.
 Converts_LaTeX_encoding_to_Unicode_characters.=Konvertiere_LaTeX_Kodierung_in_Unicode_Zeichen.
 Converts_Unicode_characters_to_LaTeX_encoding.=Konvertiere_Unicode_Zeichen_in_LaTeX_Kodierung.
 Converts_ordinals_to_LaTeX_superscripts.=Konvertiere_Ordinalzahlen_in_LaTeX_Hochstellung.
@@ -2422,3 +2423,5 @@ The_link_has_been_copied_to_the_clipboard.=Der_Link_wurde_in_die_Zwischenablage_
 Online_help_forum=Online-Hilfeforum
 
 Custom=
+
+

src/main/resources/l10n/JabRef_en.properties

@@ -2148,6 +2148,7 @@ Changes_all_letters_to_upper_case.=Changes_all_letters_to_upper_case.
 Changes_the_first_letter_of_all_words_to_capital_case_and_the_remaining_letters_to_lower_case.=Changes_the_first_letter_of_all_words_to_capital_case_and_the_remaining_letters_to_lower_case.
 Cleans_up_LaTeX_code.=Cleans_up_LaTeX_code.
 Converts_HTML_code_to_LaTeX_code.=Converts_HTML_code_to_LaTeX_code.
+Converts_HTML_code_to_Unicode.=Converts_HTML_code_to_Unicode.
 Converts_LaTeX_encoding_to_Unicode_characters.=Converts_LaTeX_encoding_to_Unicode_characters.
 Converts_Unicode_characters_to_LaTeX_encoding.=Converts_Unicode_characters_to_LaTeX_encoding.
 Converts_ordinals_to_LaTeX_superscripts.=Converts_ordinals_to_LaTeX_superscripts.

src/main/resources/l10n/JabRef_es.properties

@@ -1607,3 +1607,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_fa.properties

@@ -2393,3 +2393,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_fr.properties

@@ -1651,3 +1651,5 @@ The_link_has_been_copied_to_the_clipboard.=Le_lien_a_été_copié_dans_le_presse
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_in.properties

@@ -1626,3 +1626,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_it.properties

@@ -1726,3 +1726,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_ja.properties

@@ -2371,3 +2371,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_nl.properties

@@ -2402,3 +2402,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_no.properties

@@ -2798,3 +2798,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_pt_BR.properties

@@ -1620,3 +1620,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_ru.properties

@@ -2370,3 +2370,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_sv.properties

@@ -1421,6 +1421,7 @@ Changes_all_letters_to_upper_case.=Ändrar_alla_bokstäver_till_versaler.
 Changes_the_first_letter_of_all_words_to_capital_case_and_the_remaining_letters_to_lower_case.=
 Cleans_up_LaTeX_code.=Städar_upp_i_LaTeX-koden.
 Converts_HTML_code_to_LaTeX_code.=Konverterar_HTML-koder_till_LaTeX-kod.
+Converts_HTML_code_to_Unicode.=Konverterar_HTML-koder_till_Unicode-tecken.
 Converts_LaTeX_encoding_to_Unicode_characters.=Konverterar_Latex-koder_till_Unicode-tecken.
 Converts_Unicode_characters_to_LaTeX_encoding.=Konverterar_Unicode-tecken_till_LaTeX-kod.
 Converts_ordinals_to_LaTeX_superscripts.=Konverterar_ordningsnummer_till_LaTeX_\\superscript{}.
@@ -1562,3 +1563,5 @@ The_link_has_been_copied_to_the_clipboard.=Länken_har_kopierats_till_urklipp.
 
 Online_help_forum=Onlineforum
 Custom=Egna
+
+

src/main/resources/l10n/JabRef_tr.properties

@@ -1639,3 +1639,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_vi.properties

@@ -2394,3 +2394,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/main/resources/l10n/JabRef_zh.properties

@@ -1633,3 +1633,5 @@ The_link_has_been_copied_to_the_clipboard.=
 Online_help_forum=
 
 Custom=
+
+Converts_HTML_code_to_Unicode.=

src/test/java/net/sf/jabref/logic/formatter/FormatterTest.java

@@ -7,6 +7,7 @@
 import net.sf.jabref.JabRefPreferences;
 import net.sf.jabref.logic.formatter.bibtexfields.ClearFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.HtmlToLatexFormatter;
+import net.sf.jabref.logic.formatter.bibtexfields.HtmlToUnicodeFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.LatexCleanupFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.NormalizeDateFormatter;
 import net.sf.jabref.logic.formatter.bibtexfields.NormalizeMonthFormatter;
@@ -105,6 +106,7 @@ public static Collection<Object[]> instancesToTest() {
                 new Object[]{new CapitalizeFormatter()},
                 new Object[]{new ClearFormatter()},
                 new Object[]{new HtmlToLatexFormatter()},
+                new Object[]{new HtmlToUnicodeFormatter()},
                 new Object[]{new IdentityFormatter()},
                 new Object[]{new LatexCleanupFormatter()},
                 new Object[]{new LatexToUnicodeFormatter()},

src/test/java/net/sf/jabref/logic/formatter/IdentityFormatterTest.java

@@ -1,6 +1,6 @@
 package net.sf.jabref.logic.formatter;
 
-
+import org.junit.Before;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -10,7 +10,12 @@
  */
 public class IdentityFormatterTest {
 
-    private final IdentityFormatter formatter = new IdentityFormatter();
+    private IdentityFormatter formatter;
+
+    @Before
+    public void setUp() {
+        formatter = new IdentityFormatter();
+    }
 
     @Test
     public void formatExample() {

src/test/java/net/sf/jabref/logic/formatter/bibtexfields/ClearFormatterTest.java

@@ -4,14 +4,19 @@
  * Tests in addition to the general tests from {@link net.sf.jabref.logic.formatter.FormatterTest}
  */
 
+import org.junit.Before;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 
 public class ClearFormatterTest {
 
-    private final ClearFormatter formatter = new ClearFormatter();
+    private ClearFormatter formatter;
 
+    @Before
+    public void setUp() {
+        formatter = new ClearFormatter();
+    }
 
     /**
      * Check whether the clear formatter really returns the empty string for the empty string

src/test/java/net/sf/jabref/logic/formatter/bibtexfields/HtmlToLatexFormatterTest.java

@@ -4,6 +4,7 @@
 import net.sf.jabref.JabRefPreferences;
 
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -13,13 +14,18 @@
  */
 public class HtmlToLatexFormatterTest {
 
-    private final HtmlToLatexFormatter formatter = new HtmlToLatexFormatter();
+    private HtmlToLatexFormatter formatter;
 
-    @Before
-    public void setUp() throws Exception {
+    @BeforeClass
+    public static void setUpBeforeClass() throws Exception {
         Globals.prefs = JabRefPreferences.getInstance();
     }
 
+    @Before
+    public void setUp() {
+        formatter = new HtmlToLatexFormatter();
+    }
+
     @Test
     public void formatWithoutHtmlCharactersReturnsSameString() {
         assertEquals("abc", formatter.format("abc"));

src/test/java/net/sf/jabref/logic/formatter/bibtexfields/HtmlToUnicodeFormatterTest.java

@@ -0,0 +1,61 @@
+package net.sf.jabref.logic.formatter.bibtexfields;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class HtmlToUnicodeFormatterTest {
+
+    private HtmlToUnicodeFormatter formatter;
+
+    @Before
+    public void setUp() {
+        formatter = new HtmlToUnicodeFormatter();
+    }
+
+    @Test
+    public void formatWithoutHtmlCharactersReturnsSameString() {
+        assertEquals("abc", formatter.format("abc"));
+    }
+
+    @Test
+    public void formatMultipleHtmlCharacters() {
+        assertEquals("åäö", formatter.format("åäö"));
+    }
+
+    @Test
+    public void formatCombinedAccent() {
+        assertEquals("í", formatter.format("í"));
+    }
+
+    @Test
+    public void testBasic() {
+        assertEquals("aaa", formatter.format("aaa"));
+    }
+
+    @Test
+    public void testUmlauts() {
+        assertEquals("ä", formatter.format("ä"));
+        assertEquals("ä", formatter.format("ä"));
+        assertEquals("ä", formatter.format("ä"));
+
+    }
+
+    @Test
+    public void testGreekLetter() {
+        assertEquals("Ε", formatter.format("Ε"));
+    }
+
+    @Test
+    public void testHTMLRemoveTags() {
+        assertEquals("aaa", formatter.format("<p>aaa</p>"));
+    }
+
+    @Test
+    public void formatExample() {
+        assertEquals("bread & butter", formatter.format(formatter.getExampleInput()));
+    }
+}
+
+

src/test/java/net/sf/jabref/logic/formatter/bibtexfields/LatexCleanupFormatterTest.java

@@ -1,5 +1,6 @@
 package net.sf.jabref.logic.formatter.bibtexfields;
 
+import org.junit.Before;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -9,7 +10,12 @@
  */
 public class LatexCleanupFormatterTest {
 
-    private final LatexCleanupFormatter formatter = new LatexCleanupFormatter();
+    private LatexCleanupFormatter formatter;
+
+    @Before
+    public void setUp() {
+        formatter = new LatexCleanupFormatter();
+    }
 
     @Test
     public void test() {

src/test/java/net/sf/jabref/logic/formatter/bibtexfields/NormalizeDateFormatterTest.java

@@ -1,14 +1,20 @@
 package net.sf.jabref.logic.formatter.bibtexfields;
 
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 
 /**
  * Tests in addition to the general tests from {@link net.sf.jabref.logic.formatter.FormatterTest}
  */
 public class NormalizeDateFormatterTest {
 
-    private final NormalizeDateFormatter formatter = new NormalizeDateFormatter();
+    private NormalizeDateFormatter formatter;
+
+    @Before
+    public void setUp() {
+        formatter = new NormalizeDateFormatter();
+    }
 
     @Test
     public void formatDateYYYYMM0D() {