Merge remote-tracking branch 'upstream/master' into maintable-beta

* upstream/master:
  update gradle to 4.7 (#4049)
  Add missing translation for "HTML to Unicode" (#4046)
  New Crowdin translations (#4042)
  Use all-text-fields magic also in BibTeX cleanup. (#4039)
  Shortittle bibtexkeypattern now also discards small words (#4030)

Author: Siedlerchr

CHANGELOG.md

@@ -13,6 +13,8 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `#
 ### Changed
 - We added a text file export for 'Find Unlinked Files'. [#3341](https://github.com/JabRef/jabref/issues/3341)
 - We added a fetcher based on RFC-IDs. [#3971](https://github.com/JabRef/jabref/issues/3971)
+- We changed the implementation of the `[shorttitle]` key pattern. It now removes small words like `a`, `an`, `on`, `the` etc. Refer to the help page for a complete overview. [Feature request in the forum](http://discourse.jabref.org/t/jabref-differences-in-shorttitle-between-versions-3-8-1-and-4-not-discounting-the-a-an-of-in-titles/1147)
+- We streamlined the defaults for a [cleanup of entries](http://help.jabref.org/en/CleanupEntries) in the case of BibTeX.
 
 ### Fixed
 We fixed an issue where the export to clipboard functionality could not be invoked. [#3994](https://github.com/JabRef/jabref/issues/3994)

build.gradle

@@ -149,8 +149,8 @@ dependencies {
 
     compile 'com.github.tomtung:latex2unicode_2.12:0.2.2'
 
-    compile group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '2.1.0'
-    compile group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '2.1.0'
+    compile group: 'com.microsoft.azure', name: 'applicationinsights-core', version: '2.1.1'
+    compile group: 'com.microsoft.azure', name: 'applicationinsights-logging-log4j2', version: '2.1.1'
 
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.2.0'
     testCompile 'org.junit.jupiter:junit-jupiter-params:5.2.0'
@@ -165,7 +165,7 @@ dependencies {
     testCompile 'org.reflections:reflections:0.9.11'
     testCompile 'org.xmlunit:xmlunit-core:2.6.0'
     testCompile 'org.xmlunit:xmlunit-matchers:2.6.0'
-    testCompile 'com.tngtech.archunit:archunit-junit:0.5.0'
+    testCompile 'com.tngtech.archunit:archunit-junit:0.8.0'
     testCompile "org.testfx:testfx-core:4.0.+"
     testCompile "org.testfx:testfx-junit5:4.0.+"
 

gradle/wrapper/gradle-wrapper.jar

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.6-bin.zip

gradle/wrapper/gradle-wrapper.properties

@@ -294,7 +294,8 @@ else if (val.matches("edtr\\d+")) {
             } else if ("fulltitle".equals(val)) {
                 return entry.getResolvedFieldOrAlias(FieldName.TITLE, database).orElse("");
             } else if ("shorttitle".equals(val)) {
-                return getTitleWords(3, entry.getResolvedFieldOrAlias(FieldName.TITLE, database).orElse(""));
+                return getTitleWords(3,
+                        removeSmallWords(entry.getResolvedFieldOrAlias(FieldName.TITLE, database).orElse("")));
             } else if ("shorttitleINI".equals(val)) {
                 return keepLettersAndDigitsOnly(
                         applyModifiers(getTitleWordsWithSpaces(3, entry.getResolvedFieldOrAlias(FieldName.TITLE, database).orElse("")),

src/main/java/org/jabref/logic/bibtexkeypattern/BracketedPattern.java

@@ -36,19 +36,16 @@ public class Cleanups {
 
         List<FieldFormatterCleanup> recommendedBibTeXFormatters = new ArrayList<>();
         recommendedBibTeXFormatters.addAll(defaultFormatters);
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.TITLE, new HtmlToLatexFormatter()));
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.TITLE, new UnicodeToLatexFormatter()));
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.BOOKTITLE, new UnicodeToLatexFormatter()));
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.JOURNAL, new UnicodeToLatexFormatter()));
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.AUTHOR, new UnicodeToLatexFormatter()));
-        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.EDITOR, new UnicodeToLatexFormatter()));
+        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.INTERNAL_ALL_TEXT_FIELDS_FIELD, new HtmlToLatexFormatter()));
+        recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.INTERNAL_ALL_TEXT_FIELDS_FIELD, new UnicodeToLatexFormatter()));
         recommendedBibTeXFormatters.add(new FieldFormatterCleanup(FieldName.INTERNAL_ALL_TEXT_FIELDS_FIELD, new OrdinalsToSuperscriptFormatter()));
         RECOMMEND_BIBTEX_ACTIONS = new FieldFormatterCleanups(false, recommendedBibTeXFormatters);
 
         List<FieldFormatterCleanup> recommendedBiblatexFormatters = new ArrayList<>();
         recommendedBiblatexFormatters.addAll(defaultFormatters);
         recommendedBiblatexFormatters.add(new FieldFormatterCleanup(FieldName.TITLE, new HtmlToUnicodeFormatter()));
         recommendedBiblatexFormatters.add(new FieldFormatterCleanup(FieldName.INTERNAL_ALL_TEXT_FIELDS_FIELD, new LatexToUnicodeFormatter()));
+        // DO NOT ADD OrdinalsToSuperscriptFormatter here, because this causes issues. See https://github.com/JabRef/jabref/issues/2596.
         RECOMMEND_BIBLATEX_ACTIONS = new FieldFormatterCleanups(false, recommendedBiblatexFormatters);
     }
 

src/main/java/org/jabref/logic/cleanup/Cleanups.java

@@ -12,7 +12,7 @@ public class HtmlToUnicodeFormatter implements LayoutFormatter, Formatter {
 
     @Override
     public String getName() {
-        return "HTML to Unicode";
+        return Localization.lang("HTML to Unicode");
     }
 
     @Override

src/main/java/org/jabref/logic/formatter/bibtexfields/HtmlToUnicodeFormatter.java

@@ -1923,6 +1923,7 @@ Changes\ all\ letters\ to\ upper\ case.=Changes all letters to upper case.
 Changes\ the\ first\ letter\ of\ all\ words\ to\ capital\ case\ and\ the\ remaining\ letters\ to\ lower\ case.=Changes the first letter of all words to capital case and the remaining letters to lower case.
 Cleans\ up\ LaTeX\ code.=Cleans up LaTeX code.
 Converts\ HTML\ code\ to\ LaTeX\ code.=Converts HTML code to LaTeX code.
+HTML\ to\ Unicode=HTML to Unicode
 Converts\ HTML\ code\ to\ Unicode.=Converts HTML code to Unicode.
 Converts\ LaTeX\ encoding\ to\ Unicode\ characters.=Converts LaTeX encoding to Unicode characters.
 Converts\ Unicode\ characters\ to\ LaTeX\ encoding.=Converts Unicode characters to LaTeX encoding.

src/main/resources/l10n/JabRef_en.properties

@@ -37,6 +37,7 @@ Accept\ change=Chấp nhận thay đổi
 
 Action=Hành động
 
+What\ is\ Mr.\ DLib?=Ông DLib là gì?
 
 Add=Thêm
 
@@ -46,7 +47,9 @@ The\ path\ need\ not\ be\ on\ the\ classpath\ of\ JabRef.=Đường dẫn không
 Add\ a\ (compiled)\ custom\ Importer\ class\ from\ a\ ZIP-archive.=Thêm một lớp ĐịnhdạngNhập tùy biến (được biên dịch) từ một tập tin-zip.
 The\ ZIP-archive\ need\ not\ be\ on\ the\ classpath\ of\ JabRef.=Tập tin-zip không được trùng với đường dẫn lớp của JabRef.
 
+Add\ a\ regular\ expression\ for\ the\ key\ pattern.=Thêm một biểu thức chính quy cho mẫu chính.
 
+Add\ selected\ entries\ to\ this\ group=Thêm các mục đã chọn vào nhóm này
 
 Add\ from\ folder=Thêm từ thư mục
 
@@ -74,6 +77,7 @@ All\ entries\ of\ this\ type\ will\ be\ declared\ typeless.\ Continue?=Tất c
 
 All\ fields=Tất cả các dữ liệu
 
+Always\ reformat\ BIB\ file\ on\ save\ and\ export=Luôn luôn định dạng lại file BIB khi lưu và xuất
 
 A\ SAX\ exception\ occurred\ while\ parsing\ '%0'\:=Một lỗi SAXException xảy ra khi đang phân tách '%0'\:
 
@@ -439,6 +443,7 @@ Export\ properties=Các tính chất xuất
 
 Export\ to\ clipboard=Xuất ra bộ nhớ tạm
 
+
 Exporting=Đang xuất
 Extension=Đuôi mở rộng
 

src/main/resources/l10n/JabRef_vi.properties

@@ -767,23 +767,30 @@ public void veryShortTitle() {
      */
     @Test
     public void shortTitle() {
-        // shortTitle is getTitleWords with "3" as count
+        // shortTitle is getTitleWords with "3" as count and removed small words
         int count = 3;
         assertEquals("application migration effort",
-                BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_ALL_LOWER_FOUR_SMALL_WORDS_ONE_EN_DASH));
-        assertEquals("BPEL conformance in", BibtexKeyGenerator.getTitleWords(count,
-                TITLE_STRING_ALL_LOWER_FIRST_WORD_IN_BRACKETS_TWO_SMALL_WORDS_SMALL_WORD_AFTER_COLON));
-        assertEquals("Process Viewing Patterns", BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_CASED));
-        assertEquals("BPMN Conformance in",
-                BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_CASED_ONE_UPPER_WORD_ONE_SMALL_WORD));
-        assertEquals("The Difference Between", BibtexKeyGenerator.getTitleWords(count,
-                TITLE_STRING_CASED_TWO_SMALL_WORDS_SMALL_WORD_AT_THE_BEGINNING));
-        assertEquals("Cloud Computing: The",
-                BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_CASED_TWO_SMALL_WORDS_SMALL_WORD_AFTER_COLON));
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_ALL_LOWER_FOUR_SMALL_WORDS_ONE_EN_DASH)));
+        assertEquals("BPEL conformance open",
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_ALL_LOWER_FIRST_WORD_IN_BRACKETS_TWO_SMALL_WORDS_SMALL_WORD_AFTER_COLON)));
+        assertEquals("Process Viewing Patterns",
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED)));
+        assertEquals("BPMN Conformance Open",
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED_ONE_UPPER_WORD_ONE_SMALL_WORD)));
+        assertEquals("Difference Graph Based",
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED_TWO_SMALL_WORDS_SMALL_WORD_AT_THE_BEGINNING)));
+        assertEquals("Cloud Computing: Next",
+                BibtexKeyGenerator.getTitleWords(count,
+                        BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED_TWO_SMALL_WORDS_SMALL_WORD_AFTER_COLON)));
         assertEquals("Towards Choreography based",
-                BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_CASED_TWO_SMALL_WORDS_ONE_CONNECTED_WORD));
-        assertEquals("On the Measurement",
-                BibtexKeyGenerator.getTitleWords(count, TITLE_STRING_CASED_FOUR_SMALL_WORDS_TWO_CONNECTED_WORDS));
+                BibtexKeyGenerator.getTitleWords(count, BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED_TWO_SMALL_WORDS_ONE_CONNECTED_WORD)));
+        assertEquals("Measurement Design Time",
+                BibtexKeyGenerator.getTitleWords(count, BibtexKeyGenerator.removeSmallWords(TITLE_STRING_CASED_FOUR_SMALL_WORDS_TWO_CONNECTED_WORDS)));
     }
 
     /**
@@ -935,7 +942,7 @@ public void testApplyModifiers() {
         BibEntry entry = new BibEntry();
         entry.setField("title", "Green Scheduling of Whatever");
         assertEquals("GSo", BibtexKeyGenerator.generateKey(entry, "shorttitleINI"));
-        assertEquals("GreenSchedulingof", BibtexKeyGenerator.generateKey(entry, "shorttitle",
+        assertEquals("GreenSchedulingWhatever", BibtexKeyGenerator.generateKey(entry, "shorttitle",
                 new BibDatabase()));
     }
 
@@ -950,7 +957,7 @@ public void testcrossrefShorttitle() {
         database.insertEntry(entry1);
         entry2.setField("title", "Green Scheduling of Whatever");
 
-        assertEquals("GreenSchedulingof", BibtexKeyGenerator.generateKey(entry1, "shorttitle",
+        assertEquals("GreenSchedulingWhatever", BibtexKeyGenerator.generateKey(entry1, "shorttitle",
                 database));
     }
 

src/test/java/org/jabref/logic/bibtexkeypattern/BibtexKeyGeneratorTest.java

@@ -231,47 +231,47 @@ public void generateKeyTitleAbbr() {
     public void generateKeyShorttitle() {
         bibtexKeyPattern.setDefaultValue("[shorttitle]");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("Anawesomepaper"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("awesomepaperJabRef"), entry.getCiteKeyOptional());
     }
 
     @Test
     public void generateKeyShorttitleLowerModified() {
         bibtexKeyPattern.setDefaultValue("[shorttitle:lower]");
         entry.setField("title", "An aweSOme Paper on JabRef");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("anawesomepaper"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("awesomepaperjabref"), entry.getCiteKeyOptional());
     }
 
     @Test
     public void generateKeyShorttitleUpperModified() {
         bibtexKeyPattern.setDefaultValue("[shorttitle:upper]");
         entry.setField("title", "An aweSOme Paper on JabRef");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("ANAWESOMEPAPER"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("AWESOMEPAPERJABREF"), entry.getCiteKeyOptional());
     }
 
     @Test
     public void generateKeyShorttitleTitleCaseModified() {
         bibtexKeyPattern.setDefaultValue("[shorttitle:title_case]");
         entry.setField("title", "An aweSOme Paper on JabRef");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("AnAwesomePaper"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("AwesomePaperJabref"), entry.getCiteKeyOptional());
     }
 
     @Test
     public void generateKeyShorttitleSentenceCaseModified() {
         bibtexKeyPattern.setDefaultValue("[shorttitle:sentence_case]");
         entry.setField("title", "An aweSOme Paper on JabRef");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("Anawesomepaper"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("Awesomepaperjabref"), entry.getCiteKeyOptional());
     }
 
     @Test
     public void generateKeyShorttitleCapitalizeModified() {
         bibtexKeyPattern.setDefaultValue("[shorttitle:capitalize]");
         entry.setField("title", "An aweSOme Paper on JabRef");
         new BibtexKeyGenerator(bibtexKeyPattern, database, preferences).generateAndSetKey(entry);
-        assertEquals(Optional.of("AnAwesomePaper"), entry.getCiteKeyOptional());
+        assertEquals(Optional.of("AwesomePaperJabref"), entry.getCiteKeyOptional());
     }
 
     @Test