Enable cld3

ATTRIBUTION.md

@@ -28,4 +28,5 @@ FairEmail uses:
 * [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE).
 * [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE)
 * [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE).
+* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE).
 * [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing).

README.md

@@ -38,7 +38,7 @@ FairEmail might be for you if you value your privacy.
 * Offline storage and operations
 * Battery friendly
 * Low data usage
-* Small (~ 15 MB)
+* Small (< 20 MB)
 * Material design (including dark/black theme)
 * Maintained and supported
 

app/src/main/assets/ATTRIBUTION.md

@@ -28,4 +28,5 @@ FairEmail uses:
 * [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE).
 * [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE)
 * [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE).
+* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE).
 * [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing).

app/src/main/java/eu/faircode/email/FragmentMessages.java

@@ -3983,8 +3983,7 @@ public void onPrepareOptionsMenu(Menu menu) {
                 (viewType == AdapterMessage.ViewType.UNIFIED ||
                         (viewType == AdapterMessage.ViewType.FOLDER && !outbox));
 
-        String filter_language = prefs.getString("filter_language", null);
-        boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language));
+        boolean filter_active = (filter_seen || filter_unflagged || filter_unknown);
         MenuItem menuFilter = menu.findItem(R.id.menu_filter);
         menuFilter.setShowAsAction(folder && filter_active
                 ? MenuItem.SHOW_AS_ACTION_ALWAYS
@@ -4055,8 +4054,7 @@ else if ("snoozed".equals(sort))
 
         menu.findItem(R.id.menu_compact).setChecked(compact);
 
-        menu.findItem(R.id.menu_select_language).setVisible(
-                language_detection && folder && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q);
+        menu.findItem(R.id.menu_select_language).setVisible(language_detection && folder);
         menu.findItem(R.id.menu_select_all).setVisible(folder);
         menu.findItem(R.id.menu_select_found).setVisible(viewType == AdapterMessage.ViewType.SEARCH);
         menu.findItem(R.id.menu_mark_all_read).setVisible(folder);
@@ -4421,8 +4419,8 @@ protected Void onExecute(Context context, Bundle args) throws Throwable {
                 boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false);
                 boolean filter_unknown = prefs.getBoolean("filter_unknown", false);
                 boolean filter_snoozed = prefs.getBoolean("filter_snoozed", true);
-                String filter_language = prefs.getString("filter_language", null);
                 boolean language_detection = prefs.getBoolean("language_detection", false);
+                String filter_language = prefs.getString("filter_language", null);
 
                 DB db = DB.getInstance(context);
                 try {
@@ -4683,8 +4681,10 @@ private void updateListState(String reason, int tasks, int items) {
         boolean filter_seen = prefs.getBoolean("filter_seen", false);
         boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false);
         boolean filter_unknown = prefs.getBoolean("filter_unknown", false);
+        boolean language_detection = prefs.getBoolean("language_detection", false);
         String filter_language = prefs.getString("filter_language", null);
-        boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language));
+        boolean filter_active = (filter_seen || filter_unflagged || filter_unknown ||
+                (language_detection && !TextUtils.isEmpty(filter_language)));
 
         boolean none = (items == 0 && !loading && tasks == 0 && initialized);
         boolean filtered = (filter_active && viewType != AdapterMessage.ViewType.SEARCH);

app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java

@@ -170,7 +170,6 @@ public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
             }
         });
 
-        swLanguageDetection.setEnabled(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q);
         swLanguageDetection.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() {
             @Override
             public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {

app/src/main/java/eu/faircode/email/HtmlHelper.java

@@ -1716,7 +1716,7 @@ static String getLanguage(Context context, String body) {
             if (!language_detection)
                 return null;
 
-            String text = getPreview(body);
+            String text = getFullText(body);
             Locale locale = TextHelper.detectLanguage(context, text);
             return (locale == null ? null : locale.getLanguage());
         } catch (Throwable ex) {

app/src/main/java/eu/faircode/email/TextHelper.java

@@ -27,7 +27,6 @@
 import android.view.textclassifier.ConversationActions;
 import android.view.textclassifier.TextClassificationManager;
 import android.view.textclassifier.TextClassifier;
-import android.view.textclassifier.TextLanguage;
 
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
@@ -41,40 +40,44 @@
 import java.util.Set;
 
 public class TextHelper {
+    private static final int MAX_SAMPLE_SIZE = 8192;
+
     static {
         System.loadLibrary("fairemail");
     }
 
     private static native DetectResult jni_detect_language(byte[] octets);
 
     static Locale detectLanguage(Context context, String text) {
-        // Why not ML kit? https://developers.google.com/ml-kit/terms
+        // Why not ML kit?
+        // https://developers.google.com/ml-kit/terms
+
         if (TextUtils.isEmpty(text))
             return null;
 
-        if (BuildConfig.DEBUG) {
-            // https://github.com/google/cld3
-            DetectResult result = jni_detect_language(text.getBytes());
-            Log.i("Language=" + result);
-            return Locale.forLanguageTag(result.language);
+        byte[] octets = text.getBytes();
+        byte[] sample;
+        if (octets.length < MAX_SAMPLE_SIZE)
+            sample = octets;
+        else {
+            sample = new byte[MAX_SAMPLE_SIZE];
+            System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE);
         }
 
-        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q)
-            return null;
+        // https://github.com/google/cld3
+        Log.i("cld3 sample=" + sample.length);
+        DetectResult result = jni_detect_language(sample);
+        Log.i("Language=" + result);
 
-        // https://issuetracker.google.com/issues/173337263
-        TextClassificationManager tcm =
-                (TextClassificationManager) context.getSystemService(Context.TEXT_CLASSIFICATION_SERVICE);
-        if (tcm == null)
+        if (result.probability < 0.5)
             return null;
 
-        TextLanguage.Request request = new TextLanguage.Request.Builder(text).build();
-        TextClassifier tc = tcm.getTextClassifier();
-        TextLanguage tlanguage = tc.detectLanguage(request);
-        if (tlanguage.getLocaleHypothesisCount() > 0)
-            return tlanguage.getLocale(0).toLocale();
-
-        return null;
+        try {
+            return Locale.forLanguageTag(result.language);
+        } catch (Throwable ex) {
+            Log.w(ex);
+            return null;
+        }
     }
 
     static ConversationActions getConversationActions(

app/src/main/java/eu/faircode/email/ViewModelMessages.java

@@ -21,7 +21,6 @@
 
 import android.content.Context;
 import android.content.SharedPreferences;
-import android.os.Build;
 import android.os.Bundle;
 import android.text.TextUtils;
 import android.util.Pair;
@@ -449,10 +448,8 @@ private class Args {
             this.filter_snoozed = prefs.getBoolean("filter_snoozed", true);
 
             boolean language_detection = prefs.getBoolean("language_detection", false);
-            if (!language_detection || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q)
-                this.filter_language = null;
-            else
-                this.filter_language = prefs.getString("filter_language", null);
+            String filter_language = prefs.getString("filter_language", null);
+            this.filter_language = (language_detection ? filter_language : null);
 
             this.debug = prefs.getBoolean("debug", false);
         }

app/src/main/res/values/strings.xml

@@ -517,7 +517,7 @@
     <string name="title_advanced_shortcuts">Show frequently used contacts in Android share menu</string>
     <string name="title_advanced_conversation_actions">Suggest actions (Android 10+)</string>
     <string name="title_advanced_conversation_actions_replies">Suggest reply texts (Android 10+)</string>
-    <string name="title_advanced_language_detection">Detect message text language (Android 10+)</string>
+    <string name="title_advanced_language_detection">Detect message text language</string>
     <string name="title_advanced_fts">Build search index</string>
     <string name="title_advanced_fts_indexed">%1$d / %2$d messages indexed (%3$s)</string>
     <string name="title_advanced_classification">Classify messages</string>

metadata/en-US/full_description.txt

@@ -21,7 +21,7 @@ FairEmail might be for you if you value your privacy.
 * Offline storage and operations
 * Battery friendly
 * Low data usage
-* Small (~ 15 MB)
+* Small (< 20 MB)
 * Material design (including dark/black theme)
 * Maintained and supported