Cache and Lazy-Load Modifications

app/build.gradle

@@ -43,6 +43,8 @@ dependencies {
     testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3'
     testImplementation 'io.mockk:mockk:1.12.7'
     testImplementation 'org.json:json:20231013'
+    testImplementation 'org.mockito:mockito-core:5.5.0'
+    testImplementation 'org.json:json:20140107'
     testImplementation project(path: ':commcare-core', configuration: 'testsAsJar')
 
     androidTestImplementation 'androidx.test:runner:1.4.0'

app/res/layout/entity_select_layout.xml

@@ -53,6 +53,7 @@
         android:visibility="gone"/>
 
     <RelativeLayout
+        android:id="@+id/progress_container"
         android:layout_width="match_parent"
         android:layout_height="fill_parent"
         android:layout_below="@id/entity_select_filter_dropdown">
@@ -63,8 +64,18 @@
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
             android:layout_centerInParent="true"/>
+
+        <TextView
+            android:id="@+id/progress_text"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:layout_below="@+id/entity_select_loading"
+            android:padding="8dp"
+            android:layout_centerInParent="true" />
+
     </RelativeLayout>
 
+
     <FrameLayout
         android:layout_width="match_parent"
         android:layout_height="wrap_content"

app/res/values/strings.xml

@@ -465,5 +465,13 @@
     <string name="app_with_id_not_found">Required CommCare App is not installed on device</string>
     <string name="audio_recording_notification">Audio Recording Notification</string>
 
+    <!--    Entity List Progresss Indicators-->
+    <string name="entity_list_initializing" cc:translatable="true">Initializing list…</string>
+    <string name="entity_list_processing" cc:translatable="true">Processing cases: %s</string>
+    <string name="entity_list_loading_cache" cc:translatable="true">Loading Cache…</string>
+    <string name="entity_list_calculating" cc:translatable="true">Calculating cases %s</string>
+    <string name="entity_list_finishing" cc:translatable="true">Finishing case calculations…</string>
+
+
     <string name="connect_db_corrupt" cc:translatable="true">A problem occurred with the database, please recover your account.</string>
 </resources>

app/src/org/commcare/CommCareApp.java

@@ -25,6 +25,7 @@
 import org.commcare.resources.model.ResourceTable;
 import org.commcare.suite.model.Menu;
 import org.commcare.suite.model.Suite;
+import org.commcare.tasks.PrimeEntityCacheHelper;
 import org.commcare.util.CommCarePlatform;
 import org.commcare.util.LogTypes;
 import org.commcare.utils.AndroidCommCarePlatform;
@@ -44,6 +45,7 @@
 import org.javarosa.xpath.expr.FunctionUtils;
 import org.javarosa.xpath.expr.XPathExpression;
 import org.javarosa.xpath.parser.XPathSyntaxException;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
 import java.util.List;
@@ -76,6 +78,7 @@ public class CommCareApp implements AppFilePathBuilder {
     private static Stylizer mStylizer;
 
     private int resourceState;
+    private volatile PrimeEntityCacheHelper primeEntityCacheHelper;
 
     public CommCareApp(ApplicationRecord record) {
         this.record = record;
@@ -411,6 +414,17 @@ public SQLiteDatabase getHandle() {
         };
     }
 
+    public PrimeEntityCacheHelper getPrimeEntityCacheHelper() {
+        if (primeEntityCacheHelper == null) {
+            synchronized (this) {
+                if (primeEntityCacheHelper == null) {
+                    primeEntityCacheHelper = new PrimeEntityCacheHelper();
+                }
+            }
+        }
+        return primeEntityCacheHelper;
+    }
+
     /**
      * Initialize all of the properties that an app record should have and update it to the
      * installed state.

app/src/org/commcare/CommCareApplication.java

@@ -1,5 +1,7 @@
 package org.commcare;
 
+import static org.commcare.AppUtils.getCurrentAppId;
+
 import android.annotation.SuppressLint;
 import android.app.ActivityManager;
 import android.app.Application;
@@ -29,6 +31,7 @@
 import androidx.work.ExistingWorkPolicy;
 import androidx.work.NetworkType;
 import androidx.work.OneTimeWorkRequest;
+import androidx.work.OutOfQuotaPolicy;
 import androidx.work.PeriodicWorkRequest;
 import androidx.work.WorkManager;
 
@@ -96,7 +99,9 @@
 import org.commcare.tasks.AsyncRestoreHelper;
 import org.commcare.tasks.DataPullTask;
 import org.commcare.tasks.DeleteLogs;
+import org.commcare.tasks.EntityCacheInvalidationWorker;
 import org.commcare.tasks.LogSubmissionTask;
+import org.commcare.tasks.PrimeEntityCacheHelper;
 import org.commcare.tasks.PurgeStaleArchivedFormsTask;
 import org.commcare.tasks.templates.ManagedAsyncTask;
 import org.commcare.update.UpdateHelper;
@@ -163,6 +168,7 @@ public class CommCareApplication extends Application implements LifecycleEventOb
     private static final long BACKOFF_DELAY_FOR_UPDATE_RETRY = 5 * 60 * 1000L; // 5 mins
     private static final long BACKOFF_DELAY_FOR_FORM_SUBMISSION_RETRY = 5 * 60 * 1000L; // 5 mins
     private static final long PERIODICITY_FOR_FORM_SUBMISSION_IN_HOURS = 1;
+    private static final String ENTITY_CACHE_INVALIDATION_REQUEST = "entity-cache-invalidation-request";
     private static Markwon markwon;
 
 
@@ -388,10 +394,10 @@ public void closeUserSession() {
     }
 
     protected void cancelWorkManagerTasks() {
-        // Cancel form Submissions for this user
         if (currentApp != null) {
             WorkManager.getInstance(this).cancelUniqueWork(
                     FormSubmissionHelper.getFormSubmissionRequestName(currentApp.getUniqueId()));
+            currentApp.getPrimeEntityCacheHelper().cancelWork();
         }
     }
 
@@ -581,7 +587,7 @@ public boolean isSeated(ApplicationRecord record) {
     public void unseat(ApplicationRecord record) {
         // cancel all Workmanager tasks for the unseated record
         WorkManager.getInstance(CommCareApplication.instance())
-                .cancelAllWorkByTag(record.getApplicationId());
+                .cancelAllWorkByTag(record.getUniqueId());
         MissingMediaDownloadHelper.cancelAllDownloads();
         if (isSeated(record)) {
             this.currentApp.teardownSandbox();
@@ -801,7 +807,7 @@ public void onServiceConnected(ComponentName className, IBinder service) {
 
                         purgeLogs();
                         cleanRawMedia();
-
+                        PrimeEntityCacheHelper.schedulePrimeEntityCacheWorker();
                     }
 
                     TimedStatsTracker.registerStartSession();
@@ -854,7 +860,7 @@ private void scheduleFormSubmissions() {
 
         PeriodicWorkRequest formSubmissionRequest =
                 new PeriodicWorkRequest.Builder(FormSubmissionWorker.class, PERIODICITY_FOR_FORM_SUBMISSION_IN_HOURS, TimeUnit.HOURS)
-                        .addTag(getCurrentApp().getAppRecord().getApplicationId())
+                        .addTag(getCurrentAppId())
                         .setConstraints(constraints)
                         .setBackoffCriteria(
                                 BackoffPolicy.EXPONENTIAL,
@@ -863,7 +869,7 @@ private void scheduleFormSubmissions() {
                         .build();
 
         WorkManager.getInstance(this).enqueueUniquePeriodicWork(
-                FormSubmissionHelper.getFormSubmissionRequestName(getCurrentApp().getUniqueId()),
+                FormSubmissionHelper.getFormSubmissionRequestName(getCurrentAppId()),
                 ExistingPeriodicWorkPolicy.KEEP,
                 formSubmissionRequest
         );
@@ -880,7 +886,7 @@ private void scheduleAppUpdate() {
 
             PeriodicWorkRequest updateRequest =
                     new PeriodicWorkRequest.Builder(UpdateWorker.class, UpdateHelper.getAutoUpdatePeriodicity(), TimeUnit.HOURS)
-                            .addTag(getCurrentApp().getAppRecord().getApplicationId())
+                            .addTag(getCurrentAppId())
                             .setConstraints(constraints)
                             .setBackoffCriteria(
                                     BackoffPolicy.EXPONENTIAL,
@@ -1250,4 +1256,17 @@ public void onStateChanged(@NonNull LifecycleOwner source, @NonNull Lifecycle.Ev
                 break;
         }
     }
+
+    public void scheduleEntityCacheInvalidation() {
+        CommCareEntityStorageCache entityStorageCache = new CommCareEntityStorageCache("case");
+        if (!entityStorageCache.isEmpty()) {
+            OneTimeWorkRequest entityCacheInvalidationRequest = new OneTimeWorkRequest.Builder(
+                    EntityCacheInvalidationWorker.class)
+                    .setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST)
+                    .build();
+            WorkManager wm = WorkManager.getInstance(CommCareApplication.instance());
+            wm.enqueueUniqueWork(ENTITY_CACHE_INVALIDATION_REQUEST, ExistingWorkPolicy.REPLACE,
+                    entityCacheInvalidationRequest);
+        }
+    }
 }

app/src/org/commcare/activities/EntitySelectActivity.java

@@ -2,7 +2,6 @@
 
 import static org.commcare.activities.HomeScreenBaseActivity.RESULT_RESTART;
 
-import androidx.appcompat.app.AppCompatActivity;
 import android.content.Intent;
 import android.content.pm.ActivityInfo;
 import android.content.res.Configuration;
@@ -35,6 +34,7 @@
 import org.commcare.android.javarosa.IntentCallout;
 import org.commcare.android.logging.ReportingUtils;
 import org.commcare.cases.entity.Entity;
+import org.commcare.cases.entity.EntityLoadingProgressListener;
 import org.commcare.cases.entity.NodeEntityFactory;
 import org.commcare.dalvik.R;
 import org.commcare.fragments.ContainerFragment;
@@ -53,11 +53,13 @@
 import org.commcare.suite.model.EntityDatum;
 import org.commcare.tasks.EntityLoaderListener;
 import org.commcare.tasks.EntityLoaderTask;
+import org.commcare.tasks.PrimeEntityCacheHelper;
 import org.commcare.utils.AndroidHereFunctionHandler;
 import org.commcare.utils.AndroidInstanceInitializer;
 import org.commcare.utils.EntityDetailUtils;
 import org.commcare.utils.EntitySelectRefreshTimer;
 import org.commcare.utils.SerializationUtil;
+import org.commcare.utils.StringUtils;
 import org.commcare.views.EntityView;
 import org.commcare.views.TabbedDetailView;
 import org.commcare.views.UserfacingErrorHandling;
@@ -166,6 +168,9 @@ public class EntitySelectActivity extends SaveSessionCommCareActivity
     // Handler for displaying alert dialog when no location providers are found
     private final LocationNotificationHandler locationNotificationHandler =
             new LocationNotificationHandler(this);
+    private AdapterView visibleView;
+    private TextView progressTv;
+    private int lastProgress = 0;
 
     @Override
     public void onCreateSessionSafe(Bundle savedInstanceState) {
@@ -254,7 +259,6 @@ private void setupUI(boolean isOrientationChange) {
             setContentView(R.layout.entity_select_layout);
         }
 
-        AdapterView visibleView;
         GridView gridView = this.findViewById(R.id.screen_entity_select_grid);
         ListView listView = this.findViewById(R.id.screen_entity_select_list);
         if (shortSelect.shouldBeLaidOutInGrid()) {
@@ -268,6 +272,7 @@ private void setupUI(boolean isOrientationChange) {
             gridView.setVisibility(View.GONE);
             EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView());
         }
+        progressTv = findViewById(R.id.progress_text);
         RxAdapterView.itemClickEvents(visibleView)
                 .subscribeOn(AndroidSchedulers.mainThread())
                 .throttleFirst(CLICK_DEBOUNCE_TIME, TimeUnit.MILLISECONDS)
@@ -356,7 +361,7 @@ private void setupUIFromAdapter(AdapterView view) {
         if (view instanceof ListView) {
             EntitySelectViewSetup.setupDivider(this, (ListView)view, shortSelect.usesEntityTileView());
         }
-        findViewById(R.id.entity_select_loading).setVisibility(View.GONE);
+        findViewById(R.id.progress_container).setVisibility(View.GONE);
         entitySelectSearchUI.setSearchBannerState();
     }
 
@@ -476,14 +481,29 @@ public boolean loadEntities() {
         }
 
         if (loader == null && !EntityLoaderTask.attachToActivity(this)) {
-            EntityLoaderTask entityLoader = new EntityLoaderTask(shortSelect, evalContext());
+            setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_initializing));
+            EntityLoaderTask entityLoader = new EntityLoaderTask(shortSelect, selectDatum, evalContext());
             entityLoader.attachListener(this);
             entityLoader.executeParallel(selectDatum.getNodeset());
+            observePrimeCacheWorker();
             return true;
         }
         return false;
     }
 
+    private void observePrimeCacheWorker() {
+        if (shortSelect.shouldOptimize()) {
+            PrimeEntityCacheHelper primeEntityCacheHelper =
+                    CommCareApplication.instance().getCurrentApp().getPrimeEntityCacheHelper();
+            primeEntityCacheHelper.getProgressState().observe(this, triple -> {
+                if (triple.getFirst().contentEquals(selectDatum.getDataId()) &&
+                        triple.getSecond().contentEquals(shortSelect.getId())) {
+                    deliverProgress(triple.getThird());
+                }
+            });
+        }
+    }
+
     @Override
     public void onSaveInstanceState(Bundle savedInstanceState) {
         super.onSaveInstanceState(savedInstanceState);
@@ -852,16 +872,7 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
                                   List<TreeReference> references,
                                   NodeEntityFactory factory, int focusTargetIndex) {
         loader = null;
-
-        AdapterView visibleView;
-        if (shortSelect.shouldBeLaidOutInGrid()) {
-            visibleView = ((GridView)this.findViewById(R.id.screen_entity_select_grid));
-        } else {
-            ListView listView = this.findViewById(R.id.screen_entity_select_list);
-            EntitySelectViewSetup.setupDivider(this, listView, shortSelect.usesEntityTileView());
-            visibleView = listView;
-        }
-
+        setProgressText(StringUtils.getStringRobust(this, R.string.entity_list_finishing));
         adapter = new EntityListAdapter(this, shortSelect, references, entities, factory,
                 hideActionsFromEntityList, shortSelect.getCustomActions(evalContext()), inAwesomeMode);
         visibleView.setAdapter(adapter);
@@ -883,7 +894,7 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
             }
         }
 
-        findViewById(R.id.entity_select_loading).setVisibility(View.GONE);
+        findViewById(R.id.progress_container).setVisibility(View.GONE);
 
         if (adapter != null) {
             // filter by additional session data (search string, callout result data)
@@ -907,6 +918,10 @@ public void deliverLoadResult(List<Entity<TreeReference>> entities,
         }
     }
 
+    private void setProgressText(String message) {
+        progressTv.setText(message);
+    }
+
     private void restoreAdapterStateFromSession() {
         entitySelectSearchUI.restoreSearchString();
 
@@ -933,7 +948,7 @@ private void updateSelectedItem(TreeReference selected, boolean forceMove) {
 
     @Override
     public void attachLoader(EntityLoaderTask task) {
-        findViewById(R.id.entity_select_loading).setVisibility(View.VISIBLE);
+        findViewById(R.id.progress_container).setVisibility(View.VISIBLE);
         this.loader = task;
     }
 
@@ -994,6 +1009,27 @@ public void deliverLoadError(Exception e) {
         displayCaseListLoadException(e);
     }
 
+    @Override
+    public void deliverProgress(Integer[] values) {
+        EntityLoadingProgressListener.EntityLoadingProgressPhase phase =
+                EntityLoadingProgressListener.EntityLoadingProgressPhase.fromInt(values[0]);
+        int progress = values[1] * 100 / values[2];
+        if (progress != lastProgress) {
+            lastProgress = progress;
+            String progressDisplay = progress + "%";
+            switch (phase) {
+                case PHASE_PROCESSING -> setProgressText(
+                        StringUtils.getStringRobust(this, R.string.entity_list_processing,
+                                new String[]{progressDisplay}));
+                case PHASE_CACHING -> setProgressText(
+                        StringUtils.getStringRobust(this, R.string.entity_list_loading_cache));
+                case PHASE_UNCACHED_CALCULATION -> setProgressText(
+                        StringUtils.getStringRobust(this, R.string.entity_list_calculating,
+                                new String[]{progressDisplay}));
+            }
+        }
+    }
+
     @Override
     protected boolean onForwardSwipe() {
         // If user has picked an entity, move along to form entry

app/src/org/commcare/activities/SyncCapableCommCareActivity.java

@@ -261,6 +261,7 @@ public void handleFormUploadResult(FormUploadResult result, String formLabel, bo
                 updateUiForFormUploadResult(Localization.get(result.getLocaleKeyBase()), false);
                 break;
         }
+        CommCareApplication.instance().scheduleEntityCacheInvalidation();
     }
 
     public void updateUiForFormUploadResult(String message, boolean success) {