Expires entity cache for related cases

app/build.gradle

@@ -43,6 +43,8 @@ dependencies {
     testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.7.3'
     testImplementation 'io.mockk:mockk:1.12.7'
     testImplementation 'org.json:json:20231013'
+    testImplementation 'org.mockito:mockito-core:5.5.0'
+    testImplementation 'org.json:json:20140107'
     testImplementation project(path: ':commcare-core', configuration: 'testsAsJar')
 
     androidTestImplementation 'androidx.test:runner:1.4.0'

app/src/org/commcare/CommCareApplication.java

@@ -31,6 +31,7 @@
 import androidx.work.ExistingWorkPolicy;
 import androidx.work.NetworkType;
 import androidx.work.OneTimeWorkRequest;
+import androidx.work.OutOfQuotaPolicy;
 import androidx.work.PeriodicWorkRequest;
 import androidx.work.WorkManager;
 
@@ -98,8 +99,8 @@
 import org.commcare.tasks.AsyncRestoreHelper;
 import org.commcare.tasks.DataPullTask;
 import org.commcare.tasks.DeleteLogs;
+import org.commcare.tasks.EntityCacheInvalidationWorker;
 import org.commcare.tasks.LogSubmissionTask;
-import org.commcare.tasks.PrimeEntityCache;
 import org.commcare.tasks.PrimeEntityCacheHelper;
 import org.commcare.tasks.PurgeStaleArchivedFormsTask;
 import org.commcare.tasks.templates.ManagedAsyncTask;
@@ -167,6 +168,7 @@ public class CommCareApplication extends Application implements LifecycleEventOb
     private static final long BACKOFF_DELAY_FOR_UPDATE_RETRY = 5 * 60 * 1000L; // 5 mins
     private static final long BACKOFF_DELAY_FOR_FORM_SUBMISSION_RETRY = 5 * 60 * 1000L; // 5 mins
     private static final long PERIODICITY_FOR_FORM_SUBMISSION_IN_HOURS = 1;
+    private static final String ENTITY_CACHE_INVALIDATION_REQUEST = "entity-cache-invalidation-request";
     private static Markwon markwon;
 
 
@@ -1254,4 +1256,17 @@ public void onStateChanged(@NonNull LifecycleOwner source, @NonNull Lifecycle.Ev
                 break;
         }
     }
+
+    public void scheduleEntityCacheInvalidation() {
+        CommCareEntityStorageCache entityStorageCache = new CommCareEntityStorageCache("case");
+        if (!entityStorageCache.isEmpty()) {
+            OneTimeWorkRequest entityCacheInvalidationRequest = new OneTimeWorkRequest.Builder(
+                    EntityCacheInvalidationWorker.class)
+                    .setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST)
+                    .build();
+            WorkManager wm = WorkManager.getInstance(CommCareApplication.instance());
+            wm.enqueueUniqueWork(ENTITY_CACHE_INVALIDATION_REQUEST, ExistingWorkPolicy.REPLACE,
+                    entityCacheInvalidationRequest);
+        }
+    }
 }

app/src/org/commcare/activities/SyncCapableCommCareActivity.java

@@ -261,6 +261,7 @@ public void handleFormUploadResult(FormUploadResult result, String formLabel, bo
                 updateUiForFormUploadResult(Localization.get(result.getLocaleKeyBase()), false);
                 break;
         }
+        CommCareApplication.instance().scheduleEntityCacheInvalidation();
     }
 
     public void updateUiForFormUploadResult(String message, boolean success) {

app/src/org/commcare/engine/cases/CaseUtils.java

@@ -1,5 +1,8 @@
 package org.commcare.engine.cases;
 
+import static org.commcare.cases.model.Case.INDEX_CASE_ID;
+import static org.commcare.cases.util.CaseDBUtils.xordata;
+
 import net.sqlcipher.database.SQLiteDatabase;
 
 import org.commcare.CommCareApplication;
@@ -13,7 +16,6 @@
 import org.commcare.models.database.SqlStorage;
 import org.commcare.models.database.SqlStorageIterator;
 import org.commcare.models.database.user.models.AndroidCaseIndexTable;
-import org.commcare.modern.engine.cases.CaseIndexTable;
 import org.commcare.modern.util.Pair;
 import org.commcare.util.LogTypes;
 import org.commcare.utils.CommCareUtil;
@@ -24,16 +26,15 @@
 import org.javarosa.core.model.instance.TreeReference;
 import org.javarosa.core.services.Logger;
 import org.javarosa.core.services.storage.IStorageIterator;
-import org.javarosa.core.services.storage.IStorageUtilityIndexed;
 import org.javarosa.core.util.DAG;
 import org.javarosa.core.util.MD5;
 import org.javarosa.model.xform.XPathReference;
 
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.Vector;
 
-import static org.commcare.cases.util.CaseDBUtils.xordata;
-
 /**
  * Utilities for performing complex operations on the case database.
  *
@@ -51,8 +52,8 @@ public static String computeCaseDbHash(SqlStorage<?> storage) {
         boolean casesExist = false;
         long count = 0;
 
-        for (SqlStorageIterator i = storage.iterate(false, new String[]{Case.INDEX_CASE_ID}); i.hasMore(); ) {
-            byte[] current = MD5.hash(i.peekIncludedMetadata(Case.INDEX_CASE_ID).getBytes());
+        for (SqlStorageIterator i = storage.iterate(false, new String[]{INDEX_CASE_ID}); i.hasMore(); ) {
+            byte[] current = MD5.hash(i.peekIncludedMetadata(INDEX_CASE_ID).getBytes());
             data = xordata(data, current);
             casesExist = true;
             count++;
@@ -79,37 +80,11 @@ public static String computeCaseDbHash(SqlStorage<?> storage) {
     public static void purgeCases() throws InvalidCaseGraphException {
         long start = System.currentTimeMillis();
         //We need to determine if we're using ownership for purging. For right now, only in sync mode
-        Vector<String> owners = new Vector<>();
-        Vector<String> users = new Vector<>();
-        for (IStorageIterator<User> userIterator = CommCareApplication.instance()
-                .getUserStorage(User.STORAGE_KEY, User.class).iterate(); userIterator.hasMore(); ) {
-            String id = userIterator.nextRecord().getUniqueId();
-            owners.addElement(id);
-            users.addElement(id);
-        }
-
-        //Now add all of the relevant groups
-        //TODO: Wow. This is.... kind of megasketch
-        for (String userId : users) {
-            DataInstance instance = CommCareUtil.loadFixture("user-groups", userId);
-            if (instance == null) {
-                continue;
-            }
-            EvaluationContext ec = new EvaluationContext(instance);
-            for (TreeReference ref : ec.expandReference(XPathReference.getPathExpr("/groups/group/@id").getReference())) {
-                AbstractTreeElement<AbstractTreeElement> idelement = ec.resolveReference(ref);
-                if (idelement.getValue() != null) {
-                    owners.addElement(idelement.getValue().uncast().getString());
-                }
-            }
-        }
-
-        SQLiteDatabase db;
-        db = CommCareApplication.instance().getUserDbHandle();
-
-        db.beginTransaction();
+        Vector<String> owners = getAllOwners();
+        SQLiteDatabase db = CommCareApplication.instance().getUserDbHandle();
         int removedCaseCount;
         int removedLedgers;
+        db.beginTransaction();
         try {
             SqlStorage<ACase> storage = CommCareApplication.instance().getUserStorage(ACase.STORAGE_KEY, ACase.class);
             DAG<String, int[], String> fullCaseGraph = getFullCaseGraph(storage, new AndroidCaseIndexTable(), owners);
@@ -151,9 +126,67 @@ public static void purgeCases() throws InvalidCaseGraphException {
 
     }
 
+    private static Vector<String> getAllOwners() {
+        Vector<String> users = new Vector<>();
+        Vector<String> owners = new Vector<>();
+        SqlStorage<User> userStorage = CommCareApplication.instance().getUserStorage(User.STORAGE_KEY, User.class);
+        for (IStorageIterator<User> userIterator = userStorage.iterate(); userIterator.hasMore(); ) {
+            String id = userIterator.nextRecord().getUniqueId();
+            owners.addElement(id);
+            users.addElement(id);
+        }
+
+        //Now add all of the relevant groups
+        //TODO: Wow. This is.... kind of megasketch
+        for (String userId : users) {
+            DataInstance instance = CommCareUtil.loadFixture("user-groups", userId);
+            if (instance == null) {
+                continue;
+            }
+            EvaluationContext ec = new EvaluationContext(instance);
+            for (TreeReference ref : ec.expandReference(
+                    XPathReference.getPathExpr("/groups/group/@id").getReference())) {
+                AbstractTreeElement<AbstractTreeElement> idelement = ec.resolveReference(ref);
+                if (idelement.getValue() != null) {
+                    owners.addElement(idelement.getValue().uncast().getString());
+                }
+            }
+        }
+        return owners;
+    }
+
+    /**
+     * Get records ids for all related cases for the given set of cases including the original set of cases
+     *
+     * @param recordIds databse ids for the cases we want to find related cases
+     * @return database ids for all related cases for the given set of cases
+     */
+    public static Vector<Integer> getRelatedCases(Set<String> recordIds) {
+        if (recordIds.isEmpty()) {
+            return new Vector<>();
+        }
+
+        SqlStorage<ACase> storage = CommCareApplication.instance().getUserStorage(ACase.STORAGE_KEY, ACase.class);
+        DAG<String, int[], String> fullCaseGraph = getFullCaseGraph(storage, new AndroidCaseIndexTable(),
+                new Vector<>());
+        Set<String> caseIds = getCaseIdsFromRecordsIds(storage, recordIds);
+        Set<String> relatedCaseIds = fullCaseGraph.findConnectedRecords(caseIds);
+        return storage.getIDsForValues(new String[]{INDEX_CASE_ID}, relatedCaseIds.toArray());
+    }
+
+    private static Set<String> getCaseIdsFromRecordsIds(SqlStorage<ACase> storage, Set<String> recordIds) {
+        Set<String> caseIds = new HashSet<>(recordIds.size());
+        for (String recordId : recordIds) {
+            String[] result = storage.getMetaDataForRecord(Integer.parseInt(recordId),
+                    new String[]{INDEX_CASE_ID});
+            caseIds.add(result[0]);
+        }
+        return caseIds;
+    }
+
     public static DAG<String, int[], String> getFullCaseGraph(SqlStorage<ACase> caseStorage,
-                                                              AndroidCaseIndexTable indexTable,
-                                                              Vector<String> owners) {
+            AndroidCaseIndexTable indexTable,
+            Vector<String> owners) {
         DAG<String, int[], String> caseGraph = new DAG<>();
         Vector<Pair<String, String>> indexHolder = new Vector<>();
 
@@ -163,11 +196,11 @@ public static DAG<String, int[], String> getFullCaseGraph(SqlStorage<ACase> case
         // directed edge for each index (from the 'child' case pointing to the 'parent' case) with
         // the appropriate relationship tagged
         for (SqlStorageIterator<ACase> i = caseStorage.iterate(false, new String[]{
-                Case.INDEX_OWNER_ID, Case.INDEX_CASE_STATUS, Case.INDEX_CASE_ID}); i.hasMore(); ) {
+                Case.INDEX_OWNER_ID, Case.INDEX_CASE_STATUS, INDEX_CASE_ID}); i.hasMore(); ) {
 
             String ownerId = i.peekIncludedMetadata(Case.INDEX_OWNER_ID);
             boolean closed = i.peekIncludedMetadata(Case.INDEX_CASE_STATUS).equals("closed");
-            String caseID = i.peekIncludedMetadata(Case.INDEX_CASE_ID);
+            String caseID = i.peekIncludedMetadata(INDEX_CASE_ID);
             int caseRecordId = i.nextID();
 
 

app/src/org/commcare/models/database/user/DatabaseUserOpenHelper.java

@@ -66,9 +66,10 @@ public class DatabaseUserOpenHelper extends SQLiteOpenHelper {
      * v.26 - Adds a column for 'last_sync' in IndexedFixtureIndex
      * v.27 - Adds a column `descriptor` in FormRecord.
      * v.28 - Adds and indexes columns for Case state and category
+     * v.29 - Add columns for is_dirty and is_shallow in entity_cache table
      */
 
-    private static final int USER_DB_VERSION = 28;
+    private static final int USER_DB_VERSION = 29;
 
     private static final String USER_DB_LOCATOR = "database_sandbox_";
 

app/src/org/commcare/models/database/user/UserDatabaseUpgrader.java

@@ -230,6 +230,12 @@ public void upgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
                 oldVersion = 28;
             }
         }
+
+        if (oldVersion == 28) {
+            if (updateTwentyEightTwentyNine(db)) {
+                oldVersion = 29;
+            }
+        }
     }
 
     private boolean upgradeOneTwo(final SQLiteDatabase db) {
@@ -791,6 +797,19 @@ private boolean updateTwentySevenTwentyEight(SQLiteDatabase db) {
         }
     }
 
+    private boolean updateTwentyEightTwentyNine(SQLiteDatabase db) {
+        //drop the existing table and recreate using current definition
+        db.beginTransaction();
+        try {
+            db.execSQL("DROP TABLE IF EXISTS " + CommCareEntityStorageCache.TABLE_NAME);
+            db.execSQL(CommCareEntityStorageCache.getTableDefinition());
+            db.setTransactionSuccessful();
+            return true;
+        } finally {
+            db.endTransaction();
+        }
+    }
+
     private void migrateV2FormRecordsForSingleApp(String appId,
                                                   SqlStorage<FormRecordV2> oldStorage,
                                                   Vector<FormRecordV3> upgradedRecords) {