RN-400: Meditrak permissions based sync (#3992)

packages/central-server/package.json

@@ -22,7 +22,8 @@
     "start-dev": "yarn package:start:backend-start-dev 9999",
     "start-verbose": "LOG_LEVEL=debug yarn start-dev",
     "test": "yarn workspace @tupaia/database check-test-database-exists && DB_NAME=tupaia_test NODE_ENV=test mocha",
-    "test:coverage": "cross-env NODE_ENV=test nyc mocha"
+    "test:coverage": "cross-env NODE_ENV=test nyc mocha",
+    "create-meditrak-sync-view": "node dist/createMeditrakSyncView.js"
   },
   "dependencies": {
     "@babel/polyfill": "^7.0.0",

packages/central-server/src/apiV2/changesMetadata.js

@@ -0,0 +1,30 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2022 Beyond Essential Systems Pty Ltd
+ */
+
+import { respond } from '@tupaia/utils';
+import { buildPermissionsBasedMeditrakSyncQuery } from './utilities';
+import { allowNoPermissions } from '../permissions';
+
+/**
+ * Permissions based sync metadata
+ * {
+ *   changeCount: number of changes since last sync
+ *   countries: countries included in the sync
+ *   permissionGroups: permissions groups included in the sync
+ * }
+ * Responds to GET requests to the /changes/metadata endpoint
+ */
+export async function changesMetadata(req, res) {
+  const { models } = req;
+
+  await req.assertPermissions(allowNoPermissions);
+
+  const { query, countries, permissionGroups } = await buildPermissionsBasedMeditrakSyncQuery(req, {
+    select: 'count(*)',
+  });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
+  respond(res, { changeCount, countries, permissionGroups });
+}

packages/central-server/src/apiV2/countChanges/LegacyCountChangesHandler.js

@@ -5,7 +5,7 @@
 
 import { getKeysSortedByValues, respond, UnauthenticatedError } from '@tupaia/utils';
 import { getUniversalTypes } from '../../database/utilities';
-import { fetchRequestingMeditrakDevice, getChangesFilter } from '../utilities';
+import { fetchRequestingMeditrakDevice, buildMeditrakSyncQuery } from '../utilities';
 
 const MAX_FAILS_BEFORE_LOG_OUT = 2;
 const MAX_FAILS_BEFORE_TYPE_EXCLUSION = 5;
@@ -135,11 +135,16 @@ export class LegacyCountChangesHandler {
     const { models, query } = this.req;
 
     const universalTypes = getUniversalTypes(models);
-    const filter = await getChangesFilter({
-      ...this.req,
-      query: { ...query, recordTypes: universalTypes.join(',') },
-    });
-    const changesCount = await models.meditrakSyncQueue.count(filter);
+
+    const { query: dbQuery } = await buildMeditrakSyncQuery(
+      {
+        ...this.req,
+        query: { ...query, recordTypes: universalTypes.join(',') },
+      },
+      { select: 'count(*)' },
+    );
+    const result = await dbQuery.executeOnDatabase(models.database);
+    const changesCount = parseInt(result[0].count);
 
     return changesCount > 0;
   }
@@ -163,8 +168,9 @@ export class LegacyCountChangesHandler {
 
   async handle() {
     await this.handleSetUp();
-    const filter = await getChangesFilter(this.req);
-    const changeCount = await this.req.models.meditrakSyncQueue.count(filter);
+    const { query } = await buildMeditrakSyncQuery(this.req, { select: 'count(*)' });
+    const queryResult = await query.executeOnDatabase(this.req.models.database);
+    const changeCount = parseInt(queryResult[0].count);
     respond(this.res, { changeCount });
   }
 }

packages/central-server/src/apiV2/countChanges/countChanges.js

@@ -4,15 +4,15 @@
  */
 
 import { respond, DatabaseError, UnauthenticatedError } from '@tupaia/utils';
-import { getChangesFilter } from '../utilities';
+import { buildMeditrakSyncQuery } from '../utilities';
 import { LegacyCountChangesHandler } from './LegacyCountChangesHandler';
 import { allowNoPermissions } from '../../permissions';
 
 const handleNonLegacyRequest = async (req, res) => {
   const { models } = req;
-
-  const filter = await getChangesFilter(req);
-  const changeCount = await models.meditrakSyncQueue.count(filter);
+  const { query } = await buildMeditrakSyncQuery(req, { select: 'count(*)' });
+  const queryResult = await query.executeOnDatabase(models.database);
+  const changeCount = parseInt(queryResult[0].count);
   respond(res, { changeCount });
 };
 

packages/central-server/src/apiV2/getChanges.js

@@ -3,17 +3,24 @@
  * Copyright (c) 2017 Beyond Essential Systems Pty Ltd
  */
 
+import keyBy from 'lodash.keyby';
+import groupBy from 'lodash.groupby';
 import { respond, DatabaseError } from '@tupaia/utils';
 import { TYPES } from '@tupaia/database';
-import { getChangesFilter, getColumnsForMeditrakApp } from './utilities';
+import {
+  supportsPermissionsBasedSync,
+  buildMeditrakSyncQuery,
+  buildPermissionsBasedMeditrakSyncQuery,
+  getColumnsForMeditrakApp,
+} from './utilities';
 import { allowNoPermissions } from '../permissions';
 
 const MAX_CHANGES_RETURNED = 100;
 
 /**
  * Gets the record ready to sync down to a sync client, transforming any properties as required
  */
-async function getRecordForSync(record) {
+function getRecordForSync(record) {
   const recordWithoutNulls = {};
   // Remove null entries to a) save bandwidth and b) remain consistent with previous mongo based db
   // which simply had no key for undefined properties, whereas postgres uses null
@@ -34,45 +41,67 @@ async function getRecordForSync(record) {
  */
 export async function getChanges(req, res) {
   const { database, models } = req;
-  const { limit = MAX_CHANGES_RETURNED, offset = 0 } = req.query;
+  const { limit = MAX_CHANGES_RETURNED, offset = 0, appVersion } = req.query;
 
   await req.assertPermissions(allowNoPermissions);
 
   try {
-    const filter = await getChangesFilter(req);
-    const changes = await database.find(TYPES.MEDITRAK_SYNC_QUEUE, filter, {
-      sort: ['change_time'],
+    const queryBuilder = supportsPermissionsBasedSync(appVersion)
+      ? buildPermissionsBasedMeditrakSyncQuery
+      : buildMeditrakSyncQuery;
+    const { query } = await queryBuilder(req, {
+      select: (await models.meditrakSyncQueue.fetchFieldNames()).join(', '),
+      sort: 'change_time ASC',
       limit,
       offset,
     });
-    const changesToSend = await Promise.all(
-      changes.map(async change => {
-        const {
-          type: action,
-          record_type: recordType,
-          record_id: recordId,
-          change_time: timestamp,
-        } = change;
-        const columns = await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType));
-        const changeObject = { action, recordType, timestamp };
-        if (action === 'delete') {
-          changeObject.record = { id: recordId };
-          if (recordType === TYPES.GEOGRAPHICAL_AREA) {
-            // TODO LEGACY Deal with this bug on app end for v3 api
-            changeObject.recordType = 'area';
-          }
+    const changes = await query.executeOnDatabase(database);
+    const changesByRecordType = groupBy(changes, 'record_type');
+    const recordTypesToSync = Object.keys(changesByRecordType);
+    const columnNamesByRecordType = Object.fromEntries(
+      await Promise.all(
+        recordTypesToSync.map(async recordType => [
+          recordType,
+          await getColumnsForMeditrakApp(models.getModelForDatabaseType(recordType)),
+        ]),
+      ),
+    );
+    const changeRecords = (
+      await Promise.all(
+        Object.entries(changesByRecordType).map(async ([recordType, changesForType]) => {
+          const changeIds = changesForType.map(change => change.record_id);
+          const columns = columnNamesByRecordType[recordType];
+          return database.find(recordType, { id: changeIds }, { lean: true, columns });
+        }),
+      )
+    ).flat();
+    const changeRecordsById = keyBy(changeRecords, 'id');
+
+    const changesToSend = changes.map(change => {
+      const {
+        type: action,
+        record_type: recordType,
+        record_id: recordId,
+        change_time: timestamp,
+      } = change;
+      const changeObject = { action, recordType, timestamp };
+      if (action === 'delete') {
+        changeObject.record = { id: recordId };
+        if (recordType === TYPES.GEOGRAPHICAL_AREA) {
+          // TODO LEGACY Deal with this bug on app end for v3 api
+          changeObject.recordType = 'area';
+        }
+      } else {
+        const record = changeRecordsById[recordId];
+        if (!record) {
+          const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
+          changeObject.error = { error: errorMessage };
         } else {
-          const record = await database.findById(recordType, recordId, { lean: true, columns });
-          if (!record) {
-            const errorMessage = `Couldn't find record type ${recordType} with id ${recordId}`;
-            changeObject.error = { error: errorMessage };
-          } else {
-            changeObject.record = await getRecordForSync(record);
-          }
+          changeObject.record = getRecordForSync(record);
         }
-        return changeObject;
-      }),
-    );
+      }
+      return changeObject;
+    });
     respond(res, changesToSend);
     return;
   } catch (error) {

packages/central-server/src/apiV2/index.js

@@ -18,6 +18,7 @@ import { useRouteHandler } from './RouteHandler';
 import { exportRoutes } from './export';
 import { importRoutes } from './import';
 import { authenticate } from './authenticate';
+import { changesMetadata } from './changesMetadata';
 import { countChanges } from './countChanges';
 import { getChanges } from './getChanges';
 import { BESAdminCreateHandler } from './CreateHandler';
@@ -142,6 +143,7 @@ apiV2.use('/import', importRoutes);
  * GET routes
  */
 apiV2.get('/changes/count', catchAsyncErrors(countChanges));
+apiV2.get('/changes/metadata', catchAsyncErrors(changesMetadata));
 apiV2.get('/changes', catchAsyncErrors(getChanges));
 apiV2.get('/socialFeed', catchAsyncErrors(getSocialFeed));
 apiV2.get('/me', useRouteHandler(GETUserForMe));

packages/central-server/src/apiV2/utilities/index.js

@@ -16,10 +16,14 @@ export {
   getAdminPanelAllowedCountryCodes,
 } from './getAdminPanelAllowedEntityIds';
 export { getArrayQueryParameter } from './getArrayQueryParameter';
-export { getChangesFilter } from './getChangesFilter';
 export { getColumnsForMeditrakApp } from './getColumnsForMeditrakApp';
 export { hasAccessToEntityForVisualisation } from './hasAccessToEntityForVisualisation';
 export { hasTupaiaAdminAccessToEntityForVisualisation } from './hasTupaiaAdminAccessToEntityForVisualisation';
+export {
+  buildMeditrakSyncQuery,
+  buildPermissionsBasedMeditrakSyncQuery,
+  supportsPermissionsBasedSync,
+} from './meditrakSync';
 export { mergeFilter } from './mergeFilter';
 export { mergeMultiJoin } from './mergeMultiJoin';
 export { SurveyResponseImporter } from './SurveyResponseImporter';