Fix participant per 100k overlay (beyondessential#2975)

beyondessential · Aug 11, 2021 · 4a9ec01 · 4a9ec01
1 parent 2948289
commit 4a9ec01
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 12 deletions.
diff --git a/...ator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js b/...ator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js
@@ -25,7 +25,9 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'TO', '20200102', 3],
       ['BCD1', 'TO', '20200103', 6],
     ]);
-    expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected);
+    expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers(
+      expected,
+    );
   });
 
   it('combines by org unit and data element', () => {
@@ -51,7 +53,9 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'PG', '20200102', 10],
       ['BCD2', 'PG', '20200102', 12],
     ]);
-    expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected);
+    expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers(
+      expected,
+    );
   });
 
   it('sums with missing analytics', () => {
@@ -65,13 +69,17 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'TO', '20200102', 1],
       ['BCD1', 'TO', '20200103', 4],
     ]);
-    expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected);
+    expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers(
+      expected,
+    );
   });
 
   it('sums with no analytics', () => {
     const analytics = [];
     const expected = [];
-    expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected);
+    expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers(
+      expected,
+    );
   });
 
   it('only returns results within requestedPeriod if oldest request period > oldest data', () => {
@@ -80,7 +88,7 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'TO', '20200102', 2],
       ['BCD1', 'TO', '20200103', 3],
     ]);
-    const config = { requestedPeriod: '20200102;20200103;20200104' };
+    const config = { requestedPeriod: '20200102;20200103;20200104', sumTillLatestData: true };
     const expected = arrayToAnalytics([
       ['BCD1', 'TO', '20200102', 3],
       ['BCD1', 'TO', '20200103', 6],
@@ -94,7 +102,7 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'TO', '20200102', 2],
       ['BCD1', 'TO', '20200103', 3],
     ]);
-    const config = { requestedPeriod: '20191231;20200101;20200102' };
+    const config = { requestedPeriod: '20191231;20200101;20200102', sumTillLatestData: true };
     const expected = arrayToAnalytics([
       ['BCD1', 'TO', '20200101', 1],
       ['BCD1', 'TO', '20200102', 3],
@@ -108,14 +116,28 @@ describe('sumPreviousPerPeriod()', () => {
       // [ 'BCD1', 'TO',  '20200102',  2 ],
       ['BCD1', 'TO', '20200103', 3],
     ]);
-    const config = { requestedPeriod: '20200102;20200103' };
+    const config = { requestedPeriod: '20200102;20200103', sumTillLatestData: true };
     const expected = arrayToAnalytics([
       ['BCD1', 'TO', '20200102', 1],
       ['BCD1', 'TO', '20200103', 4],
     ]);
     expect(sumPreviousPerPeriod(analytics, config, DAY)).toIncludeSameMembers(expected);
   });
 
+  it('sum till latest period when sumTillLatestData is disable', () => {
+    const analytics = arrayToAnalytics([
+      ['BCD1', 'TO', '20200101', 1],
+      ['BCD1', 'TO', '20200102', 2],
+      ['BCD1', 'TO', '20200103', 3],
+    ]);
+    const config = { requestedPeriod: '20200105;20200106', sumTillLatestData: false };
+    const expected = arrayToAnalytics([
+      ['BCD1', 'TO', '20200105', 6],
+      ['BCD1', 'TO', '20200106', 6],
+    ]);
+    expect(sumPreviousPerPeriod(analytics, config, DAY)).toIncludeSameMembers(expected);
+  });
+
   // It won't work with non numeric periods. eg: 2020W2
   it('does everything with a different period type', () => {
     const analytics = arrayToAnalytics([
@@ -131,7 +153,7 @@ describe('sumPreviousPerPeriod()', () => {
       ['BCD1', 'PG', '2020', 7],
       ['BCD2', 'TO', '2020', 8],
     ]);
-    const config = { requestedPeriod: '2019;2020;2021;2022' };
+    const config = { requestedPeriod: '2019;2020;2021;2022', sumTillLatestData: true };
     const expected = arrayToAnalytics([
       ['BCD1', 'TO', '2019', 9],
       ['BCD2', 'TO', '2019', 2],

diff --git a/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js b/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js
@@ -9,8 +9,13 @@ import { convertToPeriod, EARLIEST_DATA_DATE_STRING, getPeriodsInRange } from '@
 import { getContinuousPeriodsForAnalytics } from './utils';
 
 export const sumPreviousPerPeriod = (analytics, aggregationConfig, aggregationPeriod) => {
-  const { requestedPeriod } = aggregationConfig;
-  const periods = calculatePeriodsFromAnalytics(analytics, aggregationPeriod, requestedPeriod);
+  const { requestedPeriod, sumTillLatestData } = aggregationConfig;
+  const periods = calculatePeriodsFromAnalytics(
+    analytics,
+    aggregationPeriod,
+    requestedPeriod,
+    sumTillLatestData,
+  );
   const analyticsByPeriod = groupBy(analytics, analytic =>
     convertToPeriod(analytic.period, aggregationPeriod),
   );
@@ -67,7 +72,12 @@ const sumByAnalytic = (previousAnalytics, currentAnalytics, period) => {
   return returnAnalytics;
 };
 
-const calculatePeriodsFromAnalytics = (analytics, aggregationPeriod, requestedPeriod) => {
+const calculatePeriodsFromAnalytics = (
+  analytics,
+  aggregationPeriod,
+  requestedPeriod,
+  sumTillLatestData = false,
+) => {
   const periodsInAnalytics = getContinuousPeriodsForAnalytics(analytics, aggregationPeriod);
   if (!requestedPeriod) {
     return periodsInAnalytics;
@@ -77,7 +87,7 @@ const calculatePeriodsFromAnalytics = (analytics, aggregationPeriod, requestedPe
     .map(period => convertToPeriod(period, aggregationPeriod));
 
   const endPeriod = Math.min(
-    Math.max(...periodsInAnalytics),
+    sumTillLatestData ? Math.max(...periodsInAnalytics) : Infinity,
     Math.max(...requestedPeriodArray),
   ).toString();
 

diff --git a/packages/database/src/migrations/20210809024412-AddConfigForSumTillLatest-modifies-data.js b/packages/database/src/migrations/20210809024412-AddConfigForSumTillLatest-modifies-data.js
@@ -0,0 +1,63 @@
+'use strict';
+
+import { arrayToDbString, insertJsonEntry, updateJsonEntry } from '../utilities';
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+const newConfig = { sumTillLatestData: true };
+const legacyReportCodes = [
+  'COVID_AU_Total_Cases_Each_State_Per_Day',
+  'COVID_Compose_Cumulative_Deaths_Vs_Cases',
+  'COVID_Tests_Per_Capita',
+];
+
+exports.up = async function (db) {
+  const regexReplaceValue = `"SUM_PREVIOUS_EACH_DAY","config":${JSON.stringify(newConfig)}`;
+  await db.runSql(`
+    update "legacy_report" dr
+    set data_builder_config = regexp_replace(dr."data_builder_config"::text, '\\"SUM_PREVIOUS_EACH_DAY\\"','${regexReplaceValue}','g')::jsonb
+    where code in (${arrayToDbString(legacyReportCodes)});
+  `);
+
+  await insertJsonEntry(
+    db,
+    'mapOverlay',
+    'measureBuilderConfig',
+    ['measureBuilders', 'denominator', 'measureBuilderConfig'],
+    {
+      aggregations: [
+        {
+          type: 'SUM_PREVIOUS_EACH_DAY',
+        },
+      ],
+    },
+    { id: 'AU_FLUTRACKING_Participants_Per_100k' },
+  );
+
+  await updateJsonEntry(
+    db,
+    'mapOverlay',
+    'measureBuilderConfig',
+    ['measureBuilders', 'denominator'],
+    { measureBuilder: 'valueForOrgGroup' },
+    { id: 'AU_FLUTRACKING_Participants_Per_100k' },
+  );
+};
+
+exports.down = function (db) {};
+
+exports._meta = {
+  version: 1,
+};
diff --git a/packages/database/src/utilities/migration.js b/packages/database/src/utilities/migration.js
@@ -142,6 +142,15 @@ export const removeJsonEntry = async (db, table, column, path, key, condition) =
   );
 };
 
+export const updateJsonEntry = async (db, table, column, path, value, condition) => {
+  assertParamsAreDefined({ db, table, column, path, value, condition }, 'updateJsonEntry');
+  Object.keys(value).map(async key => {
+    await removeJsonEntry(db, table, column, path, key, condition);
+  });
+
+  await insertJsonEntry(db, table, column, path, value, condition);
+};
+
 export async function removeArrayValue(db, table, column, value, condition) {
   assertParamsAreDefined({ db, table, column, value, condition }, 'removeArrayValue');