diff --git a/packages/aggregator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js b/packages/aggregator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js index 553b147ccc..46c99f3e95 100644 --- a/packages/aggregator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js +++ b/packages/aggregator/src/__tests__/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.test.js @@ -25,7 +25,9 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'TO', '20200102', 3], ['BCD1', 'TO', '20200103', 6], ]); - expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected); + expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers( + expected, + ); }); it('combines by org unit and data element', () => { @@ -51,7 +53,9 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'PG', '20200102', 10], ['BCD2', 'PG', '20200102', 12], ]); - expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected); + expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers( + expected, + ); }); it('sums with missing analytics', () => { @@ -65,13 +69,17 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'TO', '20200102', 1], ['BCD1', 'TO', '20200103', 4], ]); - expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected); + expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers( + expected, + ); }); it('sums with no analytics', () => { const analytics = []; const expected = []; - expect(sumPreviousPerPeriod(analytics, {}, DAY)).toIncludeSameMembers(expected); + expect(sumPreviousPerPeriod(analytics, { sumTillLatestData: true }, DAY)).toIncludeSameMembers( + expected, + ); }); it('only returns results within requestedPeriod if oldest request period > oldest data', () => { @@ -80,7 +88,7 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'TO', '20200102', 2], ['BCD1', 'TO', '20200103', 3], ]); - const config = { requestedPeriod: '20200102;20200103;20200104' }; + const config = { requestedPeriod: '20200102;20200103;20200104', sumTillLatestData: true }; const expected = arrayToAnalytics([ ['BCD1', 'TO', '20200102', 3], ['BCD1', 'TO', '20200103', 6], @@ -94,7 +102,7 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'TO', '20200102', 2], ['BCD1', 'TO', '20200103', 3], ]); - const config = { requestedPeriod: '20191231;20200101;20200102' }; + const config = { requestedPeriod: '20191231;20200101;20200102', sumTillLatestData: true }; const expected = arrayToAnalytics([ ['BCD1', 'TO', '20200101', 1], ['BCD1', 'TO', '20200102', 3], @@ -108,7 +116,7 @@ describe('sumPreviousPerPeriod()', () => { // [ 'BCD1', 'TO', '20200102', 2 ], ['BCD1', 'TO', '20200103', 3], ]); - const config = { requestedPeriod: '20200102;20200103' }; + const config = { requestedPeriod: '20200102;20200103', sumTillLatestData: true }; const expected = arrayToAnalytics([ ['BCD1', 'TO', '20200102', 1], ['BCD1', 'TO', '20200103', 4], @@ -116,6 +124,20 @@ describe('sumPreviousPerPeriod()', () => { expect(sumPreviousPerPeriod(analytics, config, DAY)).toIncludeSameMembers(expected); }); + it('sum till latest period when sumTillLatestData is disable', () => { + const analytics = arrayToAnalytics([ + ['BCD1', 'TO', '20200101', 1], + ['BCD1', 'TO', '20200102', 2], + ['BCD1', 'TO', '20200103', 3], + ]); + const config = { requestedPeriod: '20200105;20200106', sumTillLatestData: false }; + const expected = arrayToAnalytics([ + ['BCD1', 'TO', '20200105', 6], + ['BCD1', 'TO', '20200106', 6], + ]); + expect(sumPreviousPerPeriod(analytics, config, DAY)).toIncludeSameMembers(expected); + }); + // It won't work with non numeric periods. eg: 2020W2 it('does everything with a different period type', () => { const analytics = arrayToAnalytics([ @@ -131,7 +153,7 @@ describe('sumPreviousPerPeriod()', () => { ['BCD1', 'PG', '2020', 7], ['BCD2', 'TO', '2020', 8], ]); - const config = { requestedPeriod: '2019;2020;2021;2022' }; + const config = { requestedPeriod: '2019;2020;2021;2022', sumTillLatestData: true }; const expected = arrayToAnalytics([ ['BCD1', 'TO', '2019', 9], ['BCD2', 'TO', '2019', 2], diff --git a/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js b/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js index a9a5c5bcb8..959bc23b7c 100644 --- a/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js +++ b/packages/aggregator/src/analytics/aggregateAnalytics/aggregations/sumPreviousPerPeriod.js @@ -9,8 +9,13 @@ import { convertToPeriod, EARLIEST_DATA_DATE_STRING, getPeriodsInRange } from '@ import { getContinuousPeriodsForAnalytics } from './utils'; export const sumPreviousPerPeriod = (analytics, aggregationConfig, aggregationPeriod) => { - const { requestedPeriod } = aggregationConfig; - const periods = calculatePeriodsFromAnalytics(analytics, aggregationPeriod, requestedPeriod); + const { requestedPeriod, sumTillLatestData } = aggregationConfig; + const periods = calculatePeriodsFromAnalytics( + analytics, + aggregationPeriod, + requestedPeriod, + sumTillLatestData, + ); const analyticsByPeriod = groupBy(analytics, analytic => convertToPeriod(analytic.period, aggregationPeriod), ); @@ -67,7 +72,12 @@ const sumByAnalytic = (previousAnalytics, currentAnalytics, period) => { return returnAnalytics; }; -const calculatePeriodsFromAnalytics = (analytics, aggregationPeriod, requestedPeriod) => { +const calculatePeriodsFromAnalytics = ( + analytics, + aggregationPeriod, + requestedPeriod, + sumTillLatestData = false, +) => { const periodsInAnalytics = getContinuousPeriodsForAnalytics(analytics, aggregationPeriod); if (!requestedPeriod) { return periodsInAnalytics; @@ -77,7 +87,7 @@ const calculatePeriodsFromAnalytics = (analytics, aggregationPeriod, requestedPe .map(period => convertToPeriod(period, aggregationPeriod)); const endPeriod = Math.min( - Math.max(...periodsInAnalytics), + sumTillLatestData ? Math.max(...periodsInAnalytics) : Infinity, Math.max(...requestedPeriodArray), ).toString(); diff --git a/packages/database/src/migrations/20210809024412-AddConfigForSumTillLatest-modifies-data.js b/packages/database/src/migrations/20210809024412-AddConfigForSumTillLatest-modifies-data.js new file mode 100644 index 0000000000..18d1f2171c --- /dev/null +++ b/packages/database/src/migrations/20210809024412-AddConfigForSumTillLatest-modifies-data.js @@ -0,0 +1,63 @@ +'use strict'; + +import { arrayToDbString, insertJsonEntry, updateJsonEntry } from '../utilities'; + +var dbm; +var type; +var seed; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; +}; + +const newConfig = { sumTillLatestData: true }; +const legacyReportCodes = [ + 'COVID_AU_Total_Cases_Each_State_Per_Day', + 'COVID_Compose_Cumulative_Deaths_Vs_Cases', + 'COVID_Tests_Per_Capita', +]; + +exports.up = async function (db) { + const regexReplaceValue = `"SUM_PREVIOUS_EACH_DAY","config":${JSON.stringify(newConfig)}`; + await db.runSql(` + update "legacy_report" dr + set data_builder_config = regexp_replace(dr."data_builder_config"::text, '\\"SUM_PREVIOUS_EACH_DAY\\"','${regexReplaceValue}','g')::jsonb + where code in (${arrayToDbString(legacyReportCodes)}); + `); + + await insertJsonEntry( + db, + 'mapOverlay', + 'measureBuilderConfig', + ['measureBuilders', 'denominator', 'measureBuilderConfig'], + { + aggregations: [ + { + type: 'SUM_PREVIOUS_EACH_DAY', + }, + ], + }, + { id: 'AU_FLUTRACKING_Participants_Per_100k' }, + ); + + await updateJsonEntry( + db, + 'mapOverlay', + 'measureBuilderConfig', + ['measureBuilders', 'denominator'], + { measureBuilder: 'valueForOrgGroup' }, + { id: 'AU_FLUTRACKING_Participants_Per_100k' }, + ); +}; + +exports.down = function (db) {}; + +exports._meta = { + version: 1, +}; diff --git a/packages/database/src/utilities/migration.js b/packages/database/src/utilities/migration.js index a418cc9c75..7133e37d0e 100644 --- a/packages/database/src/utilities/migration.js +++ b/packages/database/src/utilities/migration.js @@ -142,6 +142,15 @@ export const removeJsonEntry = async (db, table, column, path, key, condition) = ); }; +export const updateJsonEntry = async (db, table, column, path, value, condition) => { + assertParamsAreDefined({ db, table, column, path, value, condition }, 'updateJsonEntry'); + Object.keys(value).map(async key => { + await removeJsonEntry(db, table, column, path, key, condition); + }); + + await insertJsonEntry(db, table, column, path, value, condition); +}; + export async function removeArrayValue(db, table, column, value, condition) { assertParamsAreDefined({ db, table, column, value, condition }, 'removeArrayValue');