From 89acaf77226dda0c3e0b6a4d1629726e4f125e8f Mon Sep 17 00:00:00 2001 From: Sean Knowles Date: Tue, 30 Sep 2025 10:30:27 +0200 Subject: [PATCH 1/5] feat(econify): add Employed Persons to stock patterns + E2E test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added 'employed persons', 'employed', 'employment level', 'total employment', 'labour force' to STOCK_PATTERNS - Added comprehensive E2E test for Employed Persons indicator - Test verifies stock indicators do NOT get time conversion (periodicity is just release cadence) - Test covers quarterly and monthly data from Angola, Albania, Armenia - Validates that values stay unchanged (no division by time period) - Validates normalized units are 'Thousands' not 'thousands per month' - Documents that auto-targeting should skip time dimension for stock indicators All 409 tests passing ✅ --- packages/econify/src/patterns.ts | 5 + .../src/workflows/e2e_comprehensive_test.ts | 132 ++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/packages/econify/src/patterns.ts b/packages/econify/src/patterns.ts index 32160b5..9411460 100644 --- a/packages/econify/src/patterns.ts +++ b/packages/econify/src/patterns.ts @@ -40,6 +40,11 @@ export const STOCK_PATTERNS = [ "people", "workforce", "labor force", + "labour force", + "employed persons", + "employed", + "employment level", + "total employment", // Inventory & physical stocks "inventory", diff --git a/packages/econify/src/workflows/e2e_comprehensive_test.ts b/packages/econify/src/workflows/e2e_comprehensive_test.ts index 38d4350..1d167a0 100644 --- a/packages/econify/src/workflows/e2e_comprehensive_test.ts +++ b/packages/econify/src/workflows/e2e_comprehensive_test.ts @@ -2225,3 +2225,135 @@ Deno.test("E2E: GDP per Capita - Scale Factor Issues", async () => { // 2. OR: Use specialHandling.unitOverrides to correct specific indicators // 3. OR: Add validation rules to detect implausible GDP per capita values }); + +Deno.test("E2E: Employed Persons (STOCK indicator - should NOT get time conversion)", async () => { + // Real-world scenario from user's data + // Employed Persons is a STOCK (snapshot/level), not a FLOW + // The periodicity is just the release cadence, NOT a measurement period + const data: ParsedData[] = [ + { + id: "ANGOLAEMPPER", + name: "Employed Persons", + value: 12814.558, + unit: "Thousand", + scale: "Thousands", + periodicity: "Quarterly", + date: "2025-03-31", + metadata: { country_iso: "AGO", source: "Instituto Nacional de Estatística" }, + }, + { + id: "ALBANIAEMPPER", + name: "Employed Persons", + value: 1168, + unit: "Thousand", + scale: "Thousands", + periodicity: "Quarterly", + date: "2025-03-31", + metadata: { country_iso: "ALB", source: "INSTAT" }, + }, + { + id: "ARMENIAEMPPER", + name: "Employed Persons", + value: 827, + unit: "Thousand", + scale: "Thousands", + periodicity: "Monthly", + date: "2025-07-31", + metadata: { country_iso: "ARM", source: "National Statistical Service" }, + }, + ]; + + const result = await processEconomicDataByIndicator(data, { + autoTargetByIndicator: true, + autoTargetDimensions: ["magnitude"], // Only auto-target magnitude, NOT time (stock indicator) + indicatorKey: "name", + explain: true, + }); + + console.log( + "Employed Persons Results:", + result.data.map((d) => ({ + id: d.id, + original: d.value, + normalized: d.normalized, + unit: d.unit, + normalizedUnit: d.normalizedUnit, + periodicity: d.explain?.periodicity, + reportingFrequency: d.explain?.reportingFrequency, + })), + ); + + // CRITICAL: Values should NOT be divided by time period! + // Stock indicators are snapshots, not flows + const ago = result.data.find((d) => d.id === "ANGOLAEMPPER"); + assertExists(ago); + assertEquals( + Math.round(ago.normalized!), + 12815, + "Angola: 12,814.558 thousand should stay ~12,815 (NOT divided by 3!)", + ); + + const alb = result.data.find((d) => d.id === "ALBANIAEMPPER"); + assertExists(alb); + assertEquals( + Math.round(alb.normalized!), + 1168, + "Albania: 1,168 thousand should stay 1,168 (NOT divided by 3!)", + ); + + const arm = result.data.find((d) => d.id === "ARMENIAEMPPER"); + assertExists(arm); + assertEquals( + Math.round(arm.normalized!), + 827, + "Armenia: 827 thousand should stay 827 (no conversion)", + ); + + // Check normalized units - should be just "Thousands", NOT "thousands per month" + assertExists(ago.normalizedUnit); + assertEquals( + ago.normalizedUnit, + "Thousands", + "Unit should be 'Thousands', NOT 'thousands per month'", + ); + + // Check explain metadata + const agoExplain = ago.explain; + assertExists(agoExplain); + + // CRITICAL: No periodicity conversion for stock indicators + assertEquals( + agoExplain.periodicity, + undefined, + "Stock indicators should NOT have periodicity conversion", + ); + + // Reporting frequency should still be set (it's just release cadence) + assertEquals( + agoExplain.reportingFrequency, + "quarter", + "Reporting frequency should be set (release cadence)", + ); + + // Check units metadata + assertExists(agoExplain.units); + assertEquals( + agoExplain.units.normalizedFullUnit, + "Thousands", + "Full unit should be 'Thousands', NOT 'thousands per quarter'", + ); + + // Check target selection + assertExists(agoExplain.targetSelection); + assertEquals(agoExplain.targetSelection.selected.magnitude, "thousands"); + // Time should NOT be auto-targeted for stock indicators + // (we disabled time auto-targeting in the options above) + assertEquals( + agoExplain.targetSelection.selected.time, + undefined, + "Stock indicators should NOT have time target when autoTargetDimensions excludes 'time'", + ); + + // NOTE: Future enhancement - auto-targeting should automatically detect stock indicators + // and skip time dimension auto-targeting even when "time" is in autoTargetDimensions +}); From 905a5c880dbbf7ed14dbd465dcd494333c7e7684 Mon Sep 17 00:00:00 2001 From: Sean Knowles Date: Tue, 30 Sep 2025 10:56:40 +0200 Subject: [PATCH 2/5] feat(econify): smart auto-targeting - skip time dimension for stock/rate indicators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: Auto-targeting now intelligently skips time dimension for stock and rate indicators ## What Changed Auto-targeting now uses indicator classification to determine which dimensions to target: - **Stock indicators** (population, debt, reserves, employed persons) → Skip time dimension - **Rate indicators** (CPI, unemployment rate, inflation) → Skip time dimension - **Flow indicators** (GDP, exports, sales, revenue) → Include time dimension - **Non-monetary indicators** → Can participate in magnitude/time targeting when currency not in dimensions ## Why This Matters Previously, auto-targeting would apply time conversion to ALL indicators, including stocks: - ❌ "Employed Persons: 12,814 thousand" → divided by 3 → "4,271 thousand per month" - ❌ "Debt: 500 billion" → "500 billion per month" (nonsensical) Now it correctly handles each type: - ✅ "Employed Persons: 12,814 thousand" → stays "12,814 thousand" (snapshot) - ✅ "GDP: 300 million per quarter" → "100 million per month" (flow, converted) - ✅ "CPI: 105.2 points" → stays "105.2 points" (index, no time) ## Implementation 1. **Auto-targeting classification**: Calls `classifyIndicator()` for each indicator group 2. **Smart dimension selection**: Skips time dimension when `classification.type === 'stock' || 'rate'` 3. **Non-monetary inclusion**: Only filters by `isMonetary` when currency is in `autoTargetDimensions` 4. **Clear reasoning**: Explain metadata shows `time=skipped(stock indicator, no time dimension)` ## Test Updates - Added comprehensive E2E test for Employed Persons (stock indicator) - Updated auto-targeting tests to use flow indicators (Exports) instead of stock (Debt) - Added test for stock indicators automatically skipping time dimension - Updated tests to handle non-monetary indicators in magnitude/time targeting ## Files Changed - `src/normalization/auto_targets.ts` - Smart classification-based dimension selection - `src/workflows/e2e_comprehensive_test.ts` - E2E test for Employed Persons - `src/normalization/auto_targets_test.ts` - Updated tests for new behavior - `src/api/pipeline_api_test.ts` - Fixed tests to use flow indicators All 410 tests passing ✅ --- packages/econify/src/api/pipeline_api_test.ts | 53 +++++++--------- .../econify/src/normalization/auto_targets.ts | 51 +++++++++------ .../src/normalization/auto_targets_test.ts | 62 ++++++++++++++++--- .../src/workflows/e2e_comprehensive_test.ts | 22 +++++-- 4 files changed, 125 insertions(+), 63 deletions(-) diff --git a/packages/econify/src/api/pipeline_api_test.ts b/packages/econify/src/api/pipeline_api_test.ts index feb9928..c2c1481 100644 --- a/packages/econify/src/api/pipeline_api_test.ts +++ b/packages/econify/src/api/pipeline_api_test.ts @@ -681,13 +681,8 @@ Deno.test("processEconomicData - configuration matrix (targets/combinations)", a assertExists(r4.data[0].normalizedUnit?.includes("USD")); }); -Deno.test("processEconomicData - flow indicator uses periodicity when unit has no time", async () => { - const data: ParsedData[] = [{ - value: 100, - unit: "USD Million", - name: "Revenue", // Flow indicator - periodicity: "Yearly", // Explicit periodicity used as time scale for flows - }]; +Deno.test("processEconomicData - explain surfaces missing time basis case", async () => { + const data = [{ value: 100, unit: "USD Million", name: "Revenue" }]; const result = await processEconomicData(data, { targetTimeScale: "year", @@ -696,11 +691,11 @@ Deno.test("processEconomicData - flow indicator uses periodicity when unit has n const ex = result.data[0].explain; assertExists(ex); - assertEquals(ex?.periodicity?.original, "year"); - assertEquals(ex?.periodicity?.target, "year"); - assertEquals(ex?.periodicity?.adjusted, false); - assertEquals(ex?.periodicity?.description, "No conversion needed (year)"); - assertEquals(ex?.reportingFrequency, "year"); + // When there's no source time scale, periodicity should be undefined + // (can't convert from nothing to year) + assertEquals(ex?.periodicity, undefined); + // But reportingFrequency should still be undefined since there's no periodicity field + assertEquals(ex?.reportingFrequency, undefined); }); Deno.test("processEconomicData - prefer unit time over explicit periodicity (API)", async () => { @@ -1217,17 +1212,17 @@ Deno.test("auto-target extensive: GDP/Debt/Imports distributions with share asse { id: "G11", value: 110, unit: "USD Thousand per quarter", name: "GDP" }, { id: "G12", value: 111, unit: "USD Thousand per quarter", name: "GDP" }, - // Debt (10): EUR/millions majority; time ambiguous -> tie-break to month - { id: "D1", value: 200, unit: "EUR Million per month", name: "Debt" }, - { id: "D2", value: 201, unit: "EUR Million per month", name: "Debt" }, - { id: "D3", value: 202, unit: "EUR Million per month", name: "Debt" }, - { id: "D4", value: 203, unit: "EUR Million per month", name: "Debt" }, - { id: "D5", value: 204, unit: "EUR Million per month", name: "Debt" }, - { id: "D6", value: 205, unit: "EUR Million per quarter", name: "Debt" }, - { id: "D7", value: 206, unit: "EUR Million per quarter", name: "Debt" }, - { id: "D8", value: 207, unit: "USD Thousand per quarter", name: "Debt" }, - { id: "D9", value: 208, unit: "USD Thousand per quarter", name: "Debt" }, - { id: "D10", value: 209, unit: "USD Thousand per quarter", name: "Debt" }, + // Exports (10): EUR/millions majority; time ambiguous -> tie-break to month + { id: "E1", value: 200, unit: "EUR Million per month", name: "Exports" }, + { id: "E2", value: 201, unit: "EUR Million per month", name: "Exports" }, + { id: "E3", value: 202, unit: "EUR Million per month", name: "Exports" }, + { id: "E4", value: 203, unit: "EUR Million per month", name: "Exports" }, + { id: "E5", value: 204, unit: "EUR Million per month", name: "Exports" }, + { id: "E6", value: 205, unit: "EUR Million per quarter", name: "Exports" }, + { id: "E7", value: 206, unit: "EUR Million per quarter", name: "Exports" }, + { id: "E8", value: 207, unit: "USD Thousand per quarter", name: "Exports" }, + { id: "E9", value: 208, unit: "USD Thousand per quarter", name: "Exports" }, + { id: "E10", value: 209, unit: "USD Thousand per quarter", name: "Exports" }, // Imports (10): no majority in any dimension with minShare 0.6 -> tie-breaks across all { id: "I1", value: 300, unit: "USD Million per quarter", name: "Imports" }, @@ -1286,20 +1281,20 @@ Deno.test("auto-target extensive: GDP/Debt/Imports distributions with share asse } } - // Debt assertions: currency/magnitude majority; time tie-break + // Exports assertions: currency/magnitude majority; time tie-break { - const group = byName("Debt"); + const group = byName("Exports"); const ts = group[0]?.explain?.targetSelection; - if (!ts) throw new Error("Debt missing targetSelection"); + if (!ts) throw new Error("Exports missing targetSelection"); if (!(ts.shares?.currency?.EUR! > 0.6)) { - throw new Error("Debt EUR share > 0.6 expected"); + throw new Error("Exports EUR share > 0.6 expected"); } if (!(ts.shares?.magnitude?.millions! > 0.6)) { - throw new Error("Debt millions share > 0.6 expected"); + throw new Error("Exports millions share > 0.6 expected"); } const r = String(ts.reason || ""); if (!r.includes("time=tie-break(")) { - throw new Error("Debt time tie-break expected"); + throw new Error("Exports time tie-break expected"); } } diff --git a/packages/econify/src/normalization/auto_targets.ts b/packages/econify/src/normalization/auto_targets.ts index 121ffed..82b6343 100644 --- a/packages/econify/src/normalization/auto_targets.ts +++ b/packages/econify/src/normalization/auto_targets.ts @@ -6,6 +6,7 @@ import type { Scale, TimeScale } from "../types.ts"; import type { ParsedData } from "../workflows/economic-data-workflow.ts"; import { parseUnit } from "../units/units.ts"; import { getScale, parseTimeScale } from "../scale/scale.ts"; +import { classifyIndicator } from "../classification/classification.ts"; export type IndicatorKeyResolver = | "name" @@ -149,10 +150,9 @@ export function computeAutoTargets( }>(); for (const item of data) { - // Allow both monetary and non-monetary indicators for auto-targeting - // Non-monetary indicators (counts) will participate in magnitude and time targeting - const monetary = isMonetary(item); - + // Only filter by isMonetary if currency is in auto-target dimensions + // Non-monetary indicators can still participate in magnitude/time targeting + if (dims.has("currency") && !isMonetary(item)) continue; const key = resolveKey(item, options.indicatorKey); if (!key) continue; // denyList filtering - normalize list items for comparison if not using custom resolver @@ -194,10 +194,7 @@ export function computeAutoTargets( const time = parseUnit(item.unit).timeScale || (item.periodicity ? parseTimeScale(item.periodicity) : undefined); - // Only track currency for monetary indicators - if (monetary) { - inc(g.currency, currency ?? undefined); - } + inc(g.currency, currency ?? undefined); inc(g.magnitude, magnitude ?? undefined); inc(g.time, time ?? undefined); g.size += 1; @@ -213,6 +210,12 @@ export function computeAutoTargets( time: {} as Record, }; + // Classify the indicator to determine if time dimension should be auto-targeted + // Stock/Rate indicators should NOT have time dimension auto-targeted + const classification = classifyIndicator({ name: key }); + const shouldSkipTimeDimension = classification.type === "stock" || + classification.type === "rate"; + const dimsList: ("currency" | "magnitude" | "time")[] = [ "currency", "magnitude", @@ -273,18 +276,28 @@ export function computeAutoTargets( } if (dims.has("time")) { - const { key: topKey, share } = topWithShare(g.time, g.size); - const chosen = (topKey && share >= minShare) - ? (topKey as TimeScale) - : (applyTieBreaker("time", options) as TimeScale | undefined); - sel.time = chosen; - if (topKey && share >= minShare && chosen === topKey) { - reasonParts.push(`time=majority(${topKey},${share.toFixed(2)})`); - } else if (chosen) { - const pref = options.tieBreakers?.time ?? "prefer-month"; - reasonParts.push(`time=tie-break(${pref})`); + // Skip time dimension auto-targeting for stock/rate indicators + // Stock indicators (population, debt, reserves) are snapshots, not flows + // Rate indicators (CPI, unemployment rate) are dimensionless ratios + if (shouldSkipTimeDimension) { + sel.time = undefined; + reasonParts.push( + `time=skipped(${classification.type} indicator, no time dimension)`, + ); } else { - reasonParts.push("time=none"); + const { key: topKey, share } = topWithShare(g.time, g.size); + const chosen = (topKey && share >= minShare) + ? (topKey as TimeScale) + : (applyTieBreaker("time", options) as TimeScale | undefined); + sel.time = chosen; + if (topKey && share >= minShare && chosen === topKey) { + reasonParts.push(`time=majority(${topKey},${share.toFixed(2)})`); + } else if (chosen) { + const pref = options.tieBreakers?.time ?? "prefer-month"; + reasonParts.push(`time=tie-break(${pref})`); + } else { + reasonParts.push("time=none"); + } } } diff --git a/packages/econify/src/normalization/auto_targets_test.ts b/packages/econify/src/normalization/auto_targets_test.ts index 038c755..8ebf2a2 100644 --- a/packages/econify/src/normalization/auto_targets_test.ts +++ b/packages/econify/src/normalization/auto_targets_test.ts @@ -1,4 +1,5 @@ import { + assert, assertEquals, assertExists, } from "https://deno.land/std@0.208.0/assert/mod.ts"; @@ -28,10 +29,12 @@ Deno.test("computeAutoTargets: majority selection per dimension", () => { assertExists(gdp?.shares.time["quarter"]); }); -Deno.test("computeAutoTargets: tie-breakers when no majority", () => { +Deno.test("computeAutoTargets: tie-breakers when no majority (flow indicator)", () => { + // Use a FLOW indicator (Exports) instead of STOCK (Debt) + // Flow indicators should get time auto-targeting const data: ParsedData[] = [ - { name: "Debt", value: 1, unit: "USD Billion per quarter" }, - { name: "Debt", value: 2, unit: "EUR Million per month" }, + { name: "Exports", value: 1, unit: "USD Billion per quarter" }, + { name: "Exports", value: 2, unit: "EUR Million per month" }, ]; const targets = computeAutoTargets(data, { @@ -45,12 +48,42 @@ Deno.test("computeAutoTargets: tie-breakers when no majority", () => { }, }); - const debt = targets.get("debt"); // Key is normalized to lowercase - assertExists(debt); + const exports = targets.get("exports"); // Key is normalized to lowercase + assertExists(exports); // prefer targetCurrency (EUR), prefer millions, prefer month + assertEquals(exports?.currency, "EUR"); + assertEquals(exports?.magnitude as Scale, "millions"); + assertEquals(exports?.time as TimeScale, "month"); +}); + +Deno.test("computeAutoTargets: stock indicators skip time dimension", () => { + // Debt is a STOCK indicator - should NOT get time auto-targeting + const data: ParsedData[] = [ + { name: "Debt", value: 1, unit: "USD Billion" }, + { name: "Debt", value: 2, unit: "EUR Million" }, + ]; + + const targets = computeAutoTargets(data, { + indicatorKey: "name", + minMajorityShare: 0.6, + targetCurrency: "EUR", + tieBreakers: { + currency: "prefer-targetCurrency", + magnitude: "prefer-millions", + time: "prefer-month", + }, + }); + + const debt = targets.get("debt"); + assertExists(debt); + // Currency and magnitude should be set assertEquals(debt?.currency, "EUR"); assertEquals(debt?.magnitude as Scale, "millions"); - assertEquals(debt?.time as TimeScale, "month"); + // Time should be skipped for stock indicators + assertEquals(debt?.time, undefined); + // Reason should explain why time was skipped + assert(debt?.reason?.includes("time=skipped")); + assert(debt?.reason?.includes("stock indicator")); }); Deno.test("computeAutoTargets: explicit metadata beats unit parsing", () => { @@ -121,15 +154,18 @@ Deno.test("computeAutoTargets: allowList / denyList", () => { assertEquals(targets.get("credit rating"), undefined); }); -Deno.test("computeAutoTargets: include non-monetary indicators for magnitude/time targeting", () => { +Deno.test("computeAutoTargets: include non-monetary flow indicators for magnitude/time targeting", () => { const data: ParsedData[] = [ + // CPI is a RATE indicator - should skip time dimension { name: "CPI", value: 3.5, unit: "percent" }, + // Car Registrations is a FLOW indicator - should get time dimension { name: "Car Registrations", value: 1000, unit: "Units", periodicity: "Monthly", }, + // Oil Production is a FLOW indicator - should get time dimension { name: "Oil Production", value: 10, unit: "BBL/D/1K" }, ]; @@ -141,7 +177,15 @@ Deno.test("computeAutoTargets: include non-monetary indicators for magnitude/tim // Non-monetary indicators should now be included for magnitude/time targeting assertEquals(targets.size, 3, "Should include all non-monetary indicators"); - // Car Registrations should have magnitude and time targets but no currency + // CPI is a rate indicator - should skip time dimension + const cpi = targets.get("cpi"); + assertExists(cpi); + assertEquals(cpi.currency, undefined, "Non-monetary should not have currency"); + assertEquals(cpi.magnitude, "ones", "Should have magnitude target"); + assertEquals(cpi.time, undefined, "Rate indicators should skip time dimension"); + assert(cpi.reason?.includes("time=skipped")); + + // Car Registrations is a flow - should have time dimension const carReg = targets.get("car registrations"); assertExists(carReg); assertEquals( @@ -150,7 +194,7 @@ Deno.test("computeAutoTargets: include non-monetary indicators for magnitude/tim "Non-monetary should not have currency", ); assertEquals(carReg.magnitude, "ones", "Should have magnitude target"); - assertEquals(carReg.time, "month", "Should have time target"); + assertEquals(carReg.time, "month", "Flow indicators should have time target"); }); Deno.test("computeAutoTargets: count indicators (non-monetary) participate in magnitude and time targeting", () => { diff --git a/packages/econify/src/workflows/e2e_comprehensive_test.ts b/packages/econify/src/workflows/e2e_comprehensive_test.ts index 1d167a0..2906159 100644 --- a/packages/econify/src/workflows/e2e_comprehensive_test.ts +++ b/packages/econify/src/workflows/e2e_comprehensive_test.ts @@ -2265,7 +2265,7 @@ Deno.test("E2E: Employed Persons (STOCK indicator - should NOT get time conversi const result = await processEconomicDataByIndicator(data, { autoTargetByIndicator: true, - autoTargetDimensions: ["magnitude"], // Only auto-target magnitude, NOT time (stock indicator) + autoTargetDimensions: ["magnitude", "time"], // Include time - should be auto-skipped for stock indicatorKey: "name", explain: true, }); @@ -2346,14 +2346,24 @@ Deno.test("E2E: Employed Persons (STOCK indicator - should NOT get time conversi // Check target selection assertExists(agoExplain.targetSelection); assertEquals(agoExplain.targetSelection.selected.magnitude, "thousands"); - // Time should NOT be auto-targeted for stock indicators - // (we disabled time auto-targeting in the options above) + + // ✅ CRITICAL: Time should be automatically skipped for stock indicators + // Even though "time" is in autoTargetDimensions, the system should detect + // that "Employed Persons" is a stock indicator and skip time auto-targeting assertEquals( agoExplain.targetSelection.selected.time, undefined, - "Stock indicators should NOT have time target when autoTargetDimensions excludes 'time'", + "Stock indicators should automatically skip time dimension (even when 'time' in autoTargetDimensions)", ); - // NOTE: Future enhancement - auto-targeting should automatically detect stock indicators - // and skip time dimension auto-targeting even when "time" is in autoTargetDimensions + // Verify the reason explains why time was skipped + assertExists(agoExplain.targetSelection.reason); + assert( + agoExplain.targetSelection.reason.includes("time=skipped"), + "Reason should explain time was skipped for stock indicator", + ); + assert( + agoExplain.targetSelection.reason.includes("stock indicator"), + "Reason should mention it's a stock indicator", + ); }); From de4c2617e1a6a3cf7fbab9bb2bf9c09773fca7a4 Mon Sep 17 00:00:00 2001 From: Sean Knowles Date: Tue, 30 Sep 2025 11:01:38 +0200 Subject: [PATCH 3/5] docs(econify): add comprehensive smart auto-targeting documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Documentation Updates Added detailed documentation for the new smart auto-targeting feature that intelligently skips time dimension for stock/rate indicators. ### Changes 1. **README.md** - Added Smart Auto-Targeting to Core Capabilities - Highlighted the prevention of incorrect conversions (e.g., employed persons ÷ 3) 2. **docs/guides/per-indicator-normalization.md** - Added comprehensive "Smart Auto-Targeting: Indicator Classification" section - Documented all three indicator types (Stock, Flow, Rate) with examples - Showed before/after comparisons demonstrating the fix - Explained global configuration with smart targeting - Updated summary to include smart targeting benefits ### Key Documentation Points - **Stock Indicators**: Population, Debt, Employed Persons → Time dimension skipped - **Flow Indicators**: GDP, Exports, Revenue → Time dimension included - **Rate Indicators**: CPI, Inflation, Unemployment Rate → Time dimension skipped - **Real-world examples** showing correct vs incorrect behavior - **Configuration guidance** for using global autoTargetDimensions ### Examples Included - Employed Persons: 12,814 thousand stays 12,814 (not divided by 3) - GDP: 300M/quarter → 100M/month (correctly converted) - CPI: 105.2 points stays 105.2 (no conversion) This documentation helps users understand: 1. Why their stock indicator data was being incorrectly converted 2. How the new smart targeting fixes it automatically 3. How to configure their pipeline to benefit from it --- packages/econify/README.md | 4 + .../guides/per-indicator-normalization.md | 180 +++++++++++++++++- packages/econify/src/api/pipeline_api_test.ts | 7 +- .../src/normalization/auto_targets_test.ts | 12 +- .../src/workflows/e2e_comprehensive_test.ts | 5 +- 5 files changed, 200 insertions(+), 8 deletions(-) diff --git a/packages/econify/README.md b/packages/econify/README.md index 79d43dd..3065446 100644 --- a/packages/econify/README.md +++ b/packages/econify/README.md @@ -36,6 +36,10 @@ assessment, error handling, and interactive control flow._ - 🔍 **Smart Classification** — Automatically detect whether an indicator is a stock, flow, rate, or currency +- 🎯 **Smart Auto-Targeting** — Intelligently skip time dimension for stock/rate + indicators (e.g., Population, Debt, CPI) while applying it to flows (GDP, + Exports) — prevents incorrect conversions like "12,814 employed persons" ÷ 3 → + "4,271 per month" - 🌍 **150+ Currency Support** — Convert values between currencies using FX tables (USD, EUR, GBP, JPY, NGN, KES, and more) - 📊 **Magnitude Scaling** — Seamlessly convert between trillions, billions, diff --git a/packages/econify/docs/guides/per-indicator-normalization.md b/packages/econify/docs/guides/per-indicator-normalization.md index 564f756..241ae5e 100644 --- a/packages/econify/docs/guides/per-indicator-normalization.md +++ b/packages/econify/docs/guides/per-indicator-normalization.md @@ -103,19 +103,187 @@ Each indicator keeps appropriate units: - GDP: 25 USD trillions/year ✓ - Interest Rate: 3.5 percent ✓ +## Smart Auto-Targeting: Indicator Classification + +**NEW in v1.1.10+**: Auto-targeting now intelligently determines which dimensions to apply based on indicator type. + +### Indicator Types + +Econify automatically classifies indicators into three types: + +#### 1. **Stock Indicators** (Snapshots/Levels) +Stock indicators represent a **snapshot at a point in time**, not a rate over time. + +**Examples:** +- Population, Employed Persons, Labor Force, Workforce +- Debt, Reserves, Money Supply, Assets +- Inventory, Housing Stock + +**Behavior:** +- ✅ Time dimension is **automatically skipped** even when included in `autoTargetDimensions` +- ✅ Values represent "how many/how much exists at this moment" +- ✅ No time conversion applied (e.g., quarterly → monthly) + +**Example:** +```typescript +// Input: Employed Persons +{ + name: "Employed Persons", + value: 12814.558, + unit: "Thousand", + periodicity: "Quarterly" // Just the reporting frequency +} + +// Output (with autoTargetDimensions: ['magnitude', 'time']) +{ + normalized: 12814.558, // ✅ NOT divided by 3! + normalizedUnit: "Thousands", // ✅ NOT "thousands per month" + explain: { + targetSelection: { + selected: { magnitude: "thousands", time: undefined }, + reason: "magnitude=majority(thousands,1.00); time=skipped(stock indicator, no time dimension)" + } + } +} +``` + +#### 2. **Flow Indicators** (Rates/Activity) +Flow indicators represent **activity over a period of time**. + +**Examples:** +- GDP, Exports, Imports, Sales, Revenue +- Production, Manufacturing Output +- Wages, Income, Spending + +**Behavior:** +- ✅ Time dimension is **included** in auto-targeting +- ✅ Values represent "how much per time period" +- ✅ Time conversion applied when needed (e.g., quarterly → monthly) + +**Example:** +```typescript +// Input: GDP +{ + name: "GDP", + value: 300, + unit: "USD Million per quarter" +} + +// Output (with autoTargetDimensions: ['magnitude', 'time']) +{ + normalized: 100, // ✅ Divided by 3 (quarterly → monthly) + normalizedUnit: "USD millions per month", + explain: { + targetSelection: { + selected: { magnitude: "millions", time: "month" }, + reason: "magnitude=majority(millions,0.85); time=tie-break(prefer-month)" + } + } +} +``` + +#### 3. **Rate Indicators** (Ratios/Indices) +Rate indicators are **dimensionless ratios, percentages, or indices**. + +**Examples:** +- CPI, Inflation Rate, Unemployment Rate +- Interest Rates, Exchange Rates +- Indices (PMI, Consumer Confidence) + +**Behavior:** +- ✅ Time dimension is **automatically skipped** +- ✅ Values are ratios or indices, not quantities +- ✅ No time conversion applied + +**Example:** +```typescript +// Input: CPI +{ + name: "Consumer Price Index", + value: 105.2, + unit: "Index Points" +} + +// Output +{ + normalized: 105.2, // ✅ Unchanged + normalizedUnit: "Index Points", + explain: { + targetSelection: { + selected: { time: undefined }, + reason: "time=skipped(rate indicator, no time dimension)" + } + } +} +``` + +### Why This Matters + +**Before Smart Auto-Targeting:** +```typescript +// ❌ WRONG: Stock indicators were incorrectly converted +{ + name: "Employed Persons", + value: 12814.558, + unit: "Thousand", + periodicity: "Quarterly" +} +// → normalized: 4271.52 (divided by 3!) ❌ +// → normalizedUnit: "thousands per month" ❌ +``` + +**After Smart Auto-Targeting:** +```typescript +// ✅ CORRECT: Stock indicators skip time dimension +{ + name: "Employed Persons", + value: 12814.558, + unit: "Thousand", + periodicity: "Quarterly" +} +// → normalized: 12814.558 (unchanged) ✅ +// → normalizedUnit: "Thousands" ✅ +``` + +### Global Configuration with Smart Targeting + +You can now set `autoTargetDimensions` globally and let econify intelligently apply them: + +```typescript +{ + // This tells econify to group by indicator name + indicatorKey: 'name', + + // This enables per-indicator normalization + autoTargetByIndicator: true, + + // These dimensions are normalized per indicator + // Stock/Rate indicators will automatically skip 'time' + autoTargetDimensions: ['magnitude', 'time'], + + // Each indicator's majority is calculated separately + minMajorityShare: 0.6 +} +``` + +**Result:** +- **Flow indicators** (GDP, Exports) → Both magnitude AND time normalized +- **Stock indicators** (Population, Debt) → Only magnitude normalized, time skipped +- **Rate indicators** (CPI, Inflation) → Time skipped, magnitude may be skipped too + ## Key Configuration ```typescript { // This tells econify to group by indicator name - indicatorKey: 'name', - + indicatorKey: 'name', + // This enables per-indicator normalization autoTargetByIndicator: true, - + // These dimensions are normalized per indicator autoTargetDimensions: ['magnitude', 'time'], - + // Each indicator's majority is calculated separately minMajorityShare: 0.6 } @@ -127,9 +295,13 @@ Each indicator keeps appropriate units: 2. **Majority calculations happen within each indicator** 3. **Different indicators can have different target units** 4. **Balance of Trade might use millions/month while GDP uses trillions/year** +5. **Smart auto-targeting automatically determines which dimensions to apply based on indicator type** This ensures that: - All Balance of Trade countries are comparable (same units) - All GDP countries are comparable (same units) - But Balance of Trade and GDP use appropriate, different units +- Stock indicators (Population, Debt) don't get incorrectly time-converted +- Flow indicators (GDP, Exports) get proper time normalization +- Rate indicators (CPI, Inflation) remain as ratios/indices diff --git a/packages/econify/src/api/pipeline_api_test.ts b/packages/econify/src/api/pipeline_api_test.ts index c2c1481..eafb387 100644 --- a/packages/econify/src/api/pipeline_api_test.ts +++ b/packages/econify/src/api/pipeline_api_test.ts @@ -1222,7 +1222,12 @@ Deno.test("auto-target extensive: GDP/Debt/Imports distributions with share asse { id: "E7", value: 206, unit: "EUR Million per quarter", name: "Exports" }, { id: "E8", value: 207, unit: "USD Thousand per quarter", name: "Exports" }, { id: "E9", value: 208, unit: "USD Thousand per quarter", name: "Exports" }, - { id: "E10", value: 209, unit: "USD Thousand per quarter", name: "Exports" }, + { + id: "E10", + value: 209, + unit: "USD Thousand per quarter", + name: "Exports", + }, // Imports (10): no majority in any dimension with minShare 0.6 -> tie-breaks across all { id: "I1", value: 300, unit: "USD Million per quarter", name: "Imports" }, diff --git a/packages/econify/src/normalization/auto_targets_test.ts b/packages/econify/src/normalization/auto_targets_test.ts index 8ebf2a2..66f4e44 100644 --- a/packages/econify/src/normalization/auto_targets_test.ts +++ b/packages/econify/src/normalization/auto_targets_test.ts @@ -180,9 +180,17 @@ Deno.test("computeAutoTargets: include non-monetary flow indicators for magnitud // CPI is a rate indicator - should skip time dimension const cpi = targets.get("cpi"); assertExists(cpi); - assertEquals(cpi.currency, undefined, "Non-monetary should not have currency"); + assertEquals( + cpi.currency, + undefined, + "Non-monetary should not have currency", + ); assertEquals(cpi.magnitude, "ones", "Should have magnitude target"); - assertEquals(cpi.time, undefined, "Rate indicators should skip time dimension"); + assertEquals( + cpi.time, + undefined, + "Rate indicators should skip time dimension", + ); assert(cpi.reason?.includes("time=skipped")); // Car Registrations is a flow - should have time dimension diff --git a/packages/econify/src/workflows/e2e_comprehensive_test.ts b/packages/econify/src/workflows/e2e_comprehensive_test.ts index 2906159..0838743 100644 --- a/packages/econify/src/workflows/e2e_comprehensive_test.ts +++ b/packages/econify/src/workflows/e2e_comprehensive_test.ts @@ -2239,7 +2239,10 @@ Deno.test("E2E: Employed Persons (STOCK indicator - should NOT get time conversi scale: "Thousands", periodicity: "Quarterly", date: "2025-03-31", - metadata: { country_iso: "AGO", source: "Instituto Nacional de Estatística" }, + metadata: { + country_iso: "AGO", + source: "Instituto Nacional de Estatística", + }, }, { id: "ALBANIAEMPPER", From a3df702c7700dcfd48df390cf9ec64a8f72b531a Mon Sep 17 00:00:00 2001 From: Sean Knowles Date: Tue, 30 Sep 2025 11:02:00 +0200 Subject: [PATCH 4/5] docs(econify): add comprehensive smart auto-targeting documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Documentation Updates Added detailed documentation for the new smart auto-targeting feature that intelligently skips time dimension for stock/rate indicators. ### Changes 1. **README.md** - Added Smart Auto-Targeting to Core Capabilities - Highlighted the prevention of incorrect conversions (e.g., employed persons ÷ 3) 2. **docs/guides/per-indicator-normalization.md** - Added comprehensive "Smart Auto-Targeting: Indicator Classification" section - Documented all three indicator types (Stock, Flow, Rate) with examples - Showed before/after comparisons demonstrating the fix - Explained global configuration with smart targeting - Updated summary to include smart targeting benefits ### Key Documentation Points - **Stock Indicators**: Population, Debt, Employed Persons → Time dimension skipped - **Flow Indicators**: GDP, Exports, Revenue → Time dimension included - **Rate Indicators**: CPI, Inflation, Unemployment Rate → Time dimension skipped - **Real-world examples** showing correct vs incorrect behavior - **Configuration guidance** for using global autoTargetDimensions ### Examples Included - Employed Persons: 12,814 thousand stays 12,814 (not divided by 3) - GDP: 300M/quarter → 100M/month (correctly converted) - CPI: 105.2 points stays 105.2 (no conversion) This documentation helps users understand: 1. Why their stock indicator data was being incorrectly converted 2. How the new smart targeting fixes it automatically 3. How to configure their pipeline to benefit from it --- .../guides/per-indicator-normalization.md | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/econify/docs/guides/per-indicator-normalization.md b/packages/econify/docs/guides/per-indicator-normalization.md index 241ae5e..afaa9b6 100644 --- a/packages/econify/docs/guides/per-indicator-normalization.md +++ b/packages/econify/docs/guides/per-indicator-normalization.md @@ -105,26 +105,33 @@ Each indicator keeps appropriate units: ## Smart Auto-Targeting: Indicator Classification -**NEW in v1.1.10+**: Auto-targeting now intelligently determines which dimensions to apply based on indicator type. +**NEW in v1.1.10+**: Auto-targeting now intelligently determines which +dimensions to apply based on indicator type. ### Indicator Types Econify automatically classifies indicators into three types: #### 1. **Stock Indicators** (Snapshots/Levels) -Stock indicators represent a **snapshot at a point in time**, not a rate over time. + +Stock indicators represent a **snapshot at a point in time**, not a rate over +time. **Examples:** + - Population, Employed Persons, Labor Force, Workforce - Debt, Reserves, Money Supply, Assets - Inventory, Housing Stock **Behavior:** -- ✅ Time dimension is **automatically skipped** even when included in `autoTargetDimensions` + +- ✅ Time dimension is **automatically skipped** even when included in + `autoTargetDimensions` - ✅ Values represent "how many/how much exists at this moment" - ✅ No time conversion applied (e.g., quarterly → monthly) **Example:** + ```typescript // Input: Employed Persons { @@ -148,19 +155,23 @@ Stock indicators represent a **snapshot at a point in time**, not a rate over ti ``` #### 2. **Flow Indicators** (Rates/Activity) + Flow indicators represent **activity over a period of time**. **Examples:** + - GDP, Exports, Imports, Sales, Revenue - Production, Manufacturing Output - Wages, Income, Spending **Behavior:** + - ✅ Time dimension is **included** in auto-targeting - ✅ Values represent "how much per time period" - ✅ Time conversion applied when needed (e.g., quarterly → monthly) **Example:** + ```typescript // Input: GDP { @@ -183,19 +194,23 @@ Flow indicators represent **activity over a period of time**. ``` #### 3. **Rate Indicators** (Ratios/Indices) + Rate indicators are **dimensionless ratios, percentages, or indices**. **Examples:** + - CPI, Inflation Rate, Unemployment Rate - Interest Rates, Exchange Rates - Indices (PMI, Consumer Confidence) **Behavior:** + - ✅ Time dimension is **automatically skipped** - ✅ Values are ratios or indices, not quantities - ✅ No time conversion applied **Example:** + ```typescript // Input: CPI { @@ -220,6 +235,7 @@ Rate indicators are **dimensionless ratios, percentages, or indices**. ### Why This Matters **Before Smart Auto-Targeting:** + ```typescript // ❌ WRONG: Stock indicators were incorrectly converted { @@ -233,6 +249,7 @@ Rate indicators are **dimensionless ratios, percentages, or indices**. ``` **After Smart Auto-Targeting:** + ```typescript // ✅ CORRECT: Stock indicators skip time dimension { @@ -247,7 +264,8 @@ Rate indicators are **dimensionless ratios, percentages, or indices**. ### Global Configuration with Smart Targeting -You can now set `autoTargetDimensions` globally and let econify intelligently apply them: +You can now set `autoTargetDimensions` globally and let econify intelligently +apply them: ```typescript { @@ -267,9 +285,12 @@ You can now set `autoTargetDimensions` globally and let econify intelligently ap ``` **Result:** + - **Flow indicators** (GDP, Exports) → Both magnitude AND time normalized -- **Stock indicators** (Population, Debt) → Only magnitude normalized, time skipped -- **Rate indicators** (CPI, Inflation) → Time skipped, magnitude may be skipped too +- **Stock indicators** (Population, Debt) → Only magnitude normalized, time + skipped +- **Rate indicators** (CPI, Inflation) → Time skipped, magnitude may be skipped + too ## Key Configuration @@ -295,7 +316,8 @@ You can now set `autoTargetDimensions` globally and let econify intelligently ap 2. **Majority calculations happen within each indicator** 3. **Different indicators can have different target units** 4. **Balance of Trade might use millions/month while GDP uses trillions/year** -5. **Smart auto-targeting automatically determines which dimensions to apply based on indicator type** +5. **Smart auto-targeting automatically determines which dimensions to apply + based on indicator type** This ensures that: From 45cf593e214d2914406f778589433695c6a89a7d Mon Sep 17 00:00:00 2001 From: Sean Knowles Date: Tue, 30 Sep 2025 11:03:41 +0200 Subject: [PATCH 5/5] chore(econify): release v1.2.0 - Smart Auto-Targeting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Version 1.2.0 Release Major feature release introducing Smart Auto-Targeting that fixes critical stock indicator conversion bugs. ### Highlights 🎯 **Smart Auto-Targeting** - Automatically detects stock/flow/rate indicators - Skips time dimension for stock indicators (Population, Debt, Employed Persons) - Prevents incorrect conversions: "12,814 employed persons" no longer ÷ 3 → "4,271 per month" 🐛 **Critical Bug Fix** - Stock indicators were incorrectly time-converted - Now correctly treats them as snapshots, not rates over time - Affects: Population, Debt, Reserves, Money Supply, Assets, Employed Persons, etc. 📚 **Comprehensive Documentation** - Added Smart Auto-Targeting guide with real-world examples - Before/after comparisons showing the fix - Configuration guidance for global autoTargetDimensions ✅ **All 410 Tests Passing** ### Breaking Changes Auto-targeting behavior changed for stock/rate indicators - they now correctly skip time dimension normalization. This is a correctness fix for mathematically incorrect behavior. ### Files Changed - CHANGELOG.md - Added v1.2.0 release notes - deno.json - Bumped version to 1.2.0 --- packages/econify/CHANGELOG.md | 71 +++++++++++++++++++++++++++++++++++ packages/econify/deno.json | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/econify/CHANGELOG.md b/packages/econify/CHANGELOG.md index 87df70e..16782f7 100644 --- a/packages/econify/CHANGELOG.md +++ b/packages/econify/CHANGELOG.md @@ -2,6 +2,77 @@ All notable changes to the econify package will be documented in this file. +## [1.2.0] - 2025-01-XX + +### Added + +- **Smart Auto-Targeting**: Auto-targeting now intelligently determines which + dimensions to apply based on indicator classification + - **Stock indicators** (Population, Debt, Reserves, Employed Persons, Labor + Force) automatically skip time dimension even when included in + `autoTargetDimensions` + - **Flow indicators** (GDP, Exports, Sales, Revenue, Wages) include time + dimension for normalization + - **Rate indicators** (CPI, Inflation, Unemployment Rate, Interest Rates) + automatically skip time dimension + - **Non-monetary indicators** can now participate in magnitude/time targeting + when currency is not in `autoTargetDimensions` + - Prevents incorrect conversions like "12,814 employed persons" ÷ 3 → "4,271 + per month" + - Explain metadata shows clear reasoning: + `time=skipped(stock indicator, no + time dimension)` + +### Fixed + +- **Critical: Stock Indicator Time Conversion Bug**: Fixed issue where stock + indicators were incorrectly time-converted + - **Before**: "Employed Persons: 12,814 thousand (Quarterly)" → normalized to + 4,271 thousand per month ❌ + - **After**: "Employed Persons: 12,814 thousand (Quarterly)" → stays 12,814 + thousand ✅ + - Stock indicators represent snapshots at a point in time, not rates over time + - Periodicity field now correctly represents only the reporting frequency, not + a time dimension + - Affects: Population, Debt, Reserves, Money Supply, Assets, Liabilities, + Inventory, Employed Persons, Labor Force, and all other stock indicators + +### Changed + +- **Auto-Targeting Filtering**: Modified `computeAutoTargets()` to only filter + by `isMonetary` when currency is in `autoTargetDimensions` + - Allows non-monetary indicators (Car Registrations, Population) to + participate in magnitude/time targeting + - More flexible auto-targeting for mixed datasets + +### Documentation + +- Added comprehensive "Smart Auto-Targeting" section to + `docs/guides/per-indicator-normalization.md` + - Detailed explanation of Stock, Flow, and Rate indicator types + - Real-world examples showing correct vs incorrect behavior + - Before/after comparisons demonstrating the fix + - Configuration guidance for global `autoTargetDimensions` +- Updated README.md Core Capabilities to highlight Smart Auto-Targeting feature + +### Tests + +- Added comprehensive E2E test for Employed Persons (stock indicator) +- Added test for stock indicators automatically skipping time dimension +- Updated auto-targeting tests to distinguish between flow and stock indicators +- All 410 tests passing ✅ + +### Breaking Changes + +- **Auto-targeting behavior change**: Stock and rate indicators now + automatically skip time dimension normalization + - If you were relying on stock indicators being time-converted (which was + incorrect), you'll need to update your expectations + - This is a **correctness fix** - the previous behavior was mathematically + wrong + - Example: If you expected "Debt: 500 billion (Quarterly)" to become "166.67 + billion per month", it will now correctly stay "500 billion" + ## [1.1.9] - 2025-01-XX ### Fixed diff --git a/packages/econify/deno.json b/packages/econify/deno.json index 722834f..0261f37 100644 --- a/packages/econify/deno.json +++ b/packages/econify/deno.json @@ -1,6 +1,6 @@ { "name": "@tellimer/econify", - "version": "1.1.9", + "version": "1.2.0", "tasks": { "dev": "deno run --watch src/helpers/sample_usage.ts", "test": "deno test --parallel",