Fix/10 bug fixes excel viewer

backend/app/api/ingestion.py

@@ -79,6 +79,8 @@ def _persist_anomalies(
         "cashflow": "cashflow_anomaly",
         "synthetic": "synthetic_pattern",
         "transactions": "data_quality",
+        "date_sequence": "date_sequence_anomaly",
+        "metadata_integrity": "metadata_integrity_failure",
     }
     for w in validation_warnings:
         field = w.get("field", "unknown")
@@ -194,7 +196,8 @@ async def ingest_documents(
             continue
 
         classification = analysis.get("classification", {})
-        extracted_fields = analysis.get("extracted_fields", {})
+        # Ensure extracted_fields is a mutable plain dict (some backends return special objects)
+        extracted_fields = dict(analysis.get("extracted_fields", {}))
         remote_layout = analysis.get("layout", {})
         layout_flags = {**local_layout, **remote_layout}
         backboard_thread_id = analysis.get("document_id")
@@ -224,6 +227,10 @@ async def ingest_documents(
             if excel_result.closing_balance is not None:
                 extracted_fields["closing_balance"] = excel_result.closing_balance
 
+            # Pass metadata discrepancy to frontend for dynamic integrity display
+            if excel_result.metadata_discrepancy:
+                extracted_fields["metadata_discrepancy"] = excel_result.metadata_discrepancy
+
         if classification.get("type") == "bank_statement":
             opening_balance = extracted_fields.get("opening_balance")
             closing_balance = extracted_fields.get("closing_balance")
@@ -265,6 +272,28 @@ async def ingest_documents(
 
         confidence = classification.get("confidence") or 0.0
 
+        # ── Metadata Integrity Check: massive penalty for header fraud ──
+        if excel_result and excel_result.metadata_discrepancy:
+            disc = excel_result.metadata_discrepancy
+            debug_log.append(
+                f"METADATA INTEGRITY FAILURE: header_closing={disc['header_closing']}, "
+                f"calculated_closing={disc['calculated_closing']}, "
+                f"discrepancy={disc['discrepancy']:,.2f}, ratio={disc['ratio']:.1f}x"
+            )
+            # This is a fraud signal — slam confidence to near zero
+            confidence = min(confidence, 0.12)
+            debug_log.append(f"confidence_slammed_to: {confidence} (metadata fraud)")
+            # Also inject as a validation error so it shows in the UI
+            errors.append({
+                "field": "metadata_integrity",
+                "message": (
+                    f"FRAUD: Header closing balance ({disc['header_closing']:,.2f}) "
+                    f"does not match calculated balance ({disc['calculated_closing']:,.2f}). "
+                    f"Discrepancy: {disc['discrepancy']:,.2f}"
+                ),
+                "severity": "critical",
+            })
+
         # ── FIX #9: Penalize confidence based on validation issues ──
         confidence_penalty = 0.0
         for e in errors:

backend/app/services/excel_normalizer.py

@@ -35,6 +35,8 @@ class NormalizedStatement:
     repair_log: List[str] = field(default_factory=list)
     raw_headers: List[str] = field(default_factory=list)
     detected_anomalies: List[Dict[str, Any]] = field(default_factory=list)
+    # Metadata integrity: stores header vs calculated discrepancy for fraud detection
+    metadata_discrepancy: Optional[Dict[str, Any]] = None
 
 
 # Common header aliases for bank statements
@@ -241,12 +243,12 @@ def normalize_excel_statement(content: bytes, filename: str = "") -> NormalizedS
     """
     Normalize a bank statement Excel file into a clean, structured format.
 
-    Handles the real-world messiness found in the dataset:
-    - Account 3: Multi-section file with summary + transaction sections, -61M closing balance
-    - Account 4: Mixed date formats, OCR garbage rows ("unrings ICEASE")
-    - Account 5: Comma-formatted numbers, different header names
-    - Account 8: Summary at top before transactions, synthetic text
-    - Account 9: Simple format with transaction totals embedded
+    Handles real-world messiness found in financial documents:
+    - Multi-section files with summary + transaction sections
+    - Mixed date formats, OCR garbage rows
+    - Comma-formatted numbers, varying header names
+    - Summary sections at top before transactions
+    - Embedded transaction totals and metadata rows
     """
     result = NormalizedStatement()
     result.repair_log.append(f"normalizing: {filename}")
@@ -455,30 +457,79 @@ def normalize_excel_statement(content: bytes, filename: str = "") -> NormalizedS
         result.closing_balance = summary_closing
         result.repair_log.append(f"closing_from_summary: {summary_closing}")
 
-    # Step 5: Detect summary injection anomaly (Account 3 pattern)
-    if result.closing_balance is not None and result.transactions:
-        last_balance = None
+    # Step 5: Metadata Integrity Check
+    # Compare the header/summary closing balance against the actual last
+    # transaction balance.  A massive discrepancy (>1.0 AND >50x) means the
+    # header is lying — possible data tampering / summary injection.
+    if result.transactions:
+        last_tx_balance = None
         for tx in reversed(result.transactions):
             if tx.get("balance") is not None:
-                last_balance = tx["balance"]
+                last_tx_balance = tx["balance"]
                 break
-        if last_balance is not None and result.closing_balance is not None:
-            if abs(result.closing_balance) > abs(last_balance) * 50 and abs(last_balance) > 0:
+
+        # Also compare against the raw summary_closing extracted from the
+        # header section at the top of the sheet.
+        header_closing = summary_closing  # from Step 1
+        calculated_closing = last_tx_balance
+
+        if header_closing is not None and calculated_closing is not None:
+            discrepancy = abs(header_closing - calculated_closing)
+            if discrepancy > 1.0 and abs(calculated_closing) > 0 and abs(header_closing) > abs(calculated_closing) * 5:
+                result.metadata_discrepancy = {
+                    "header_closing": header_closing,
+                    "calculated_closing": calculated_closing,
+                    "discrepancy": discrepancy,
+                    "ratio": round(abs(header_closing / calculated_closing), 2) if calculated_closing != 0 else 999999999,
+                }
                 result.detected_anomalies.append({
-                    "type": "summary_injection",
+                    "type": "metadata_integrity_failure",
                     "severity": "critical",
                     "description": (
-                        f"Closing balance ({result.closing_balance}) is wildly inconsistent "
-                        f"with last transaction balance ({last_balance}). "
-                        "Possible summary injection or data tampering."
+                        f"FRAUD SIGNAL: Header closing balance ({header_closing:,.2f}) "
+                        f"does not match calculated row balance ({calculated_closing:,.2f}). "
+                        f"Discrepancy: {discrepancy:,.2f}. "
+                        "The document header is inconsistent with the transaction data — "
+                        "possible summary injection or data tampering."
                     ),
+                    "header_closing": header_closing,
+                    "calculated_closing": calculated_closing,
+                    "discrepancy": discrepancy,
+                    "ratio": round(abs(header_closing / calculated_closing), 2) if calculated_closing != 0 else 999999999,
                 })
                 result.repair_log.append(
-                    f"CRITICAL: summary injection detected - "
-                    f"closing={result.closing_balance} vs last_row={last_balance}"
+                    f"CRITICAL: metadata integrity failure - "
+                    f"header_closing={header_closing} vs calculated={calculated_closing} "
+                    f"(discrepancy={discrepancy:,.2f})"
                 )
-                # Override with the transaction-level closing balance
-                result.closing_balance = last_balance
+                # DO NOT override closing_balance — keep the fraudulent header
+                # value so the validation layer also catches the mismatch
+                # and fires its own balance-continuity error.
+
+        # Separate check: if closing_balance was set from rows (closing_from_row)
+        # but differs wildly from the last transaction balance, flag it too
+        if result.closing_balance is not None and last_tx_balance is not None:
+            if abs(result.closing_balance) > abs(last_tx_balance) * 50 and abs(last_tx_balance) > 0:
+                if result.metadata_discrepancy is None:
+                    result.metadata_discrepancy = {
+                        "header_closing": result.closing_balance,
+                        "calculated_closing": last_tx_balance,
+                        "discrepancy": abs(result.closing_balance - last_tx_balance),
+                        "ratio": round(abs(result.closing_balance / last_tx_balance), 2) if last_tx_balance != 0 else 999999999,
+                    }
+                    result.detected_anomalies.append({
+                        "type": "metadata_integrity_failure",
+                        "severity": "critical",
+                        "description": (
+                            f"FRAUD SIGNAL: Closing balance ({result.closing_balance:,.2f}) is wildly inconsistent "
+                            f"with last transaction balance ({last_tx_balance:,.2f}). "
+                            "Possible summary injection or data tampering."
+                        ),
+                    })
+                    result.repair_log.append(
+                        f"CRITICAL: summary injection detected - "
+                        f"closing={result.closing_balance} vs last_row={last_tx_balance}"
+                    )
 
     # Step 6: Check for merge artifact (mixed date formats)
     # Only flag if there are truly multiple REAL date formats (ignore single-count outliers)

backend/app/services/validation.py

@@ -121,6 +121,7 @@ def run_validations(
         if opening is not None and closing is not None and transactions:
             tx_total = 0.0
             last_date = None
+            date_sequence_violations = 0
             for tx in transactions:
                 amount = _safe_number(tx.get("amount"))
                 if amount is not None:
@@ -132,13 +133,18 @@ def run_validations(
                 tx_date_raw.append(str(tx.get("date") or ""))
                 tx_descriptions.append(str(tx.get("description") or ""))
                 if last_date and tx_date and tx_date < last_date:
-                    warnings.append({
-                        "field": "transactions",
-                        "message": "Transaction dates are not in sequence.",
-                        "severity": "info",
-                    })
+                    date_sequence_violations += 1
                 if tx_date:
                     last_date = tx_date
+            # Emit date-sequence warning (once, with count)
+            if date_sequence_violations > 0:
+                sev = "critical" if date_sequence_violations >= 5 else "warning"
+                warnings.append({
+                    "field": "date_sequence",
+                    "message": f"Transaction dates are not in chronological order ({date_sequence_violations} violations).",
+                    "severity": sev,
+                    "count": date_sequence_violations,
+                })
             if not _compare_close(opening + tx_total, closing, tolerance=0.05):
                 issue = {
                     "field": "closing_balance",