Fix mismatch records treatment (RACO-5100)

rapo/algorithms/reconciliation/s01_correlate.sql

@@ -9,6 +9,8 @@ select {parallelism}
        time_shift_rank_a,
        time_shift_rank_b,
        {discrepancy_fields}
+       distance_rank_a,
+       distance_rank_b,       
        discrepancy_rank_a,
        discrepancy_rank_b,
        discrepancy_time_a,
@@ -37,6 +39,8 @@ select {parallelism}
                 time_shift_rank_a,
                 time_shift_rank_b,
                 {discrepancy_fields}
+                distance_rank_a,
+                distance_rank_b,
                 discrepancy_rank_a,
                 discrepancy_rank_b,
                 discrepancy_time_a,
@@ -55,6 +59,8 @@ select {parallelism}
                          time_shift_rank_a,
                          time_shift_rank_b,
                          {discrepancy_fields}
+                         dense_rank() over (partition by a_id order by (time_distance_ratio + 1) * (discrepancy_distance_ratio + 1), discrepancy_rank_a, time_shift_rank_a) as distance_rank_a,
+                         dense_rank() over (partition by b_id order by (time_distance_ratio + 1) * (discrepancy_distance_ratio + 1), discrepancy_rank_b, time_shift_rank_b) as distance_rank_b,
                          discrepancy_rank_a,
                          discrepancy_rank_b,
                          discrepancy_time_a,
@@ -75,6 +81,8 @@ select {parallelism}
                                   {discrepancy_formulas}
                                   dense_rank() over (partition by a_id order by {discrepancy_order_a}) as discrepancy_rank_a,
                                   dense_rank() over (partition by b_id order by {discrepancy_order_b}) as discrepancy_rank_b,
+			             time_distance_ratio,
+                                  {discrepancy_order_a} discrepancy_distance_ratio,
                                   discrepancy_time_a,
                                   discrepancy_time_b,
                                   discrepancy_time_value
@@ -92,6 +100,10 @@ select {parallelism}
                                            {discrepancy_rules}
                                            case when 86400*({date_field_a}-{date_field_b}) not between {time_tolerance_from} and {time_tolerance_to} then '{date_field_name_a}' end as discrepancy_time_a,
                                            case when 86400*({date_field_a}-{date_field_b}) not between {time_tolerance_from} and {time_tolerance_to} then '{date_field_name_b}' end as discrepancy_time_b,
+                                           case 
+                                                 when ({time_tolerance_to})-({time_tolerance_from}) = 0 then abs(86400*({date_field_a}-{date_field_b}))
+                                                 else power(86400*({date_field_a}-{date_field_b})/(({time_tolerance_to})-({time_tolerance_from})), 2)
+                                           end as time_distance_ratio,
                                            86400*({date_field_a}-{date_field_b}) as discrepancy_time_value
                                       from rapo_temp_source_a_{process_id} a join rapo_temp_source_b_{process_id} b
                                            on {key_rules}

rapo/algorithms/reconciliation/s06_prepare_conflicts.sql

@@ -2,8 +2,8 @@ create materialized view rapo_temp_t05_mac_{process_id} refresh on demand as
 select {parallelism} a_id, b_id
 from (
   select m.a_id, m.b_id,
-         row_number() over (partition by m.b_id order by m.discrepancy_rank_b, m.time_shift_rank_b, m.a_id) as match_position_a,
-         row_number() over (partition by m.a_id order by m.discrepancy_rank_a, m.time_shift_rank_a, m.b_id) as match_position_b
+         row_number() over (partition by m.b_id order by m.distance_rank_b, m.discrepancy_rank_b, m.time_shift_rank_b, m.a_id) as match_position_a,
+         row_number() over (partition by m.a_id order by m.distance_rank_a, m.discrepancy_rank_a, m.time_shift_rank_a, m.b_id) as match_position_b
     from rapo_temp_t01_mod_{process_id} m
          left join rapo_temp_t02_org_a_{process_id} a on a.a_id = m.a_id
          left join rapo_temp_t02_org_b_{process_id} b on b.b_id = m.b_id

rapo/algorithms/reconciliation/s08_save_error_a.sql

@@ -2,25 +2,32 @@ create table rapo_temp_error_a_{process_id} as
 select {parallelism} *
   from (
          select {parallelism} a.*,
-                case when o.correlation_type in ('F', 'A', 'B', 'M') then 'Duplicate' else 'Loss' end as rapo_result_type,
-                cast(null as varchar2(4000)) as rapo_discrepancy_id,
-                cast(null as varchar2(4000)) as rapo_discrepancy_description
+                case
+                     when o.correlation_indicator is not null and (m.discrepancy_time_a is not null {discrepancy_filters_a}) then 'Discrepancy'
+                     when o.correlation_indicator is null then 'Loss'
+                end as rapo_result_type,
+                cast(m.b_id as varchar2(4000)) as rapo_discrepancy_id,
+                case when m.discrepancy_time_a is not null then m.discrepancy_time_a||'|'||m.discrepancy_time_value||';' end{discrepancy_descriptions_a} as rapo_discrepancy_description
            from rapo_temp_source_a_{process_id} a
+                left join rapo_temp_t01_mod_{process_id} m on {key_field_a} = m.a_id and m.correlation_indicator is not null
                 left join rapo_temp_t02_org_a_{process_id} o on {key_field_a} = o.a_id
           where {date_field_a} between to_date('{date_from:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS') and to_date('{date_to:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS')
-            and o.correlation_indicator is null
+            and (o.correlation_type = 'O' or o.correlation_type is null)
 
          union all
 
          select {parallelism} a.*,
-                case when o.correlation_type in ('F', 'A', 'B', 'M') then 'Duplicate' else 'Discrepancy' end as rapo_result_type,
+                case
+                     when o.correlation_indicator is not null and (m.discrepancy_time_a is not null {discrepancy_filters_a}) then 'Discrepancy'
+                     when o.correlation_indicator is null and (m.discrepancy_time_a is not null {discrepancy_filters_a}) then 'Loss'
+                     when o.correlation_indicator is null then 'Duplicate'
+                end as rapo_result_type,
                 cast(m.b_id as varchar2(4000)) as rapo_discrepancy_id,
                 case when m.discrepancy_time_a is not null then m.discrepancy_time_a||'|'||m.discrepancy_time_value||';' end{discrepancy_descriptions_a} as rapo_discrepancy_description
-           from rapo_temp_source_a_{process_id} a
-                left join rapo_temp_t01_mod_{process_id} m on {key_field_a} = m.a_id and m.correlation_indicator is not null
-                left join rapo_temp_t02_org_a_{process_id} o on {key_field_a} = o.a_id
+           from rapo_temp_source_a_{process_id} a 
+           left join rapo_temp_t02_org_a_{process_id} o on {key_field_a} = o.a_id
+           left join rapo_temp_t01_mod_{process_id} m on {key_field_a} = m.a_id and m.distance_rank_a = 1
           where {date_field_a} between to_date('{date_from:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS') and to_date('{date_to:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS')
-            and o.correlation_indicator is not null
-            and (m.discrepancy_time_a is not null {discrepancy_filters_a})
+            and o.correlation_type in ('F', 'A', 'B', 'M')
        )
  where rapo_result_type in ({target_error_types_a})

rapo/algorithms/reconciliation/s08_save_error_b.sql

@@ -2,25 +2,32 @@ create table rapo_temp_error_b_{process_id} as
 select {parallelism} *
   from (
          select {parallelism} b.*,
-                case when o.correlation_type in ('F', 'A', 'B', 'M') then 'Duplicate' else 'Loss' end as rapo_result_type,
-                cast(null as varchar2(4000)) as rapo_discrepancy_id,
-                cast(null as varchar2(4000)) as rapo_discrepancy_description
+                case
+                     when o.correlation_indicator is not null and (m.discrepancy_time_b is not null {discrepancy_filters_b}) then 'Discrepancy'
+                     when o.correlation_indicator is null then 'Loss'
+                end as rapo_result_type,
+                cast(m.b_id as varchar2(4000)) as rapo_discrepancy_id,
+                case when m.discrepancy_time_b is not null then m.discrepancy_time_b||'|'||m.discrepancy_time_value||';' end{discrepancy_descriptions_b} as rapo_discrepancy_description
            from rapo_temp_source_b_{process_id} b
+                left join rapo_temp_t01_mod_{process_id} m on {key_field_b} = m.b_id and m.correlation_indicator is not null
                 left join rapo_temp_t02_org_b_{process_id} o on {key_field_b} = o.b_id
           where {date_field_b} between to_date('{date_from:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS') and to_date('{date_to:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS')
-            and o.correlation_indicator is null
+            and (o.correlation_type = 'O' or o.correlation_type is null)
 
          union all
 
          select {parallelism} b.*,
-                case when o.correlation_type in ('F', 'A', 'B', 'M') then 'Duplicate' else 'Discrepancy' end as rapo_result_type,
+                case
+                     when o.correlation_indicator is not null and (m.discrepancy_time_b is not null {discrepancy_filters_b}) then 'Discrepancy'
+                     when o.correlation_indicator is null and (m.discrepancy_time_b is not null {discrepancy_filters_b}) then 'Loss'
+                     when o.correlation_indicator is null then 'Duplicate'
+                end as rapo_result_type,
                 cast(m.b_id as varchar2(4000)) as rapo_discrepancy_id,
                 case when m.discrepancy_time_b is not null then m.discrepancy_time_b||'|'||m.discrepancy_time_value||';' end{discrepancy_descriptions_b} as rapo_discrepancy_description
-           from rapo_temp_source_b_{process_id} b
-                left join rapo_temp_t01_mod_{process_id} m on {key_field_b} = m.b_id and m.correlation_indicator is not null
-                left join rapo_temp_t02_org_b_{process_id} o on {key_field_b} = o.b_id
+           from rapo_temp_source_b_{process_id} b 
+           left join rapo_temp_t02_org_b_{process_id} o on {key_field_b} = o.b_id
+           left join rapo_temp_t01_mod_{process_id} m on {key_field_b} = m.b_id and m.distance_rank_b = 1
           where {date_field_b} between to_date('{date_from:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS') and to_date('{date_to:%Y-%m-%d %H:%M:%S}', 'YYYY-MM-DD HH24:MI:SS')
-            and o.correlation_indicator is not null
-            and (m.discrepancy_time_b is not null {discrepancy_filters_b})
+            and o.correlation_type in ('F', 'A', 'B', 'M')
        )
  where rapo_result_type in ({target_error_types_b})

rapo/core/control.py

@@ -2615,7 +2615,10 @@ def is_sequence(obj):
             )
             discrepancy_fields.append(discrepancy_field)
 
-            discrepancy_sum = f'abs(discrepancy_{discrepancy_number}_value)'
+            if abs(tolerance_to-tolerance_from) > 0:
+                discrepancy_sum = f'power(abs(discrepancy_{discrepancy_number}_value)/{abs(tolerance_to-tolerance_from)}, 2)'
+            else:
+                discrepancy_sum = f'abs(discrepancy_{discrepancy_number}_value)'
             discrepancy_sums.append(discrepancy_sum)
 
             discrepancy_description_a = discrepancy_description_form.format(
@@ -2644,8 +2647,8 @@ def is_sequence(obj):
         discrepancy_rules = ''.join(discrepancy_rules)
         discrepancy_formulas = ''.join(discrepancy_formulas)
         discrepancy_fields = ''.join(discrepancy_fields)
-        discrepancy_order_a = '+'.join(discrepancy_sums) or 'b_id'
-        discrepancy_order_b = '+'.join(discrepancy_sums) or 'a_id'
+        discrepancy_order_a = '+'.join(discrepancy_sums) or '1'
+        discrepancy_order_b = '+'.join(discrepancy_sums) or '1'
         discrepancy_descriptions_a = ''.join(discrepancy_descriptions_a)
         discrepancy_descriptions_b = ''.join(discrepancy_descriptions_b)
         discrepancy_filters_a = ''.join(discrepancy_filters_a)

rapo/reader.py

@@ -165,7 +165,7 @@ def read_control_recent_logs(self, control_name):
     def read_running_controls(self):
         """Get list of running controls."""
         table = db.tables.log
-        select = table.select().where(table.c.status == 'P')
+        select = table.select().where(table.c.status.in_(['I', 'P', 'W', 'S', 'F']))
         answerset = db.execute(select, as_table=True)
         return answerset
 

rapo/web/api/ui/index.html

@@ -1 +1 @@
-<!doctype html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/favicon.ico"><title>RAPO</title><script defer="defer" src="/js/chunk-vendors.8f4fec03.js"></script><script defer="defer" src="/js/app.ce3f764b.js"></script><link href="/css/chunk-vendors.2bf7b79d.css" rel="stylesheet"><link href="/css/app.347180f0.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but rapo-ui doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>
+<!doctype html><html lang=""><head><meta charset="utf-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/favicon.ico"><title>RAPO</title><script defer="defer" src="/js/chunk-vendors.8f4fec03.js"></script><script defer="defer" src="/js/app.3f41888e.js"></script><link href="/css/chunk-vendors.2bf7b79d.css" rel="stylesheet"><link href="/css/app.76be9651.css" rel="stylesheet"></head><body><noscript><strong>We're sorry but rapo-ui doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id="app"></div></body></html>