feat: Add more powerful drift windowing strategies, warmup and dynamic thresholds

analytics/app/pages/plots/eval_over_time.py

@@ -26,7 +26,12 @@ class _PageState:
 
 
 def gen_figure(
-    page: str, multi_pipeline_mode: bool, patch_yearbook: bool, eval_handler: str, dataset_id: str, metric: str
+    page: str,
+    multi_pipeline_mode: bool,
+    patch_yearbook: bool,
+    eval_handler: str,
+    dataset_id: str,
+    metric: str,
 ) -> go.Figure:
     """
     Create the evaluation over time figure with a line plot.
@@ -58,7 +63,7 @@ def gen_figure(
         # we only want the pipeline performance (composed of the models active periods stitched together)
         df_adjusted = df_adjusted[df_adjusted[composite_model_variant]]
     else:
-        assert df_adjusted["pipeline_ref"].nunique() == 1
+        assert df_adjusted["pipeline_ref"].nunique() <= 1
         # add the pipeline time series which is the performance of different models stitched together dep.
         # w.r.t which model was active
         pipeline_composite_model = df_adjusted[df_adjusted[composite_model_variant]]
@@ -106,7 +111,10 @@ def section_metricovertime(
     composite_model_variant: CompositeModelOptions,
 ) -> html.Div:
     if page not in _shared_data:
-        _shared_data[page] = _PageState(composite_model_variant=composite_model_variant, df_eval_single=df_eval_single)
+        _shared_data[page] = _PageState(
+            composite_model_variant=composite_model_variant,
+            df_eval_single=df_eval_single,
+        )
     _shared_data[page].composite_model_variant = composite_model_variant
     _shared_data[page].df_eval_single = df_eval_single
 
@@ -118,7 +126,14 @@ def section_metricovertime(
         Input(f"{page}-evalovertime-evaluation-metric", "value"),
     )
     def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str, metric: str) -> go.Figure:
-        return gen_figure(page, multi_pipeline_mode, patch_yearbook, eval_handler_ref, dataset_id, metric)
+        return gen_figure(
+            page,
+            multi_pipeline_mode,
+            patch_yearbook,
+            eval_handler_ref,
+            dataset_id,
+            metric,
+        )
 
     eval_handler_refs = list(df_eval_single["eval_handler"].unique())
     eval_datasets = list(df_eval_single["dataset_id"].unique())
@@ -139,7 +154,7 @@ def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str,
                             dcc.RadioItems(
                                 id=f"{page}-evalovertime-evaluation-handler",
                                 options=eval_handler_refs,
-                                value=eval_handler_refs[0] if len(eval_handler_refs) > 0 else None,
+                                value=(eval_handler_refs[0] if len(eval_handler_refs) > 0 else None),
                                 persistence=True,
                             ),
                         ],
@@ -155,7 +170,7 @@ def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str,
                             dcc.RadioItems(
                                 id=f"{page}-evalovertime-dataset-id",
                                 options=eval_datasets,
-                                value=eval_datasets[0] if len(eval_datasets) > 0 else None,
+                                value=(eval_datasets[0] if len(eval_datasets) > 0 else None),
                                 persistence=True,
                             ),
                         ],
@@ -173,7 +188,7 @@ def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str,
                                 options=[
                                     {"label": metric, "value": metric, "disabled": True} for metric in eval_metrics
                                 ],
-                                value=eval_metrics[0] if len(eval_metrics) > 0 else None,
+                                value=(eval_metrics[0] if len(eval_metrics) > 0 else None),
                                 persistence=True,
                             ),
                         ],
@@ -189,8 +204,14 @@ def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str,
                             dcc.RadioItems(
                                 id=f"{page}-evalovertime-radio-time-patch-yearbook",
                                 options=[
-                                    {"label": "yes (convert day based timestamps to years)", "value": True},
-                                    {"label": "no (use timestamps as they are)", "value": False},
+                                    {
+                                        "label": "yes (convert day based timestamps to years)",
+                                        "value": True,
+                                    },
+                                    {
+                                        "label": "no (use timestamps as they are)",
+                                        "value": False,
+                                    },
                                 ],
                                 value=False,
                                 persistence=True,
@@ -206,7 +227,7 @@ def update_figure(patch_yearbook: bool, eval_handler_ref: str, dataset_id: str,
                     page,
                     multi_pipeline_mode,
                     False,
-                    eval_handler=eval_handler_refs[0] if len(eval_handler_refs) > 0 else None,
+                    eval_handler=(eval_handler_refs[0] if len(eval_handler_refs) > 0 else None),
                     dataset_id=eval_datasets[0] if len(eval_datasets) > 0 else None,
                     metric=eval_metrics[0] if len(eval_metrics) > 0 else None,
                 ),

benchmark/huffpost_kaggle/README.md

@@ -7,7 +7,6 @@ In this directory, you can find the files necessary to run experiments using the
 The goal is to predict the tag of news given headlines.
 The dataset contains more than 60k samples collected from 2012 to 2018.
 Titles belonging to the same year are grouped into the same CSV file and stored together.
-Each year is mapped to a year starting from 1/1/1970.
 There is a total of 42 categories/classes.
 
 > Note: The wild-time variant of the huffpost dataset has only 11 classes. This is due to the fact that

benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/arxiv_datadrift.yaml

@@ -45,7 +45,9 @@ trigger:
   metrics:
     ev_mmd:
       id: EvidentlyModelDriftMetric
-      threshold: 0.7
+      decision_criterion:
+        id: ThresholdDecisionCriterion
+        threshold: 0.7
   aggregation_strategy:
     id: MajorityVote
 selection_strategy:

benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/huffpost_datadrift.yaml

@@ -47,6 +47,9 @@ trigger:
     ev_mmd:
       id: AlibiDetectMmdDriftMetric
       num_permutations: 1000
+      decision_criterion:
+        id: DynamicThresholdCriterion
+
   aggregation_strategy:
     id: MajorityVote
 selection_strategy:

benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/yearbook_datadrift.yaml

@@ -48,7 +48,9 @@ trigger:
   metrics:
     ev_mmd:
       id: EvidentlyModelDriftMetric
-      threshold: 0.7
+      decision_criterion:
+        id: ThresholdDecisionCriterion
+        threshold: 0.7
   aggregation_strategy:
     id: MajorityVote
 selection_strategy:

docs/pipeline/TRIGGERING.md

@@ -25,36 +25,36 @@ classDiagram
         class TimeTrigger {
         }
 
-        class DataAmount {
+        class DataAmountTrigger {
         }
 
     }
 
     namespace complex_triggers {
 
-        class DataDrift {
+        class DataDriftTrigger {
         }
 
-        class CostBased {
+        class CostBasedTrigger {
         }
 
         class _BatchedTrigger {
             <<Abstract>>
         }
 
-        class EnsemblePolicy {
+        class EnsembleTrigger {
         }
 
     }
 
     Trigger <|-- _BatchedTrigger
-    Trigger <|-- EnsemblePolicy
+    Trigger <|-- EnsembleTrigger
 
     Trigger <|-- TimeTrigger
-    Trigger <|-- DataAmount
+    Trigger <|-- DataAmountTrigger
 
-    _BatchedTrigger <|-- DataDrift
-    _BatchedTrigger <|-- CostBased
+    _BatchedTrigger <|-- DataDriftTrigger
+    _BatchedTrigger <|-- CostBasedTrigger
 
-    EnsemblePolicy *-- "n" Trigger
+    EnsembleTrigger *-- "n" Trigger
 ```