feat: Add more powerful drift windowing strategies, warmup and dynamic thresholds

modyn/config/schema/pipeline/trigger/__init__.py

@@ -4,16 +4,14 @@
 
 from pydantic import Field
 
-from .data_amount import *  # noqa
-from .data_amount import DataAmountTriggerConfig
 from .drift import *  # noqa
 from .drift import DataDriftTriggerConfig
 from .ensemble import *  # noqa
 from .ensemble import EnsembleTriggerConfig
-from .time import *  # noqa
-from .time import TimeTriggerConfig
+from .simple import *  # noqa
+from .simple import SimpleTriggerConfig
 
 TriggerConfig = Annotated[
-    TimeTriggerConfig | DataAmountTriggerConfig | DataDriftTriggerConfig | EnsembleTriggerConfig,
+    SimpleTriggerConfig | DataDriftTriggerConfig | EnsembleTriggerConfig,
     Field(discriminator="id"),
 ]

modyn/config/schema/pipeline/trigger/drift/alibi_detect.py

@@ -62,9 +62,6 @@ def validate_threshold_permutations(self) -> "AlibiDetectMmdDriftMetric":
                 + "or threshold comparison for making drift decisions."
             )
 
-        if self.threshold is None and self.num_permutations is None:
-            raise ValueError("Please specify either threshold or num_permutations")
-
         return self
 
 

modyn/config/schema/pipeline/trigger/drift/config.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
-from typing import Annotated, ForwardRef, Literal
+from typing import Annotated, ForwardRef, Literal, Self
 
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from modyn.config.schema.base_model import ModynBaseModel
 from modyn.config.schema.pipeline.trigger.drift.detection_window import (
     AmountWindowingStrategy,
     DriftWindowingStrategy,
 )
+from modyn.config.schema.pipeline.trigger.simple import SimpleTriggerConfig
 
 from .aggregation import DriftAggregationStrategy, MajorityVoteDriftAggregationStrategy
 from .alibi_detect import AlibiDetectDriftMetric
@@ -39,6 +40,7 @@ class DataDriftTriggerConfig(ModynBaseModel):
         AmountWindowingStrategy(),
         description="Which windowing strategy to use for current and reference data",
     )
+
     warmup_intervals: int | None = Field(
         None,
         description=(
@@ -47,6 +49,13 @@ class DataDriftTriggerConfig(ModynBaseModel):
             "a trigger."
         ),
     )
+    warmup_policy: SimpleTriggerConfig | None = Field(
+        None,
+        description=(
+            "The policy to use for triggering during the warmup phase of the drift policy. "
+            "Metrics that don't need calibration can ignore this."
+        ),
+    )
 
     metrics: dict[str, DriftMetric] = Field(
         min_length=1,
@@ -56,3 +65,12 @@ class DataDriftTriggerConfig(ModynBaseModel):
         MajorityVoteDriftAggregationStrategy(),
         description="The strategy to aggregate the decisions of the individual drift metrics.",
     )
+
+    @model_validator(mode="after")
+    def warmup_policy_requirement(self) -> Self:
+        """Assert whether the warmup policy is set when a metric needs
+        calibration."""
+        for metric in self.metrics.values():
+            if metric.decision_criterion.needs_calibration and self.warmup_policy is None:
+                raise ValueError("A warmup policy is required for metrics that need calibration.")
+        return self

modyn/config/schema/pipeline/trigger/simple/__init__.py

@@ -0,0 +1,11 @@
+from typing import Annotated
+
+from pydantic import Field
+
+from .data_amount import DataAmountTriggerConfig  # noqa
+from .time import TimeTriggerConfig  # noqa
+
+SimpleTriggerConfig = Annotated[
+    TimeTriggerConfig | DataAmountTriggerConfig,
+    Field(discriminator="id"),
+]