Linting

experiments/models.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 
 from modyn.config.schema.pipeline.evaluation.handler import EvalHandlerConfig
-from modyn.config.schema.pipeline.evaluation.metrics import MetricConfig
 from modyn.config.schema.pipeline.trigger.cost.cost import CostTriggerConfig
 from modyn.config.schema.pipeline.trigger.drift.config import DataDriftTriggerConfig
 from modyn.config.schema.pipeline.trigger.ensemble import EnsembleTriggerConfig

experiments/yearbook/compare_trigger_policies/run.py

@@ -1,7 +1,4 @@
 import os
-from turtle import backward
-
-from sklearn.linear_model import ridge_regression
 
 from experiments.models import Experiment
 from experiments.utils.experiment_runner import run_multiple_pipelines
@@ -13,7 +10,6 @@
     ModynPipelineConfig,
     TimeTriggerConfig,
 )
-from modyn.config.schema.pipeline import trigger
 from modyn.config.schema.pipeline.evaluation.config import EvalDataConfig
 from modyn.config.schema.pipeline.evaluation.handler import EvalHandlerConfig
 from modyn.config.schema.pipeline.evaluation.metrics import AccuracyMetricConfig
@@ -26,15 +22,11 @@
 from modyn.config.schema.pipeline.evaluation.strategy.slicing import (
     SlicingEvalStrategyConfig,
 )
-from modyn.config.schema.pipeline.trigger import DataDriftTriggerConfig, TriggerConfig
+from modyn.config.schema.pipeline.trigger import DataDriftTriggerConfig
 from modyn.config.schema.pipeline.trigger.cost.cost import (
     AvoidableMisclassificationCostTriggerConfig,
-    CostTriggerConfig,
     DataIncorporationLatencyCostTriggerConfig,
 )
-from modyn.config.schema.pipeline.trigger.drift.aggregation import (
-    MajorityVoteDriftAggregationStrategy,
-)
 from modyn.config.schema.pipeline.trigger.drift.alibi_detect import (
     AlibiDetectMmdDriftMetric,
 )
@@ -152,7 +144,7 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
         gpu_device="cuda:1",
     ),
     # -------------------------------- Drift triggers -------------------------------- #
-    # Static treshold drift
+    # Static threshold drift
     3: Experiment(
         name="yb-baseline-datadrift-static",
         eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
@@ -328,9 +320,8 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
         },
         gpu_device="cuda:1",
     ),
-
-    # ------------------------------- Ensemble triggers ------------------------------ 
-    # with best working preivous triggers
+    # ------------------------------- Ensemble triggers ------------------------------
+    # with best working previous triggers
     20: Experiment(
         name="yb-ensemble",
         eval_handlers=[construct_slicing_eval_handler(), construct_between_trigger_eval_handler()],
@@ -339,15 +330,15 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                 subtriggers={
                     "drift1": DataDriftTriggerConfig(
                         evaluation_interval_data_points=500,
-                        windowing_strategy=TimeWindowingStrategy(limit_ref=f"4d", limit_cur=f"4d"),
+                        windowing_strategy=TimeWindowingStrategy(limit_ref="4d", limit_cur="4d"),
                         warmup_intervals=10,
                         warmup_policy=TimeTriggerConfig(every="3d", start_timestamp=_FIRST_TIMESTAMP),
                         metrics={
                             "mmd": AlibiDetectMmdDriftMetric(
                                 device="cuda:0",
                                 decision_criterion=DynamicRollingAverageThresholdCriterion(
                                     deviation=0.1, absolute=False, window_size=15
-                                )
+                                ),
                             )
                         },
                     ),
@@ -363,25 +354,24 @@ def construct_pipelines(experiment: Experiment) -> list[ModynPipelineConfig]:
                                 batch_size=64,
                                 dataloader_workers=1,
                                 metrics=[
-                                    AccuracyMetricConfig(evaluation_transformer_function=yb_evaluation_transformer_function),
+                                    AccuracyMetricConfig(
+                                        evaluation_transformer_function=yb_evaluation_transformer_function
+                                    ),
                                 ],
                             ),
                         ),
                         mode="hindsight",  # TODO: lookahead
                         forecasting_method="ridge_regression",
                         decision_criteria={
-                            "static-0.8": StaticPerformanceThresholdCriterion(
-                                metric="Accuracy", metric_threshold=0.8
-                            )    
+                            "static-0.8": StaticPerformanceThresholdCriterion(metric="Accuracy", metric_threshold=0.8)
                         },
                     ),
                 },
-                ensemble_strategy=AtLeastNEnsembleStrategy(n=1)
+                ensemble_strategy=AtLeastNEnsembleStrategy(n=1),
             )
         },
         gpu_device="cuda:0",
     ),
-
     # ----------------------------- Evaluation intervals ----------------------------- #
     # 100: Experiment(
     #     name="yb-timetrigger1y-periodic-eval-intervals",

modyn/supervisor/internal/pipeline_executor/evaluation_executor.py

@@ -396,9 +396,8 @@ def get_failure_reason(eval_aborted_reason: EvaluationAbortedReason) -> str:
 #                                       DevTools                                       #
 # ------------------------------------------------------------------------------------ #
 
-def eval_executor_single_pipeline(
-    pipeline_dir: Path
-) -> None:
+
+def eval_executor_single_pipeline(pipeline_dir: Path) -> None:
     # restart evaluation executor
     ex = EvaluationExecutor.init_from_path(pipeline_dir)
 
@@ -412,11 +411,10 @@ def eval_executor_single_pipeline(
 
     logs_.supervisor_logs = ex.run_post_pipeline_evaluations(manual_run=True)
     logs_.materialize(pipeline_dir, mode="final")
-    logger.info(f"Done with manual evaluation.")
+    logger.info("Done with manual evaluation.")
+
 
-def eval_executor_multi_pipeline(
-    pipelines_dir: Path
-) -> None:
+def eval_executor_multi_pipeline(pipelines_dir: Path) -> None:
     """Run the evaluation executor for multiple pipelines."""
     pipeline_dirs = [p for p in pipelines_dir.iterdir() if p.is_dir()]
     for p_dir in pipeline_dirs:
@@ -427,10 +425,11 @@ def eval_executor_multi_pipeline(
             faulty_dir.mkdir(exist_ok=True)
             os.rename(p_dir, faulty_dir / p_dir.name)
             continue
-            
+
         eval_executor_single_pipeline(p_dir)
         logger.info(f"Done with pipeline {p_dir.name}")
 
+
 if __name__ == "__main__":
     single_pipeline_mode = input("Run evaluation executor for single pipeline? (y/n): ")
     userpath = Path(input("Enter pipeline log directory path to (re)run evaluation executor: "))
@@ -445,5 +444,5 @@ def eval_executor_multi_pipeline(
     else:
         print("Invalid input")
         sys.exit(1)
-        
+
     sys.exit(0)

modyn/supervisor/internal/triggers/__init__.py

@@ -6,13 +6,13 @@
 import os
 
 from .amounttrigger import DataAmountTrigger  # noqa: F401
+from .avoidablemissclassification_costtrigger import AvoidableMisclassificationCostTrigger  # noqa: F401
 from .datadrifttrigger import DataDriftTrigger  # noqa: F401
-from .timetrigger import TimeTrigger  # noqa: F401
-from .trigger import Trigger  # noqa: F401
-from .performancetrigger import PerformanceTrigger  # noqa: F401
 from .dataincorporationlatency_costtrigger import DataIncorporationLatencyCostTrigger  # noqa: F401
-from .avoidablemissclassification_costtrigger import AvoidableMisclassificationCostTrigger  # noqa: F401
 from .ensembletrigger import EnsembleTrigger  # noqa: F401
+from .performancetrigger import PerformanceTrigger  # noqa: F401
+from .timetrigger import TimeTrigger  # noqa: F401
+from .trigger import Trigger  # noqa: F401
 
 files = os.listdir(os.path.dirname(__file__))
 files.remove("__init__.py")