refactor: Prepare performance aware triggers (#588)

eth-easl · Aug 19, 2024 · 5d1b088 · 5d1b088
1 parent 3da3b5c
commit 5d1b088
Show file tree

Hide file tree

Showing 69 changed files with 602 additions and 405 deletions.
diff --git a/benchmark/cloc/pipelines/exp0_finetune.yml b/benchmark/cloc/pipelines/exp0_finetune.yml
@@ -62,7 +62,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp10_triggerbalanced.yml b/benchmark/cloc/pipelines/exp10_triggerbalanced.yml
@@ -65,7 +65,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp1_random_ct.yml b/benchmark/cloc/pipelines/exp1_random_ct.yml
@@ -65,7 +65,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp2_dlis_gradnorm_bts_ct.yml b/benchmark/cloc/pipelines/exp2_dlis_gradnorm_bts_ct.yml
@@ -66,7 +66,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp3_dlis_loss_bts_ct.yml b/benchmark/cloc/pipelines/exp3_dlis_loss_bts_ct.yml
@@ -66,7 +66,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp4_triggerbalanced_tail3.yml b/benchmark/cloc/pipelines/exp4_triggerbalanced_tail3.yml
@@ -66,7 +66,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp5_current_trigger_only.yml b/benchmark/cloc/pipelines/exp5_current_trigger_only.yml
@@ -62,7 +62,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp6_retrain_keep_model.yml b/benchmark/cloc/pipelines/exp6_retrain_keep_model.yml
@@ -62,7 +62,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp7_retrain_new_model.yml b/benchmark/cloc/pipelines/exp7_retrain_new_model.yml
@@ -62,7 +62,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp8_dlis_gradnorm_bts_full.yml b/benchmark/cloc/pipelines/exp8_dlis_gradnorm_bts_full.yml
@@ -66,7 +66,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/cloc/pipelines/exp9_random_full.yml b/benchmark/cloc/pipelines/exp9_random_full.yml
@@ -65,7 +65,6 @@ trigger:
     trigger_every: "52w"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: cloc
       transformations:

diff --git a/benchmark/criteo_1TB/pipelines/exp0_finetune.yml b/benchmark/criteo_1TB/pipelines/exp0_finetune.yml
@@ -117,7 +117,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp10_triggerbalanced.yml b/benchmark/criteo_1TB/pipelines/exp10_triggerbalanced.yml
@@ -122,7 +122,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp1_random_ct.yml b/benchmark/criteo_1TB/pipelines/exp1_random_ct.yml
@@ -122,7 +122,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp2_dlis_gradnorm_bts_ct.yml b/benchmark/criteo_1TB/pipelines/exp2_dlis_gradnorm_bts_ct.yml
@@ -123,7 +123,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp3_dlis_loss_bts_ct.yml b/benchmark/criteo_1TB/pipelines/exp3_dlis_loss_bts_ct.yml
@@ -123,7 +123,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp4_triggerbalanced_tail3.yml b/benchmark/criteo_1TB/pipelines/exp4_triggerbalanced_tail3.yml
@@ -123,7 +123,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp5_current_day_only.yml b/benchmark/criteo_1TB/pipelines/exp5_current_day_only.yml
@@ -117,7 +117,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp6_retrain_keep_model.yml b/benchmark/criteo_1TB/pipelines/exp6_retrain_keep_model.yml
@@ -117,7 +117,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp7_retrain_new_model.yml b/benchmark/criteo_1TB/pipelines/exp7_retrain_new_model.yml
@@ -117,7 +117,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp8_dlis_gradnorm_bts_full.yml b/benchmark/criteo_1TB/pipelines/exp8_dlis_gradnorm_bts_full.yml
@@ -123,7 +123,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/criteo_1TB/pipelines/exp9_random_full.yml b/benchmark/criteo_1TB/pipelines/exp9_random_full.yml
@@ -122,7 +122,6 @@ trigger:
     trigger_every: "1d"
 evaluation:
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: criteo
       bytes_parser_function: |

diff --git a/benchmark/sigmod/selection/arxiv_config.py b/benchmark/sigmod/selection/arxiv_config.py
@@ -113,7 +113,6 @@ def gen_arxiv_config(
             after_pipeline_evaluation_workers=2,
             after_training_evaluation_workers=2,
             device=gpu_device,
-            result_writers=["json"],
             datasets=[
                 EvalDataConfig(
                     dataset_id=dataset,

diff --git a/benchmark/sigmod/selection/cglm_config.py b/benchmark/sigmod/selection/cglm_config.py
@@ -118,7 +118,6 @@ def gen_cglm_config(
             after_pipeline_evaluation_workers=4,
             after_training_evaluation_workers=4,
             device=gpu_device,
-            result_writers=["json"],
             datasets=[
                 EvalDataConfig(
                     dataset_id=_dataset,

diff --git a/benchmark/sigmod/selection/yearbook_config.py b/benchmark/sigmod/selection/yearbook_config.py
@@ -139,7 +139,6 @@ def gen_yearbook_config(
             after_pipeline_evaluation_workers=12,
             after_training_evaluation_workers=12,
             device=gpu_device,
-            result_writers=["json"],
             datasets=[
                 EvalDataConfig(
                     dataset_id=dataset,

diff --git a/benchmark/sigmod/triggering/yearbook_triggering_config.py b/benchmark/sigmod/triggering/yearbook_triggering_config.py
@@ -80,7 +80,6 @@ def gen_yearbook_triggering_config(
             after_pipeline_evaluation_workers=12,
             after_training_evaluation_workers=12,
             device=gpu_device,
-            result_writers=["json"],
             datasets=[
                 EvalDataConfig(
                     dataset_id=dataset,

diff --git a/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/arxiv_datadrift.yaml b/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/arxiv_datadrift.yaml
@@ -67,7 +67,6 @@ evaluation:
       models: matrix
       datasets: ["arxiv_test"]
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: arxiv_test
       bytes_parser_function: |

diff --git a/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/huffpost_datadrift.yaml b/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/huffpost_datadrift.yaml
@@ -70,7 +70,6 @@ evaluation:
       models: matrix
       datasets: ["huffpost_test"]
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: huffpost_test
       bytes_parser_function: |

diff --git a/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/yearbook_datadrift.yaml b/benchmark/wildtime_benchmarks/example_pipelines/data_drift_trigger/yearbook_datadrift.yaml
@@ -70,7 +70,6 @@ evaluation:
       models: matrix
       datasets: ["yearbook_test"]
   device: "cuda:0"
-  result_writers: ["json"]
   datasets:
     - dataset_id: yearbook_test
       bytes_parser_function: |

diff --git a/docs/pipeline/TRIGGERING.md b/docs/pipeline/TRIGGERING.md
@@ -17,7 +17,7 @@ classDiagram
         <<Abstract>>
         +init_trigger()
         +inform()
-        +inform_previous_model()
+        +inform_new_model()
     }
 
     namespace simple_triggers {

diff --git a/docs/pipeline/triggering/DRIFT_TRIGGERS.md b/docs/pipeline/triggering/DRIFT_TRIGGERS.md
@@ -20,7 +20,7 @@ classDiagram
         <<abstract>>
         +void init_trigger(TriggerContext context)
         +Generator[Triggers] inform(new_data)
-        +void inform_previous_model(int previous_model_id)
+        +void inform_new_model(int previous_model_id)
     }
 
     class TimeTrigger {
@@ -41,7 +41,7 @@ classDiagram
 
         +void init_trigger(TriggerContext context)
         +Generator[triggers] inform(new_data)
-        +void inform_previous_model(int previous_model_id)
+        +void inform_new_model(int previous_model_id)
     }
 
     class DriftDecisionPolicy {

diff --git a/experiments/arxiv/compare_trigger_policies/pipeline_config.py b/experiments/arxiv/compare_trigger_policies/pipeline_config.py
@@ -63,7 +63,6 @@ def gen_pipeline_config(
         evaluation=EvaluationConfig(
             handlers=eval_handlers,
             device="cuda:0",
-            result_writers=["json"],
             datasets=[
                 EvalDataConfig(
                     dataset_id=yb_dataset_name,

diff --git a/experiments/huffpost/compare_trigger_policies/pipeline_config.py b/experiments/huffpost/compare_trigger_policies/pipeline_config.py
@@ -64,7 +64,6 @@ def gen_pipeline_config(
         evaluation=EvaluationConfig(
             handlers=eval_handlers,
             device="cuda:0",
-            result_writers=["json"],
             after_training_evaluation_workers=10,
             after_pipeline_evaluation_workers=10,
             datasets=[

diff --git a/experiments/yearbook/compare_trigger_policies/pipeline_config.py b/experiments/yearbook/compare_trigger_policies/pipeline_config.py
@@ -67,7 +67,6 @@ def gen_pipeline_config(
         evaluation=EvaluationConfig(
             handlers=eval_handlers,
             device=device,
-            result_writers=["json"],
             after_pipeline_evaluation_workers=20,
             after_training_evaluation_workers=20,
             datasets=[

diff --git a/modyn/config/examples/example-pipeline.yaml b/modyn/config/examples/example-pipeline.yaml
@@ -78,7 +78,6 @@ evaluation:
       models: matrix
       datasets: ["mnist"]
   device: "cpu"
-  result_writers: ["json", "tensorboard"]
   datasets:
     - dataset_id: mnist
       transformations:

diff --git a/modyn/config/schema/pipeline/evaluation/config.py b/modyn/config/schema/pipeline/evaluation/config.py
@@ -40,14 +40,6 @@ class EvaluationConfig(ModynBaseModel):
         min_length=1,
     )
     device: str = Field(description="The device the model should be put on.")
-    result_writers: list[ResultWriterType] = Field(
-        ["json"],
-        description=(
-            "List of names that specify in which formats to store the evaluation results. We currently support "
-            "json and tensorboard."
-        ),
-        min_length=1,
-    )
     datasets: list[EvalDataConfig] = Field(
         description="An array of all datasets on which the model is evaluated.",
         min_length=1,

diff --git a/modyn/config/schema/pipeline/trigger/simple/time.py b/modyn/config/schema/pipeline/trigger/simple/time.py
@@ -1,7 +1,7 @@
 from functools import cached_property
 from typing import Literal
 
-from pydantic import Field
+from pydantic import Field, field_validator
 
 from modyn.config.schema.base_model import ModynBaseModel
 from modyn.const.regex import REGEX_TIME_UNIT
@@ -27,3 +27,12 @@ def every_seconds(self) -> int:
         unit = str(self.every)[-1:]
         num = int(str(self.every)[:-1])
         return num * SECONDS_PER_UNIT[unit]
+
+    @field_validator("every")
+    @classmethod
+    def validate_every(cls, value: str) -> str:
+        unit = str(value)[-1:]
+        num = int(str(value)[:-1])
+        assert num * SECONDS_PER_UNIT[unit] >= 1, "trigger_every must be > 0, but is {self.config.every_seconds}"
+
+        return value