Implement ratio_max scaling for downsamplers

modyn/config/schema/pipeline/config.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
-from typing import Optional
+from typing import Optional, Self
 
 from modyn.config.schema.base_model import ModynBaseModel
-from pydantic import Field
+from pydantic import Field, model_validator
 
 from .data import DataConfig
 from .evaluation.config import EvaluationConfig
 from .model import ModelConfig
 from .model_storage import PipelineModelStorageConfig
-from .sampling.config import SelectionStrategy
+from .sampling.config import CoresetStrategyConfig, SelectionStrategy
+from .sampling.downsampling_config import MultiDownsamplingConfig
 from .training import TrainingConfig
 from .trigger import TriggerConfig
 
@@ -32,3 +33,25 @@ class ModynPipelineConfig(ModynBaseModel):
     trigger: TriggerConfig
     selection_strategy: SelectionStrategy
     evaluation: EvaluationConfig | None = Field(None)
+
+    @model_validator(mode="after")
+    def validate_bts_training_selection_works(self) -> Self:
+        # Validates that when using Downsampling with BtS, we choose a functional ratio
+        if isinstance(self.selection_strategy, CoresetStrategyConfig) and not isinstance(
+            self.selection_strategy.downsampling_config, MultiDownsamplingConfig
+        ):
+            if not self.selection_strategy.downsampling_config.sample_then_batch:  # bts
+                ratio = self.selection_strategy.downsampling_config.ratio
+                ratio_max = self.selection_strategy.downsampling_config.ratio_max
+                batch_size = self.training.batch_size
+
+                post_downsampling_size = max((ratio * batch_size) // ratio_max, 1)
+                if batch_size % post_downsampling_size != 0:
+                    raise ValueError(
+                        f"The target batch size of {batch_size} is not a multiple of the batch size "
+                        + f"after downsampling with ratio {ratio} a batch in BtS mode ({post_downsampling_size}). "
+                        + "We cannot accumulate batches. "
+                        + "Please choose the downsampling ratio and batch size such that this is possible."
+                    )
+
+        return self

modyn/config/schema/pipeline/sampling/downsampling_config.py

@@ -20,9 +20,20 @@ class BaseDownsamplingConfig(ModynBaseModel):
         ),
     )
     ratio: int = Field(
-        description="Ratio post_sampling_size/pre_sampling_size. E.g. with 160 records and a ratio of 50 we keep 80.",
+        description=(
+            "Ratio post_sampling_size/pre_sampling_size * ratio_max. "
+            "For the default of ratio_max of 100, this implies percent, "
+            "e.g., with 160 records and a ratio of 50 we keep 80."
+        ),
         min=0,
-        max=100,
+    )
+    ratio_max: int = Field(
+        description=(
+            "Reference maximum ratio value. Defaults to 100, which implies percent."
+            " If you set this to 1000, ratio describes promille instead."
+        ),
+        default=100,
+        min=1,
     )
     period: int = Field(
         1,
@@ -34,6 +45,12 @@ class BaseDownsamplingConfig(ModynBaseModel):
         min=0,
     )
 
+    @model_validator(mode="after")
+    def validate_ratio(self) -> Self:
+        if self.ratio > self.ratio_max:
+            raise ValueError("ratio cannot be greater than ratio_max.")
+        return self
+
 
 class UncertaintyDownsamplingConfig(BaseDownsamplingConfig):
     """Config for the Craig downsampling strategy."""

modyn/selector/internal/selector_strategies/downsampling_strategies/abstract_downsampling_strategy.py

@@ -38,6 +38,7 @@ def __init__(
 
         self.downsampling_period = downsampling_config.period
         self.downsampling_ratio = downsampling_config.ratio
+        self.ratio_max = downsampling_config.ratio_max
 
         self.requires_remote_computation = True
         self.maximum_keys_in_memory = maximum_keys_in_memory
@@ -60,6 +61,7 @@ def _compute_status_bar_scale(self) -> int:
     def downsampling_params(self) -> dict:
         config = {
             "downsampling_ratio": self.downsampling_ratio,
+            "ratio_max": self.ratio_max,
             "maximum_keys_in_memory": self.maximum_keys_in_memory,
             "sample_then_batch": self.downsampling_mode == DownsamplingMode.SAMPLE_THEN_BATCH,
         }

modyn/tests/selector/internal/selector_strategies/downsampling_strategies/test_rho_loss_downsampling_strategy.py

@@ -301,6 +301,7 @@ def test_downsampling_params(il_training_config: ILTrainingConfig, data_config:
 
     expected = {
         "downsampling_ratio": 60,
+        "ratio_max": 100,
         "maximum_keys_in_memory": maximum_keys_in_memory,
         "sample_then_batch": False,
         "il_model_id": 3,

modyn/tests/selector/internal/selector_strategies/downsampling_strategies/test_scheduler.py

@@ -86,6 +86,7 @@ def test_switch_functions():
             "downsampling_ratio": 50,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": True,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteLossDownsampling"
         assert downs.training_status_bar_scale == 50
@@ -98,6 +99,7 @@ def test_switch_functions():
             "downsampling_ratio": 25,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": False,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteGradNormDownsampling"
         assert downs.training_status_bar_scale == 100
@@ -140,6 +142,7 @@ def test_double_threshold():
             "downsampling_ratio": 50,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": True,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteLossDownsampling"
         assert downs.training_status_bar_scale == 50
@@ -152,6 +155,7 @@ def test_double_threshold():
             "downsampling_ratio": 25,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": False,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteGradNormDownsampling"
         assert downs.training_status_bar_scale == 100
@@ -179,6 +183,7 @@ def test_wrong_trigger():
             "downsampling_ratio": 50,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": True,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteLossDownsampling"
         assert downs.training_status_bar_scale == 50
@@ -195,6 +200,7 @@ def test_wrong_trigger():
             "downsampling_ratio": 25,
             "maximum_keys_in_memory": 1000,
             "sample_then_batch": False,
+            "ratio_max": 100,
         }
         assert downs.downsampling_strategy == "RemoteGradNormDownsampling"
         assert downs.training_status_bar_scale == 100

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_abstract_matrix_downsampling_strategy.py

@@ -19,6 +19,7 @@ def get_sampler_config(dummy_system_config: ModynConfig, balance=False):
         "sample_then_batch": False,
         "args": {},
         "balance": balance,
+        "ratio_max": 100,
     }
     return (
         0,

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_abstract_remote_downsampling_strategy.py

@@ -11,7 +11,7 @@
 def test_batch_then_sample_general(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "ratio_max": 100}
     sampler = AbstractRemoteDownsamplingStrategy(
         154, 128, 64, params_from_selector, dummy_system_config.model_dump(by_alias=True), "cpu"
     )

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_craig_remote_downsampling.py

@@ -20,6 +20,7 @@ def get_sampler_config(modyn_config, balance=False):
         "balance": balance,
         "selection_batch": 64,
         "greedy": "NaiveGreedy",
+        "ratio_max": 100,
     }
     return 0, 0, 0, params_from_selector, modyn_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
 
@@ -347,7 +348,7 @@ def test_matching_results_with_deepcore(dummy_system_config: ModynConfig):
         0,
         0,
         5,
-        {"downsampling_ratio": 20, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy"},
+        {"downsampling_ratio": 20, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy", "ratio_max": 100},
         dummy_system_config.model_dump(by_alias=True),
         BCEWithLogitsLoss(reduction="none"),
         "cpu",
@@ -402,7 +403,7 @@ def test_matching_results_with_deepcore_permutation(dummy_system_config: ModynCo
         0,
         0,
         5,
-        {"downsampling_ratio": 30, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy"},
+        {"downsampling_ratio": 30, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy", "ratio_max": 100},
         dummy_system_config.model_dump(by_alias=True),
         BCEWithLogitsLoss(reduction="none"),
         "cpu",
@@ -461,7 +462,7 @@ def test_matching_results_with_deepcore_permutation_fancy_ids(dummy_system_confi
         0,
         0,
         5,
-        {"downsampling_ratio": 50, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy"},
+        {"downsampling_ratio": 50, "balance": False, "selection_batch": 64, "greedy": "NaiveGreedy", "ratio_max": 100},
         dummy_system_config.model_dump(by_alias=True),
         BCEWithLogitsLoss(reduction="none"),
         "cpu",

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_gradmatch_downsampling_strategy.py

@@ -19,6 +19,7 @@ def get_sampler_config(modyn_config: ModynConfig, balance=False):
         "sample_then_batch": False,
         "args": {},
         "balance": balance,
+        "ratio_max": 100,
     }
     return 0, 0, 0, params_from_selector, modyn_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
 
@@ -185,7 +186,7 @@ def test_matching_results_with_deepcore(dummy_system_config: ModynConfig):
             0,
             0,
             5,
-            {"downsampling_ratio": 10 * num_of_target_samples, "balance": False},
+            {"downsampling_ratio": 10 * num_of_target_samples, "balance": False, "ratio_max": 100},
             dummy_system_config.model_dump(by_alias=True),
             BCEWithLogitsLoss(reduction="none"),
             "cpu",
@@ -237,7 +238,7 @@ def test_matching_results_with_deepcore_permutation_fancy_ids(dummy_system_confi
         0,
         0,
         5,
-        {"downsampling_ratio": 50, "balance": False},
+        {"downsampling_ratio": 50, "balance": False, "ratio_max": 100},
         dummy_system_config.model_dump(by_alias=True),
         BCEWithLogitsLoss(reduction="none"),
         "cpu",

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_gradnorm_downsample.py

@@ -14,7 +14,7 @@ def test_sample_shape_ce(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteGradNormDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )
@@ -45,7 +45,7 @@ def test_sample_shape_other_losses(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
     per_sample_loss_fct = torch.nn.BCEWithLogitsLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteGradNormDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )
@@ -84,6 +84,7 @@ def test_sampling_crossentropy(dummy_system_config: ModynConfig):
         "downsampling_ratio": downsampling_ratio,
         "replacement": False,
         "sample_then_batch": False,
+        "ratio_max": 100,
     }
 
     # Here we use autograd since the number of classes is not provided
@@ -135,7 +136,7 @@ def test_sample_dict_input(dummy_system_config: ModynConfig):
     model = DictLikeModel()
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": 50, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": 50, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteGradNormDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_kcenter_downsampling_strategy.py

@@ -17,6 +17,7 @@ def get_sampler_config(modyn_config: ModynConfig, balance=False):
         "sample_then_batch": False,
         "args": {},
         "balance": balance,
+        "ratio_max": 100,
     }
     return 0, 0, 0, params_from_selector, modyn_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
 
@@ -137,7 +138,7 @@ def test_matching_results_with_deepcore(dummy_system_config: ModynConfig):
             0,
             0,
             5,
-            {"downsampling_ratio": 10 * num_of_target_samples, "balance": False},
+            {"downsampling_ratio": 10 * num_of_target_samples, "balance": False, "ratio_max": 100},
             dummy_system_config.model_dump(by_alias=True),
             BCEWithLogitsLoss(reduction="none"),
             "cpu",
@@ -166,7 +167,7 @@ def test_matching_results_with_deepcore_permutation_fancy_ids(dummy_system_confi
         0,
         0,
         5,
-        {"downsampling_ratio": 50, "balance": False},
+        {"downsampling_ratio": 50, "balance": False, "ratio_max": 100},
         dummy_system_config.model_dump(by_alias=True),
         BCEWithLogitsLoss(reduction="none"),
         "cpu",

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_loss_downsample.py

@@ -12,7 +12,7 @@ def test_sample_shape(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteLossDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )
@@ -39,7 +39,7 @@ def test_sample_weights(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteLossDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )
@@ -67,7 +67,7 @@ def test_sample_loss_dependent_sampling(dummy_system_config: ModynConfig):
     downsampling_ratio = 50
     per_sample_loss_fct = torch.nn.MSELoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": downsampling_ratio, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteLossDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )
@@ -116,7 +116,7 @@ def test_sample_dict_input(dummy_system_config: ModynConfig):
     mymodel = DictLikeModel()
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
 
-    params_from_selector = {"downsampling_ratio": 50, "sample_then_batch": False}
+    params_from_selector = {"downsampling_ratio": 50, "sample_then_batch": False, "ratio_max": 100}
     sampler = RemoteLossDownsampling(
         0, 0, 0, params_from_selector, dummy_system_config.model_dump(by_alias=True), per_sample_loss_fct, "cpu"
     )

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_rho_loss_downsampling.py

@@ -29,6 +29,7 @@ def dummy_init_params(dummy_system_config: ModynConfig):
         "il_model_id": 2,
         "downsampling_ratio": 50,
         "sample_then_batch": False,
+        "ratio_max": 100,
     }
     modyn_config = dummy_system_config.model_dump(by_alias=True)
     per_sample_loss_fct = torch.nn.CrossEntropyLoss(reduction="none")

modyn/tests/trainer_server/internal/trainer/remote_downsamplers/test_remote_rs2_downsampling.py

@@ -10,7 +10,7 @@ def test_init(dummy_system_config: ModynConfig):
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": True, "downsampling_ratio": 50}
+    params_from_selector = {"replacement": True, "downsampling_ratio": 50, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"
 
@@ -41,7 +41,7 @@ def test_inform_samples(dummy_system_config: ModynConfig):
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": True, "downsampling_ratio": 50}
+    params_from_selector = {"replacement": True, "downsampling_ratio": 50, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"
 
@@ -76,7 +76,7 @@ def test_multiple_epochs_with_replacement(dummy_system_config: ModynConfig):
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": True, "downsampling_ratio": 50}
+    params_from_selector = {"replacement": True, "downsampling_ratio": 50, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"
 
@@ -110,7 +110,7 @@ def test_multiple_epochs_without_replacement(dummy_system_config: ModynConfig):
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": False, "downsampling_ratio": 50}
+    params_from_selector = {"replacement": False, "downsampling_ratio": 50, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"
 
@@ -171,7 +171,7 @@ def test_multiple_epochs_without_replacement_leftover_data(dummy_system_config:
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": False, "downsampling_ratio": 40}
+    params_from_selector = {"replacement": False, "downsampling_ratio": 40, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"
 
@@ -207,7 +207,7 @@ def test_multiple_epochs_empty_without_replacement_leftover_data(dummy_system_co
     pipeline_id = 0
     trigger_id = 0
     batch_size = 32
-    params_from_selector = {"replacement": False, "downsampling_ratio": 40}
+    params_from_selector = {"replacement": False, "downsampling_ratio": 40, "ratio_max": 100}
     per_sample_loss = None
     device = "cpu"