Skip to content

Commit

Permalink
support other ratios than 50%
Browse files Browse the repository at this point in the history
  • Loading branch information
@MaxiBoether
MaxiBoether committed Jun 20, 2024
1 parent 53004e2 commit 3c8091d
Showing 1 changed file with 23 additions and 19 deletions.
42 changes: 23 additions & 19 deletions benchmark/sigmod/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@


def gen_selection_strategies(
warmup_triggers: int, num_classes: int, training_config: TrainingConfig, small_run: bool = False
ratio: int, warmup_triggers: int, num_classes: int, training_config: TrainingConfig, small_run: bool = False
) -> list[tuple[str, SelectionStrategy]]:
strategies = []

Expand All @@ -65,7 +65,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
presampling_config=PresamplingConfig(strategy="Random", ratio=50),
presampling_config=PresamplingConfig(strategy="Random", ratio=ratio),
),
)
)
Expand All @@ -80,7 +80,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
presampling_config=PresamplingConfig(strategy="LabelBalanced", ratio=50),
presampling_config=PresamplingConfig(strategy="LabelBalanced", ratio=ratio),
),
)
)
Expand All @@ -96,7 +96,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=RS2DownsamplingConfig(ratio=50, with_replacement=True),
downsampling_config=RS2DownsamplingConfig(ratio=ratio, with_replacement=True),
),
)
)
Expand All @@ -111,7 +111,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=RS2DownsamplingConfig(ratio=50, with_replacement=False),
downsampling_config=RS2DownsamplingConfig(ratio=ratio, with_replacement=False),
),
)
)
Expand All @@ -126,7 +126,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=LossDownsamplingConfig(ratio=50, sample_then_batch=True, period=1),
downsampling_config=LossDownsamplingConfig(ratio=ratio, sample_then_batch=True, period=1),
),
)
)
Expand All @@ -141,7 +141,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=LossDownsamplingConfig(ratio=50, sample_then_batch=False),
downsampling_config=LossDownsamplingConfig(ratio=ratio, sample_then_batch=False),
),
)
)
Expand All @@ -157,7 +157,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=GradNormDownsamplingConfig(ratio=50, sample_then_batch=True, period=1),
downsampling_config=GradNormDownsamplingConfig(ratio=ratio, sample_then_batch=True, period=1),
),
)
)
Expand All @@ -172,7 +172,7 @@ def gen_selection_strategies(
tail_triggers=0,
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=GradNormDownsamplingConfig(ratio=50, sample_then_batch=False),
downsampling_config=GradNormDownsamplingConfig(ratio=ratio, sample_then_batch=False),
),
)
)
Expand All @@ -195,7 +195,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=RHOLossDownsamplingConfig(
ratio=50,
ratio=ratio,
sample_then_batch=False,
period=1,
holdout_set_ratio=10,
Expand All @@ -217,7 +217,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=True, period=1, score_metric="Margin"
ratio=ratio, sample_then_batch=True, period=1, score_metric="Margin"
),
),
)
Expand All @@ -234,7 +234,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=False, period=1, score_metric="Margin"
ratio=ratio, sample_then_batch=False, period=1, score_metric="Margin"
),
),
)
Expand All @@ -252,7 +252,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=True, period=1, score_metric="LeastConfidence"
ratio=ratio, sample_then_batch=True, period=1, score_metric="LeastConfidence"
),
),
)
Expand All @@ -269,7 +269,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=False, period=1, score_metric="LeastConfidence"
ratio=ratio, sample_then_batch=False, period=1, score_metric="LeastConfidence"
),
),
)
Expand All @@ -287,7 +287,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=True, period=1, score_metric="Entropy"
ratio=ratio, sample_then_batch=True, period=1, score_metric="Entropy"
),
),
)
Expand All @@ -304,7 +304,7 @@ def gen_selection_strategies(
limit=-1,
warmup_triggers=warmup_triggers,
downsampling_config=UncertaintyDownsamplingConfig(
ratio=50, sample_then_batch=False, period=1, score_metric="Entropy"
ratio=ratio, sample_then_batch=False, period=1, score_metric="Entropy"
),
),
)
Expand Down Expand Up @@ -355,6 +355,7 @@ def run_experiment() -> None:
warmup_triggers = 1 # default value, for CGLM/arxiv/yearbook see below
disable_scheduling = True # For our baselines, scheduling was mostly meaningless.
seeds = [42, 99, 12] # set to [None] to disable, should be 0-100
ratios = [50, 10, 30]
num_gpus = 1 # to parallelize across gpus
gpu_id = 0
small_run = True
Expand All @@ -377,7 +378,8 @@ def run_experiment() -> None:
lr_sched_id,
num_epochs,
warmup_triggers,
dataset: f"{model}_{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}"
ratio,
dataset: f"{model}_{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}_r{ratio}"
)
train_conf_func = gen_yearbook_training_conf
elif pipeline_gen_func == gen_arxiv_config:
Expand All @@ -392,7 +394,8 @@ def run_experiment() -> None:
lr_sched_id,
num_epochs,
warmup_triggers,
dataset: f"{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}"
ratio,
dataset: f"{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}_r{ratio}"
)
train_conf_func = gen_arxiv_training_conf

Expand All @@ -407,7 +410,8 @@ def run_experiment() -> None:
lr_sched_id,
num_epochs,
warmup_triggers,
dataset: f"{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}_ds{dataset}"
ratio,
dataset: f"{selection_strategy_id}_{lr_sched_id}_epoch{num_epochs}_warm{warmup_triggers}_ds{dataset}_r{ratio}"
)
ds_class_map = {"cglm_landmark_min25": 6404, "cglm_hierarchical_min25": 79}
num_classes = ds_class_map[dataset]
Expand Down

0 comments on commit 3c8091d

Please sign in to comment.