Prepared scripts to run models for new MVI techniques

cluster/run_exp1/run-exp1-models-template.sbatch

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 #SBATCH --nodes=1
-#SBATCH --mem=120GB
-#SBATCH --cpus-per-task=24
+#SBATCH --mem=32GB
+#SBATCH --cpus-per-task=12
 #SBATCH --time=48:00:00
 #SBATCH --mail-type=END
 #SBATCH --mail-user=dh3553@nyu.edu

cluster/run_exp1/run-exp1-models.sh

@@ -1,7 +1,304 @@
 # Define the list of tuples (dataset_name, model_name, run_nums)
 declare -a job_configs=(
-#     "folk_emp automl exp1_mnar3 gandalf_clf 2"
-    "folk_emp boost_clean exp1_mnar3 gandalf_clf 2"
+     "diabetes notmiwae exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "diabetes notmiwae exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "diabetes notmiwae exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "diabetes notmiwae mixed_exp rf_clf 1,2,3,4,5,6"
+     "diabetes gain exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "diabetes gain exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "diabetes gain exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "diabetes gain mixed_exp rf_clf 1,2,3,4,5,6"
+     "diabetes tdm exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "diabetes tdm exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "diabetes tdm exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "diabetes tdm mixed_exp rf_clf 1,2,3,4,5,6"
+     "diabetes nomi exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "diabetes nomi exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "diabetes nomi exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "diabetes nomi mixed_exp rf_clf 1,2,3,4,5,6"
+
+     "german notmiwae exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "german notmiwae exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "german notmiwae exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "german notmiwae mixed_exp rf_clf 1,2,3,4,5,6"
+     "german gain exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "german gain exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "german gain exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "german gain mixed_exp rf_clf 1,2,3,4,5,6"
+     "german tdm exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "german tdm exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "german tdm exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "german tdm mixed_exp rf_clf 1,2,3,4,5,6"
+     "german nomi exp1_mcar3 rf_clf 1,2,3,4,5,6"
+     "german nomi exp1_mar3 rf_clf 1,2,3,4,5,6"
+     "german nomi exp1_mnar3 rf_clf 1,2,3,4,5,6"
+     "german nomi mixed_exp rf_clf 1,2,3,4,5,6"
+
+#     "law_school notmiwae exp1_mcar3 lr_clf 1,2,3,4,5,6"
+#     "law_school notmiwae exp1_mar3 lr_clf 1,2,3,4,5,6"
+#     "law_school notmiwae exp1_mnar3 lr_clf 1,2,3,4,5,6"
+#     "law_school notmiwae mixed_exp lr_clf 1,2,3,4,5,6"
+#     "law_school gain exp1_mcar3 lr_clf 1,2,3,4,5,6"
+#     "law_school gain exp1_mar3 lr_clf 1,2,3,4,5,6"
+#     "law_school gain exp1_mnar3 lr_clf 1,2,3,4,5,6"
+#     "law_school gain mixed_exp lr_clf 1,2,3,4,5,6"
+#     "law_school tdm exp1_mcar3 lr_clf 1,2,3,4,5,6"
+#     "law_school tdm exp1_mar3 lr_clf 1,2,3,4,5,6"
+#     "law_school tdm exp1_mnar3 lr_clf 1,2,3,4,5,6"
+#     "law_school tdm mixed_exp lr_clf 1,2,3,4,5,6"
+#     "law_school nomi exp1_mcar3 lr_clf 1,2,3,4,5,6"
+#     "law_school nomi exp1_mar3 lr_clf 1,2,3,4,5,6"
+#     "law_school nomi exp1_mnar3 lr_clf 1,2,3,4,5,6"
+#     "law_school nomi mixed_exp lr_clf 1,2,3,4,5,6"
+#
+#     "bank notmiwae exp1_mcar3 lgbm_clf 1,2"
+#     "bank notmiwae exp1_mcar3 lgbm_clf 3,4"
+#     "bank notmiwae exp1_mcar3 lgbm_clf 5,6"
+#     "bank notmiwae exp1_mar3 lgbm_clf 1,2"
+#     "bank notmiwae exp1_mar3 lgbm_clf 3,4"
+#     "bank notmiwae exp1_mar3 lgbm_clf 5,6"
+#     "bank notmiwae exp1_mnar3 lgbm_clf 1,2"
+#     "bank notmiwae exp1_mnar3 lgbm_clf 3,4"
+#     "bank notmiwae exp1_mnar3 lgbm_clf 5,6"
+#     "bank notmiwae mixed_exp lgbm_clf 1,2"
+#     "bank notmiwae mixed_exp lgbm_clf 3,4"
+#     "bank notmiwae mixed_exp lgbm_clf 5,6"
+#     "bank gain exp1_mcar3 lgbm_clf 1,2"
+#     "bank gain exp1_mcar3 lgbm_clf 3,4"
+#     "bank gain exp1_mcar3 lgbm_clf 5,6"
+#     "bank gain exp1_mar3 lgbm_clf 1,2"
+#     "bank gain exp1_mar3 lgbm_clf 3,4"
+#     "bank gain exp1_mar3 lgbm_clf 5,6"
+#     "bank gain exp1_mnar3 lgbm_clf 1,2"
+#     "bank gain exp1_mnar3 lgbm_clf 3,4"
+#     "bank gain exp1_mnar3 lgbm_clf 5,6"
+#     "bank gain mixed_exp lgbm_clf 1,2"
+#     "bank gain mixed_exp lgbm_clf 3,4"
+#     "bank gain mixed_exp lgbm_clf 5,6"
+#     "bank tdm exp1_mcar3 lgbm_clf 1,2"
+#     "bank tdm exp1_mcar3 lgbm_clf 3,4"
+#     "bank tdm exp1_mcar3 lgbm_clf 5,6"
+#     "bank tdm exp1_mar3 lgbm_clf 1,2"
+#     "bank tdm exp1_mar3 lgbm_clf 3,4"
+#     "bank tdm exp1_mar3 lgbm_clf 5,6"
+#     "bank tdm exp1_mnar3 lgbm_clf 1,2"
+#     "bank tdm exp1_mnar3 lgbm_clf 3,4"
+#     "bank tdm exp1_mnar3 lgbm_clf 5,6"
+#     "bank tdm mixed_exp lgbm_clf 1,2"
+#     "bank tdm mixed_exp lgbm_clf 3,4"
+#     "bank tdm mixed_exp lgbm_clf 5,6"
+#     "bank nomi exp1_mcar3 lgbm_clf 1,2"
+#     "bank nomi exp1_mcar3 lgbm_clf 3,4"
+#     "bank nomi exp1_mcar3 lgbm_clf 5,6"
+#     "bank nomi exp1_mar3 lgbm_clf 1,2"
+#     "bank nomi exp1_mar3 lgbm_clf 3,4"
+#     "bank nomi exp1_mar3 lgbm_clf 5,6"
+#     "bank nomi exp1_mnar3 lgbm_clf 1,2"
+#     "bank nomi exp1_mnar3 lgbm_clf 3,4"
+#     "bank nomi exp1_mnar3 lgbm_clf 5,6"
+#     "bank nomi mixed_exp lgbm_clf 1,2"
+#     "bank nomi mixed_exp lgbm_clf 3,4"
+#     "bank nomi mixed_exp lgbm_clf 5,6"
+#
+#     "folk notmiwae exp1_mcar3 mlp_clf 1,2"
+#     "folk notmiwae exp1_mcar3 mlp_clf 3,4"
+#     "folk notmiwae exp1_mcar3 mlp_clf 5,6"
+#     "folk notmiwae exp1_mar3 mlp_clf 1,2"
+#     "folk notmiwae exp1_mar3 mlp_clf 3,4"
+#     "folk notmiwae exp1_mar3 mlp_clf 5,6"
+#     "folk notmiwae exp1_mnar3 mlp_clf 1,2"
+#     "folk notmiwae exp1_mnar3 mlp_clf 3,4"
+#     "folk notmiwae exp1_mnar3 mlp_clf 5,6"
+#     "folk notmiwae mixed_exp mlp_clf 1,2"
+#     "folk notmiwae mixed_exp mlp_clf 3,4"
+#     "folk notmiwae mixed_exp mlp_clf 5,6"
+#     "folk gain exp1_mcar3 mlp_clf 1,2"
+#     "folk gain exp1_mcar3 mlp_clf 3,4"
+#     "folk gain exp1_mcar3 mlp_clf 5,6"
+#     "folk gain exp1_mar3 mlp_clf 1,2"
+#     "folk gain exp1_mar3 mlp_clf 3,4"
+#     "folk gain exp1_mar3 mlp_clf 5,6"
+#     "folk gain exp1_mnar3 mlp_clf 1,2"
+#     "folk gain exp1_mnar3 mlp_clf 3,4"
+#     "folk gain exp1_mnar3 mlp_clf 5,6"
+#     "folk gain mixed_exp mlp_clf 1,2"
+#     "folk gain mixed_exp mlp_clf 3,4"
+#     "folk gain mixed_exp mlp_clf 5,6"
+#     "folk tdm exp1_mcar3 mlp_clf 1,2"
+#     "folk tdm exp1_mcar3 mlp_clf 3,4"
+#     "folk tdm exp1_mcar3 mlp_clf 5,6"
+#     "folk tdm exp1_mar3 mlp_clf 1,2"
+#     "folk tdm exp1_mar3 mlp_clf 3,4"
+#     "folk tdm exp1_mar3 mlp_clf 5,6"
+#     "folk tdm exp1_mnar3 mlp_clf 1,2"
+#     "folk tdm exp1_mnar3 mlp_clf 3,4"
+#     "folk tdm exp1_mnar3 mlp_clf 5,6"
+#     "folk tdm mixed_exp mlp_clf 1,2"
+#     "folk tdm mixed_exp mlp_clf 3,4"
+#     "folk tdm mixed_exp mlp_clf 5,6"
+#     "folk nomi exp1_mcar3 mlp_clf 1,2"
+#     "folk nomi exp1_mcar3 mlp_clf 3,4"
+#     "folk nomi exp1_mcar3 mlp_clf 5,6"
+#     "folk nomi exp1_mar3 mlp_clf 1,2"
+#     "folk nomi exp1_mar3 mlp_clf 3,4"
+#     "folk nomi exp1_mar3 mlp_clf 5,6"
+#     "folk nomi exp1_mnar3 mlp_clf 1,2"
+#     "folk nomi exp1_mnar3 mlp_clf 3,4"
+#     "folk nomi exp1_mnar3 mlp_clf 5,6"
+#     "folk nomi mixed_exp mlp_clf 1,2"
+#     "folk nomi mixed_exp mlp_clf 3,4"
+#     "folk nomi mixed_exp mlp_clf 5,6"
+#
+#     "heart notmiwae exp1_mcar3 gandalf_clf 1,2"
+#     "heart notmiwae exp1_mcar3 gandalf_clf 3,4"
+#     "heart notmiwae exp1_mcar3 gandalf_clf 5,6"
+#     "heart notmiwae exp1_mar3 gandalf_clf 1,2"
+#     "heart notmiwae exp1_mar3 gandalf_clf 3,4"
+#     "heart notmiwae exp1_mar3 gandalf_clf 5,6"
+#     "heart notmiwae exp1_mnar3 gandalf_clf 1,2"
+#     "heart notmiwae exp1_mnar3 gandalf_clf 3,4"
+#     "heart notmiwae exp1_mnar3 gandalf_clf 5,6"
+#     "heart notmiwae mixed_exp gandalf_clf 1,2"
+#     "heart notmiwae mixed_exp gandalf_clf 3,4"
+#     "heart notmiwae mixed_exp gandalf_clf 5,6"
+#     "heart gain exp1_mcar3 gandalf_clf 1,2"
+#     "heart gain exp1_mcar3 gandalf_clf 3,4"
+#     "heart gain exp1_mcar3 gandalf_clf 5,6"
+#     "heart gain exp1_mar3 gandalf_clf 1,2"
+#     "heart gain exp1_mar3 gandalf_clf 3,4"
+#     "heart gain exp1_mar3 gandalf_clf 5,6"
+#     "heart gain exp1_mnar3 gandalf_clf 1,2"
+#     "heart gain exp1_mnar3 gandalf_clf 3,4"
+#     "heart gain exp1_mnar3 gandalf_clf 5,6"
+#     "heart gain mixed_exp gandalf_clf 1,2"
+#     "heart gain mixed_exp gandalf_clf 3,4"
+#     "heart gain mixed_exp gandalf_clf 5,6"
+#     "heart tdm exp1_mcar3 gandalf_clf 1,2"
+#     "heart tdm exp1_mcar3 gandalf_clf 3,4"
+#     "heart tdm exp1_mcar3 gandalf_clf 5,6"
+#     "heart tdm exp1_mar3 gandalf_clf 1,2"
+#     "heart tdm exp1_mar3 gandalf_clf 3,4"
+#     "heart tdm exp1_mar3 gandalf_clf 5,6"
+#     "heart tdm exp1_mnar3 gandalf_clf 1,2"
+#     "heart tdm exp1_mnar3 gandalf_clf 3,4"
+#     "heart tdm exp1_mnar3 gandalf_clf 5,6"
+#     "heart tdm mixed_exp gandalf_clf 1,2"
+#     "heart tdm mixed_exp gandalf_clf 3,4"
+#     "heart tdm mixed_exp gandalf_clf 5,6"
+#     "heart nomi exp1_mcar3 gandalf_clf 1,2"
+#     "heart nomi exp1_mcar3 gandalf_clf 3,4"
+#     "heart nomi exp1_mcar3 gandalf_clf 5,6"
+#     "heart nomi exp1_mar3 gandalf_clf 1,2"
+#     "heart nomi exp1_mar3 gandalf_clf 3,4"
+#     "heart nomi exp1_mar3 gandalf_clf 5,6"
+#     "heart nomi exp1_mnar3 gandalf_clf 1,2"
+#     "heart nomi exp1_mnar3 gandalf_clf 3,4"
+#     "heart nomi exp1_mnar3 gandalf_clf 5,6"
+#     "heart nomi mixed_exp gandalf_clf 1,2"
+#     "heart nomi mixed_exp gandalf_clf 3,4"
+#     "heart nomi mixed_exp gandalf_clf 5,6"
+#
+#
+#
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 1"
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 2"
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 3"
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 4"
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 5"
+#     "folk_emp notmiwae exp1_mcar3 gandalf_clf 6"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 1"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 2"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 3"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 4"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 5"
+#     "folk_emp notmiwae exp1_mar3 gandalf_clf 6"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 1"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 2"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 3"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 4"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 5"
+#     "folk_emp notmiwae exp1_mnar3 gandalf_clf 6"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 1"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 2"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 3"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 4"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 5"
+#     "folk_emp notmiwae mixed_exp gandalf_clf 6"
+#
+#     "folk_emp gain exp1_mcar3 gandalf_clf 1"
+#     "folk_emp gain exp1_mcar3 gandalf_clf 2"
+#     "folk_emp gain exp1_mcar3 gandalf_clf 3"
+#     "folk_emp gain exp1_mcar3 gandalf_clf 4"
+#     "folk_emp gain exp1_mcar3 gandalf_clf 5"
+#     "folk_emp gain exp1_mcar3 gandalf_clf 6"
+#     "folk_emp gain exp1_mar3 gandalf_clf 1"
+#     "folk_emp gain exp1_mar3 gandalf_clf 2"
+#     "folk_emp gain exp1_mar3 gandalf_clf 3"
+#     "folk_emp gain exp1_mar3 gandalf_clf 4"
+#     "folk_emp gain exp1_mar3 gandalf_clf 5"
+#     "folk_emp gain exp1_mar3 gandalf_clf 6"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 1"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 2"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 3"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 4"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 5"
+#     "folk_emp gain exp1_mnar3 gandalf_clf 6"
+#     "folk_emp gain mixed_exp gandalf_clf 1"
+#     "folk_emp gain mixed_exp gandalf_clf 2"
+#     "folk_emp gain mixed_exp gandalf_clf 3"
+#     "folk_emp gain mixed_exp gandalf_clf 4"
+#     "folk_emp gain mixed_exp gandalf_clf 5"
+#     "folk_emp gain mixed_exp gandalf_clf 6"
+#
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 1"
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 2"
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 3"
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 4"
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 5"
+#     "folk_emp tdm exp1_mcar3 gandalf_clf 6"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 1"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 2"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 3"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 4"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 5"
+#     "folk_emp tdm exp1_mar3 gandalf_clf 6"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 1"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 2"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 3"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 4"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 5"
+#     "folk_emp tdm exp1_mnar3 gandalf_clf 6"
+#     "folk_emp tdm mixed_exp gandalf_clf 1"
+#     "folk_emp tdm mixed_exp gandalf_clf 2"
+#     "folk_emp tdm mixed_exp gandalf_clf 3"
+#     "folk_emp tdm mixed_exp gandalf_clf 4"
+#     "folk_emp tdm mixed_exp gandalf_clf 5"
+#     "folk_emp tdm mixed_exp gandalf_clf 6"
+#
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 1"
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 2"
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 3"
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 4"
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 5"
+#     "folk_emp nomi exp1_mcar3 gandalf_clf 6"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 1"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 2"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 3"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 4"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 5"
+#     "folk_emp nomi exp1_mar3 gandalf_clf 6"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 1"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 2"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 3"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 4"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 5"
+#     "folk_emp nomi exp1_mnar3 gandalf_clf 6"
+#     "folk_emp nomi mixed_exp gandalf_clf 1"
+#     "folk_emp nomi mixed_exp gandalf_clf 2"
+#     "folk_emp nomi mixed_exp gandalf_clf 3"
+#     "folk_emp nomi mixed_exp gandalf_clf 4"
+#     "folk_emp nomi mixed_exp gandalf_clf 5"
+#     "folk_emp nomi mixed_exp gandalf_clf 6"
 )
 
 TEMPLATE_FILE="../cluster/run_exp1/run-exp1-models-template.sbatch"

source/utils/model_tuning_utils.py

@@ -172,8 +172,8 @@ def validate_pytorch_tabular_model(model_config: ModelConfig, optimizer_config:
             validation=val,
             search_space=search_space,
             strategy="random_search",
-            # n_trials=25 if base_flow_dataset.X_train_val.shape[0] > 20_000 or 'folk' in saved_models_prefix else 100,
-            n_trials=100,
+            n_trials=25 if base_flow_dataset.X_train_val.shape[0] > 20_000 or 'folk' in saved_models_prefix else 100,
+            # n_trials=100,
             metric=macro_f1_score,
             mode="max",
             progress_bar=True,