results: update GMM

.gitignore

@@ -1,4 +1,4 @@
 *.idea/
 *__pycache__/
 /examples/paper_results/gmm/models/
-/examples/paper_results/many_well/models/
+/examples/many_well/models/

README.md

@@ -22,7 +22,7 @@ python examples/gmm/run.py training.use_buffer=True training.prioritised_buffer=
 To run the full set of experiments see the [README](../examples/gmm/README.md) for the GMM experiments. 
 
 The below plot shows samples from various trained models, with the GMM problem target contours in the background.
-![Gaussian Mixture Model samples vs contours](./examples/paper_results/gmm/plots/MoG.png)
+![Gaussian Mixture Model samples vs contours](examples/gmm/plots/MoG.png)
 
 ### Many Well distribution
 The 32 Many Well distribution is made up of 16 repeats of the Double Well distribution, 
@@ -37,7 +37,7 @@ To run the full set of experiments see the [README](./examples/many_well/README.
 The below plot shows samples for our model (FAB) vs training a flow by reverse KL divergence 
 minimisation, with the Many Well problem target contours in the background. 
 This visualisation is for the marginal pairs of the distributions for the first four elements of the x.
-![Many Well distribution FAB vs training by KL divergence minimisation](./examples/paper_results/many_well/plots/many_well.png)
+![Many Well distribution FAB vs training by KL divergence minimisation](examples/many_well/plots/many_well.png)
 
 **Alanine Dipeptide distribution**
 

examples/gmm/README.md

@@ -0,0 +1,33 @@
+# GMM Problem
+## Experiments
+The following commands can each be used to train the methods from the paper:
+```
+# FAB with prioritised buffer.
+python examples/gmm/run.py training.seed=0,1,2 training.use_buffer=True training.prioritised_buffer=True 
+
+# FAB without the prioritised buffer.
+python examples/gmm/run.py training.seed=0,1,2 fab.loss_type=p2_over_q_alpha_2_div 
+
+# Flow using ground truth samples, training by maximum likelihood/forward KL divergence minimiation.
+python examples/gmm/run.py training.seed=0,1,2 fab.loss_type=target_forward_kl
+
+# Flow using alpha-divergence, with alpha=2
+python examples/gmm/run.py training.seed=0,1,2 fab.loss_type=flow_alpha_2_div_nis
+
+# Flow using reverse KL divergence
+python examples/gmm/run.py training.seed=0,1,2 fab.loss_type=flow_reverse_kld
+
+# SNF using reverse KLD
+python examples/gmm/run.py training.seed=0,1,2 flow.use_snf=True
+```
+
+**Further notes** This will use hydra-multirun to run the random seeds in parallel. 
+However, if you just want to run locally and get a general idea of the results, 
+you can run a single random seed for a much lower number of iterations. 
+The config file for this experiment is [here](../config/gmm.yaml), where you can change the hyper-parameters.
+
+## Evaluation
+Trained models may be evaluated using the code in
+[`evaluation.py`](evaluation.py) and [`evaluation_expectation_quadratic_func.py`](evaluation_expectation_quadratic_func.py).
+Furthermore [`results_vis.py`](results_vis.py) may be used to obtain the plot from the paper
+visualising each of the modes. 

examples/paper_results/gmm/evaluation.py → examples/gmm/evaluation.py

@@ -63,7 +63,7 @@ def evaluate(cfg: DictConfig, model_name: str, num_samples=int(1e4)):
     return eval
 
 
-@hydra.main(config_path="../../config", config_name="gmm.yaml")
+@hydra.main(config_path="../config", config_name="gmm.yaml")
 def main(cfg: DictConfig):
     model_names = ["fab_buffer", "fab_no_buffer", "flow_kld", "flow_nis", "target_kld", "snf"]
     seeds = [1, 2, 3]
@@ -89,7 +89,7 @@ def main(cfg: DictConfig):
     results.to_csv(open(FILENAME_EVAL_INFO, "w"))
 
 
-FILENAME_EVAL_INFO = "/home/laurence/work/code/FAB-TORCH/examples/paper_results/gmm/gmm_results.csv"
+FILENAME_EVAL_INFO = "/examples/paper_results/gmm/gmm_results.csv"
 
 if __name__ == '__main__':
     main()

examples/paper_results/gmm/evaluation_expectation_quadratic_func.py → examples/gmm/evaluation_expectation_quadratic_func.py

@@ -94,7 +94,7 @@ def evaluate(cfg: DictConfig, model_name: str, num_samples=int(1e3), n_repeats=1
     return info
 
 
-@hydra.main(config_path="../../config", config_name="gmm.yaml")
+@hydra.main(config_path="../config", config_name="gmm.yaml")
 def main(cfg: DictConfig):
     model_names = ["target", "fab_buffer", "fab_no_buffer", "flow_kld", "flow_nis", "target_kld", "snf"]
     seeds = [1, 2, 3]
@@ -118,7 +118,7 @@ def main(cfg: DictConfig):
     print(results.groupby("model_name").std()[fields])
     results.to_csv(open(FILENAME_EXPECTATION_INFO, "w"))
 
-FILENAME_EXPECTATION_INFO = "/home/laurence/work/code/FAB-TORCH/examples/paper_results/gmm/gmm_results_expectation.csv"
+FILENAME_EXPECTATION_INFO = "/examples/paper_results/gmm/gmm_results_expectation.csv"
 
 if __name__ == '__main__':
     if True:

examples/paper_results/gmm/latex_table.py → examples/gmm/latex_table.py

@@ -1,6 +1,6 @@
 import pandas as pd
-from examples.paper_results.gmm.evaluation import FILENAME_EVAL_INFO
-from examples.paper_results.gmm.evaluation_expectation_quadratic_func import FILENAME_EXPECTATION_INFO
+from examples.gmm.evaluation import FILENAME_EVAL_INFO
+from examples.gmm.evaluation_expectation_quadratic_func import FILENAME_EXPECTATION_INFO
 
 
 

examples/paper_results/gmm/results_vis.py → examples/gmm/results_vis.py

@@ -48,7 +48,7 @@ def plot_result(cfg: DictConfig, ax: plt.axes, target, model_name: Optional[str]
     plot_marginal_pair(samples_flow, ax=ax, bounds=plotting_bounds, alpha=alpha)
 
 
-@hydra.main(config_path="../../config", config_name="gmm.yaml")
+@hydra.main(config_path="../config", config_name="gmm.yaml")
 def run(cfg: DictConfig):
     appendix = True
     if appendix:

examples/paper_results/many_well/comparing_modes.py → examples/many_well/comparing_modes.py

@@ -8,7 +8,7 @@
 
 PATH = os.getcwd()
 
-@hydra.main(config_path="./", config_name="config.yaml")
+@hydra.main(config_path="/", config_name="config.yaml")
 def run(cfg: DictConfig):
     dim = cfg.target.dim
     target = ManyWellEnergy(cfg.target.dim, a=-0.5, b=-6, use_gpu=False)

examples/paper_results/many_well/evaluation.py → examples/many_well/evaluation.py

@@ -2,13 +2,13 @@
 from examples.setup_run_snf import make_normflow_snf_model, SNFModel
 
 from fab.target_distributions.many_well import ManyWellEnergy
-from examples.paper_results.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
+from examples.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
 import pandas as pd
 import os
 from omegaconf import DictConfig
 import torch
 
-from fab import FABModel, HamiltonianMonteCarlo, Metropolis
+from fab import FABModel, HamiltonianMonteCarlo
 from examples.make_flow import make_wrapped_normflowdist
 
 
@@ -87,7 +87,7 @@ def evaluate(cfg: DictConfig, model_name: str, target, num_samples=int(5e4)):
     return eval
 
 
-@hydra.main(config_path="../../config", config_name="many_well.yaml")
+@hydra.main(config_path="../config", config_name="many_well.yaml")
 def main(cfg: DictConfig):
     model_names = ["target_kld"] # ["fab_buffer", "fab_no_buffer", "flow_kld", "flow_nis", "snf"]
     seeds = [1, 2, 3]
@@ -121,7 +121,7 @@ def main(cfg: DictConfig):
     print("overall results")
     print(results[["model_name", "seed"] + keys])
 
-FILENAME_EVAL_INFO = "/home/laurence/work/code/FAB-TORCH/examples/paper_results/many_well/many_well_results_ml_only.csv"
+FILENAME_EVAL_INFO = "/examples/many_well/many_well_results_ml_only.csv"
 
 
 if __name__ == '__main__':

examples/paper_results/many_well/latex_table.py → examples/many_well/latex_table.py

@@ -1,5 +1,5 @@
 import pandas as pd
-from examples.paper_results.many_well.evaluation import FILENAME_EVAL_INFO
+from examples.many_well.evaluation import FILENAME_EVAL_INFO
 
 
 

examples/many_well.py → examples/many_well/many_well.py

@@ -69,7 +69,7 @@ def _run(cfg: DictConfig):
         setup_trainer_and_run_snf(cfg, setup_many_well_plotter, target)
 
 
-@hydra.main(config_path="./config", config_name="many_well.yaml")
+@hydra.main(config_path="../config", config_name="many_well.yaml")
 def run(cfg: DictConfig):
     _run(cfg)
 

examples/paper_results/many_well/results_vis.py → examples/many_well/results_vis.py

@@ -1,12 +1,11 @@
 import os
-from typing import Optional
 import hydra
 import matplotlib.pyplot as plt
 from matplotlib import rc
 import matplotlib as mpl
 from omegaconf import DictConfig
 from examples.make_flow import make_wrapped_normflowdist
-from examples.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
+from examples.many_well.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
 from fab.utils.plotting import plot_contours, plot_marginal_pair
 from fab.target_distributions.many_well import ManyWellEnergy
 import torch
@@ -55,7 +54,7 @@ def plot_marginals(cfg: DictConfig, supfig, model_name, plot_y_label):
 
 
 
-@hydra.main(config_path="./", config_name="config.yaml")
+@hydra.main(config_path="/", config_name="config.yaml")
 def run(cfg: DictConfig):
     mpl.rcParams['figure.dpi'] = 300
     rc('font', **{'family': 'serif', 'serif': ['Times']})

examples/paper_results/many_well/results_vis_appendix.py → examples/many_well/results_vis_appendix.py

@@ -1,15 +1,14 @@
 import os
-from typing import Optional
 import hydra
 import matplotlib.pyplot as plt
 from matplotlib import rc
 import matplotlib as mpl
 from omegaconf import DictConfig
 from examples.make_flow import make_wrapped_normflowdist
-from examples.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
+from examples.many_well.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
 from fab.utils.plotting import plot_contours, plot_marginal_pair
 from fab.target_distributions.many_well import ManyWellEnergy
-from examples.paper_results.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
+from examples.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
 import torch
 from examples.setup_run_snf import make_normflow_snf_model, SNFModel
 
@@ -82,7 +81,7 @@ def plot_marginals(cfg: DictConfig, supfig, model_name, plot_y_label):
 
 
 
-@hydra.main(config_path="./", config_name="config.yaml")
+@hydra.main(config_path="/", config_name="config.yaml")
 def run(cfg: DictConfig):
     mpl.rcParams['figure.dpi'] = 300
     rc('font', **{'family': 'serif', 'serif': ['Times']})

examples/paper_results/many_well/results_vis_appendix_init.py → examples/many_well/results_vis_appendix_init.py

@@ -1,15 +1,14 @@
 import os
-from typing import Optional
 import hydra
 import matplotlib.pyplot as plt
 from matplotlib import rc
 import matplotlib as mpl
 from omegaconf import DictConfig
 from examples.make_flow import make_wrapped_normflowdist
-from examples.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
+from examples.many_well.many_well_visualise_all_marginal_pairs import get_target_log_prob_marginal_pair
 from fab.utils.plotting import plot_contours, plot_marginal_pair
 from fab.target_distributions.many_well import ManyWellEnergy
-from examples.paper_results.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
+from examples.many_well.old_target_many_well import ManyWellEnergy as OldManyWellEnergy
 import torch
 from examples.setup_run_snf import make_normflow_snf_model, SNFModel
 
@@ -83,7 +82,7 @@ def plot_marginals(cfg: DictConfig, supfig, model_name, plot_y_label):
 
 
 
-@hydra.main(config_path="./", config_name="config.yaml")
+@hydra.main(config_path="/", config_name="config.yaml")
 def run(cfg: DictConfig):
     mpl.rcParams['figure.dpi'] = 300
     rc('font', **{'family': 'serif', 'serif': ['Times']})

examples/paper_results/many_well/testing_snf.py → examples/many_well/testing_snf.py

@@ -48,7 +48,7 @@ def evaluate(cfg: DictConfig, target, num_samples=int(100), log_prob_scale_facto
     return eval
 
 
-@hydra.main(config_path="../../config", config_name="many_well.yaml")
+@hydra.main(config_path="../config", config_name="many_well.yaml")
 def main(cfg: DictConfig):
     model_names = ["snf"]
     log_prob_scale_factors = [0, 1, 10, 10, 70]