another attempt at the MOG prior

Author: DeNeutoy

bayesian_rnn.py

@@ -5,7 +5,7 @@
 from tensorflow.contrib.rnn import static_rnn, LSTMStateTuple
 
 from stochastic_variables import get_random_normal_variable, ExternallyParameterisedLSTM
-from stochastic_variables import log_gaussian_mixture_sample_probabilities, log_gaussian_sample_probabilities
+from stochastic_variables import gaussian_mixture_nll
 import logging
 
 logger = logging.getLogger(__name__)
@@ -145,21 +145,17 @@ def build_rnn(self):
                                   [softmax_b, softmax_b_mean, softmax_b_std]]:
 
             # TODO(Mark): get this to work with the MOG prior using sampling.
-            # bernoulli_samples = tf.floor(0.8 + tf.random_uniform(tf.shape(weight), minval=0.0, maxval=1.0))
-            # mean1 = mean2 = tf.zeros_like(mean)
-            # # Very pointy one:
-            # std1 = 0.0009 * tf.ones_like(std)
-            # # Flatter one:
-            # std2 = 0.15 * tf.ones_like(std)
-            # phi_log_probs = log_gaussian_sample_probabilities(weight, mean, std)
-            # phi_mixture_log_probs = \
-            #     log_gaussian_mixture_sample_probabilities(weight, bernoulli_samples, mean1, mean2, std1, std2)
-            # kl = tf.exp(phi_log_probs) * (phi_log_probs - phi_mixture_log_probs)
-            # phi_kl += kl
+            mean1 = mean2 = tf.zeros_like(mean)
+            # Very pointy one:
+            std1 = 0.0009 * tf.ones_like(std)
+            # Flatter one:
+            std2 = 0.15 * tf.ones_like(std)
+            phi_mixture_nll = gaussian_mixture_nll(weight, [0.6, 0.4], mean1, mean2, std1, std2)
+            phi_kl += phi_mixture_nll
 
             # This is different from the paper - just using a univariate gaussian
             # prior so that the KL has a closed form.
-            phi_kl += self.compute_kl_divergence((mean, std), (tf.zeros_like(mean), tf.ones_like(std) * 0.01))
+            #phi_kl += self.compute_kl_divergence((mean, std), (tf.zeros_like(mean), tf.ones_like(std) * 0.01))
 
         tf.summary.scalar("phi_kl", phi_kl)
 

stochastic_variables.py

@@ -7,29 +7,17 @@
 from tensorflow.contrib.rnn.python.ops.core_rnn_cell_impl import _checked_scope
 
 
-def log_gaussian_sample_probabilities(samples, mean, std):
+def gaussian_mixture_nll(samples, mixing_weights, mean1, mean2, std1, std2):
     """
-    Computes the log probability that the samples were drawn from a gaussian distribution
-    with the given mean and standard deviation.
-    """
-    pi_sigma = - 0.5 * tf.log(2.0 * std * math.pi)
-    mean_shift = tf.square(samples - mean) / (2.0 * std)
-
-    return pi_sigma - mean_shift
-
-
-def log_gaussian_mixture_sample_probabilities(samples, bernouli_samples, mean1, mean2, std1, std2):
-    """
-    Computes the log probability that the samples were drawn from a mixture of two gaussian distributions
-    with the given means and standard deviations, along with precomputed bernouli samples to compute the
-    mixture.
+    Computes the NLL from a mixture of two gaussian distributions with the given
+    means and standard deviations, mixing weights and samples.
     """
     gaussian1 = (1.0/tf.sqrt(2.0 * std1 * math.pi)) * tf.exp(- tf.square(samples - mean1) / (2.0 * std1))
     gaussian2 = (1.0/tf.sqrt(2.0 * std2 * math.pi)) * tf.exp(- tf.square(samples - mean2) / (2.0 * std2))
 
-    mixture = bernouli_samples * gaussian1 + (1.0 - bernouli_samples) * gaussian2
+    mixture = (mixing_weights[0] * gaussian1) + (mixing_weights[1] * gaussian2)
 
-    return tf.log(mixture)
+    return - tf.log(mixture)
 
 
 def get_random_normal_variable(name, mean, standard_dev, shape, dtype):