Prettify TensorBoard scalars and histogram summaries

edward/inferences/gan_inference.py

@@ -103,9 +103,9 @@ def initialize(self, optimizer=None, optimizer_d=None,
 
     if self.logging:
       summary_key = 'summaries_' + str(id(self))
-      tf.summary.scalar('loss_discriminative', self.loss_d,
+      tf.summary.scalar("loss/discriminative", self.loss_d,
                         collections=[summary_key])
-      tf.summary.scalar('loss_generative', self.loss,
+      tf.summary.scalar("loss/generative", self.loss,
                         collections=[summary_key])
       self.summarize = tf.summary.merge_all(key=summary_key)
 
@@ -120,7 +120,8 @@ def build_loss_and_gradients(self, var_list):
 
     if self.logging:
       summary_key = 'summaries_' + str(id(self))
-      tf.summary.histogram('disc_outputs', tf.concat(d_true, d_fake, axis=0),
+      tf.summary.histogram("discriminator_outputs",
+                           tf.concat(d_true, d_fake, axis=0),
                            collections=[summary_key])
 
     loss_d = tf.nn.sigmoid_cross_entropy_with_logits(

edward/inferences/inference.py

@@ -159,7 +159,7 @@ def initialize(self, n_iter=1000, n_print=None, scale=None, logdir=None,
       Number of iterations for algorithm.
     n_print : int, optional
       Number of iterations for each print progress. To suppress print
-      progress, then specify 0. Default is ``int(n_iter / 10)``.
+      progress, then specify 0. Default is ``int(n_iter / 100)``.
     scale : dict of RandomVariable to tf.Tensor, optional
       A tensor to scale computation for any random variable that it is
       binded to. Its shape must be broadcastable; it is multiplied
@@ -176,7 +176,7 @@ def initialize(self, n_iter=1000, n_print=None, scale=None, logdir=None,
     """
     self.n_iter = n_iter
     if n_print is None:
-      self.n_print = int(n_iter / 10)
+      self.n_print = int(n_iter / 100)
     else:
       self.n_print = n_print
 

edward/inferences/klpq.py

@@ -131,8 +131,10 @@ def build_loss_and_gradients(self, var_list):
 
     if self.logging:
       summary_key = 'summaries_' + str(id(self))
-      tf.summary.histogram("p_log_prob", p_log_prob, collections=[summary_key])
-      tf.summary.histogram("q_log_prob", q_log_prob, collections=[summary_key])
+      tf.summary.scalar("loss/p_log_prob", tf.reduce_mean(p_log_prob),
+                        collections=[summary_key])
+      tf.summary.scalar("loss/q_log_prob", tf.reduce_mean(q_log_prob),
+                        collections=[summary_key])
 
     log_w = p_log_prob - q_log_prob
     log_w_norm = log_w - tf.reduce_logsumexp(log_w)

edward/inferences/klqp.py

@@ -388,15 +388,15 @@ def build_reparam_loss_and_gradients(inference, var_list):
         p_log_prob[s] += tf.reduce_sum(
             inference.scale.get(x, 1.0) * x_copy.log_prob(dict_swap[x]))
 
-  p_log_prob = tf.stack(p_log_prob)
-  q_log_prob = tf.stack(q_log_prob)
+  p_log_prob = tf.reduce_mean(p_log_prob)
+  q_log_prob = tf.reduce_mean(q_log_prob)
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram("p_log_prob", p_log_prob, collections=[summary_key])
-    tf.summary.histogram("q_log_prob", q_log_prob, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_prob", p_log_prob, collections=[summary_key])
+    tf.summary.scalar("loss/q_log_prob", q_log_prob, collections=[summary_key])
 
-  loss = -tf.reduce_mean(p_log_prob - q_log_prob)
+  loss = -(p_log_prob - q_log_prob)
 
   grads = tf.gradients(loss, var_list)
   grads_and_vars = list(zip(grads, var_list))
@@ -444,18 +444,18 @@ def build_reparam_kl_loss_and_gradients(inference, var_list):
         p_log_lik[s] += tf.reduce_sum(
             inference.scale.get(x, 1.0) * x_copy.log_prob(dict_swap[x]))
 
-  p_log_lik = tf.stack(p_log_lik)
+  p_log_lik = tf.reduce_mean(p_log_lik)
 
-  kl = tf.reduce_sum([
+  kl_penalty = tf.reduce_sum([
       inference.kl_scaling.get(z, 1.0) * tf.reduce_sum(ds.kl(qz, z))
       for z, qz in six.iteritems(inference.latent_vars)])
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram('p_log_lik', p_log_lik, collections=[summary_key])
-    tf.summary.scalar('kl', kl, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_lik", p_log_lik, collections=[summary_key])
+    tf.summary.scalar("loss/kl_penalty", kl_penalty, collections=[summary_key])
 
-  loss = -(tf.reduce_mean(p_log_lik) - kl)
+  loss = -(p_log_lik - kl_penalty)
 
   grads = tf.gradients(loss, var_list)
   grads_and_vars = list(zip(grads, var_list))
@@ -508,17 +508,17 @@ def build_reparam_entropy_loss_and_gradients(inference, var_list):
         p_log_prob[s] += tf.reduce_sum(
             inference.scale.get(x, 1.0) * x_copy.log_prob(dict_swap[x]))
 
-  p_log_prob = tf.stack(p_log_prob)
+  p_log_prob = tf.reduce_mean(p_log_prob)
 
   q_entropy = tf.reduce_sum([
       qz.entropy() for z, qz in six.iteritems(inference.latent_vars)])
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram('p_log_prob', p_log_prob, collections=[summary_key])
-    tf.summary.scalar('q_entropy', q_entropy, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_prob", p_log_prob, collections=[summary_key])
+    tf.summary.scalar("loss/q_entropy", q_entropy, collections=[summary_key])
 
-  loss = -(tf.reduce_mean(p_log_prob) + q_entropy)
+  loss = -(p_log_prob + q_entropy)
 
   grads = tf.gradients(loss, var_list)
   grads_and_vars = list(zip(grads, var_list))
@@ -571,8 +571,10 @@ def build_score_loss_and_gradients(inference, var_list):
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram('p_log_prob', p_log_prob, collections=[summary_key])
-    tf.summary.scalar('q_log_prob', q_log_prob, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_prob", tf.reduce_mean(p_log_prob),
+                      collections=[summary_key])
+    tf.summary.scalar("loss/q_log_prob", tf.reduce_mean(q_log_prob),
+                      collections=[summary_key])
 
   losses = p_log_prob - q_log_prob
   loss = -tf.reduce_mean(losses)
@@ -625,19 +627,19 @@ def build_score_kl_loss_and_gradients(inference, var_list):
   p_log_lik = tf.stack(p_log_lik)
   q_log_prob = tf.stack(q_log_prob)
 
-  kl = tf.reduce_sum([
+  kl_penalty = tf.reduce_sum([
       inference.kl_scaling.get(z, 1.0) * tf.reduce_sum(ds.kl(qz, z))
       for z, qz in six.iteritems(inference.latent_vars)])
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram('p_log_lik', p_log_lik, collections=[summary_key])
-    tf.summary.histogram('q_log_prob', q_log_prob, collections=[summary_key])
-    tf.summary.scalar('kl', kl, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_lik", tf.reduce_mean(p_log_lik),
+                      collections=[summary_key])
+    tf.summary.scalar("loss/kl_penalty", kl_penalty, collections=[summary_key])
 
-  loss = -(tf.reduce_mean(p_log_lik) - kl)
+  loss = -(tf.reduce_mean(p_log_lik) - kl_penalty)
   grads = tf.gradients(
-      -(tf.reduce_mean(q_log_prob * tf.stop_gradient(p_log_lik)) - kl),
+      -(tf.reduce_mean(q_log_prob * tf.stop_gradient(p_log_lik)) - kl_penalty),
       var_list)
   grads_and_vars = list(zip(grads, var_list))
   return loss, grads_and_vars
@@ -694,9 +696,11 @@ def build_score_entropy_loss_and_gradients(inference, var_list):
 
   if inference.logging:
     summary_key = 'summaries_' + str(id(inference))
-    tf.summary.histogram('p_log_prob', p_log_prob, collections=[summary_key])
-    tf.summary.histogram('q_log_prob', q_log_prob, collections=[summary_key])
-    tf.summary.scalar('q_entropy', q_entropy, collections=[summary_key])
+    tf.summary.scalar("loss/p_log_prob", tf.reduce_mean(p_log_prob),
+                      collections=[summary_key])
+    tf.summary.scalar("loss/q_log_prob", tf.reduce_mean(q_log_prob),
+                      collections=[summary_key])
+    tf.summary.scalar("loss/q_entropy", q_entropy, collections=[summary_key])
 
   loss = -(tf.reduce_mean(p_log_prob) + q_entropy)
   grads = tf.gradients(

edward/inferences/monte_carlo.py

@@ -99,7 +99,7 @@ def initialize(self, *args, **kwargs):
 
     if self.logging:
       summary_key = 'summaries_' + str(id(self))
-      tf.summary.scalar('n_accept', self.n_accept, collections=[summary_key])
+      tf.summary.scalar("n_accept", self.n_accept, collections=[summary_key])
       self.summarize = tf.summary.merge_all(key=summary_key)
 
   def update(self, feed_dict=None):

edward/inferences/variational_inference.py

@@ -76,44 +76,17 @@ def initialize(self, optimizer=None, var_list=None, use_prettytensor=False,
     if self.logging:
       summary_key = 'summaries_' + str(id(self))
       tf.summary.scalar("loss", self.loss, collections=[summary_key])
-      with tf.name_scope('variational'):
-        for grad, var in grads_and_vars:
-          if var in latent_var_list:
-            tf.summary.histogram("parameter_" +
-                                 var.name.replace(':', '_'),
-                                 var, collections=[summary_key])
-            tf.summary.histogram("gradient_" +
-                                 var.name.replace(':', '_'),
-                                 grad, collections=[summary_key])
-            tf.summary.scalar("gradient_norm_" +
-                              var.name.replace(':', '_'),
-                              tf.norm(grad), collections=[summary_key])
-      # replace : with _ because tf does not allow : in var names in summaries
-
-      with tf.name_scope('model'):
-        for grad, var in grads_and_vars:
-          if var in data_var_list:
-            tf.summary.histogram("parameter_" + var.name.replace(':', '_'),
-                                 var, collections=[summary_key])
-            tf.summary.histogram("gradient_" +
-                                 var.name.replace(':', '_'),
-                                 grad, collections=[summary_key])
-            tf.summary.scalar("gradient_norm_" +
-                              var.name.replace(':', '_'),
-                              tf.norm(grad), collections=[summary_key])
-
-      # when var_list is not initialized with None
-      with tf.name_scope(''):
-        for grad, var in grads_and_vars:
-          if var not in latent_var_list and var not in data_var_list:
-            tf.summary.histogram("parameter_" + var.name.replace(':', '_'),
-                                 var, collections=[summary_key])
-            tf.summary.histogram("gradient_" +
-                                 var.name.replace(':', '_'),
-                                 grad, collections=[summary_key])
-            tf.summary.scalar("gradient_norm_" +
-                              var.name.replace(':', '_'),
-                              tf.norm(grad), collections=[summary_key])
+      for grad, var in grads_and_vars:
+        # replace colons which are an invalid character
+        tf.summary.histogram("parameter/" +
+                             var.name.replace(':', '/'),
+                             var, collections=[summary_key])
+        tf.summary.histogram("gradient/" +
+                             var.name.replace(':', '/'),
+                             grad, collections=[summary_key])
+        tf.summary.scalar("gradient_norm/" +
+                          var.name.replace(':', '/'),
+                          tf.norm(grad), collections=[summary_key])
 
       self.summarize = tf.summary.merge_all(key=summary_key)
 

examples/bayesian_nn.py

@@ -3,6 +3,8 @@
 (see, e.g., Blundell et al. (2015); Kucukelbir et al. (2016)).
 
 Inspired by autograd's Bayesian neural network example.
+This example prettifies some of the tensor naming for visualization in
+TensorBoard. To view TensorBoard, run `tensorboard --logdir=log`.
 
 References
 ----------
@@ -45,31 +47,45 @@ def neural_network(X):
 X_train, y_train = build_toy_dataset(N)
 
 # MODEL
-W_0 = Normal(loc=tf.zeros([D, 10]), scale=tf.ones([D, 10]))
-W_1 = Normal(loc=tf.zeros([10, 10]), scale=tf.ones([10, 10]))
-W_2 = Normal(loc=tf.zeros([10, 1]), scale=tf.ones([10, 1]))
-b_0 = Normal(loc=tf.zeros(10), scale=tf.ones(10))
-b_1 = Normal(loc=tf.zeros(10), scale=tf.ones(10))
-b_2 = Normal(loc=tf.zeros(1), scale=tf.ones(1))
+with tf.name_scope("model"):
+  W_0 = Normal(loc=tf.zeros([D, 10]), scale=tf.ones([D, 10]), name="W_0")
+  W_1 = Normal(loc=tf.zeros([10, 10]), scale=tf.ones([10, 10]), name="W_1")
+  W_2 = Normal(loc=tf.zeros([10, 1]), scale=tf.ones([10, 1]), name="W_2")
+  b_0 = Normal(loc=tf.zeros(10), scale=tf.ones(10), name="b_0")
+  b_1 = Normal(loc=tf.zeros(10), scale=tf.ones(10), name="b_1")
+  b_2 = Normal(loc=tf.zeros(1), scale=tf.ones(1), name="b_2")
 
-X = tf.placeholder(tf.float32, [N, D])
-y = Normal(loc=neural_network(X), scale=0.1 * tf.ones(N))
+  X = tf.placeholder(tf.float32, [N, D], name="X")
+  y = Normal(loc=neural_network(X), scale=0.1 * tf.ones(N), name="y")
 
 # INFERENCE
-qW_0 = Normal(loc=tf.Variable(tf.random_normal([D, 10])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([D, 10]))))
-qW_1 = Normal(loc=tf.Variable(tf.random_normal([10, 10])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([10, 10]))))
-qW_2 = Normal(loc=tf.Variable(tf.random_normal([10, 1])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([10, 1]))))
-qb_0 = Normal(loc=tf.Variable(tf.random_normal([10])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([10]))))
-qb_1 = Normal(loc=tf.Variable(tf.random_normal([10])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([10]))))
-qb_2 = Normal(loc=tf.Variable(tf.random_normal([1])),
-              scale=tf.nn.softplus(tf.Variable(tf.random_normal([1]))))
+with tf.name_scope("posterior"):
+  with tf.name_scope("qW_0"):
+    qW_0 = Normal(loc=tf.Variable(tf.random_normal([D, 10]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([D, 10]), name="scale")))
+  with tf.name_scope("qW_1"):
+    qW_1 = Normal(loc=tf.Variable(tf.random_normal([10, 10]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([10, 10]), name="scale")))
+  with tf.name_scope("qW_2"):
+    qW_2 = Normal(loc=tf.Variable(tf.random_normal([10, 1]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([10, 1]), name="scale")))
+  with tf.name_scope("qb_0"):
+    qb_0 = Normal(loc=tf.Variable(tf.random_normal([10]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([10]), name="scale")))
+  with tf.name_scope("qb_1"):
+    qb_1 = Normal(loc=tf.Variable(tf.random_normal([10]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([10]), name="scale")))
+  with tf.name_scope("qb_2"):
+    qb_2 = Normal(loc=tf.Variable(tf.random_normal([1]), name="loc"),
+                  scale=tf.nn.softplus(
+                      tf.Variable(tf.random_normal([1]), name="scale")))
 
 inference = ed.KLqp({W_0: qW_0, b_0: qb_0,
                      W_1: qW_1, b_1: qb_1,
                      W_2: qW_2, b_2: qb_2}, data={X: X_train, y: y_train})
-inference.run()
+inference.run(logdir='log')

examples/normal_normal_tensorboard.py

@@ -26,4 +26,4 @@
 
 # analytic solution: N(loc=0.0, scale=\sqrt{1/51}=0.140)
 inference = ed.KLqp({mu: qmu}, data={x: x_data})
-inference.run(logdir='train')
+inference.run(logdir='log')