avoid 0

Author: morvanzhou

contents/12_Proximal_Policy_Optimization/discrete_DPPO.py

@@ -61,7 +61,7 @@ def __init__(self):
         a_indices = tf.stack([tf.range(tf.shape(self.tfa)[0], dtype=tf.int32), self.tfa], axis=1)
         pi_prob = tf.gather_nd(params=self.pi, indices=a_indices)   # shape=(None, )
         oldpi_prob = tf.gather_nd(params=oldpi, indices=a_indices)  # shape=(None, )
-        ratio = pi_prob/oldpi_prob
+        ratio = pi_prob/(oldpi_prob + 1e-5)
         surr = ratio * self.tfadv                       # surrogate loss
 
         self.aloss = -tf.reduce_mean(tf.minimum(        # clipped surrogate objective

contents/12_Proximal_Policy_Optimization/simply_PPO.py

@@ -60,7 +60,7 @@ def __init__(self):
         with tf.variable_scope('loss'):
             with tf.variable_scope('surrogate'):
                 # ratio = tf.exp(pi.log_prob(self.tfa) - oldpi.log_prob(self.tfa))
-                ratio = pi.prob(self.tfa) / oldpi.prob(self.tfa)
+                ratio = pi.prob(self.tfa) / (oldpi.prob(self.tfa) + 1e-5)
                 surr = ratio * self.tfadv
             if METHOD['name'] == 'kl_pen':
                 self.tflam = tf.placeholder(tf.float32, None, 'lambda')