different optimizer

Author: lazyprogrammer

rl2/cartpole/dqn_tf.py

@@ -74,13 +74,13 @@ def __init__(self, D, K, hidden_layer_sizes, gamma, max_experiences=10000, min_e
     )
 
     cost = tf.reduce_sum(tf.square(self.G - selected_action_values))
-    # self.train_op = tf.train.AdamOptimizer(10e-3).minimize(cost)
-    self.train_op = tf.train.AdagradOptimizer(10e-3).minimize(cost)
+    self.train_op = tf.train.AdamOptimizer(10e-3).minimize(cost)
+    # self.train_op = tf.train.AdagradOptimizer(10e-3).minimize(cost)
     # self.train_op = tf.train.MomentumOptimizer(10e-4, momentum=0.9).minimize(cost)
     # self.train_op = tf.train.GradientDescentOptimizer(10e-5).minimize(cost)
 
     # create replay memory
-    self.experience = {'s': [], 'a': [], 'r': [], 's2': []}
+    self.experience = {'s': [], 'a': [], 'r': [], 's2': [], 'done': []}
     self.max_experiences = max_experiences
     self.min_experiences = min_experiences
     self.batch_sz = batch_sz
@@ -118,8 +118,9 @@ def train(self, target_network):
     actions = [self.experience['a'][i] for i in idx]
     rewards = [self.experience['r'][i] for i in idx]
     next_states = [self.experience['s2'][i] for i in idx]
+    dones = [self.experience['done'][i] for i in idx]
     next_Q = np.max(target_network.predict(next_states), axis=1)
-    targets = [r + self.gamma*next_q for r, next_q in zip(rewards, next_Q)]
+    targets = [r + self.gamma*next_q if not done else r for r, next_q, done in zip(rewards, next_Q, dones)]
 
     # call optimizer
     self.session.run(
@@ -131,16 +132,18 @@ def train(self, target_network):
       }
     )
 
-  def add_experience(self, s, a, r, s2):
+  def add_experience(self, s, a, r, s2, done):
     if len(self.experience['s']) >= self.max_experiences:
       self.experience['s'].pop(0)
       self.experience['a'].pop(0)
       self.experience['r'].pop(0)
       self.experience['s2'].pop(0)
+      self.experience['done'].pop(0)
     self.experience['s'].append(s)
     self.experience['a'].append(a)
     self.experience['r'].append(r)
     self.experience['s2'].append(s2)
+    self.experience['done'].append(done)
 
   def sample_action(self, x, eps):
     if np.random.random() < eps:
@@ -167,7 +170,7 @@ def play_one(env, model, tmodel, eps, gamma, copy_period):
       reward = -200
 
     # update the model
-    model.add_experience(prev_observation, action, reward, observation)
+    model.add_experience(prev_observation, action, reward, observation, done)
     model.train(tmodel)
 
     iters += 1