Feat: Apex Dqn

Author: choru-k

README.md

@@ -22,20 +22,22 @@ So you can run this example in your computer(maybe it take just only 1~2 minitue
 - [x] GAE(Generalized Advantage Estimation) [[12]](#reference)
 - [x] TNPG [[20]](#reference)
 - [x] TRPO [[13]](#reference)
-- [ ] PPO [[14]](#reference)
+- [x] PPO [[14]](#reference)
 
 ## Parallel
 - [x] Asynchronous Q-learning [[11]](#reference)
 - [x] A3C (Asynchronous Advange Actor Critice) [[11]](#reference)
 - [x] ACER [[21]](#reference)
-- [ ] APE-X [[15]](#reference)
+- [ ] PPO [[14]](#reference)
+- [x] APE-X DQN [[15]](#reference)
+- [ ] IMPALA [[23]](#reference)
+- [ ] R2D2 [[16]](#reference)
 
 ## Will
-- [ ] R2D2 [[16]](#reference)
 - [ ] RND [[17]](#reference)
+- [ ] ICM [[22]](#refercence)
 - [ ] QRDQN [[18]](#reference)
 - [ ] IQN [[19]](#reference)
-- [ ] PAAC
 
 
 ## Reference
@@ -60,6 +62,8 @@ So you can run this example in your computer(maybe it take just only 1~2 minitue
 [19][Implicit Quantile Networks for Distributional Reinforcement Learning](https://arxiv.org/pdf/1806.06923.pdf)  
 [20][A Natural Policy Gradient](https://papers.nips.cc/paper/2073-a-natural-policy-gradient.pdf)  
 [21][SAMPLE EFFICIENT ACTOR-CRITIC WITH EXPERIENCE REPLAY](https://arxiv.org/pdf/1611.01224.pdf)  
+[22][Curiosity-driven Exploration by Self-supervised Prediction](https://arxiv.org/pdf/1705.05363.pdf)
+[23][IMPALA: Scalable Distributed Deep-RL with Importance Weighted Actor-Learner Architectures](https://arxiv.org/pdf/1802.01561.pdf)
 
 
 ## Acknowledgements

parallel/1-Async-Q-Learning/worker.py

@@ -43,7 +43,6 @@ def get_action(self, state, epsilon):
             return self.target_net.get_action(state)
 
     def run(self):
-        running_score = 0
         epsilon = 1.0
         steps = 0
         while self.global_ep.value < max_episode:
@@ -84,7 +83,7 @@ def run(self):
                     loss = QNet.train_model(self.online_net, self.target_net, self.optimizer, batch)
                     memory = Memory(async_update_step)
                     if done:
-                        running_score = self.record(score, epsilon, loss)
+                        self.record(score, epsilon, loss)
                         break
                 if steps % update_target == 0:
                     self.update_target_model()

parallel/5-ApeX/config.py

@@ -0,0 +1,19 @@
+import torch
+
+env_name = 'CartPole-v1'
+gamma = 0.99
+lr = 0.002
+goal_score = 200
+log_interval = 10
+max_episode = 30000
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+
+replay_memory_capacity = 10000
+n_step = 3 
+local_mini_batch = 32
+batch_size = 32
+alpha = 0.5
+beta = 0.4

parallel/5-ApeX/memory.py

@@ -0,0 +1,98 @@
+import random
+import numpy as np
+import torch
+from collections import namedtuple, deque
+
+from config import gamma, batch_size, alpha, beta
+
+Transition = namedtuple('Transition', ('state', 'next_state', 'action', 'reward', 'mask', 'step'))
+
+class N_Step_Buffer(object):
+    def __init__(self): 
+        self.memory = []
+        self.step = 0
+
+    def push(self, state, next_state, action, reward, mask):
+        self.step += 1
+        self.memory.append([state, next_state, action, reward, mask])        
+
+    def sample(self):
+        [state, _, action, _, _] = self.memory[0]
+        [_, next_state, _, _, mask] = self.memory[-1]
+
+        sum_reward = 0
+        for t in reversed(range(len(self.memory))):
+            [_, _, _, reward, _] = self.memory[t]
+            sum_reward += reward + gamma * sum_reward
+        reward = sum_reward
+        step = self.step
+        self.reset()
+
+        return [state, next_state, action, reward, mask, step]
+
+    def reset(self):
+        self.memory = []
+        self.step = 0
+    
+    def __len__(self):
+        return len(self.memory)
+
+
+class LocalBuffer(object):
+    def __init__(self):
+        self.memory = []
+    
+    def push(self, state, next_state, action, reward, mask, step):
+        self.memory.append(Transition(state, next_state, action, reward, mask, step))
+    
+    def sample(self):
+        transitions = self.memory
+        batch = Transition(*zip(*transitions))
+        return batch
+    
+    def reset(self):
+        self.memory = []
+    
+    def __len__(self):
+        return len(self.memory)
+
+class Memory(object):
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.memory = deque(maxlen=capacity)
+        self.memory_probability = deque(maxlen=capacity)
+    
+    def push(self, state, next_state, action, reward, mask, step, prior):
+        self.memory.append(Transition(state, next_state, action, reward, mask, step))
+        self.memory_probability.append(prior)
+
+    def sample(self):
+        probaility = torch.Tensor(self.memory_probability)
+        probaility = probaility.pow(alpha)
+        probaility = probaility / probaility.sum()
+
+        p = probaility.numpy()
+
+        indexes = np.random.choice(range(len(self.memory_probability)), batch_size, p=p)
+        
+        transitions = [self.memory[idx] for idx in indexes]
+        transitions_p = torch.Tensor([self.memory_probability[idx] for idx in indexes])
+        
+        batch = Transition(*zip(*transitions))
+
+        weights = (self.capacity * transitions_p).pow(-beta)
+        weights = weights / weights.max()
+
+        return indexes, batch, weights
+
+    def update_prior(self, indexes, priors):
+        priors_idx = 0
+        for idx in indexes:
+            self.memory_probability[idx] = priors[priors_idx]
+            priors_idx += 1
+    
+    def __len__(self):
+        return len(self.memory)
+        
+    
+    

parallel/5-ApeX/model.py

@@ -0,0 +1,40 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+class Model(nn.Module):
+    def __init__(self, num_inputs, num_outputs):
+        super(Model, self).__init__()
+        self.num_inputs = num_inputs
+        self.num_outputs = num_outputs
+
+        self.fc = nn.Linear(num_inputs, 128)
+        self.fc_adv = nn.Linear(128, num_outputs)
+        self.fc_val = nn.Linear(128, 1)
+
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.xavier_uniform(m.weight)
+
+    def forward(self, x):
+        x = F.relu(self.fc(x))
+        adv = self.fc_adv(x)
+        adv = adv.view(-1, self.num_outputs)
+        val = self.fc_val(x)
+        val = val.view(-1, 1)
+
+        qvalue = val + (adv - adv.mean(dim=1, keepdim=True))
+        return qvalue
+
+class LocalModel(Model):
+    def __init__(self, num_inputs, num_outputs):
+        super(LocalModel, self).__init__(num_inputs, num_outputs)
+
+    def pull_from_global_model(self, global_model):
+        self.load_state_dict(global_model.state_dict())
+
+    def get_action(self, input):
+        qvalue = self.forward(input)
+        _, action = torch.max(qvalue, 1)
+        return action.numpy()[0]

parallel/5-ApeX/train.py

@@ -0,0 +1,58 @@
+import gym
+import torch
+
+from model import Model
+from worker import Actor, Learner
+import torch.multiprocessing as mp
+from tensorboardX import SummaryWriter
+
+from memory import Memory
+from config import env_name, lr, replay_memory_capacity
+
+def main():
+    env = gym.make(env_name)
+    env.seed(500)
+    torch.manual_seed(500)
+
+    num_inputs = env.observation_space.shape[0]
+    num_actions = env.action_space.n
+    env.close()
+
+    global_target_model = Model(num_inputs, num_actions)
+    global_online_model = Model(num_inputs, num_actions)
+    global_target_model.train()
+    global_online_model.train()
+    
+    global_target_model.load_state_dict(global_online_model.state_dict())
+    global_target_model.share_memory()
+    global_online_model.share_memory()
+    
+    global_memory = Memory(replay_memory_capacity)
+    
+    
+    global_ep, global_ep_r, res_queue, global_memory_pipe = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue(), mp.Queue()
+
+    writer = SummaryWriter('logs')
+
+    n = 2 
+    epsilons = [(i * 0.05 + 0.1) for i in range(n)]
+
+    actors = [Actor(global_target_model, global_memory_pipe, global_ep, global_ep_r, epsilons[i], i) for i in range(n)]
+    [w.start() for w in actors]
+    learner = Learner(global_online_model, global_target_model, global_memory, global_memory_pipe, res_queue)
+    learner.start()
+
+    res = []
+    while True:
+        r = res_queue.get()
+        if r is not None:
+            res.append(r)
+            [ep, loss] = r
+            # writer.add_scalar('log/score', float(ep_r), ep)
+            writer.add_scalar('log/loss', float(loss), ep)
+        else:
+            break
+    [w.join() for w in actors]
+
+if __name__=="__main__":
+    main()