moved actor, critic networks into model directory

Author: alvkao58

python/ray/rllib/ddpg/models.py

@@ -4,9 +4,9 @@
 
 import numpy as np
 import tensorflow as tf
-import tensorflow.contrib.slim as slim
 
 from ray.experimental.tfutils import TensorFlowVariables
+from ray.rllib.models.ddpgnet import DDPGActor, DDPGCritic
 from ray.rllib.ddpg.random_process import OrnsteinUhlenbeckProcess
 
 
@@ -202,50 +202,20 @@ def _setup_critic_loss(self, action_space):
     def _setup_critic_network(self, obs_space, ac_space):
         """Sets up Q network."""
         with tf.variable_scope("critic", reuse=tf.AUTO_REUSE):
-            self.critic_eval = self._create_critic_network(
-                              self.obs, self.act)
+            self.critic_network = DDPGCritic((self.obs, self.act), 1, {})
+            self.critic_eval = self.critic_network.outputs
 
         with tf.variable_scope("critic", reuse=True):
-            tf.get_variable_scope().reuse_variables()
-            self.cn_for_loss = self._create_critic_network(
-                              self.obs, self.output_action)
-
-    def _create_critic_network(self, obs, action):
-        """Network for critic."""
-        w_normal = tf.truncated_normal_initializer()
-        w_init = tf.random_uniform_initializer(minval=-0.0003, maxval=0.0003)
-        net = slim.fully_connected(
-             obs, 400, activation_fn=tf.nn.relu, weights_initializer=w_normal)
-        t1 = slim.fully_connected(
-            net, 300, activation_fn=None, biases_initializer=None,
-            weights_initializer=w_normal)
-        t2 = slim.fully_connected(
-            action, 300, activation_fn=None, weights_initializer=w_normal)
-        net = tf.nn.relu(tf.add(t1, t2))
-
-        out = slim.fully_connected(
-             net, 1, activation_fn=None, weights_initializer=w_init)
-        return out
+            self.cn_for_loss = DDPGCritic(
+                              (self.obs, self.output_action), 1, {}).outputs
 
     def _setup_actor_network(self, obs_space, ac_space):
         """Sets up actor network."""
         with tf.variable_scope("actor", reuse=tf.AUTO_REUSE):
-            self.output_action = self._create_actor_network(self.obs)
-
-    def _create_actor_network(self, obs):
-        """Network for actor."""
-        w_normal = tf.truncated_normal_initializer()
-        w_init = tf.random_uniform_initializer(minval=-0.003, maxval=0.003)
-
-        net = slim.fully_connected(
-             obs, 400, activation_fn=tf.nn.relu, weights_initializer=w_normal)
-        net = slim.fully_connected(
-             net, 300, activation_fn=tf.nn.relu, weights_initializer=w_normal)
-        out = slim.fully_connected(
-             net, self.ac_size, activation_fn=tf.nn.tanh,
-             weights_initializer=w_init)
-        scaled_out = tf.multiply(out, self.action_bound)
-        return scaled_out
+            self.actor_network = DDPGActor(
+                                self.obs, self.ac_size,
+                                options={"action_bound": self.action_bound})
+            self.output_action = self.actor_network.outputs
 
     def get_weights(self):
         """Returns critic weights, actor weights."""

python/ray/rllib/models/ddpgnet.py

@@ -0,0 +1,49 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+import tensorflow.contrib.slim as slim
+
+from ray.rllib.models.model import Model
+
+
+class DDPGActor(Model):
+    """Actor network for DDPG."""
+
+    def _init(self, inputs, num_outputs, options):
+        w_normal = tf.truncated_normal_initializer()
+        w_init = tf.random_uniform_initializer(minval=-0.003, maxval=0.003)
+        ac_bound = options["action_bound"]
+
+        net = slim.fully_connected(
+             inputs, 400, activation_fn=tf.nn.relu,
+             weights_initializer=w_normal)
+        net = slim.fully_connected(
+             net, 300, activation_fn=tf.nn.relu, weights_initializer=w_normal)
+        out = slim.fully_connected(
+             net, num_outputs, activation_fn=tf.nn.tanh,
+             weights_initializer=w_init)
+        scaled_out = tf.multiply(out, ac_bound)
+        return scaled_out, net
+
+
+class DDPGCritic(Model):
+    """Critic network for DDPG."""
+
+    def _init(self, inputs, num_outputs, options):
+        obs, action = inputs
+        w_normal = tf.truncated_normal_initializer()
+        w_init = tf.random_uniform_initializer(minval=-0.0003, maxval=0.0003)
+        net = slim.fully_connected(
+             obs, 400, activation_fn=tf.nn.relu, weights_initializer=w_normal)
+        t1 = slim.fully_connected(
+            net, 300, activation_fn=None, biases_initializer=None,
+            weights_initializer=w_normal)
+        t2 = slim.fully_connected(
+            action, 300, activation_fn=None, weights_initializer=w_normal)
+        net = tf.nn.relu(tf.add(t1, t2))
+
+        out = slim.fully_connected(
+             net, 1, activation_fn=None, weights_initializer=w_init)
+        return out, net