Optimize actor before the critic (#109)

The actor should be optimized first. Otherwise, the weights of the cirtic would have been changed when back propagating the gradients for the actor. The latter will create an error in pytorch 1.5.0

Author: Xingyu Lin

rlkit/torch/sac/sac.py

@@ -132,6 +132,10 @@ def train_from_torch(self, batch):
         """
         Update networks
         """
+        self.policy_optimizer.zero_grad()
+        policy_loss.backward()
+        self.policy_optimizer.step()
+        
         self.qf1_optimizer.zero_grad()
         qf1_loss.backward()
         self.qf1_optimizer.step()
@@ -140,10 +144,6 @@ def train_from_torch(self, batch):
         qf2_loss.backward()
         self.qf2_optimizer.step()
 
-        self.policy_optimizer.zero_grad()
-        policy_loss.backward()
-        self.policy_optimizer.step()
-
         """
         Soft Updates
         """