[bug-fix] Fix non-LSTM SeparateActorCritic (#4306)

Author: Ervin T

ml-agents/mlagents/trainers/tests/torch/test_networks.py

@@ -188,7 +188,9 @@ def test_actor_critic(ac_type, lstm):
         )
     else:
         sample_obs = torch.ones((1, obs_size))
-        memories = None
+        memories = torch.tensor([])
+        # memories isn't always set to None, the network should be able to
+        # deal with that.
     # Test critic pass
     value_out = actor.critic_pass([sample_obs], [], memories=memories)
     for stream in stream_names:

ml-agents/mlagents/trainers/torch/networks.py

@@ -428,7 +428,7 @@ def critic_pass(
         vis_inputs: List[torch.Tensor],
         memories: Optional[torch.Tensor] = None,
     ) -> Dict[str, torch.Tensor]:
-        if memories is not None:
+        if self.use_lstm:
             # Use only the back half of memories for critic
             _, critic_mem = torch.split(memories, self.half_mem_size, -1)
         else:
@@ -446,7 +446,7 @@ def get_dist_and_value(
         memories: Optional[torch.Tensor] = None,
         sequence_length: int = 1,
     ) -> Tuple[List[DistInstance], Dict[str, torch.Tensor], torch.Tensor]:
-        if memories is not None:
+        if self.use_lstm:
             # Use only the back half of memories for critic and actor
             actor_mem, critic_mem = torch.split(memories, self.half_mem_size, dim=-1)
         else: