Tic tac toe can now learn, but slow.

Author: ManuelMeraz

rl/agents/agent_builder.py

@@ -1,4 +1,5 @@
 #! /usr/bin/env python3
+
 from rl import agents
 
 
@@ -35,18 +36,24 @@ def add(self, agent_type: str):
     def set(self, *args, **kwargs):
         self.args = args
         self.kwargs = kwargs
-
-    def make(self) -> agents.Agent:
         policy_name = self.registry[self.policy_agent].__name__
         learning_name = self.registry[self.learning_agent].__name__
+
         exec(f"""
+global {policy_name}{learning_name} 
 class {policy_name}{learning_name}(agents.{policy_name} ,agents.{learning_name}):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    """)
+
+    def make(self) -> agents.Agent:
+        policy_name = self.registry[self.policy_agent].__name__
+        learning_name = self.registry[self.learning_agent].__name__
+
+        exec(f"""
 global agent
-agent = {policy_name}{learning_name}(*self.args, **self.kwargs)
-""")
+agent = {policy_name}{learning_name}(*self.args, **self.kwargs) """)
 
         return agent
 
@@ -55,4 +62,3 @@ def __init__(self, *args, **kwargs):
     builder = AgentBuilder(policy="EGreedy", learning="TemporalDifference")
     builder.set(exploratory_rate=0.1, learning_rate=0.5)
     agent = builder.make()
-    print()

rl/agents/learning/learning_agent.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 from abc import abstractmethod
 from collections import defaultdict, Counter
+from typing import Tuple
 
 import numpy
 
@@ -79,38 +80,24 @@ def merge(self, agent):
         for state_action, counts in agent.transitions.items():
             self.transitions[state_action] += counts
 
-    def transition_model(self, state: numpy.ndarray, action: int, copy: bool = False) -> numpy.ndarray:
+    def transition_model(self, state: numpy.ndarray, action: int) -> Tuple[numpy.ndarray, numpy.ndarray]:
         """
         State transition model that describes how the environment state changes when the
         agent performs an action depending on the action and the current state.
         :param state: The state of the environment
         :param action: An action available to the agent
-        :param copy: When applying the action to the state, do so with a copy or apply it directly
         """
-        if copy:
-            next_state = state.copy()
-        else:
-            next_state = state
-
-        state_counts = self.transitions[(*next_state, action)]
+        state_action_pair = (*state, action)
+        state_counts = self.transitions[state_action_pair]
 
         if not state_counts:
-            return state
+            return numpy.array([]), numpy.array([])
 
-        states = list(state_counts.keys())
+        states = numpy.array(list(state_counts.keys()))
         counts = numpy.array(list(state_counts.values()))
 
-        counts = numpy.maximum(counts, 0)
-        sum = counts.sum()
-        probabilities = counts / sum
-
-        # values = []
-        # for p, s in zip(probabilities, states):
-        #     values.append(self.state_values[s].value)
-
-        index = numpy.random.choice(numpy.arange(len(state_counts)), p=probabilities)
-        # return states[numpy.argmax(numpy.array(values))]
-        return states[index]
+        probabilities = counts / counts.sum()
+        return probabilities, states
 
     def value_model(self, state: numpy.ndarray, action: int) -> float:
         """

rl/agents/policy/decaying_egreedy_policy_agent.py

@@ -51,7 +51,12 @@ def greedy_action(self, state: numpy.ndarray, available_actions: numpy.ndarray)
         max_index: int = 0
 
         for index, action in enumerate(available_actions):
-            next_state: numpy.ndarray = self.transition_model(state, action, copy=True)
+            probabilities: numpy.ndarray
+            states: numpy.ndarray
+            probabilities, states = self.transition_model(state.copy(), action)
+            index = numpy.random.choice(numpy.arange(len(states), p=probabilities))
+
+            next_state: numpy.ndarray = states[index]
             next_value: float = self.value_model(next_state, action)
 
             if next_value > max_value:

rl/agents/policy/egreedy_policy_agent.py

@@ -50,8 +50,16 @@ def greedy_action(self, state: numpy.ndarray, available_actions: numpy.ndarray)
         max_index: int = 0
 
         for index, action in enumerate(available_actions):
-            next_state: numpy.ndarray = self.transition_model(state, action, copy=True)
-            next_value: float = self.value_model(next_state, action)
+            probabilities: numpy.ndarray
+            states: numpy.ndarray
+            probabilities, states = self.transition_model(state.copy(), action)
+
+            if probabilities.any():
+                index = numpy.random.choice(numpy.arange(len(states), p=probabilities))
+                next_state: numpy.ndarray = states[index]
+                next_value: float = self.value_model(next_state, action)
+            else:
+                continue
 
             if next_value > max_value:
                 max_index: int = index

rl/agents/policy/human_policy_agent.py

@@ -18,7 +18,7 @@ def act(self, state: numpy.ndarray, available_actions: numpy.ndarray) -> int:
         :return: The action selected
         """
         while True:
-            user_input: str = input(f"available actions: {available_actions}")
+            user_input: str = input(f"available actions: {[action + 1 for action in available_actions]}")
 
             if user_input.startswith("q") or "quit" in user_input:
                 print("quitting!")

rl/tictactoe/main.py

@@ -80,8 +80,6 @@ def learn_from_game(args):
     env = gym.make("TicTacToe-v0")
     obs: numpy.ndarray = env.reset()
 
-    builder = AgentBuilder(policy="EGreedy", learning="TemporalDifference")
-
     players: Dict[Mark, Agent] = {
         Mark.X: builder.make(),
         Mark.O: builder.make(),