ML_RL_Blackjack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PokerAgent\n",
    "The following code is the agent of Poker Game. There are two different way to draw cards.<br>\n",
    "(1)  Uniform:  Each card has the same probability.<br>\n",
    "(2) Optional:  The cards with 10 points have less probability.<p>\n",
    "There are three different action.<br>\n",
    "<b>Hit Stand Double<b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     123
    ]
   },
   "outputs": [],
   "source": [
    "# Envrionment\n",
    "# The code is form gym PokerAgent example\n",
    "# Chin-Wei Wang add function double and split card 2019/5/22\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 1 = Ace, 2-10 = Number cards, Jack/Queen/King = 10\n",
    "# 卡牌點數\n",
    "deck = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]\n",
    "p=0.09\n",
    "probability = [p,p,p,p,p,p,p,p,p,p,0.10,0,0]\n",
    "\n",
    "# 比大小\n",
    "def cmp(a, b):\n",
    "    # when a > b return 1; when a < b return -1;when a == b return 0\n",
    "    return int((a > b)) - int((a < b))\n",
    "\n",
    "# 給牌\n",
    "def draw_card_uniform():\n",
    "    return random.sample(deck, 1)[0]\n",
    "\n",
    "def draw_card_non_uniform():\n",
    "    return np.random.choice(deck, 1, p=probability)[0]\n",
    "\n",
    "# Ace 當1或11\n",
    "# return true if the ace is usable\n",
    "def usable_ace(hand):\n",
    "    return 1 in hand and sum(hand) + 10 <= 21\n",
    "\n",
    "# 算牌\n",
    "def sum_hand(hand):\n",
    "    if usable_ace(hand):\n",
    "        return sum(hand) + 10\n",
    "    return sum(hand)\n",
    "\n",
    "# 爆掉\n",
    "def is_bust(hand):\n",
    "    return sum_hand(hand) > 21\n",
    "\n",
    "# 算分數\n",
    "def score(hand):\n",
    "    return 0 if is_bust(hand) else sum_hand(hand)\n",
    "\n",
    "# 直接21點\n",
    "def is_natural(hand):\n",
    "    return sorted(hand) == [1, 10]\n",
    "\n",
    "# 判斷兩張相同時可分牌\n",
    "def is_duplicated(hand):\n",
    "    if np.size(hand) == 2:\n",
    "        return hand[0] == hand[1]\n",
    "    return False\n",
    "\n",
    "# action(0, 1, 2)     -> (stay, hit, double)\n",
    "# action(0, 1, 2, 3) -> (stay, hit, double, surrender)\n",
    "\n",
    "class PokerAgent:\n",
    "    def __init__(self, natural=True):\n",
    "        self.action_size = 1\n",
    "        self.state_size = [1, 3]\n",
    "        self.natural = natural\n",
    "        self.uniform = True\n",
    "        self.duplicated = False\n",
    "        self.Surrender = False\n",
    "        self.player = []\n",
    "        self.dealer = []\n",
    "        self.reset()\n",
    "\n",
    "    def step(self, action):\n",
    "        # Split\n",
    "        if is_duplicated(self.player) and self.duplicated:\n",
    "            duplicate = self.player.pop(1)\n",
    "            if self.uniform:\n",
    "                self.player.append(draw_card_uniform())\n",
    "            else:\n",
    "                self.player.append(draw_card_non_uniform())\n",
    "            done = True\n",
    "            \n",
    "            rewards = 0\n",
    "            if is_bust(self.player):   # 第一組牌\n",
    "                reward = -1\n",
    "            else:\n",
    "                while sum_hand(self.dealer) < 17:\n",
    "                    if self.uniform:\n",
    "                        self.dealer.append(draw_card_uniform())\n",
    "                    else:\n",
    "                        self.dealer.append(draw_card_non_uniform())\n",
    "                reward = cmp(score(self.player), score(self.dealer))     \n",
    "            \n",
    "            self.player=[]\n",
    "            self.player.append(duplicate)\n",
    "            \n",
    "            if self.uniform:\n",
    "                self.player.append(draw_card_uniform())\n",
    "            else:\n",
    "                self.player.append(draw_card_non_uniform())\n",
    "                \n",
    "            if is_bust(self.player):   # 第二組牌\n",
    "                reward = reward-1\n",
    "            else:\n",
    "                while sum_hand(self.dealer) < 17:\n",
    "                    if self.uniform:\n",
    "                        self.dealer.append(draw_card_uniform())\n",
    "                    else:\n",
    "                        self.dealer.append(draw_card_non_uniform())\n",
    "                reward = cmp(score(self.player), score(self.dealer)) + reward\n",
    "        else:\n",
    "            # stand\n",
    "            if action == 0:\n",
    "                done = True\n",
    "                # dealer's turn\n",
    "                while sum_hand(self.dealer) < 17:\n",
    "                    if self.uniform:\n",
    "                        self.dealer.append(draw_card_uniform())\n",
    "                    else:\n",
    "                        self.dealer.append(draw_card_non_uniform())\n",
    "                reward = cmp(score(self.player), score(self.dealer))\n",
    "                if self.natural and is_natural(self.player) and reward == 1:\n",
    "                    reward = 1.5\n",
    "\n",
    "            # hit\n",
    "            elif action == 1:\n",
    "                if self.uniform:\n",
    "                    self.player.append(draw_card_uniform())\n",
    "                else:\n",
    "                    self.player.append(draw_card_non_uniform())               \n",
    "\n",
    "                if is_bust(self.player):\n",
    "                    done = True\n",
    "                    reward = -1\n",
    "                else:\n",
    "                    done = False\n",
    "                    reward = 0\n",
    "\n",
    "            # double down\n",
    "            elif action == 2:\n",
    "                if self.uniform:\n",
    "                    self.player.append(draw_card_uniform())\n",
    "                else:\n",
    "                    self.player.append(draw_card_non_uniform()) \n",
    "                done = True\n",
    "                if is_bust(self.player):\n",
    "                    reward = -1 * 2\n",
    "                # dealer's turn\n",
    "                else:\n",
    "                    while sum_hand(self.dealer) < 17:\n",
    "                        if self.uniform:\n",
    "                            self.dealer.append(draw_card_uniform())\n",
    "                        else:\n",
    "                            self.dealer.append(draw_card_non_uniform())\n",
    "                    reward = cmp(score(self.player), score(self.dealer)) * 2\n",
    "        \n",
    "            # Surrender\n",
    "            else:\n",
    "                done = True\n",
    "                reward = -0.5\n",
    "                \n",
    "        return self.get_obs(), reward, done, {}\n",
    "\n",
    "    def get_obs(self):\n",
    "        return [sum_hand(self.player), self.dealer[0], usable_ace(self.player)]\n",
    "\n",
    "    def reset(self):\n",
    "        self.player = []\n",
    "        self.dealer = []\n",
    "        if self.uniform:\n",
    "            self.dealer.append(draw_card_uniform())\n",
    "            self.player.append(draw_card_uniform())\n",
    "            self.player.append(draw_card_uniform())\n",
    "        else:\n",
    "            self.dealer.append(draw_card_non_uniform())\n",
    "            self.player.append(draw_card_non_uniform())\n",
    "            self.player.append(draw_card_non_uniform())\n",
    "        return self.get_obs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Human Policy\n",
    "In geneal, when the total points of hand cards reach to 17, we don't ask card anymore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Policy\n",
    "# action(0, 1, 2) -> (stay, hit, double)\n",
    "def human_policy(env):\n",
    "    if sum_hand(env.player) < 17 and sum(env.dealer) > 8:\n",
    "        action = 1   # double\n",
    "    elif sum_hand(env.player) < 17:\n",
    "        action = 1   # hit\n",
    "    else:\n",
    "        action = 0  # stay\n",
    "    observation,reward,done,_ = env.step(action)\n",
    "    return action,observation,reward,done\n",
    "\n",
    "def random_policy(env):\n",
    "    action = np.random.randint(0,4)\n",
    "    observation,reward,done,_ = env.step(action)\n",
    "    return action,observation,reward,done\n",
    "\n",
    "def optimize_policy(env):\n",
    "    if sum_hand(env.player) < 8:\n",
    "        action = 1   # hit\n",
    "    elif sum_hand(env.player) < 17 and sum(env.dealer) > 6:\n",
    "        action = 1   # hit\n",
    "    elif sum_hand(env.player) < 17 and sum(env.dealer) == 1:\n",
    "        action = 1   # hit\n",
    "    elif sum_hand(env.player) < 12 and sum(env.dealer) < 4 :\n",
    "        action = 1   # hit\n",
    "    elif sum_hand(env.player) == 11:\n",
    "        action = 2   # double\n",
    "    elif sum_hand(env.player) == 10 and sum(env.dealer) < 10 and sum(env.dealer) > 1:\n",
    "        action = 2   # double\n",
    "    elif sum_hand(env.player) == 9 and sum(env.dealer) < 7 and sum(env.dealer) > 2:\n",
    "        action = 2   # double\n",
    "    else:\n",
    "        action = 0   # stay\n",
    "\n",
    "    observation,reward,done,_ = env.step(action)\n",
    "    return action,observation,reward,done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_card(i_time,env):\n",
    "    print(f\"Game Set : {i_time}\")\n",
    "    print(f\"Player Card : {env.player}\")\n",
    "    print(f\"Is Bust : {is_bust(env.player)}\")\n",
    "    print(f\"Dealer Card : {env.dealer}\")\n",
    "\n",
    "def print_result(i_times,actions,results,money_):\n",
    "    print(f\"Total : {i_times} times\")\n",
    "    print(f\"stay : {actions.count(0)} , hit : {actions.count(1)} , double : {actions.count(2)}\")\n",
    "    print(f\"Player win: {results.count(2)+results.count(1.5)+results.count(1)} Player double win: {results.count(2)}\")\n",
    "    print(f\"Dealer win: {results.count(-1)+results.count(-2)} Dealer double win: {results.count(-2)}\")\n",
    "    print(f\"Tie : {results.count(0)}\")\n",
    "    print(f\"Money : {money_} \")\n",
    "    print(f\"Expection value : {money_/i_times} \")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Human Policy Game Loop\n",
    "There are three different policy.<br>\n",
    "(1) Random Policy<br>\n",
    "(2) Human Policy<br>\n",
    "(3) Optimize Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "each_print = False\n",
    "# Play the game\n",
    "def game_loop(game_times,step_times):\n",
    "    env = PokerAgent()\n",
    "    results = []\n",
    "    actions = []\n",
    "    money = []\n",
    "    money_ = 0\n",
    "    for i in range(1,game_times+1):\n",
    "        for j in range(step_times):\n",
    "            state = env.reset()\n",
    "            done = False\n",
    "            while done == False:\n",
    "                #action,observation,reward,done = human_policy(env)\n",
    "                #action,observation,reward,done = random_policy(env)\n",
    "                action,observation,reward,done = optimize_policy(env)\n",
    "                actions.append(action)\n",
    "            results.append(reward)\n",
    "            money_ += reward\n",
    "            money.append(money_)\n",
    "        if each_print and i % step_times == 0:\n",
    "            print_result(i*step_times,actions,results,money_)\n",
    "\n",
    "    print_result(i*step_times,actions,results,money_)\n",
    "    fig = plt.figure()\n",
    "    plt.plot(np.arange(1,game_times),money[1:game_times])\n",
    "    plt.ylabel('Money')\n",
    "    plt.xlabel('Episode')\n",
    "    plt.grid()\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "game_loop(10000,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Deep Q-Learning Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Dense, Input\n",
    "from keras.optimizers import Adam\n",
    "from collections import deque\n",
    "\n",
    "# Deep Q-learning Agent\n",
    "class DQNAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.batch_size = 64\n",
    "        self.memory = deque(maxlen=10000)\n",
    "        self.gamma = 0.95         # discount rate\n",
    "        self.epsilon = 1.0            # exploration rate\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.995\n",
    "        self.learning_rate = 0.001\n",
    "        self.model = self.build_model()\n",
    "        self.target_model = self.build_model()\n",
    "        \n",
    "    def build_model(self):\n",
    "        # Neural Net for Deep-Q learning Model\n",
    "        model = Sequential()\n",
    "        model.add(Dense(128, input_dim=self.state_size, activation='relu'))\n",
    "        model.add(Dense(64, activation='relu'))\n",
    "        model.add(Dense(32, activation='relu'))\n",
    "        model.add(Dense(self.action_size, activation='linear'))\n",
    "        model.compile(loss='mse',optimizer=Adam(lr=self.learning_rate))\n",
    "        return model\n",
    "    \n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "        \n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.action_size)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])  # returns action\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def replay(self):\n",
    "        minibatch = self.sample(self.batch_size)\n",
    "        for state, action, reward, next_state, done in minibatch:\n",
    "            target = reward\n",
    "            if not done:\n",
    "                target = (reward + self.gamma *\n",
    "                          np.amax(self.target_model.predict(next_state)[0]))           \n",
    "            target_f = self.model.predict(state)\n",
    "            target_f[0][action] = target\n",
    "            self.model.fit(state, target_f, epochs=1, verbose=0)\n",
    "        if self.epsilon > self.epsilon_min:\n",
    "            self.epsilon *= self.epsilon_decay\n",
    "    \n",
    "    def update_model(self, total_step):\n",
    "          if total_step % 10 == 0:      # %10\n",
    "                weights = self.model.get_weights()\n",
    "                target_weights = self.target_model.get_weights()\n",
    "                for i in range(len(target_weights)):\n",
    "                    target_weights[i] = weights[i]\n",
    "                self.target_model.set_weights(target_weights)\n",
    "                \n",
    "    def save(self, file):\n",
    "        self.model.save_weights(file)\n",
    "\n",
    "    def load(self, file):\n",
    "        self.model.load_weights(file)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Double Deep Q-Learning Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Dense, Input\n",
    "from keras.optimizers import Adam\n",
    "from collections import deque\n",
    "\n",
    "# Double Deep Q-learning Agent\n",
    "class DDQNAgent:\n",
    "    def __init__(self, state_size, action_size):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.batch_size = 64\n",
    "        self.memory = deque(maxlen=10000)\n",
    "        self.gamma = 0.95         # discount rate\n",
    "        self.epsilon = 1.0        # exploration rate\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.995\n",
    "        self.learning_rate = 0.001\n",
    "        self.model = self.build_model()\n",
    "        self.target_model = self.build_model()\n",
    "        \n",
    "    def build_model(self):\n",
    "        # Neural Net for Double Deep-Q learning Model\n",
    "        model = Sequential()\n",
    "        model.add(Dense(128, input_dim=self.state_size, activation='relu'))\n",
    "        model.add(Dense(64, activation='relu'))\n",
    "        model.add(Dense(32, activation='relu'))\n",
    "        model.add(Dense(self.action_size, activation='linear'))\n",
    "        model.compile(loss='mse',optimizer=Adam(lr=self.learning_rate))\n",
    "        return model\n",
    "    \n",
    "    def remember(self, state, action, reward, next_state, done):\n",
    "        self.memory.append((state, action, reward, next_state, done))\n",
    "        \n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return random.randrange(self.action_size)\n",
    "        act_values = self.model.predict(state)\n",
    "        return np.argmax(act_values[0])  # returns action\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def replay(self):\n",
    "        minibatch = self.sample(self.batch_size)\n",
    "        for state, action, reward, next_state, done in minibatch:\n",
    "            target = reward\n",
    "            if not done:\n",
    "                predicted_action = np.argmax(self.model.predict(next_state)[0])                \n",
    "                target = reward + self.gamma * self.target_model.predict(next_state)[0][predicted_action]         \n",
    "            target_f = self.model.predict(state)\n",
    "            target_f[0][action] = target\n",
    "            self.model.fit(state, target_f, epochs=1, verbose=0)\n",
    "        if self.epsilon > self.epsilon_min:\n",
    "            self.epsilon *= self.epsilon_decay\n",
    "    \n",
    "    def update_model(self, total_step):\n",
    "          if total_step % 10 == 0:      # %10\n",
    "                weights = self.model.get_weights()\n",
    "                target_weights = self.target_model.get_weights()\n",
    "                for i in range(len(target_weights)):\n",
    "                    target_weights[i] = weights[i]\n",
    "                self.target_model.set_weights(target_weights)\n",
    "                \n",
    "    def save(self, file):\n",
    "        self.model.save_weights(file)\n",
    "\n",
    "    def load(self, file):\n",
    "        self.model.load_weights(file)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "env = PokerAgent()\n",
    "state_size_,action_size_ = 3, 4    # 3,4\n",
    "dqn_agent = DDQNAgent(state_size_, action_size_)\n",
    "#dqn_agent = DDQNAgent(state_size_, action_size_)\n",
    "# dqn_agent.load('poke_model_per_d.h5')\n",
    "n_episode = 15000\n",
    "n_steps = 10\n",
    "stand, hit, double, surrender = 0, 0, 0, 0\n",
    "bj_win = 0\n",
    "money = 0\n",
    "capacity = 10000\n",
    "cur_money = money\n",
    "total_step = 0\n",
    "mon = []\n",
    "# start training\n",
    "for episode_i in range(n_episode):\n",
    "    cur_state_ = env.reset()\n",
    "    for step_i in range(n_steps):\n",
    "        total_step += 1\n",
    "        cur_state_ = np.reshape(cur_state_, [1, dqn_agent.state_size])\n",
    "        if total_step > capacity:\n",
    "            action = dqn_agent.act(cur_state_)\n",
    "        else:\n",
    "            action = np.random.randint(0, 4)    # (0,3)\n",
    "        if action == 0:\n",
    "            stand += 1\n",
    "        elif action == 1:\n",
    "            hit += 1\n",
    "        elif action == 2:\n",
    "            double += 1\n",
    "        elif action ==3:\n",
    "            surrender +=1\n",
    "            \n",
    "        observation, reward_, done_, _ = env.step(action)\n",
    "        observation = np.reshape(observation, [1, dqn_agent.state_size])\n",
    "        dqn_agent.remember(cur_state_, action, reward_ , observation, done_)\n",
    "        if total_step > capacity:\n",
    "            dqn_agent.replay()\n",
    "            dqn_agent.update_model(total_step)\n",
    "        cur_state_ = observation\n",
    "        if done_:\n",
    "            if reward_ == 1.5:\n",
    "                bj_win += 1\n",
    "            money += reward_\n",
    "            break\n",
    "\n",
    "    if episode_i % 100 == 0:\n",
    "        money_change = money - cur_money\n",
    "        mon.append(money_change)\n",
    "        dqn_agent.save('poke_model_dd_u_2.h5')\n",
    "        print(f\"Stand {stand}\")\n",
    "        print(f\"Hit {hit}\")\n",
    "        print(f\"Double {double}\")\n",
    "        print(f\"Surrender {surrender}\")\n",
    "\n",
    "        stand, hit, double, surrender = 0, 0, 0, 0\n",
    "        print(\"Episode {}/{}, Money:{}, Epsilon:{}\".format(episode_i, n_episode, money_change, dqn_agent.epsilon))\n",
    "        cur_money = money\n",
    "        print()\n",
    "        print()\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = np.linspace(0,15000,num = 150)\n",
    "plt.plot(ax,mon)\n",
    "plt.title('Rewards',fontsize=12)\n",
    "plt.grid()\n",
    "plt.show()\n",
    "print(bj_win)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reinforcement Learning Game Loop\n",
    "The following code is using the Policy after Q-Learning.<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "env = PokerAgent()\n",
    "state_size , action_size = 3, 4    # 3,3\n",
    "dqn_agent = DDQNAgent(state_size, action_size)\n",
    "#dqn_agent = DDQNAgent(state_size, action_size)\n",
    "\n",
    "dqn_agent.load('poke_model_dd_u_2.h5') # Uniform DQNAgent\n",
    "\n",
    "dqn_agent.epsilon  = 0\n",
    "n_episode = 100000\n",
    "n_steps = 5\n",
    "stand, hit, double, surrender, bj_win = 0, 0, 0, 0, 0\n",
    "money = 0\n",
    "cur_money = money\n",
    "game = []\n",
    "moneys = []\n",
    "for episode_i in range(n_episode):\n",
    "    cur_state = env.reset()\n",
    "    for step_i in range(n_steps):\n",
    "        cur_state = np.reshape(cur_state, [1, dqn_agent.state_size])\n",
    "        action = dqn_agent.act(cur_state)\n",
    "\n",
    "        if action == 0:\n",
    "             stand += 1\n",
    "        elif action == 1:\n",
    "             hit += 1\n",
    "        elif action == 2:\n",
    "             double += 1\n",
    "        elif action == 3:\n",
    "             surrender +=1\n",
    "                \n",
    "        observation, reward_, done_, _ = env.step(action)\n",
    "        observation = np.reshape(observation, [1, dqn_agent.state_size])\n",
    "        cur_state = observation\n",
    "        if done_:\n",
    "            if reward_ == 1.5:\n",
    "                bj_win += 1\n",
    "            money += reward_\n",
    "            break\n",
    "    if episode_i % 1000 == 0:\n",
    "        money_change = money - cur_money\n",
    "        game.append(money_change)\n",
    "        cur_money = money\n",
    "        moneys.append(money)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Result\n",
    "Show the result with "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "print(f\"Stand Rate : {stand / (stand + hit + double+surrender)}\")\n",
    "print(f\"Hit Rate : {hit / (stand + hit + double+surrender)}\")\n",
    "print(f\"Double Rate : {double / (stand + hit + double+surrender)}\")\n",
    "print(f\"Surrender Rate : {surrender / (stand + hit + double+surrender)}\")\n",
    "\n",
    "\n",
    "print(f\"Total Times  : {stand + hit + double+surrender}\")\n",
    "print(f\"Money : {money}\")\n",
    "print(f\"BJ Win : {bj_win}\")\n",
    "print(f\"Expect Value of each action: {money/(stand + hit + double+surrender+bj_win)}\")\n",
    "print(f\"Expect Value of each game: {money/(100000)}\")\n",
    "\n",
    "ax = np.linspace(0,100000,num = 100)\n",
    "plt.plot(ax,game)\n",
    "plt.title('Game per 1000',fontsize=12)\n",
    "plt.grid()\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.plot(ax,moneys)\n",
    "plt.title('Total Money',fontsize=12)\n",
    "plt.grid()\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Strategy Table\n",
    "Consider the steady and return the maximum probability of action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_episode = 50\n",
    "Memory = []\n",
    "Ace = 0\n",
    "for player_card in range(2,22):\n",
    "    cur_state = env.reset()\n",
    "    for dealer_card in range(1,11):\n",
    "        stand, hit, double, surrender = 0, 0, 0, 0\n",
    "        for episode in range(n_episode):\n",
    "            cur_state = np.array([[player_card, dealer_card,  0]])\n",
    "            action = dqn_agent.act(cur_state)\n",
    "            if action == 0:\n",
    "                stand += 1\n",
    "            elif action == 1:\n",
    "                hit += 1\n",
    "            elif action == 2:\n",
    "                double += 1\n",
    "            elif action == 3:\n",
    "                surrender +=1\n",
    "            \n",
    "        steady_action = np.argmax([stand,hit,double,surrender])\n",
    "        if steady_action == 0:\n",
    "            Memory.append([cur_state,\"Stand\"])\n",
    "        elif steady_action == 1:\n",
    "            Memory.append([cur_state,\"Hit\"])\n",
    "        elif steady_action == 2:\n",
    "            Memory.append([cur_state,\"Double\"])\n",
    "        elif steady_action == 3:\n",
    "            Memory.append([cur_state,\"Surrender\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "memory_dict = {}\n",
    "for mem in Memory:\n",
    "    row,col = (mem[0][0][:2])\n",
    "    if not col in memory_dict.keys():\n",
    "        memory_dict[col] = []\n",
    "    else:\n",
    "        memory_dict[col].append(mem[1])\n",
    "df = pd.DataFrame()\n",
    "for col in memory_dict.keys():\n",
    "    df[col] = memory_dict[col]\n",
    "df['Player Card'] = np.arange(3, 22)\n",
    "\n",
    "df.set_index(['Player Card'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}