material/deep_learning/rnn_next_character_prediction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "rnn_next_character_prediction.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gv9nJ_bMt3uy"
      },
      "source": [
        "copy from https://machinelearningmastery.com/text-generation-lstm-recurrent-neural-networks-python-keras/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1NdSvZiy9pi"
      },
      "source": [
        "import numpy\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Dense, Dropout, LSTM\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swqyBux_9q15"
      },
      "source": [
        "# 데이터"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aklnTJzD7r78"
      },
      "source": [
        "import string\n",
        "\n",
        "all_char = string.ascii_lowercase + string.digits\n",
        "# all_char = 'abcdefghijklmnopqrstuvwxyz0123456789'\n",
        "# len(all_char) = 36\n",
        "\n",
        "raw_text = all_char*100\n",
        "# len(raw_text) = 3600\n",
        "\n",
        "SEQ_LENGTH = 10"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0vkRiI88-nHf"
      },
      "source": [
        "## 문자 인덱스 <--> 문자 변환"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XD50z_lKy9kC"
      },
      "source": [
        "chars = sorted(list(set(raw_text)))\n",
        "# chars = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
        "\n",
        "char_to_int = dict((c, i) for i, c in enumerate(chars))\n",
        "# char_to_int = {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, 'a': 10, 'b': 11, 'c': 12, ... }\n",
        "int_to_char = dict((i, c) for i, c in enumerate(chars))\n",
        "# int_to_char = {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9', 10: 'a', 11: 'b', 12: 'c', ... }"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3XBiD7uF-107"
      },
      "source": [
        "## text 길이, 문자 수"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gwJSCX5Uy9g_",
        "outputId": "152922f8-314d-4345-c2b4-a0b8476a6c65"
      },
      "source": [
        "n_chars = len(raw_text)\n",
        "n_vocab = len(chars)\n",
        "print(\"Total Characters: \", n_chars)\n",
        "print(\"Total Vocab: \", n_vocab)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Characters:  3600\n",
            "Total Vocab:  36\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kVrZBkCr-6EM"
      },
      "source": [
        "## 입출력 데이터 만들기"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eUNUvjdU0bc2"
      },
      "source": [
        "```\n",
        "# seq_length=20\n",
        "\n",
        "         1         2         3         4         5         6    \n",
        "123456789012345678901234567890123456789012345678901234567890\n",
        "alice's adventures in wonderland\n",
        "<----- seq_in------>\n",
        "                    ^\n",
        "                    |\n",
        "                seq_out\n",
        "```\n",
        "\n",
        "```\n",
        "seq_in = 'alice's adventures i'\n",
        "seq_out = 'n'\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YJzkgzsky9c-",
        "outputId": "44135fd6-e9a4-450a-d7b1-abeb350003f8"
      },
      "source": [
        "seq_length = SEQ_LENGTH\n",
        "dataX = []\n",
        "dataY = []\n",
        "for i in range(0, n_chars - seq_length, 1):\n",
        "\tseq_in = raw_text[i:i + seq_length]\n",
        "\tseq_out = raw_text[i + seq_length]\n",
        "\tdataX.append([char_to_int[char] for char in seq_in])\n",
        "\tdataY.append(char_to_int[seq_out])\n",
        "n_patterns = len(dataX)\n",
        "print(\"Total Patterns: \", n_patterns)\n",
        "\n",
        "X = numpy.reshape(dataX, (n_patterns, seq_length, 1))\n",
        "# X.shape = (148474, 100, 1)   # 데이터 148,474개, 1개 데이터는 길이 100 \n",
        "\n",
        "# normalize\n",
        "X = X / float(n_vocab)\n",
        "y = numpy.array(dataY)\n",
        "# y = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 31, 24, 42, 28, 38, 1, 22, 20, 37, 37 ... ]\n",
        "# y.shape = (148474, )  # 데이터 148,474개. "
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Patterns:  3590\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V7uVKU983Fbq",
        "outputId": "ff6c36d8-4da5-463e-c225-c73410561a8b"
      },
      "source": [
        "print(y.shape)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(3590,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vN1VzhDg-9x_"
      },
      "source": [
        "# 학습"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KzeG6Rna--58"
      },
      "source": [
        "## 모델 정의"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BEBL8mb32mJV"
      },
      "source": [
        "\n",
        "# define the LSTM model\n",
        "model = Sequential()\n",
        "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(LSTM(256))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(n_vocab, activation='softmax'))\n",
        "\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G6RsdFof_AIb"
      },
      "source": [
        "## 학습 실행"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m8w06r4Rt0mX",
        "outputId": "4c3244ee-6f4c-49ca-bb7a-46362a368017"
      },
      "source": [
        "model.fit(X, y, epochs=50, batch_size=256)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "15/15 [==============================] - 7s 9ms/step - loss: 3.4963\n",
            "Epoch 2/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 2.9719\n",
            "Epoch 3/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 2.3492\n",
            "Epoch 4/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 1.8684\n",
            "Epoch 5/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 1.6444\n",
            "Epoch 6/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 1.2970\n",
            "Epoch 7/50\n",
            "15/15 [==============================] - 0s 20ms/step - loss: 1.2344\n",
            "Epoch 8/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.9682\n",
            "Epoch 9/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.8686\n",
            "Epoch 10/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.7430\n",
            "Epoch 11/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.7137\n",
            "Epoch 12/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.6256\n",
            "Epoch 13/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.5607\n",
            "Epoch 14/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.4815\n",
            "Epoch 15/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.4632\n",
            "Epoch 16/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.4310\n",
            "Epoch 17/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.4505\n",
            "Epoch 18/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 1.7549\n",
            "Epoch 19/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 1.3365\n",
            "Epoch 20/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.7521\n",
            "Epoch 21/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.6114\n",
            "Epoch 22/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.7111\n",
            "Epoch 23/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.4495\n",
            "Epoch 24/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.3627\n",
            "Epoch 25/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.3276\n",
            "Epoch 26/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2942\n",
            "Epoch 27/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2709\n",
            "Epoch 28/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2493\n",
            "Epoch 29/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.2384\n",
            "Epoch 30/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.2141\n",
            "Epoch 31/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2152\n",
            "Epoch 32/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.3176\n",
            "Epoch 33/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 1.4623\n",
            "Epoch 34/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 1.0317\n",
            "Epoch 35/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.5211\n",
            "Epoch 36/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.3598\n",
            "Epoch 37/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.3067\n",
            "Epoch 38/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.2758\n",
            "Epoch 39/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2557\n",
            "Epoch 40/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2180\n",
            "Epoch 41/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.1987\n",
            "Epoch 42/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.1919\n",
            "Epoch 43/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.2131\n",
            "Epoch 44/50\n",
            "15/15 [==============================] - 0s 9ms/step - loss: 0.1973\n",
            "Epoch 45/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.1575\n",
            "Epoch 46/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.1382\n",
            "Epoch 47/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.1826\n",
            "Epoch 48/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 1.0363\n",
            "Epoch 49/50\n",
            "15/15 [==============================] - 0s 8ms/step - loss: 0.6845\n",
            "Epoch 50/50\n",
            "15/15 [==============================] - 0s 7ms/step - loss: 0.3997\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f428015ea90>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JMMGAXnu_BuK"
      },
      "source": [
        "## 예측"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uqeSToC93co5",
        "outputId": "389ebdda-81b1-4dbf-b2b6-4c7c2c656a43"
      },
      "source": [
        "start = numpy.random.randint(0, len(dataX)-1)\n",
        "pattern = dataX[start]\n",
        "print(\"Seed:\")\n",
        "print(\"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
        "\n",
        "# generate characters\n",
        "for i in range(100):\n",
        "\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n",
        "\tx = x / float(n_vocab)\n",
        "\tprediction = model.predict(x, verbose=0)\n",
        "\tindex = numpy.argmax(prediction)\n",
        "\tresult = int_to_char[index]\n",
        "\tprint(result, end=\"\")\n",
        "\tpattern.append(index)\n",
        "\tpattern = pattern[1:len(pattern)]\n",
        "print(\"\\nDone.\")"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Seed:\n",
            "\" 789abcdefg \"\n",
            "hijklmnopqrstuvwxyz0123456789bccdffhiiklmmopprssuvwwyzz1223456789cccdffhijklmmopqrstuvwxyz0123456789\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jkx2qWMh_KDx"
      },
      "source": [
        "# GRU로\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7uRF1rD_M4d"
      },
      "source": [
        "from tensorflow.keras.layers import GRU\n",
        "\n",
        "# define the LSTM model\n",
        "model = Sequential()\n",
        "# model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
        "model.add(GRU(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
        "model.add(Dropout(0.2))\n",
        "# model.add(LSTM(256))\n",
        "model.add(GRU(256))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(n_vocab, activation='softmax'))\n",
        "\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EiBh7NRm_UVw",
        "outputId": "9e6a103d-44d3-4da1-d11c-0fb62ccbc79a"
      },
      "source": [
        "model.fit(X, y, epochs=50, batch_size=256)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "15/15 [==============================] - 3s 6ms/step - loss: 3.4841\n",
            "Epoch 2/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 2.7317\n",
            "Epoch 3/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 2.0706\n",
            "Epoch 4/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.6820\n",
            "Epoch 5/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.4387\n",
            "Epoch 6/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.2141\n",
            "Epoch 7/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.0592\n",
            "Epoch 8/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.1264\n",
            "Epoch 9/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.0305\n",
            "Epoch 10/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.8324\n",
            "Epoch 11/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.7032\n",
            "Epoch 12/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.6047\n",
            "Epoch 13/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.5384\n",
            "Epoch 14/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4881\n",
            "Epoch 15/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4502\n",
            "Epoch 16/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4726\n",
            "Epoch 17/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.5241\n",
            "Epoch 18/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4904\n",
            "Epoch 19/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.7090\n",
            "Epoch 20/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4452\n",
            "Epoch 21/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.3106\n",
            "Epoch 22/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2722\n",
            "Epoch 23/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2415\n",
            "Epoch 24/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2260\n",
            "Epoch 25/50\n",
            "15/15 [==============================] - 0s 5ms/step - loss: 0.1953\n",
            "Epoch 26/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2063\n",
            "Epoch 27/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1893\n",
            "Epoch 28/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2067\n",
            "Epoch 29/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1438\n",
            "Epoch 30/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1335\n",
            "Epoch 31/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1226\n",
            "Epoch 32/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1111\n",
            "Epoch 33/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1048\n",
            "Epoch 34/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1361\n",
            "Epoch 35/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.0921\n",
            "Epoch 36/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.0970\n",
            "Epoch 37/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.1396\n",
            "Epoch 38/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.4886\n",
            "Epoch 39/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 4.2691\n",
            "Epoch 40/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 1.7398\n",
            "Epoch 41/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.9879\n",
            "Epoch 42/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.6657\n",
            "Epoch 43/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4963\n",
            "Epoch 44/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4035\n",
            "Epoch 45/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.3362\n",
            "Epoch 46/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.3242\n",
            "Epoch 47/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.2888\n",
            "Epoch 48/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.4256\n",
            "Epoch 49/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.6789\n",
            "Epoch 50/50\n",
            "15/15 [==============================] - 0s 6ms/step - loss: 0.6088\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4cb0193750>"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z-8Jb_4J_f9-",
        "outputId": "06c5e2e3-ec09-46b8-82ff-84c6eee29cce"
      },
      "source": [
        "start = numpy.random.randint(0, len(dataX)-1)\n",
        "pattern = dataX[start]\n",
        "print(\"Seed:\")\n",
        "print(\"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
        "\n",
        "# generate characters\n",
        "for i in range(100):\n",
        "\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n",
        "\tx = x / float(n_vocab)\n",
        "\tprediction = model.predict(x, verbose=0)\n",
        "\tindex = numpy.argmax(prediction)\n",
        "\tresult = int_to_char[index]\n",
        "\tprint(result, end=\"\")\n",
        "\tpattern.append(index)\n",
        "\tpattern = pattern[1:len(pattern)]\n",
        "print(\"\\nDone.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Seed:\n",
            "\" 3456789abc \"\n",
            "cdefghijklmnopqrstuvwwxzz0123455789aaacdefghijjkmnopqqrstuwwxyzz0123456789aaacdefghijjkmnopqqrstuwwx\n",
            "Done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZvT6nvam_o3t"
      },
      "source": [
        "# Bidirectional 로"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qiEqBaRd_uvp"
      },
      "source": [
        "from tensorflow.keras.layers import Bidirectional\n",
        "\n",
        "# define the LSTM model\n",
        "model = Sequential()\n",
        "# model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
        "model.add(Bidirectional(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True)))\n",
        "model.add(Dropout(0.2))\n",
        "# model.add(LSTM(256))\n",
        "model.add(Bidirectional(LSTM(256)))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Dense(n_vocab, activation='softmax'))\n",
        "\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GAaSO86R_uv1",
        "outputId": "b47c3e53-2135-4046-e3dc-221a9cef47fd"
      },
      "source": [
        "model.fit(X, y, epochs=50, batch_size=256)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "15/15 [==============================] - 6s 13ms/step - loss: 3.4383\n",
            "Epoch 2/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 2.4754\n",
            "Epoch 3/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 1.5842\n",
            "Epoch 4/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 1.1516\n",
            "Epoch 5/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.8496\n",
            "Epoch 6/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.7526\n",
            "Epoch 7/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.6296\n",
            "Epoch 8/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.5683\n",
            "Epoch 9/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.4108\n",
            "Epoch 10/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.4342\n",
            "Epoch 11/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.3621\n",
            "Epoch 12/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.3792\n",
            "Epoch 13/50\n",
            "15/15 [==============================] - 0s 14ms/step - loss: 0.2346\n",
            "Epoch 14/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.1891\n",
            "Epoch 15/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.1387\n",
            "Epoch 16/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.1213\n",
            "Epoch 17/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0990\n",
            "Epoch 18/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.2649\n",
            "Epoch 19/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 1.3228\n",
            "Epoch 20/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.6148\n",
            "Epoch 21/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.3343\n",
            "Epoch 22/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.2108\n",
            "Epoch 23/50\n",
            "15/15 [==============================] - 0s 14ms/step - loss: 0.1657\n",
            "Epoch 24/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.3595\n",
            "Epoch 25/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.3594\n",
            "Epoch 26/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.6055\n",
            "Epoch 27/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.4205\n",
            "Epoch 28/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.2305\n",
            "Epoch 29/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.1278\n",
            "Epoch 30/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0880\n",
            "Epoch 31/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0658\n",
            "Epoch 32/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0552\n",
            "Epoch 33/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0511\n",
            "Epoch 34/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0435\n",
            "Epoch 35/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0372\n",
            "Epoch 36/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0343\n",
            "Epoch 37/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0312\n",
            "Epoch 38/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0293\n",
            "Epoch 39/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0286\n",
            "Epoch 40/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0254\n",
            "Epoch 41/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0246\n",
            "Epoch 42/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0213\n",
            "Epoch 43/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0194\n",
            "Epoch 44/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0192\n",
            "Epoch 45/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0176\n",
            "Epoch 46/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0162\n",
            "Epoch 47/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0155\n",
            "Epoch 48/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0147\n",
            "Epoch 49/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0141\n",
            "Epoch 50/50\n",
            "15/15 [==============================] - 0s 13ms/step - loss: 0.0136\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f4c29680410>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6L-SvFeOAK5z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b991fa9b-95f5-4153-f655-f409868e84e5"
      },
      "source": [
        "start = numpy.random.randint(0, len(dataX)-1)\n",
        "pattern = dataX[start]\n",
        "print(\"Seed:\")\n",
        "print(\"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
        "\n",
        "# generate characters\n",
        "for i in range(100):\n",
        "\tx = numpy.reshape(pattern, (1, len(pattern), 1))\n",
        "\tx = x / float(n_vocab)\n",
        "\tprediction = model.predict(x, verbose=0)\n",
        "\tindex = numpy.argmax(prediction)\n",
        "\tresult = int_to_char[index]\n",
        "\tprint(result, end=\"\")\n",
        "\tpattern.append(index)\n",
        "\tpattern = pattern[1:len(pattern)]\n",
        "print(\"\\nDone.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Seed:\n",
            "\" hijklmnopq \"\n",
            "rstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghi\n",
            "Done.\n"
          ]
        }
      ]
    }
  ]
}