gru-scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7dbf52e2",
   "metadata": {
    "papermill": {
     "duration": 0.003579,
     "end_time": "2024-06-11T07:02:53.444787",
     "exception": false,
     "start_time": "2024-06-11T07:02:53.441208",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"padding:10px; \n",
    "            color:#FF9F00;\n",
    "            margin:10px;\n",
    "            font-size:150%;\n",
    "            display:fill;\n",
    "            border-radius:1px;\n",
    "            border-style: solid;\n",
    "            border-color:#FF9F00;\n",
    "            background-color:#3E3D53;\n",
    "            overflow:hidden;\">\n",
    "    <center>\n",
    "        <a id='top'></a>\n",
    "        <b>Table of Contents</b>\n",
    "    </center>\n",
    "    <br>\n",
    "    <ul>\n",
    "        <li>\n",
    "            <a href=\"#1\">1 -  Overview and Imports</a>\n",
    "        </li>\n",
    "        <li>\n",
    "            <a href=\"#2\">2 - Data Preparation</a>\n",
    "        </li>\n",
    "        <li>\n",
    "            <a href=\"#3\">3 - GRU Implementation</a>\n",
    "        <li>\n",
    "            <a href=\"#4\">4 - Thank you</a>\n",
    "        </li> \n",
    "    </ul>\n",
    "</div>\n",
    "<a id=\"1\"></a>\n",
    "\n",
    "<h1 style='background:#FF9F00;border:0; color:black;\n",
    "    box-shadow: 10px 10px 5px 0px rgba(0,0,0,0.75);\n",
    "    transform: rotateX(10deg);\n",
    "    '><center style='color: #3E3D53;'>Overview and Imports</center></h1>\n",
    "    \n",
    "# Overview and Imports\n",
    "\n",
    "**Long Short-Term Memory (LSTM) networks are a type of Recurrent Neural Network (RNN) that are designed to handle sequential data, such as time series or natural language text. They achieve this by using a memory cell that can selectively remember or forget information at each time step of the input sequence, allowing the network to maintain a memory of previous inputs over a longer period of time.**\n",
    "\n",
    "**This notebook contains an implementation of an LSTM that can be used for language modeling. The self takes in a sequence of characters and outputs the probability distribution over the next character in the sequence. The network is trained on a corpus of text and then used to generate new text that has a similar distribution of characters as the training corpus.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ca403ede",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-11T07:02:53.452793Z",
     "iopub.status.busy": "2024-06-11T07:02:53.452410Z",
     "iopub.status.idle": "2024-06-11T07:02:53.483757Z",
     "shell.execute_reply": "2024-06-11T07:02:53.482666Z"
    },
    "papermill": {
     "duration": 0.038396,
     "end_time": "2024-06-11T07:02:53.486450",
     "exception": false,
     "start_time": "2024-06-11T07:02:53.448054",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import scipy as sp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a509751d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-11T07:02:53.495200Z",
     "iopub.status.busy": "2024-06-11T07:02:53.494236Z",
     "iopub.status.idle": "2024-06-11T07:02:53.505775Z",
     "shell.execute_reply": "2024-06-11T07:02:53.504690Z"
    },
    "papermill": {
     "duration": 0.018193,
     "end_time": "2024-06-11T07:02:53.507972",
     "exception": false,
     "start_time": "2024-06-11T07:02:53.489779",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class DataGenerator:\n",
    "    \"\"\"\n",
    "    A class for reading and preprocessing text data.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, path: str, sequence_length: int):\n",
    "        \"\"\"\n",
    "        Initializes a DataReader object with the path to a text file and the desired sequence length.\n",
    "\n",
    "        Args:\n",
    "            path (str): The path to the text file.\n",
    "            sequence_length (int): The length of the sequences that will be fed to the self.\n",
    "        \"\"\"\n",
    "        with open(path) as f:\n",
    "            # Read the contents of the file\n",
    "            self.data = f.read()\n",
    "\n",
    "        # Find all unique characters in the text\n",
    "        chars = list(set(self.data))\n",
    "\n",
    "        # Create dictionaries to map characters to indices and vice versa\n",
    "        self.char_to_idx = {ch: i for (i, ch) in enumerate(chars)}\n",
    "        self.idx_to_char = {i: ch for (i, ch) in enumerate(chars)}\n",
    "\n",
    "        # Store the size of the text data and the size of the vocabulary\n",
    "        self.data_size = len(self.data)\n",
    "        self.vocab_size = len(chars)\n",
    "\n",
    "        # Initialize the pointer that will be used to generate sequences\n",
    "        self.pointer = 0\n",
    "\n",
    "        # Store the desired sequence length\n",
    "        self.sequence_length = sequence_length\n",
    "\n",
    "\n",
    "    def next_batch(self):\n",
    "        \"\"\"\n",
    "        Generates a batch of input and target sequences.\n",
    "\n",
    "        Returns:\n",
    "            inputs_one_hot (np.ndarray): A numpy array with shape `(batch_size, vocab_size)` where each row is a one-hot encoded representation of a character in the input sequence.\n",
    "            targets (list): A list of integers that correspond to the indices of the characters in the target sequence, which is the same as the input sequence shifted by one position to the right.\n",
    "        \"\"\"\n",
    "        input_start = self.pointer\n",
    "        input_end = self.pointer + self.sequence_length\n",
    "\n",
    "        # Get the input sequence as a list of integers\n",
    "        inputs = [self.char_to_idx[ch] for ch in self.data[input_start:input_end]]\n",
    "\n",
    "        # One-hot encode the input sequence\n",
    "        inputs_one_hot = np.zeros((len(inputs), self.vocab_size))\n",
    "        inputs_one_hot[np.arange(len(inputs)), inputs] = 1\n",
    "\n",
    "        # Get the target sequence as a list of integers\n",
    "        targets = [self.char_to_idx[ch] for ch in self.data[input_start + 1:input_end + 1]]\n",
    "\n",
    "        # Update the pointer\n",
    "        self.pointer += self.sequence_length\n",
    "\n",
    "        # Reset the pointer if the next batch would exceed the length of the text data\n",
    "        if self.pointer + self.sequence_length + 1 >= self.data_size:\n",
    "            self.pointer = 0\n",
    "\n",
    "        return inputs_one_hot, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0acd187f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-11T07:02:53.516199Z",
     "iopub.status.busy": "2024-06-11T07:02:53.515841Z",
     "iopub.status.idle": "2024-06-11T07:02:53.574516Z",
     "shell.execute_reply": "2024-06-11T07:02:53.573488Z"
    },
    "papermill": {
     "duration": 0.065885,
     "end_time": "2024-06-11T07:02:53.577011",
     "exception": false,
     "start_time": "2024-06-11T07:02:53.511126",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class GRU:\n",
    "    \"\"\"\n",
    "    A class used to represent a Recurrent Neural Network (GRU).\n",
    "\n",
    "    Attributes\n",
    "    ----------\n",
    "    hidden_size : int\n",
    "        The number of hidden units in the GR.\n",
    "    vocab_size : int\n",
    "        The size of the vocabulary used by the GRU.\n",
    "    sequence_length : int\n",
    "        The length of the input sequences fed to the GRU.\n",
    "    self.learning_rate : float\n",
    "        The learning rate used during training.\n",
    "\n",
    "    Methods\n",
    "    -------\n",
    "    __init__(hidden_size, vocab_size, sequence_length, self.learning_rate)\n",
    "        Initializes an instance of the GRU class.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, hidden_size, vocab_size, sequence_length, learning_rate):\n",
    "        \"\"\"\n",
    "        Initializes an instance of the GRU class.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        hidden_size : int\n",
    "            The number of hidden units in the GRU.\n",
    "        vocab_size : int\n",
    "            The size of the vocabulary used by the GRU.\n",
    "        sequence_length : int\n",
    "            The length of the input sequences fed to the GRU.\n",
    "        learning_rate : float\n",
    "            The learning rate used during training.\n",
    "        \"\"\"\n",
    "        # hyper parameters\n",
    "        self.hidden_size = hidden_size\n",
    "        self.vocab_size = vocab_size\n",
    "        self.sequence_length = sequence_length\n",
    "        self.learning_rate = learning_rate\n",
    "\n",
    "        # model parameters\n",
    "        self.Wz = np.random.uniform(-np.sqrt(1. / hidden_size), np.sqrt(1. / hidden_size),\n",
    "                                    (hidden_size, hidden_size + vocab_size))\n",
    "        self.bz = np.zeros((hidden_size, 1))\n",
    "\n",
    "        self.Wr = np.random.uniform(-np.sqrt(1. / hidden_size), np.sqrt(1. / hidden_size),\n",
    "                                    (hidden_size, hidden_size + vocab_size))\n",
    "        self.br = np.zeros((hidden_size, 1))\n",
    "\n",
    "        self.Wa = np.random.uniform(-np.sqrt(1. / hidden_size), np.sqrt(1. / hidden_size),\n",
    "                                    (hidden_size, hidden_size + vocab_size))\n",
    "        self.ba = np.zeros((hidden_size, 1))\n",
    "\n",
    "        self.Wy = np.random.uniform(-np.sqrt(1. / hidden_size), np.sqrt(1. / hidden_size),\n",
    "                                    (vocab_size, hidden_size))\n",
    "        self.by = np.zeros((vocab_size, 1))\n",
    "\n",
    "        # initialize gradients for each parameter\n",
    "        self.dWz, self.dWr, self.dWa, self.dWy = np.zeros_like(self.Wz), np.zeros_like(self.Wr), np.zeros_like(\n",
    "            self.Wa), np.zeros_like(self.Wy)\n",
    "        self.dbz, self.dbr, self.dba, self.dby = np.zeros_like(self.bz), np.zeros_like(self.br), np.zeros_like(\n",
    "            self.bz), np.zeros_like(self.by)\n",
    "\n",
    "        # initialize parameters for adamw optimizer\n",
    "        self.mWz = np.zeros_like(self.Wz)\n",
    "        self.vWz = np.zeros_like(self.Wz)\n",
    "        self.mWr = np.zeros_like(self.Wr)\n",
    "        self.vWr = np.zeros_like(self.Wr)\n",
    "        self.mWa = np.zeros_like(self.Wa)\n",
    "        self.vWa = np.zeros_like(self.Wa)\n",
    "        self.mWy = np.zeros_like(self.Wy)\n",
    "        self.vWy = np.zeros_like(self.Wy)\n",
    "        self.mbz = np.zeros_like(self.bz)\n",
    "        self.vbz = np.zeros_like(self.bz)\n",
    "        self.mbr = np.zeros_like(self.br)\n",
    "        self.vbr = np.zeros_like(self.br)\n",
    "        self.mba = np.zeros_like(self.ba)\n",
    "        self.vba = np.zeros_like(self.ba)\n",
    "        self.mby = np.zeros_like(self.by)\n",
    "        self.vby = np.zeros_like(self.by)\n",
    "\n",
    "    def sigmoid(self, x):\n",
    "        \"\"\"\n",
    "        Computes the sigmoid activation function for a given input array.\n",
    "\n",
    "        Parameters:\n",
    "            x (ndarray): Input array.\n",
    "\n",
    "        Returns:\n",
    "            ndarray: Array of the same shape as `x`, containing the sigmoid activation values.\n",
    "        \"\"\"\n",
    "        return 1 / (1 + np.exp(-x))\n",
    "\n",
    "    def softmax(self, x):\n",
    "        \"\"\"\n",
    "        Computes the softmax activation function for a given input array.\n",
    "\n",
    "        Parameters:\n",
    "            x (ndarray): Input array.\n",
    "\n",
    "        Returns:\n",
    "            ndarray: Array of the same shape as `x`, containing the softmax activation values.\n",
    "        \"\"\"\n",
    "        # shift the input to prevent overflow when computing the exponentials\n",
    "        x = x - np.max(x)\n",
    "        # compute the exponentials of the shifted input\n",
    "        p = np.exp(x)\n",
    "        # normalize the exponentials by dividing by their sum\n",
    "        return p / np.sum(p)\n",
    "\n",
    "    def forward(self, X, c_prev, a_prev):\n",
    "        \"\"\"\n",
    "        Performs forward propagation for a simple GRU model.\n",
    "\n",
    "        Args:\n",
    "            X (numpy array): Input sequence, shape (sequence_length, input_size)\n",
    "            c_prev (numpy array): Previous cell state, shape (hidden_size, 1)\n",
    "            a_prev (numpy array): Previous hidden state, shape (hidden_size, 1)\n",
    "\n",
    "        Returns: X (numpy array): Input sequence, shape (sequence_length, input_size) c (dictionary): Cell state for\n",
    "        each time step, keys = time step, values = numpy array shape (hidden_size, 1) r (dictionary): Reset gate for\n",
    "        each time step, keys = time step, values = numpy array shape (hidden_size, 1) z (dictionary): Update gate for\n",
    "        each time step, keys = time step, values = numpy array shape (hidden_size, 1) cc (dictionary): Candidate cell\n",
    "        state for each time step, keys = time step, values = numpy array shape (hidden_size, 1) a (dictionary):\n",
    "        Hidden state for each time step, keys = time step, values = numpy array shape (hidden_size, 1) y_pred (\n",
    "        dictionary): Output probability vector for each time step, keys = time step, values = numpy array shape (\n",
    "        output_size, 1)\n",
    "        \"\"\"\n",
    "\n",
    "        # initialize dictionaries for backpropagation\n",
    "        # initialize dictionaries for backpropagation\n",
    "        r, z, c, cc, a, y_pred = {}, {}, {}, {}, {}, {}\n",
    "        c[-1] = np.copy(c_prev)  # store the initial cell state in the dictionary\n",
    "        a[-1] = np.copy(a_prev)  # store the initial hidden state in the dictionary\n",
    "\n",
    "        # iterate over each time step in the input sequence\n",
    "        for t in range(X.shape[0]):\n",
    "            # concatenate the input and hidden state\n",
    "            xt = X[t, :].reshape(-1, 1)\n",
    "            concat = np.vstack((a[t - 1], xt))\n",
    "\n",
    "            # compute the reset gate\n",
    "            r[t] = self.sigmoid(np.dot(self.Wr, concat) + self.br)\n",
    "\n",
    "            # compute the update gate\n",
    "            z[t] = self.sigmoid(np.dot(self.Wz, concat) + self.bz)\n",
    "\n",
    "            # compute the candidate cell state\n",
    "            cc[t] = np.tanh(np.dot(self.Wa, np.vstack((r[t] * a[t - 1], xt))) + self.ba)\n",
    "\n",
    "            # compute the cell state\n",
    "            c[t] = z[t] * cc[t] + (1 - z[t]) * c[t - 1]\n",
    "\n",
    "            # compute the hidden state\n",
    "            a[t] = c[t]\n",
    "\n",
    "            # compute the output probability vector\n",
    "            y_pred[t] = self.softmax(np.dot(self.Wy, a[t]) + self.by)\n",
    "\n",
    "        # return the output probability vectors, cell state, hidden state and gate vectors\n",
    "        return X, r, z, c, cc, a, y_pred\n",
    "\n",
    "    def backward(self, X, a_prev, c_prev, r, z, c, cc, a, y_pred, targets):\n",
    "        \"\"\"\n",
    "        Performs backward propagation through time for a GRU network.\n",
    "\n",
    "        Args:\n",
    "            X (numpy array): Input sequence, shape (sequence_length, input_size)\n",
    "            a_prev (numpy array): Previous hidden state, shape (hidden_size, 1)\n",
    "            r (dictionary): Reset gate for each time step, keys = time step, values = numpy array shape (hidden_size, 1)\n",
    "            z (dictionary): Update gate for each time step, keys = time step, values = numpy array shape (hidden_size, 1)\n",
    "            c (dictionary): Cell state for each time step, keys = time step, values = numpy array shape (hidden_size, 1)\n",
    "            cc (dictionary): Candidate cell state for each time step, keys = time step, values = numpy array shape (hidden_size, 1)\n",
    "            a (dictionary): Hidden state for each time step, keys = time step, values = numpy array shape (hidden_size, 1)\n",
    "            y_pred (dictionary): Output probability vector for each time step, keys = time step, values = numpy array shape (output_size, 1)\n",
    "            targets (numpy array): Target outputs for each time step, shape (sequence_length, output_size)\n",
    "\n",
    "        Returns:\n",
    "            None       \n",
    "        \"\"\"\n",
    "        # Initialize gradients for hidden state\n",
    "        dc_next = np.zeros_like(c_prev)\n",
    "        da_next = np.zeros_like(a_prev)\n",
    "\n",
    "        # Iterate backwards through time steps\n",
    "        for t in reversed(range(X.shape[0])):\n",
    "            # compute the gradient of the output probability vector\n",
    "            dy = np.copy(y_pred[t])\n",
    "            dy[targets[t]] -= 1\n",
    "\n",
    "            # compute the gradient of the output layer weights and biases\n",
    "            self.dWy += np.dot(dy, a[t].T)\n",
    "            self.dby += dy\n",
    "\n",
    "            # compute the gradient of the hidden state\n",
    "            da = np.dot(self.Wy.T, dy) + da_next\n",
    "\n",
    "            # compute the gradient of the update gate\n",
    "            xt = X[t, :].reshape(-1, 1)\n",
    "            concat = np.vstack((a_prev, xt))\n",
    "            dz = da * (a[t] - c[t])\n",
    "            self.dWz += np.dot(dz, concat.T)\n",
    "            self.dbz += dz\n",
    "\n",
    "            # compute the gradient of the reset gate\n",
    "            dr = da * np.dot(self.Wz[:, :self.hidden_size].T, dz) * (1 - r[t]) * r[t]\n",
    "            self.dWr += np.dot(dr, concat.T)\n",
    "            self.dbr += dr\n",
    "\n",
    "            # compute the gradient of the current hidden state\n",
    "            da = np.dot(self.Wa[:, :self.hidden_size].T, dr) + np.dot(self.Wz[:, :self.hidden_size].T, dz)\n",
    "            self.dWa += np.dot(da * (1 - a[t]**2), concat.T)\n",
    "            self.dba += da * (1 - a[t]**2)\n",
    "\n",
    "            # compute the gradient of the input to the next hidden state\n",
    "            da_next = np.dot(self.Wr[:, :self.hidden_size].T, dr) \\\n",
    "                      + np.dot(self.Wz[:, :self.hidden_size].T, dz) \\\n",
    "                      + np.dot(self.Wa[:, :self.hidden_size].T, da)\n",
    "        # clip gradients to avoid exploding gradients\n",
    "        for grad in [self.dWz, self.dWr, self.dWa, self.dWy, self.dbz, self.dbr, self.dba, self.dby]:\n",
    "            np.clip(grad, -1, 1)\n",
    "\n",
    "    def loss(self, y_preds, targets):\n",
    "        \"\"\"\n",
    "        Computes the cross-entropy loss for a given sequence of predicted probabilities and true targets.\n",
    "\n",
    "        Parameters:\n",
    "            y_preds (ndarray): Array of shape (sequence_length, vocab_size) containing the predicted probabilities for each time step.\n",
    "            targets (ndarray): Array of shape (sequence_length, 1) containing the true targets for each time step.\n",
    "\n",
    "        Returns:\n",
    "            float: Cross-entropy loss.\n",
    "        \"\"\"\n",
    "        # calculate cross-entropy loss\n",
    "        return sum(-np.log(y_preds[t][targets[t], 0]) for t in range(self.sequence_length))\n",
    "\n",
    "    def adamw(self, beta1=0.9, beta2=0.999, epsilon=1e-8, L2_reg=1e-4):\n",
    "        \"\"\"\n",
    "        Updates the GRU's parameters using the AdamW optimization algorithm.\n",
    "        \"\"\"\n",
    "        \n",
    "        # AdamW update for Wz\n",
    "        self.mWz = beta1 * self.mWz + (1 - beta1) * self.dWz\n",
    "        self.vWz = beta2 * self.vWz + (1 - beta2) * np.square(self.dWz)\n",
    "        m_hat = self.mWz / (1 - beta1)\n",
    "        v_hat = self.vWz / (1 - beta2)\n",
    "        self.Wz -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.Wz)\n",
    "\n",
    "        # AdamW update for bu\n",
    "        self.mbz = beta1 * self.mbz + (1 - beta1) * self.dbz\n",
    "        self.vbz = beta2 * self.vbz + (1 - beta2) * np.square(self.dbz)\n",
    "        m_hat = self.mbz / (1 - beta1)\n",
    "        v_hat = self.vbz / (1 - beta2)\n",
    "        self.bz -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.bz)\n",
    "\n",
    "        # AdamW update for Wr\n",
    "        self.mWr = beta1 * self.mWr + (1 - beta1) * self.dWr\n",
    "        self.vWr = beta2 * self.vWr + (1 - beta2) * np.square(self.dWr)\n",
    "        m_hat = self.mWr / (1 - beta1)\n",
    "        v_hat = self.vWr / (1 - beta2)\n",
    "        self.Wr -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.Wr)\n",
    "\n",
    "        # AdamW update for br\n",
    "        self.mbr = beta1 * self.mbr + (1 - beta1) * self.dbr\n",
    "        self.vbr = beta2 * self.vbr + (1 - beta2) * np.square(self.dbr)\n",
    "        m_hat = self.mbr / (1 - beta1)\n",
    "        v_hat = self.vbr / (1 - beta2)\n",
    "        self.br -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.br)\n",
    "\n",
    "        # AdamW update for Wa\n",
    "        self.mWa = beta1 * self.mWa + (1 - beta1) * self.dWa\n",
    "        self.vWa = beta2 * self.vWa + (1 - beta2) * np.square(self.dWa)\n",
    "        m_hat = self.mWa / (1 - beta1)\n",
    "        v_hat = self.vWa / (1 - beta2)\n",
    "        self.Wa -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.Wa)\n",
    "\n",
    "        # AdamW update for br\n",
    "        self.mba = beta1 * self.mba + (1 - beta1) * self.dba\n",
    "        self.vba = beta2 * self.vba + (1 - beta2) * np.square(self.dba)\n",
    "        m_hat = self.mba / (1 - beta1)\n",
    "        v_hat = self.vba / (1 - beta2)\n",
    "        self.ba -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.ba)\n",
    "\n",
    "        # AdamW update for Wy\n",
    "        self.mWy = beta1 * self.mWy + (1 - beta1) * self.dWy\n",
    "        self.vWy = beta2 * self.vWy + (1 - beta2) * np.square(self.dWy)\n",
    "        m_hat = self.mWy / (1 - beta1)\n",
    "        v_hat = self.vWy / (1 - beta2)\n",
    "        self.Wy -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.Wy)\n",
    "\n",
    "        # AdamW update for by\n",
    "        self.mby = beta1 * self.mby + (1 - beta1) * self.dby\n",
    "        self.vby = beta2 * self.vby + (1 - beta2) * np.square(self.dby)\n",
    "        m_hat = self.mby / (1 - beta1)\n",
    "        v_hat = self.vby / (1 - beta2)\n",
    "        self.by -= self.learning_rate * (m_hat / (np.sqrt(v_hat) + epsilon) + L2_reg * self.by)\n",
    "\n",
    "    def train(self, data_generator,iterations):\n",
    "        \"\"\"\n",
    "        Train the GRU on a dataset using backpropagation through time.\n",
    "\n",
    "        Args:\n",
    "            data_generator: An instance of DataGenerator containing the training data.\n",
    "\n",
    "        Returns:\n",
    "            None\n",
    "        \"\"\"\n",
    "        iter_num = 0\n",
    "        # stopping criterion for training\n",
    "        threshold = 50\n",
    "    \n",
    "        smooth_loss = -np.log(1.0 / data_generator.vocab_size) * self.sequence_length  # initialize loss\n",
    "        while (iter_num < iterations):\n",
    "            # initialize hidden state at the beginning of each sequence\n",
    "            if data_generator.pointer == 0:\n",
    "                c_prev = np.zeros((self.hidden_size, 1))\n",
    "                a_prev = np.zeros((self.hidden_size, 1))\n",
    "\n",
    "            # get a batch of inputs and targets\n",
    "            inputs, targets = data_generator.next_batch()\n",
    "\n",
    "            # forward pass\n",
    "            X, r, z, c, cc, a, y_pred = self.forward(inputs, c_prev, a_prev)\n",
    "\n",
    "            # backward pass\n",
    "            self.backward(X, a_prev, c_prev, r, z, c, cc, a, y_pred, targets)\n",
    "\n",
    "            # calculate and update loss\n",
    "            loss = self.loss(y_pred, targets)\n",
    "            self.adamw()\n",
    "            smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
    "            # update previous hidden state for the next batch\n",
    "            a_prev = a[self.sequence_length - 1]\n",
    "            c_prev = c[self.sequence_length - 1]\n",
    "#             if iter_num == 5900 or iter_num == 30000:\n",
    "#                         self.learning_rate *= 0.1\n",
    "            # print progress every 100 iterations\n",
    "            if iter_num % 100 == 0:\n",
    "#                 self.learning_rate *= 0.99\n",
    "                sample_idx = self.sample(c_prev, a_prev, inputs[0, :], 200)\n",
    "                print(''.join(data_generator.idx_to_char[idx] for idx in sample_idx))\n",
    "                print(\"\\n\\niter :%d, loss:%f\" % (iter_num, smooth_loss))\n",
    "            iter_num += 1\n",
    "\n",
    "    def sample(self, c_prev, a_prev, seed_idx, n):\n",
    "        \"\"\"\n",
    "        Sample a sequence of integers from the model.\n",
    "\n",
    "        Args:\n",
    "            c_prev (numpy.ndarray): Previous cell state, a numpy array of shape (hidden_size, 1).\n",
    "            a_prev (numpy.ndarray): Previous hidden state, a numpy array of shape (hidden_size, 1).\n",
    "            seed_idx (numpy.ndarray): Seed letter from the first time step, a numpy array of shape (vocab_size, 1).\n",
    "            n (int): Number of characters to generate.\n",
    "\n",
    "        Returns:\n",
    "            list: A list of integers representing the generated sequence.\n",
    "\n",
    "        \"\"\"\n",
    "        # initialize input and seed_idx\n",
    "        x = np.zeros((self.vocab_size, 1))\n",
    "        # convert one-hot encoding to integer index\n",
    "        seed_idx = np.argmax(seed_idx, axis=-1)\n",
    "\n",
    "        # set the seed letter as the input for the first time step\n",
    "        x[seed_idx] = 1\n",
    "\n",
    "        # generate sequence of characters\n",
    "        idxes = []\n",
    "        c = np.copy(c_prev)\n",
    "        a = np.copy(a_prev)\n",
    "        for t in range(n):\n",
    "            # compute the hidden state and cell state\n",
    "            concat = np.vstack((a, x))\n",
    "            z = self.sigmoid(np.dot(self.Wz, concat) + self.bz)\n",
    "            r = self.sigmoid(np.dot(self.Wr, concat) + self.br)\n",
    "            cc = np.tanh(np.dot(self.Wa, np.vstack((r * a, x))) + self.ba)\n",
    "            c = z * c + (1 - z) * cc\n",
    "            a = c\n",
    "            # compute the output probabilities\n",
    "            y = self.softmax(np.dot(self.Wy, a) + self.by)\n",
    "\n",
    "            # sample the next character from the output probabilities\n",
    "            idx = np.random.choice(range(self.vocab_size), p=y.ravel())\n",
    "\n",
    "            # set the input for the next time step\n",
    "            x = np.zeros((self.vocab_size, 1))\n",
    "            x[idx] = 1\n",
    "\n",
    "            # append the sampled character to the sequence\n",
    "            idxes.append(idx)\n",
    "\n",
    "        # return the generated sequence\n",
    "        return idxes\n",
    "\n",
    "    def predict(self, data_generator, start, n):\n",
    "        \"\"\"\n",
    "        Generate a sequence of n characters using the trained GRU model, starting from the given start sequence.\n",
    "\n",
    "        Args:\n",
    "        - data_generator: an instance of DataGenerator\n",
    "        - start: a string containing the start sequence\n",
    "        - n: an integer indicating the length of the generated sequence\n",
    "\n",
    "        Returns:\n",
    "        - txt: a string containing the generated sequence\n",
    "        \"\"\"\n",
    "        # initialize input sequence\n",
    "        x = np.zeros((self.vocab_size, 1))\n",
    "        chars = [ch for ch in start]\n",
    "        idxes = []\n",
    "        for i in range(len(chars)):\n",
    "            idx = data_generator.char_to_idx[chars[i]]\n",
    "            x[idx] = 1\n",
    "            idxes.append(idx)\n",
    "        # initialize cell state and hidden state\n",
    "        a = np.zeros((self.hidden_size, 1))\n",
    "        c = np.zeros((self.hidden_size, 1))\n",
    "\n",
    "        # generate new sequence of characters\n",
    "        for t in range(n):\n",
    "            # compute the hidden state and cell state\n",
    "            concat = np.vstack((a, x))\n",
    "\n",
    "            # compute the reset gate\n",
    "            r = self.sigmoid(np.dot(self.Wr, concat) + self.br)\n",
    "\n",
    "            # compute the update gate\n",
    "            z = self.sigmoid(np.dot(self.Wz, concat) + self.bz)\n",
    "\n",
    "            # compute the candidate cell state\n",
    "            cc = np.tanh(np.dot(self.Wa, np.vstack((r * a, x))) + self.ba)\n",
    "\n",
    "            # compute the cell state\n",
    "            c = z * cc + (1 - z) * c\n",
    "\n",
    "            # compute the hidden state\n",
    "            a = c\n",
    "\n",
    "            # compute the output probability vector\n",
    "            y_pred = self.softmax(np.dot(self.Wy, a) + self.by)\n",
    "            # sample the next character from the output probabilities\n",
    "            idx = np.random.choice(range(self.vocab_size), p=y_pred.ravel())\n",
    "            x = np.zeros((self.vocab_size, 1))\n",
    "            x[idx] = 1\n",
    "            idxes.append(idx)\n",
    "        txt = ''.join(data_generator.idx_to_char[i] for i in idxes)\n",
    "        return txt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a9b617d1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-11T07:02:53.584791Z",
     "iopub.status.busy": "2024-06-11T07:02:53.584417Z",
     "iopub.status.idle": "2024-06-11T07:04:27.330696Z",
     "shell.execute_reply": "2024-06-11T07:04:27.329516Z"
    },
    "papermill": {
     "duration": 93.754168,
     "end_time": "2024-06-11T07:04:27.334392",
     "exception": false,
     "start_time": "2024-06-11T07:02:53.580224",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ":Hua:vGI3eZeYpNvFgz$Yff'ySa:dFgXZJuGIJW$ ipCGRoZIsXT;zkuDh-?e!i$-F.btpbUraeGkhYS;b&UEJytc'PubecXpYXm,3&!!Dw\n",
      ":l$JSp?VttZplx Su;uANtN;s\n",
      "V$oG&FBVWUyfQitXskrLaDhTDgrZ:sFPVDSOF&fYdV!.-EiqY'\n",
      ",dnfO?OGXW-KrKI\n",
      "\n",
      "\n",
      "iter :0, loss:100.185202\n",
      ".e&o,CBy$luhK,tysG it ceer;i c mlaktoehaSelulleas,sO ut oao aw htiaenenhceq&haEll\n",
      "BiMhlT\n",
      "mNc qHctC\n",
      "\n",
      "l\n",
      "q?yDdr.ugse tasordp CpEstt hth tscdnocScLeG!3y,drsdc odc ury\n",
      "\n",
      "q$SntoawyYreto trCudD pnhmdi. pt\n",
      "\n",
      "ze\n",
      "\n",
      "\n",
      "iter :100, loss:98.317961\n",
      "n'un oureend ys tiit taheedd ailnd tie te se t ta osourn auarit sh,dmsou, ayizel wh yr hemstaoss?t ten gg n omigr hhe tes bacrsyu,th y eayle\n",
      "Ien elllo dhueh ot halyte my, mIdcouom hae mns theelnoe\n",
      "v m\n",
      "\n",
      "\n",
      "iter :200, loss:95.657128\n",
      "alvt imse hed itrsare ats bo he tatad thea m ffrriern weg  co mw\n",
      "SAlsFast he andsfo,:\n",
      "iztal sfe t oures th epy ouad tgllllll\n",
      "\n",
      "Frittliznnd wharee mne foren le fyfre ter,- athe alllomritn athe y aftlreE\n",
      "\n",
      "\n",
      "iter :300, loss:92.904172\n",
      " ot,o\n",
      "el yoo, gheas te o whanhet,thelu be moe,\n",
      "Whe ve p can wre'oue heallv gasbd, algfo,aell s, mthezakti,o\n",
      " ayu sca bce codo lydt  fcoro eo Iis,t ou p\n",
      "Whael,e cmb fcod owhe c t irae arherease.ndg ind\n",
      "\n",
      "\n",
      "iter :400, loss:90.512794\n",
      "s, pfing rahtine es co wict\n",
      "herhat hes thare bthave s cyg\n",
      "izheesl, orou\n",
      "Wsapst roie  syohres hatende ong y wih th pe ansl oath aesyot in, o aev ic coup ithns thesoy ins at co\n",
      "EirFs,th athar the s wath\n",
      "\n",
      "\n",
      "iter :500, loss:88.580773\n",
      "e thtin soue.\n",
      "\n",
      "AS:\n",
      "MYThhoirses thru erst aly' indoston\n",
      "T'o\n",
      "\n",
      "TThthedeon.\n",
      "Ielasl whan:\n",
      "AGr rous s athres  wiraten s isve thaitheste m t atheer, he v hp ehine'ed yorsrou puser prou,in dooun haned pirs in\n",
      "\n",
      "\n",
      "iter :600, loss:86.525404\n",
      "IUSS:\n",
      "TTon:g\n",
      "\n",
      "Ther ymin\n",
      "Asien:'u\n",
      "o\n",
      "MENIUS:irsupur ousoun:\n",
      "A\n",
      "Thto\n",
      "Irore wean'.\n",
      "\n",
      "TIUS-he fno'ds wh' mec mus msmancaknnkoI wpellies y co dsm yinte in:thihenso fourU thein sci thitize:\n",
      "IIUS:\n",
      "S:o\n",
      "MNMENIUNI\n",
      "\n",
      "\n",
      "iter :700, loss:85.043075\n",
      " dsy\n",
      "Airu e'\n",
      "Ait pall yauk mondaind ed filone: u\n",
      "IUSlSe: ours asd tind yoe me\n",
      "To ms' war whee mad warde.s;\n",
      "ieant e t my;ak de ffr Iy u as.\n",
      "B pnod de-e ak lyound.\n",
      "\n",
      "MENIUS:\n",
      "\n",
      "ATn\n",
      "Te mp ald ve'd n'k\n",
      "IUSn:\n",
      "\n",
      "\n",
      "iter :800, loss:83.899490\n",
      " utr adre,:\n",
      "S:\n",
      "ENIIUS:\n",
      "Ft heand ofn weakle y, bllyothe' fowedo.\n",
      "I:\n",
      "ARI bandou hable.\n",
      "Vd IUS:\n",
      "S:\n",
      "OS:\n",
      "NIUS:p s.\n",
      "A gad wad fomyo.\n",
      "B\n",
      "Hu bwh:\n",
      "Alre am blofn yllldMart ge an ady ya thbe and\n",
      "hI':\n",
      "OB\n",
      "AleIa my'\n",
      "\n",
      "\n",
      "iter :900, loss:82.081073\n",
      "'\n",
      "O\n",
      "RHe; thole ble fommUS:I:'dIIUS:C\n",
      "A\n",
      "NIan:\n",
      "\n",
      "Ohe b lupr s latrecud y uss.\n",
      "IIUS::S:\n",
      "S: ofyo mwthe' beld das mmaeirme, beelllld.\n",
      "I\n",
      "Are s ug t, oe y, owurik whoit, flot go whor art huflel fIo whaldilobu\n",
      "\n",
      "\n",
      "iter :1000, loss:80.442129\n",
      " hthey gov weser thisu thonorccIur folrud Ciuse.\n",
      "\n",
      "H\n",
      "AIt,\n",
      "IUS:\n",
      "S:HNIUS:ecouul l ono whor thto olf owwh themI thee.\n",
      "o r hus, ond allitrane sese,\n",
      "Tsak hote st w heol:\n",
      "VL a gh\n",
      "A torc teprto t hIe  asith e\n",
      "\n",
      "\n",
      "iter :1100, loss:78.811536\n",
      "nd, ithyo, I!\n",
      "YRCA ourto hm ieccouup this t' oferyot tour.\n",
      "\n",
      "CIUS:\n",
      "H:\n",
      "MNII:\n",
      "ARCIATu ses th ius\n",
      "cto, geatrcceaccus,\n",
      "A CIIUS:\n",
      "Whiemesiur thy g hatriken;,\n",
      "INIUS:S:\n",
      "B\n",
      "C\n",
      "ARC:\n",
      "CINa Ipofr' garemo, fald o tor,\n",
      "\n",
      "\n",
      "iter :1200, loss:77.323965\n",
      "et,\n",
      "C\n",
      "F tes' bud sout CIour thith\n",
      ":\n",
      "MARt sgo, mavever venses? panc;e fa;dan kus powirecus! ther yu bus!\n",
      "BLINIUS:\n",
      "WCNIUS::\n",
      "LRAyu thalllshine sihusus ustea do pare hnide she cius pee pitet entor thas oe\n",
      "\n",
      "\n",
      "iter :1300, loss:75.738777\n",
      "S:or te.e \n",
      "Hen goet,o wheann, ou\n",
      "t\n",
      "hay  chand thanse theutri I:e uhst\n",
      " thand ithiuse masin sete, owi ugt ion?\n",
      "Ve,\n",
      "ARken si gass!\n",
      "Vancit hamende and,\n",
      "Wi gh\n",
      "aincde se ntor otithane yu the womaende, dart\n",
      "\n",
      "\n",
      "iter :1400, loss:74.606535\n",
      "e frldad,\n",
      "WARCNIUS:\n",
      "L\n",
      "YVUS:C\n",
      "Heun fin soo waonded te weliberu, whet!\n",
      "\n",
      "BC\n",
      "CRMCIUS:\n",
      "MARCne w wiglus ofreillad for forme wo senon me ange s'fe orbu fore tr aldo menk ist cino thite thte-,\n",
      "\n",
      "BLRC ein yoo w\n",
      "\n",
      "\n",
      "iter :1500, loss:73.574237\n",
      "al reramu\n",
      "Wank iny omd svepare ad gonoutrt,\n",
      "Wan cok odi mogonke anded ten dery one me g averin mory ofmy for atnoer sav ngon oor befre-------------------------------------------------------- bmeso w h\n",
      "\n",
      "\n",
      "iter :1600, loss:72.628815\n",
      "s piend oTgve pereart doreing coor t thin utec frec,a mopp ram y owufr the yy omof oind omye tr obles pofropr thaly lin govet; wear che lld pai nwhaithi me am and yo t my our fol oupf oufr youl bllat \n",
      "\n",
      "\n",
      "iter :1700, loss:71.846101\n",
      "ret ert hebret yiugad ingant.\n",
      "\n",
      "MRCUS:\n",
      "VOLMaver omablll\n",
      "B\n",
      "OLUS:elalinok bend theefrey port whagill andy er tourth your that! rthallll ingo' s'loburs twitheri ys odiko bl,uprtuss angeny ol pers, mcingo \n",
      "\n",
      "\n",
      "iter :1800, loss:71.112346\n",
      "orever cth anth thwealll\n",
      "iMVLARThars polhee the nobey ugsriget.\n",
      "y\n",
      "VLRIARUS:e,\n",
      "MARCous, y ht omter cont h thefart havet hmet he fict.\n",
      "ThUS:\n",
      "MaNpENIUS:bUey uts hgereagere fih laveneterciuran, thu cingor\n",
      "\n",
      "\n",
      "iter :1900, loss:70.429889\n",
      "g head an citim,\n",
      "TUS:\n",
      "VOLatyarallse worin igatheres,\n",
      "oucingies,\n",
      "\n",
      "OM, wesers poursin goll ivelitosucllin hersed the wioni thalitshen kensy ofuarsi\n",
      "VOMal:\n",
      "MNENENENENIUS:\n",
      "\n",
      "BRCO, tinous avlisthe ter.\n",
      "\n",
      "MRU\n",
      "\n",
      "\n",
      "iter :2000, loss:69.698235\n",
      "wheisss, wow\n",
      "Vhe whes theesipast.\n",
      "\n",
      "Th coheis noout\n",
      "oul thed tantinn, winoo\n",
      "VARTUSepsind us\n",
      "'\n",
      "The pole WhiMIA:\n",
      "Tles.\n",
      "\n",
      "BRCUS:\n",
      "TUSUS:on twind at.\n",
      "Horets cheme pthes wesin, aserasp sot h's th home pre tth\n",
      "\n",
      "\n",
      "iter :2100, loss:69.064900\n",
      "iOMENIA:\n",
      "Tel:\n",
      "\n",
      "Fis\n",
      "owir soulls\n",
      "\n",
      "Thomeve ns non owod he shan ceano b deingo, fanke nse ek\n",
      "iu nchincok tinof IA thou nt soh h\n",
      "MENENINUS:\n",
      "OLMA:\n",
      "Fo thall, nosTh ho thes, wimve,\n",
      "\n",
      "TMENENEENENENEENENEN\n",
      "ENUS:\n",
      "\n",
      "\n",
      "iter :2200, loss:68.608172\n",
      "me beron ode cofer ass\n",
      "ENENENENENINENENA:\n",
      "Fir ust rerod 's eth an; leirivend s, to cceorof sites nan's twolinod soucs sat to norem, f noond tor mke kon tthevespit so whe ndi ss\n",
      "MENENENENENENENENENI:\n",
      "O\n",
      "\n",
      "\n",
      "iter :2300, loss:67.800775\n",
      "if 'd be cve ur ntol praitres rpe wourl e cod ake nu bere ifto meer wainspe tanod tin ofer cmin.\n",
      "\n",
      "FRGIe bel andeerve tte rovre te hir pot ofdi Rowo nder'se, is be aried se iturcoom che anve rdir mand \n",
      "\n",
      "\n",
      "iter :2400, loss:67.408757\n",
      "pousandireve nde thivereved and it gabare ceorwe 'd vepend me bevd arve darit rucis urild,\n",
      "Yorve pe hue thime rtid, u nom ben, veere rit noco wGli fcheis his ac'dits andes ris wol 'thi nom.\n",
      "\n",
      "FI:\n",
      "IL IA\n",
      "\n",
      "\n",
      "iter :2500, loss:66.810755\n",
      "ne rig he wolerit aiced tit thim, ht on tireve nwitut; satinigth him iciomo nd eatils wer gethe im mIak IAnd I Ar: uca panins:\n",
      "Hu.\n",
      "\n",
      "Fince, re chimble iravit; berepit rvet mend andos\n",
      "Thip\n",
      "eul.\n",
      "\n",
      "Hov.\n",
      "ea\n",
      "\n",
      "\n",
      "iter :2600, loss:66.307699\n",
      " wlow wor fatm asrmy, ver le hvetulcke yu tcho the yirould it Iem co\n",
      "and tim youtr.\n",
      "\n",
      "GINUS:\n",
      "H rhaveravis uradyur athe roud wec,o hm eter:\n",
      "ay, I th\n",
      "ol'ed, ay surckonm, oul\n",
      "and.\n",
      "Wit stecour pend Wheetmi\n",
      "\n",
      "\n",
      "iter :2700, loss:65.889164\n",
      "od owllice handist; hiscesto-?\n",
      "usl ov the have\n",
      "yor hiarveeceme s:\n",
      "COINUS:\n",
      "Yehi my, wol:\n",
      "INI:\n",
      "ININI: INI I ntcoug herood my ourh ountowhem yor do.\n",
      "\n",
      "Frotur; hak yuun tillal. h\n",
      "Yopute:\n",
      "CINI:\n",
      "\n",
      "INI tomense\n",
      "\n",
      "\n",
      "iter :2800, loss:65.450410\n",
      "he yourly; yow\n",
      "olt hams\n",
      "\n",
      "CINI: f ong\n",
      "oont-\n",
      "aconco hamans tusS:eece ol athem I: h hakend, neulle.\n",
      "\n",
      "CORILA:\n",
      "OUS:\n",
      "He ly toonm young; higatit;\n",
      "enwele chanemy, ount\n",
      "oret; hus\n",
      "e ongoul sthe, rises.\n",
      "\n",
      "RGI A:\n",
      "\n",
      "\n",
      "\n",
      "iter :2900, loss:64.664483\n",
      "nendis yhurl.\n",
      "HRong\n",
      "CINIUS:\n",
      "COLotu chak, nghe hante ang hantepethun\n",
      "lyo, n\n",
      "Whal.\n",
      "\n",
      "BRI: thesesour.\n",
      "\n",
      "COOLAUS:\n",
      "NINUS:\n",
      "CINUS:\n",
      "y\n",
      "HH I Cak nt wheles\n",
      "aUS: I'l\n",
      "ININI I hally re, h oplily caew lhilf code?\n",
      "\n",
      "G\n",
      "B\n",
      "\n",
      "\n",
      "iter :3000, loss:64.356022\n",
      "e t hure'ss hay asolend se\n",
      "mour CAun\n",
      "Ton, ryople\n",
      "antorf ong bat thaspe\n",
      "Th fo ferady, \n",
      "Whans s yoof ot obrege.\n",
      "own ont he:\n",
      "\n",
      "BRUS:\n",
      "WINI:\n",
      "G, han dhany,\n",
      "Wheres das plalles yould,\n",
      "Whet hasmO far sant\n",
      "yoppl\n",
      "\n",
      "\n",
      "iter :3100, loss:64.145914\n",
      "ory were.sonf so srecethen youtrt he\n",
      "C\n",
      "BUS:\n",
      "Selcerer: hopalt tha rave ond\n",
      "Whemizes stha fyor you bys\n",
      "Then.\n",
      "S:\n",
      "IOLANUS:\n",
      "Th hadi si soerth foron sope wh hairs so fof the hanst hastimy,\n",
      "An wetre now one \n",
      "\n",
      "\n",
      "iter :3200, loss:63.910229\n",
      "es betere?\n",
      "A:\n",
      "COOfffrizeng.\n",
      "Towil, thelirav b yon gizest oft ow\n",
      "Whe sow res ffor; an bot re'd stens oon\n",
      "Tit, nfospe potwen,- when woneerte sat yo wour tCon to wonte orbe te roneg hse fofr fcees torave\n",
      "\n",
      "\n",
      "iter :3300, loss:63.527096\n",
      "t wintourtt bees.\n",
      "WAdt mantrist owe, popere t weristhif sourt om ben ors epesl-e tat ceant 's owon manbe rot the bot od itretil'd betiseur: mt atirt borg aft cer'sift fr owistasir, dtee sibtd diat sbe\n",
      "\n",
      "\n",
      "iter :3400, loss:63.088516\n",
      " thate atre'd sad tize bitult thean timou.\n",
      "\n",
      "BRUSTUS:\n",
      "T fofr efise wister, for ateakr burbere ghal fore!\n",
      "aes tafizethe mit\n",
      "batritrat comak,\n",
      "\n",
      "BUSorbe ff teiserad plits os ece por at igthe wof wat on ofe\n",
      "\n",
      "\n",
      "iter :3500, loss:62.881928\n",
      "al'dt othieor, tit ho't atd ceoth,\n",
      "\n",
      "WAncourat peanse fople cherave ckite obed ad,\n",
      "any to dis on tth imanicnot r'd the\n",
      "SICOOLAll ietobe! ro' dYour, coer, 'd dHe!\n",
      "es oud thimes,\n",
      "\n",
      "Sar bewere dan ribt iri\n",
      "\n",
      "\n",
      "iter :3600, loss:62.525906\n",
      "hat vithas, nar te, inoat th asil lak dead wonur, ldo tou do'd sousr' du dad ffur cous weldicer'd oun.\n",
      "Fri thecman\n",
      "MENENI:\n",
      "\n",
      "SIOLANUSeecoun got hamintledes dooi cat he rast wecher\n",
      "opl, wan tetinar, bu\n",
      "\n",
      "\n",
      "\n",
      "iter :3700, loss:62.444984\n",
      "LANUS:\n",
      "Thast oure.\n",
      "IUS:\n",
      "OMENENIUSUS:\n",
      "CiORIUSLANUSoin tha\n",
      "willld t\n",
      "Thay,\n",
      "\n",
      "ENIUS:\n",
      "SID thin talll' du ter,\n",
      "\n",
      "Weing nat perask,a ncemen!\n",
      "\n",
      "COul,\n",
      "\n",
      "Whains plads me\n",
      "Tho wan do d Yomasincus, Youn;\n",
      "\n",
      "Aseuds meake\n",
      "\n",
      "\n",
      "iter :3800, loss:62.362802\n",
      "cear nd atuchim.\n",
      "\n",
      "You'd dinos, pateo wed emang ghe'se\n",
      "Anasc mak nepores.\n",
      "\n",
      "PMortis poun thang thra vces.\n",
      "\n",
      "Fcounme!\n",
      "\n",
      "OMENIUS:\n",
      "Of alds,\n",
      "Seex\n",
      "MENID Ithanus thewererisin.\n",
      "\n",
      "MENIUSIOLAns, minmbetr.\n",
      "\n",
      "PMNII IL\n",
      "\n",
      "\n",
      "iter :3900, loss:61.976163\n",
      "\n",
      "Th gure gho sheagritn toh,\n",
      "\n",
      "MIIOLANIUSI:\n",
      "Th tha verighes,\n",
      "ILANUSUS:\n",
      "Whidin de mans udmeas my pithurcensag\n",
      "ry, bu doftn fre\n",
      "Wassy, nu'ds nand d You cervis chemeceredill meance.\n",
      "il, noth mast, henor pe\n",
      "\n",
      "\n",
      "iter :4000, loss:61.840595\n",
      " ce omere'sthe rors.\n",
      "\n",
      "CIOLr de um cnoind.\n",
      "CIOLANIUS:\n",
      "MII ILANIUUS:\n",
      "You'd whearst haslld int hore\n",
      "Whor, Mand th her on me\n",
      "no'ss tore mbeigg hrs:\n",
      "Whteree!\n",
      "\n",
      "CORIOL MIIOLNIUS:\n",
      "I ICORILAnd il viend re yo t\n",
      "\n",
      "\n",
      "iter :4100, loss:61.882617\n",
      "t hogers of theous ty ns folince?e\n",
      "\n",
      "MENIUSIUSesex ICORIILAs.\n",
      "\n",
      "MEENENI'USI COMINIILANIUS:\n",
      "Macan thor, Heonge;\n",
      "it honme sho hastus meall, I the nakten: toa veeng wlith beris, the par ysi angut hil vecth\n",
      "\n",
      "\n",
      "iter :4200, loss:61.582568\n",
      "s go bu ste rok, lit o ftho the orge beasto poswhand who she cvos las, ofe tho hithe sos yows thost has winseas tis hes,end y ow leet ags atry ible hat ives, whirercorimiz wo ferithow tis atrk poert: \n",
      "\n",
      "\n",
      "iter :4300, loss:61.780059\n",
      " yono?\n",
      "Bf\n",
      "Ate ower\n",
      "int issou feasr vos ake u nabefortil bleas woint to to af woe aty o weod me by itay rsefereke, whonust mors whondes,----e war yo sh fe ormme ay sy btrikne as obay wond tat hero sow \n",
      "\n",
      "\n",
      "iter :4400, loss:61.744010\n",
      "oud at breanf dofre.\n",
      "o yo fit here they ined man aty ban om thay thaistehay wole avefe fomien:\n",
      "WHilel, wo byu thoist have avbevi yoand bea you fary nofog tanefity e foft held ibens loat hasirtos, on n\n",
      "\n",
      "\n",
      "iter :4500, loss:61.792672\n",
      "hal a burt makinf, ant yeled ist igy o thant yous.\n",
      "\n",
      "Cithe le't siby lis ofl?\n",
      "atd ille ivex\n",
      "cemendoe; yo fothy, byou shm.\n",
      "Thit foravend ils at.ibe fal fearumos; thorut braty nsaty!\n",
      "Thas,\n",
      "ri fl yofis: H\n",
      "\n",
      "\n",
      "iter :4600, loss:61.887179\n",
      "y und yus wende?\n",
      "Atind whe ads tris,e'd, thy erithisem;\n",
      "Ast any!\n",
      "\n",
      "Firinit stil!\n",
      "Any thy he ay buft yomu\n",
      "BUTh yo ise,\n",
      "y, eland i fiexe, adroche tyu wi nend ist hanbat yu youtry bul teronceres, ty itout\n",
      "\n",
      "\n",
      "iter :4700, loss:61.996062\n",
      "haravend bit ittyol.\n",
      "'sh wirh d bit rer:\n",
      "\n",
      "Citrinmen:\n",
      "Wther\n",
      "An:\n",
      "Aluse?\n",
      "And y anoctr't hand ty andea, corve?\n",
      "\n",
      "Arim yow, wrin trim.\n",
      "CIOLNUS:\n",
      "ein ty y qumend,e nomy t eanlo,f time nlarvis:\n",
      "Hom wilferaris \n",
      "\n",
      "\n",
      "iter :4800, loss:62.036948\n",
      "rany nondoe; je ndeme',---------\n",
      "wy, u nomen nman tore,--------------------------------thy seal!\n",
      "ull weave ldomen?\n",
      "\n",
      "Fint thme, naty oferus;\n",
      "'t sawnot hy hay mefNy.\n",
      "Wind jerothe:\n",
      "\n",
      "Cing wist\n",
      "an wis im t\n",
      "\n",
      "\n",
      "iter :4900, loss:61.896314\n",
      "nd swanwird meprende 's wervigs himy su whin garucsun mande, sthere mell, Hene than: erd oken yours?\n",
      "Afru shes cre old ferveigg\n",
      "arv:\n",
      "When:\n",
      "chmend incoul:\n",
      "\n",
      "CiOROLUNS:\n",
      "The lmeppeterecen:\n",
      "Then: Haver fre\n",
      "\n",
      "\n",
      "iter :5000, loss:61.759006\n",
      "He by anse': tound ingoe the halu whisigde nce.\n",
      "\n",
      "Dwond mes,\n",
      "\n",
      "EENEENEENENEINUS:\n",
      "Whatri nmeoed\n",
      "meneved neme snd\n",
      "A:\n",
      "\n",
      "RORIOLNUS:\n",
      "is willt hed heave: man w\n",
      "hlu nes pold\n",
      "Se ndoo tiken:\n",
      "\n",
      "Fr rvie'lex\n",
      "\n",
      "RTogr:e\n",
      "\n",
      "\n",
      "iter :5100, loss:61.754423\n",
      "'du shefris irde rquptis sande Iwslus hmes sall owirs hals hg\n",
      "a spres reve ssals,\n",
      "Sous: Hacud,\n",
      " cupiracn mno oe pacpancels ore's woculu st he whot hacor balld some pooken\n",
      "Seppelde th.\n",
      "\n",
      "nderscend\n",
      "\n",
      "Thi \n",
      "\n",
      "\n",
      "iter :5200, loss:61.883614\n",
      "avis pok,an plous ped courlig, he si silag sh out oke he al sod buper's; is Sexav withis sparu soacus bis pous his\n",
      "S:\n",
      "H hancimas, oldo cen mok.\n",
      "\n",
      "HRORU:\n",
      "The Thig. s\n",
      "Servigong puprtik igno:\n",
      "Heel se ello\n",
      "\n",
      "\n",
      "iter :5300, loss:61.931866\n",
      "oken oak usce fof is ouppre,\n",
      "\n",
      "Be't haicst hout he ridout he:\n",
      "Wethe's:\n",
      "\n",
      "Hefid a nouts hamels shan ofarus thas lof acond ad ouperes loalf toris han od\n",
      "ar wiss fourm am anfolutid IUS:\n",
      "Whits ofo bealld\n",
      "\n",
      "G\n",
      "\n",
      "\n",
      "iter :5400, loss:61.708048\n",
      "\n",
      "Hameld oan:\n",
      "Whimon,\n",
      "\n",
      "Your th asire had otolk-\n",
      "es fon t\n",
      "rar tha\n",
      "cke\n",
      "ndo otu.\n",
      "\n",
      "BRUFUS:\n",
      "Col on apre. stoes;\n",
      "\n",
      "Wilolick', I'US:\n",
      "Separe faacnoat, bolfow, th tiredot othe acall-is he aclofte, oro't gthat ol\n",
      "\n",
      "\n",
      "iter :5500, loss:61.705710\n",
      "Sert the tolol; hapieste\n",
      "Whcon at focoreme.\n",
      "Ha lowoth, adest oede\n",
      "rsote, tore thabelfe.\n",
      "\n",
      "Set:\n",
      "Whio tas beft fore othal, I Sethet\n",
      "Seal batin tinme es\n",
      "Sallo uki gonog, acenot alud boafrilo onte toll ofo\n",
      "\n",
      "\n",
      "iter :5600, loss:61.861054\n",
      "'ts hiricno, balloct thyo wal, foul the I Serere, foall kieno, th\n",
      "Herene htie emees.\n",
      "\n",
      "Scam to therve itheren nor bere, outrede talef ot he avefe,\n",
      "Aino ndoe,\n",
      "\n",
      "Yoont,\n",
      "\n",
      "He fo fre,e as beld\n",
      "ofemet asfto. \n",
      "\n",
      "\n",
      "iter :5700, loss:62.372404\n",
      "oe ade toll Raef ofrrerads,\n",
      "Seveclor, I't\n",
      "Serrast, ake fod Seaend\n",
      "Woot henot, I Ser\n",
      "armeerif of tore the ronco thim inals bofre torod;\n",
      "\n",
      "Thied I'ld,\n",
      "\n",
      "Yow the he eram mabe fixe firerneve ttarersew thien\n",
      "\n",
      "\n",
      "iter :5800, loss:62.473962\n",
      "l bils Romeld on th ish onl bere hod Seex\n",
      "ANUS:\n",
      "Have th\n",
      "mare mand hamu by oren tsin praf toor siken, afer icki. o thear one, orancd slam nber in mentoren tthe in, Seeld pindelp soren sorals baet, ere \n",
      "\n",
      "\n",
      "iter :5900, loss:62.549312\n"
     ]
    }
   ],
   "source": [
    "sequence_length = 24\n",
    "#read text from the \"input.txt\" file\n",
    "data_generator = DataGenerator('/kaggle/input/shakespeare-text/text.txt', sequence_length)\n",
    "gru =  GRU(hidden_size=100, vocab_size=data_generator.vocab_size,sequence_length=sequence_length,learning_rate=0.005)\n",
    "\n",
    "gru.train(data_generator,iterations=6000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7c5a3cb6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-11T07:04:27.362478Z",
     "iopub.status.busy": "2024-06-11T07:04:27.361813Z",
     "iopub.status.idle": "2024-06-11T07:04:27.594538Z",
     "shell.execute_reply": "2024-06-11T07:04:27.593389Z"
    },
    "papermill": {
     "duration": 0.250386,
     "end_time": "2024-06-11T07:04:27.597996",
     "exception": false,
     "start_time": "2024-06-11T07:04:27.347610",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"can muermay, I IUS:\\nWthe's Sest hart cormansth an ndesles ton bo wandls wsthe docrumakn! so whices allod renmes llk manw sdoble fit h or owanid, whi ns cour manon rivou nilfr was ndo meand renw moy upre went hteres iken mmy amaked, wello mekin my montat th ongan th ake fou neram nmo traende t aerd amesrst-\\nSed Ser\\nthins mmanre ncamam bly wor Seiuras thand er our ign ferer nannt wiss MI I Iw h'ds in, in masl four wherthaid onnnterem ake wdo Said woir monnnt hisecram mnad, agr wet hinn wornak in'sewirst bs ofres Serve corns enows, wind, hanmenes inowthi winn; urik'ls efiurord; bat inon wis cour enfourtll ef\\nAtr incomar inon voor nnive mamomen towrshe smame thes win\\nond ROLANUS:\\nYouer nonf leve is benor ckes wor ake wod ingneerse frincurs tren, bamy, he willd wor inodures wharus eman, to nd moe nmane wourn ntor whe. erw hirons, on woud ils, abut nithor bus ifenfor od befr nwaldo n for may far ino nma blli npakinon; rante wont an bin wthine son ls bone win terraet himand;\\nYou rvour w ath mo\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gru.predict(data_generator, \"c\", 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f21cb11b",
   "metadata": {
    "papermill": {
     "duration": 0.012533,
     "end_time": "2024-06-11T07:04:27.624077",
     "exception": false,
     "start_time": "2024-06-11T07:04:27.611544",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<a id=\"4\"></a>\n",
    "<h1 style='background:#FF9F00;border:0; color:black;\n",
    "    box-shadow: 10px 10px 5px 0px rgba(0,0,0,0.75);\n",
    "    transform: rotateX(10deg);\n",
    "    '><center style='color: #3E3D53;'>Thank you</center></h1>\n",
    "\n",
    "# Thank you\n",
    "\n",
    "**Thank you for going through this notebook**\n",
    "\n",
    "**If you have any suggestions please let me know**\n"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "datasetId": 759820,
     "sourceId": 1311864,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30732,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 97.289646,
   "end_time": "2024-06-11T07:04:28.059216",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-06-11T07:02:50.769570",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}