test_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokens: [1, 2, 3, 4, 5]\n"
     ]
    }
   ],
   "source": [
    "vocab = [\"<pad>\", 'El', 'perro', 'persigue', 'al', 'gato', 'y', 'se', 'lo', 'come', \"<end>\"]\n",
    "\n",
    "def text_to_tokens(text, vocab):\n",
    "    tokens = text.split(\" \")\n",
    "    return [vocab.index(token) if token in vocab else vocab.index(\"<pad>\") for token in tokens]\n",
    "\n",
    "frase = \"El perro persigue al gato\"\n",
    "tokens = text_to_tokens(frase, vocab)\n",
    "print(f\"Tokens: {tokens}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from transformer import *\n",
    "import torch\n",
    "from Transformer import TransformerEncoder, TransformerDecoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# covertimos a un tensor los tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = torch.tensor(tokens, dtype=torch.long)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parametros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(vocab)\n",
    "d_model = 512\n",
    "num_layers = 6\n",
    "num_heads = 8\n",
    "d_ff = 2048\n",
    "dropout = 0.1\n",
    "max_seq_length = 5 # este tiene que tener el mismo tamaño de los tokens (checar tokens.shape[1] si es de una dimension, tokens.shape[1] si son dos dimensiones)\n",
    "batch_size = 1\n",
    "\n",
    "src_mask = torch.ones(batch_size, 1, max_seq_length, max_seq_length) # mascara que es una matriz de solos 1s porque no tenemos tokens con padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokens.unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3, 4, 5]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = TransformerEncoder(vocab_size, d_model, num_layers, num_heads, d_ff, dropout, max_seq_length)\n",
    "output_encoder = encoder(tokens, src_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5, 512])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_encoder.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-1.4020,  0.3544,  1.4220,  ...,  1.3007, -0.2529,  1.6627],\n",
       "         [-0.5963, -0.0571, -1.3349,  ..., -0.9605, -0.5553,  0.6529],\n",
       "         [-0.8698,  0.4888, -0.5234,  ...,  2.2267,  0.4915,  0.3641],\n",
       "         [-0.4291,  0.1886, -0.6805,  ...,  1.6035,  1.0038,  0.3653],\n",
       "         [-1.7163,  0.9961, -0.1121,  ..., -2.2682,  0.3475, -0.3644]]],\n",
       "       grad_fn=<NativeLayerNormBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 0., 0., 0.],\n",
      "        [1., 1., 0., 0.],\n",
      "        [1., 1., 1., 0.],\n",
      "        [1., 1., 1., 1.]])\n",
      "torch.Size([4, 4])\n"
     ]
    }
   ],
   "source": [
    "tgt = torch.tril(torch.ones(4, 4))\n",
    "print(tgt)\n",
    "print(tgt.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1, 2, 3, 4]])\n"
     ]
    }
   ],
   "source": [
    "x = tokens[:, :-1]\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder = TransformerDecoder(vocab_size, d_model, num_layers, num_heads, d_ff, dropout, max_seq_length)\n",
    "output_decoder = decoder(x, output_encoder, tgt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-1.0115,  0.0571,  1.8621,  ..., -1.8423,  2.1023,  0.9634],\n",
      "         [ 0.6030,  0.0345, -0.1469,  ..., -0.0258,  0.7944,  1.8433],\n",
      "         [ 0.6364, -1.8943,  0.4000,  ..., -0.3813,  2.7131,  0.8408],\n",
      "         [-0.7746, -0.8679,  0.1132,  ..., -1.3419,  1.8634,  2.0326]]],\n",
      "       grad_fn=<NativeLayerNormBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(output_decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "adding a dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-1.6646,  0.7465, -0.7856,  0.2578,  0.3722, -0.8806,  0.1744,\n",
      "          -0.6289,  0.3803, -0.2897,  0.7315],\n",
      "         [ 0.2475, -0.0300, -0.1901,  0.7969,  0.2784, -0.4264, -0.0627,\n",
      "           0.0933,  0.5479, -0.2422, -0.0995],\n",
      "         [-0.5493,  0.9710, -0.3939, -0.1515, -0.0917, -0.6521, -0.1032,\n",
      "          -0.9892,  0.4811, -0.3036,  0.0455],\n",
      "         [-0.6274,  0.4968,  0.1906, -0.4895,  0.2001, -1.2997, -0.0390,\n",
      "           1.0698,  0.5241,  0.4972,  1.1193]]], grad_fn=<ViewBackward0>)\n"
     ]
    }
   ],
   "source": [
    "layer = torch.nn.Linear(d_model, vocab_size)\n",
    "salida_probabilistica = layer(output_decoder)\n",
    "print(salida_probabilistica)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 4, 11])\n"
     ]
    }
   ],
   "source": [
    "print(salida_probabilistica.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 4, 11])\n",
      "tensor([[[0.0159, 0.1769, 0.0382, 0.1085, 0.1216, 0.0348, 0.0998, 0.0447,\n",
      "          0.1226, 0.0628, 0.1742],\n",
      "         [0.1006, 0.0762, 0.0650, 0.1743, 0.1038, 0.0513, 0.0738, 0.0863,\n",
      "          0.1359, 0.0617, 0.0711],\n",
      "         [0.0532, 0.2431, 0.0621, 0.0791, 0.0840, 0.0480, 0.0830, 0.0342,\n",
      "          0.1489, 0.0680, 0.0964],\n",
      "         [0.0339, 0.1042, 0.0767, 0.0389, 0.0775, 0.0173, 0.0610, 0.1849,\n",
      "          0.1071, 0.1043, 0.1943]]], grad_fn=<SoftmaxBackward0>)\n"
     ]
    }
   ],
   "source": [
    "probabilidades = torch.softmax(salida_probabilistica, dim=-1)\n",
    "print(probabilidades.shape)\n",
    "print(probabilidades)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 4])\n",
      "tensor([[ 1,  3,  1, 10]])\n"
     ]
    }
   ],
   "source": [
    "predicciones = torch.argmax(probabilidades, dim=-1)\n",
    "print(predicciones.shape)\n",
    "print(predicciones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Secuencia generada:  ['El', 'persigue', 'El', '<end>']\n"
     ]
    }
   ],
   "source": [
    "secuencia_generada = [vocab[idx] for idx in predicciones[0].tolist()]\n",
    "print(\"Secuencia generada: \", secuencia_generada)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deeplearning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}