Bert/06-Bert-add-norm-residual-connection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert 中的 add & norm 和残差连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertConfig {\n",
       "  \"_name_or_path\": \"bert-base-uncased\",\n",
       "  \"architectures\": [\n",
       "    \"BertForMaskedLM\"\n",
       "  ],\n",
       "  \"attention_probs_dropout_prob\": 0.1,\n",
       "  \"classifier_dropout\": null,\n",
       "  \"gradient_checkpointing\": false,\n",
       "  \"hidden_act\": \"gelu\",\n",
       "  \"hidden_dropout_prob\": 0.1,\n",
       "  \"hidden_size\": 768,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 3072,\n",
       "  \"layer_norm_eps\": 1e-12,\n",
       "  \"max_position_embeddings\": 512,\n",
       "  \"model_type\": \"bert\",\n",
       "  \"num_attention_heads\": 12,\n",
       "  \"num_hidden_layers\": 12,\n",
       "  \"output_hidden_states\": true,\n",
       "  \"pad_token_id\": 0,\n",
       "  \"position_embedding_type\": \"absolute\",\n",
       "  \"transformers_version\": \"4.33.2\",\n",
       "  \"type_vocab_size\": 2,\n",
       "  \"use_cache\": true,\n",
       "  \"vocab_size\": 30522\n",
       "}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers.models.bert import BertModel, BertTokenizer\n",
    "\n",
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertModel.from_pretrained(model_name, output_hidden_states=True)\n",
    "# \"intermediate_size\": 3072,\n",
    "model.config\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertModel(\n",
       "  (embeddings): BertEmbeddings(\n",
       "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "    (position_embeddings): Embedding(512, 768)\n",
       "    (token_type_embeddings): Embedding(2, 768)\n",
       "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoder): BertEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0-11): 12 x BertLayer(\n",
       "        (attention): BertAttention(\n",
       "          (self): BertSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "          (output): BertSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): BertIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          (intermediate_act_fn): GELUActivation()\n",
       "        )\n",
       "        (output): BertOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pooler): BertPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sent = 'this is a test sentence'\n",
    "\n",
    "model_input = tokenizer(test_sent, return_tensors='pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1556, -0.0080, -0.0707,  ...,  0.0786,  0.0213,  0.0616],\n",
       "         [-0.5333,  0.5799,  0.1044,  ...,  0.0241,  0.4888,  0.0161],\n",
       "         [-1.0609, -0.3058, -0.5043,  ...,  0.1874,  0.2874,  0.4032],\n",
       "         ...,\n",
       "         [ 0.8206, -0.6656, -0.7054,  ...,  0.1347,  0.1117, -1.9040],\n",
       "         [ 1.1128,  0.6603, -0.1509,  ...,  0.3253, -1.0006, -1.9106],\n",
       "         [-0.0736,  0.0346,  0.0376,  ..., -0.4506,  0.6585, -0.0502]]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    output = model(**model_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1686, -0.2858, -0.3261,  ..., -0.0276,  0.0383,  0.1640],\n",
       "         [-0.6485,  0.6739, -0.0932,  ...,  0.4475,  0.6696,  0.1820],\n",
       "         [-0.6270, -0.0633, -0.3143,  ...,  0.3427,  0.4636,  0.4594],\n",
       "         ...,\n",
       "         [ 0.6010, -0.6970, -0.2001,  ...,  0.2960,  0.2060, -1.7181],\n",
       "         [ 0.8323,  0.2878,  0.0021,  ...,  0.2628, -1.1310, -1.2708],\n",
       "         [-0.1481, -0.2948, -0.1690,  ..., -0.5009,  0.2544, -0.0700]]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# embeddings\n",
    "output[2][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1556, -0.0080, -0.0707,  ...,  0.0786,  0.0213,  0.0616],\n",
       "         [-0.5333,  0.5799,  0.1044,  ...,  0.0241,  0.4888,  0.0161],\n",
       "         [-1.0609, -0.3058, -0.5043,  ...,  0.1874,  0.2874,  0.4032],\n",
       "         ...,\n",
       "         [ 0.8206, -0.6656, -0.7054,  ...,  0.1347,  0.1117, -1.9040],\n",
       "         [ 1.1128,  0.6603, -0.1509,  ...,  0.3253, -1.0006, -1.9106],\n",
       "         [-0.0736,  0.0346,  0.0376,  ..., -0.4506,  0.6585, -0.0502]]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first bert layer output\n",
    "output[2][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## from scratch\n",
    "\n",
    "- BertLayer\n",
    "  - attention: BertAttention\n",
    "    - self: BertSelfAttention\n",
    "    - output: BertSelfOutput\n",
    "  - intermediate: BertIntermediate, 768=>4*768\n",
    "  - output: BertOutput"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = output[2][0]\n",
    "layer = model.encoder.layer[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第一次 add & norm，发生在 Multihead-Attention 内部"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mha_output = layer.attention.self(embeddings)\n",
    "mha_output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn_output = layer.attention.output(mha_output[0], embeddings)\n",
    "attn_output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 第二次 add & norm，发生在 mlp 内部"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 3072])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp1 = layer.intermediate(attn_output)\n",
    "mlp1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 7, 768])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp2 = layer.output(mlp1, attn_output)\n",
    "mlp2.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lightning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}