demo_notebooks/nllb_en_bn_translator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pip install transformers==4.15.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: /home/apsisdev/anaconda3/envs/eblict/lib/libtinfo.so.6: no version information available (required by /bin/bash)\r\n",
      "Mon Aug 29 14:39:23 2022       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 510.85.02    Driver Version: 510.85.02    CUDA Version: 11.6     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  NVIDIA GeForce ...  Off  | 00000000:01:00.0  On |                  N/A |\r\n",
      "| 30%   51C    P0   111W / 370W |    139MiB / 24576MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|    0   N/A  N/A      1334      G   /usr/lib/xorg/Xorg                 17MiB |\r\n",
      "|    0   N/A  N/A      1932      G   /usr/lib/xorg/Xorg                 55MiB |\r\n",
      "|    0   N/A  N/A      2070      G   /usr/bin/gnome-shell               18MiB |\r\n",
      "|    0   N/A  N/A    925401      G   ...392055696706106309,131072       31MiB |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/apsisdev/mobassir'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/bash: /home/apsisdev/anaconda3/envs/eblict/lib/libtinfo.so.6: no version information available (required by /bin/bash)\r\n",
      " \u001b[0m\u001b[01;34maaivu-tamizhi-net-OCR-master\u001b[0m/\r\n",
      " \u001b[01;34mArabic_Fonts_Check\u001b[0m/\r\n",
      " \u001b[01;34mbanglaSpeech\u001b[0m/\r\n",
      " best-transformer-representations.ipynb\r\n",
      " bn_old_testament_of_kjv.csv\r\n",
      " \u001b[01;34mbnsynthtiger\u001b[0m/\r\n",
      " bn_translated_tafseer.csv\r\n",
      " \u001b[01;34mdata\u001b[0m/\r\n",
      "'data downloader.ipynb'\r\n",
      " \u001b[01;34mdatasets\u001b[0m/\r\n",
      " deberta-base.py\r\n",
      " deberta.py\r\n",
      " deberta-train.ipynb\r\n",
      " \u001b[01;34mdocUnet.pytorch-master\u001b[0m/\r\n",
      " \u001b[01;34men_bn_bukhari_muslim\u001b[0m/\r\n",
      " en_bn_bukhari_muslim.csv\r\n",
      " en_bn_translated_sahih_hadiths.csv\r\n",
      " EN_old_testament_of_kjv.csv\r\n",
      "'feedback prize baseline.ipynb'\r\n",
      " \u001b[01;34mfonts\u001b[0m/\r\n",
      " get_oof_csv.ipynb\r\n",
      " \u001b[01;34mhandwriting_line_generation\u001b[0m/\r\n",
      " \u001b[01;34mHandwriting-Transformers-main\u001b[0m/\r\n",
      " infer_1.ipynb\r\n",
      " molat_pdf_to_imgs.ipynb\r\n",
      " \u001b[01;34mNID-PORICHOY\u001b[0m/\r\n",
      " \u001b[01;34mnllb-200-1.3B\u001b[0m/\r\n",
      " \u001b[01;34mnllb-200-3.3B\u001b[0m/\r\n",
      " \u001b[01;34mnllb-200-distilled-1.3B\u001b[0m/\r\n",
      " \u001b[01;34mnllb-200-distilled-600M\u001b[0m/\r\n",
      " nllb_en_bn_translator.ipynb\r\n",
      " nllb_en_bn_translator.py\r\n",
      " nlp_profanity_data_generator.ipynb\r\n",
      " pppm5class.ipynb\r\n",
      "'pppm awp.py'\r\n",
      " pppm_awp.py\r\n",
      " \u001b[01;34mprofanity\u001b[0m/\r\n",
      " pytorch-feedback-deberta-v3-baseline.ipynb\r\n",
      " pytorch_roberta_large_finetuning_part2.py\r\n",
      " \u001b[01;34mquran_en\u001b[0m/\r\n",
      " \u001b[01;34mSmartPatch-main\u001b[0m/\r\n",
      " \u001b[01;34mspelling-correction-english-base\u001b[0m/\r\n",
      " \u001b[01;34mt5-base-spellchecker\u001b[0m/\r\n",
      " \u001b[01;34mTextRecognitionDataGenerator\u001b[0m/\r\n",
      "'tf test gpu.ipynb'\r\n",
      "'torch roberta custom  fine tuner for clrp.ipynb'\r\n",
      " train_1.ipynb\r\n",
      " train-deberta-v3-large-with-optimization-approach.ipynb\r\n",
      " train_fpe.py\r\n",
      " train_pytorch-feedback-deberta-v3-baseline.ipynb\r\n",
      " train_xlarge.ipynb\r\n",
      " undervolt.py\r\n",
      " \u001b[01;34mwandb\u001b[0m/\r\n",
      "'Within-task pre-training _deberta.ipynb'\r\n"
     ]
    }
   ],
   "source": [
    "ls  '/home/apsisdev/mobassir/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-29 14:39:25.279458: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'4.21.2'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import transformers\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import os\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline\n",
    "import torch\n",
    "import gc\n",
    "import re\n",
    "tqdm.pandas()\n",
    "print(torch.__version__)\n",
    "transformers.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.6 s, sys: 3.27 s, total: 20.8 s\n",
      "Wall time: 12.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# !pip install -q transformers\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(\"/home/apsisdev/mobassir/nllb-200-1.3B\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"/home/apsisdev/mobassir/nllb-200-1.3B\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n",
      "CPU times: user 1.35 s, sys: 743 ms, total: 2.1 s\n",
      "Wall time: 2.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "translation_pipeline = pipeline('translation', \n",
    "                                model=model, \n",
    "                                tokenizer=tokenizer, \n",
    "                                src_lang=\"eng_latn\",  #arb_Arab,eng_latn\n",
    "                                tgt_lang='ben_Beng',\n",
    "                                device = device,\n",
    "                                max_length = 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translation_pipeline.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hadith_id</th>\n",
       "      <th>source</th>\n",
       "      <th>chapter_no</th>\n",
       "      <th>hadith_no</th>\n",
       "      <th>chapter</th>\n",
       "      <th>chain_indx</th>\n",
       "      <th>text_ar</th>\n",
       "      <th>text_en</th>\n",
       "      <th>narrators</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Revelation - كتاب بدء الوحى</td>\n",
       "      <td>30418, 20005, 11062, 11213, 11042, 3</td>\n",
       "      <td>حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...</td>\n",
       "      <td>Narrated 'Umar bin Al-Khattab:          ...</td>\n",
       "      <td>'Abdullah bin al-Zubair bin 'Isa عبد الله بن ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Revelation - كتاب بدء الوحى</td>\n",
       "      <td>30355, 20001, 11065, 10511, 53</td>\n",
       "      <td>حدثنا عبد الله بن يوسف، قال أخبرنا مالك، عن هش...</td>\n",
       "      <td>Narrated 'Aisha:                        ...</td>\n",
       "      <td>'Abdullah bin Yusuf al-Tunaysi عبد الله بن يوس...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  hadith_id           source  chapter_no  hadith_no  \\\n",
       "0   0          1   Sahih Bukhari            1          1   \n",
       "1   1          2   Sahih Bukhari            1          2   \n",
       "\n",
       "                       chapter                            chain_indx  \\\n",
       "0  Revelation - كتاب بدء الوحى  30418, 20005, 11062, 11213, 11042, 3   \n",
       "1  Revelation - كتاب بدء الوحى        30355, 20001, 11065, 10511, 53   \n",
       "\n",
       "                                             text_ar  \\\n",
       "0  حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...   \n",
       "1  حدثنا عبد الله بن يوسف، قال أخبرنا مالك، عن هش...   \n",
       "\n",
       "                                             text_en  \\\n",
       "0        Narrated 'Umar bin Al-Khattab:          ...   \n",
       "1        Narrated 'Aisha:                        ...   \n",
       "\n",
       "                                           narrators  \n",
       "0  'Abdullah bin al-Zubair bin 'Isa عبد الله بن ا...  \n",
       "1  'Abdullah bin Yusuf al-Tunaysi عبد الله بن يوس...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_bn_bukhari_muslim = pd.read_csv('/home/apsisdev/mobassir/en_bn_bukhari_muslim/en_bukhari_muslim.csv')\n",
    "en_bn_bukhari_muslim.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85207931c9f1469ea9c5eede5b8d68b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14966 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7672644247284e419d88626b29336622",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14966 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78fedbcfb20542cd8e6a09f3a0779df7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14966 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 199 ms, sys: 4.62 ms, total: 203 ms\n",
      "Wall time: 200 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"'Abdullah bin al Zubair bin 'Isa ,Sufyan bin 'Uyaynah ,Yahya bin Sa'id al Ansari ,Muhammad bin Ibrahim bin al Harith ,'Alqama bin Waqqas ,'Umar ibn al Khattab \""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def remove_non_english(mixed_text,only_special = False):\n",
    "    if(only_special):\n",
    "        return mixed_text.replace(\"\\'\",\"\")\n",
    "    return re.sub(\"[^a-zA-Z0-9,']+\", \" \",str(mixed_text))\n",
    "\n",
    "\n",
    "en_bn_bukhari_muslim[\"narrators\"]=en_bn_bukhari_muslim.narrators.progress_apply(lambda mixed_text: remove_non_english(mixed_text))\n",
    "en_bn_bukhari_muslim[\"chapter\"]=en_bn_bukhari_muslim.chapter.progress_apply(lambda mixed_text: remove_non_english(mixed_text))\n",
    "\n",
    "en_bn_bukhari_muslim[\"text_en\"]=en_bn_bukhari_muslim.text_en.progress_apply(lambda mixed_text: remove_non_english(mixed_text,only_special = True))\n",
    "\n",
    "en_bn_bukhari_muslim.narrators[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hadith_id</th>\n",
       "      <th>source</th>\n",
       "      <th>chapter_no</th>\n",
       "      <th>hadith_no</th>\n",
       "      <th>chapter</th>\n",
       "      <th>chain_indx</th>\n",
       "      <th>text_ar</th>\n",
       "      <th>text_en</th>\n",
       "      <th>narrators</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>30418, 20005, 11062, 11213, 11042, 3</td>\n",
       "      <td>حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...</td>\n",
       "      <td>Narrated Umar bin Al-Khattab:           ...</td>\n",
       "      <td>'Abdullah bin al Zubair bin 'Isa ,Sufyan bin '...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  hadith_id           source  chapter_no  hadith_no      chapter  \\\n",
       "0   0          1   Sahih Bukhari            1          1  Revelation    \n",
       "\n",
       "                             chain_indx  \\\n",
       "0  30418, 20005, 11062, 11213, 11042, 3   \n",
       "\n",
       "                                             text_ar  \\\n",
       "0  حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...   \n",
       "\n",
       "                                             text_en  \\\n",
       "0        Narrated Umar bin Al-Khattab:           ...   \n",
       "\n",
       "                                           narrators  \n",
       "0  'Abdullah bin al Zubair bin 'Isa ,Sufyan bin '...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_bn_bukhari_muslim.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Narrated Aisha:                       (the mother of the faithful believers) The commencement of the Divine Inspiration to Allahs Apostle was in the form of good dreams which came true like bright daylight, and then the love of seclusion was bestowed upon him. He used to go in seclusion in the cave of Hira where he used to worship (Allah alone) continuously for many days before his desire to see his family. He used to take with him the journey food for the stay and then come back to (his wife) Khadija to take his food likewise again till suddenly the Truth descended upon him while he was in the cave of Hira. The angel came to him and asked him to read. The Prophet replied, \"I do not know how to read.\" The Prophet added, \"The angel caught me (forcefully) and pressed me so hard that I could not bear it any more. He then released me and again asked me to read and I replied, I do not know how to read. Thereupon he caught me again and pressed me a second time till I could not bear it any more. He then released me and again asked me to read but again I replied, I do not know how to read (or what shall I read)? Thereupon he caught me for the third time and pressed me, and then released me and said, Read in the name of your Lord, who has created (all that exists), created man from a clot. Read! And your Lord is the Most Generous.\" (96.1, 96.2, 96.3) Then Allahs Apostle returned with the Inspiration and with his heart beating severely. Then he went to Khadija bint Khuwailid and said, \"Cover me! Cover me!\" They covered him till his fear was over and after that he told her everything that had happened and said, \"I fear that something may happen to me.\" Khadija replied, \"Never! By Allah, Allah will never disgrace you. You keep good relations with your kith and kin, help the poor and the destitute, serve your guests generously and assist the deserving calamity-afflicted ones.\"  Khadija then accompanied him to her cousin Waraqa bin Naufal bin Asad bin Abdul Uzza, who, during the pre-Islamic Period became a Christian and used to write the writing with Hebrew letters. He would write from the Gospel in Hebrew as much as Allah wished him to write. He was an old man and had lost his eyesight. Khadija said to Waraqa, \"Listen to the story of your nephew, O my cousin!\" Waraqa asked, \"O my nephew! What have you seen?\" Allahs Apostle described whatever he had seen. Waraqa said, \"This is the same one who keeps the secrets (angel Gabriel) whom Allah had sent to Moses. I wish I were young and could live up to the time when your people would turn you out.\" Allahs Apostle asked, \"Will they drive me out?\" Waraqa replied in the affirmative and said, \"Anyone (man) who came with something similar to what you have brought was treated with hostility; and if I should remain alive till the day when you will be turned out then I would support you strongly.\" But after a few days Waraqa died and the Divine Inspiration was also paused for a while.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_bn_bukhari_muslim.text_en[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14966\n",
      "CPU times: user 5.58 s, sys: 204 ms, total: 5.78 s\n",
      "Wall time: 6.17 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'তিনি বলেন, ইবনে জুমাইর বলেনঃ ইবনে আব্বাস আল্লার এই বক্তব্যের ব্যাখ্যা বর্ণনায় ইবনে আব্বাস বলেন, \"তোমরা কুরআনের ব্যাপারে তোমাদের জিহ্বাকে না সরিয়ে ফেলো, যাতে তোমরা তা তাড়াতাড়ি করতে পারো।\" (৭৫ঃ১৬) তিনি বলেন, \"আল্লাহর রসূল (সাঃ) খুব কষ্টে অবতীর্ণ অবতীর্ণ অবতীর্ণ করেন এবং প্রেরণ দিয়ে তাঁর ঠোঁট (দ্রুত) সরিয়ে দেন।\" ইবনে আব্বাস তাঁর ঠোঁট সরিয়ে বলেন, \"আমি আপনার সামনে আমার ঠোঁট সরিয়ে দিচ্ছি যেমন আল্লাহর রসূল (সাঃ) তাঁর ঠোঁট সরিয়ে দিয়েছিলেন।\" তিনি বলেন, \"আমি আমার ঠোঁট সরিয়ে দিচ্ছি, যেমন আমি ইবনে আব্বাসকে তাঁর ঠোঁট সরিয়ে দেখেছিলাম।\" ইবনে আব্বাস যোগ করেন, \"সুতরাং আল্লাহ্ অবতীর্ণ করেছেন, যাতে তোমরা কুরআনের ব্যাপারে তোমাদের জিহ্বাকে না সরিয়ে ফেলো যাতে তোমরা তা তাড়াতাড়ি করতে পারো।\" আর আমাদেরই দায়িত্ব তা সংগ্রহ করা এবং তোমার (হে মুহাম্মদ) কে তা পাঠ করার ক্ষমতা দেয়া। আর যখন আমি এটি তোমার কাছে পাঠ করি তখন তুমি এর পাঠ অনুসরণ কর। অতঃপর আমিই তা সুস্পষ্ট করে দেব, অতঃপর আমিই তা পাঠ করব। এর পরে যখনই রসূল (সাঃ) এসেছিলেন, তিনি জিব্রাইলের কথা শুনতেন, অতঃপর যখন তিনি চলে যান তখন তিনি জিব্রাইলের মতই তা পাঠ করতেন।'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "print(len(en_bn_bukhari_muslim))\n",
    "sentences = re.split(r'(?<=\\w\\.)\\s', en_bn_bukhari_muslim.text_en[4])\n",
    "bn = []\n",
    "for i in range(len(sentences)):\n",
    "    result = translation_pipeline(sentences[i])\n",
    "    bn.append(result[0]['translation_text'])\n",
    "bn = \" \".join(bn)\n",
    "bn = re.sub(' ্ ','',bn)\n",
    "bn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# en_bn_bukhari_muslim['Count'] = en_bn_bukhari_muslim.text_ar.apply(lambda x: len(x.split(\" \")))\n",
    "# en_bn_bukhari_muslim = en_bn_bukhari_muslim.sort_values(['Count'], ascending=False)\n",
    "# print(en_bn_bukhari_muslim.iloc[0]) #1924"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7421cc84c1004c9883c290861bb26b5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14966 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/apsisdev/anaconda3/envs/kaggle/lib/python3.8/site-packages/transformers/pipelines/base.py:1036: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11h 25min 30s, sys: 2min 39s, total: 11h 28min 10s\n",
      "Wall time: 11h 28min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def en_to_bn_translator(en_text):\n",
    "    \n",
    "#     sentences = re.split(r'(?<=\\w\\.)\\s', str(en_text))\n",
    "    \n",
    "    sentenceEnders = re.compile('[.!?]')\n",
    "    sentences = sentenceEnders.split(str(en_text))\n",
    "\n",
    "    bn = []\n",
    "    for i in range(len(sentences)):\n",
    "        result = translation_pipeline(sentences[i])\n",
    "        bn.append(result[0]['translation_text'])\n",
    "    res = \" \".join(bn)\n",
    "    res = re.sub(' ্ ','',res)\n",
    "    return res\n",
    "    \n",
    "en_bn_bukhari_muslim[\"text_bn\"]=en_bn_bukhari_muslim.text_en.progress_apply(lambda en_text: en_to_bn_translator(en_text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "en_bn_bukhari_muslim.to_csv('./en_bn_bukhari_muslim.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>hadith_id</th>\n",
       "      <th>source</th>\n",
       "      <th>chapter_no</th>\n",
       "      <th>hadith_no</th>\n",
       "      <th>chapter</th>\n",
       "      <th>chain_indx</th>\n",
       "      <th>text_ar</th>\n",
       "      <th>text_en</th>\n",
       "      <th>narrators</th>\n",
       "      <th>text_bn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>30418, 20005, 11062, 11213, 11042, 3</td>\n",
       "      <td>حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...</td>\n",
       "      <td>Narrated Umar bin Al-Khattab:           ...</td>\n",
       "      <td>'Abdullah bin al Zubair bin 'Isa ,Sufyan bin '...</td>\n",
       "      <td>উমর বিন আল-খাত্তাবের বর্ণনায় উমর বিন আল-খাত্ত...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>30355, 20001, 11065, 10511, 53</td>\n",
       "      <td>حدثنا عبد الله بن يوسف، قال أخبرنا مالك، عن هش...</td>\n",
       "      <td>Narrated Aisha:                         ...</td>\n",
       "      <td>'Abdullah bin Yusuf al Tunaysi ,Imam Maalik ,H...</td>\n",
       "      <td>আয়েশা (রাঃ) এর কাহিনী বর্ণিত হয়েছে: (বিশ্বাস...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>30399, 20023, 11207, 11013, 10511, 53</td>\n",
       "      <td>حدثنا يحيى بن بكير، قال حدثنا الليث، عن عقيل، ...</td>\n",
       "      <td>Narrated Aisha:                       (the mo...</td>\n",
       "      <td>Yahya bin Yahya bin Bukayr ,al Laith bin Sa'd ...</td>\n",
       "      <td>আয়েশা (রাঃ) এর বানীঃ (বিশ্বাসী মুমিনদের মা) আ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>11013, 10567, 34</td>\n",
       "      <td>قال ابن شهاب وأخبرني أبو سلمة بن عبد الرحمن، أ...</td>\n",
       "      <td>Narrated Jabir bin Abdullah Al-Ansari while t...</td>\n",
       "      <td>al Zuhri , ,Abu Salama bin 'Abdur Rahman ,Jabi...</td>\n",
       "      <td>বর্ণিত জাবির বিন আবদুল্লাহ আল-আনসারি যখন প্রকা...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Sahih Bukhari</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>Revelation</td>\n",
       "      <td>20040, 20469, 11399, 11050, 17</td>\n",
       "      <td>حدثنا موسى بن إسماعيل، قال حدثنا أبو عوانة، قا...</td>\n",
       "      <td>Narrated Said bin Jubair:               ...</td>\n",
       "      <td>Musa bin Isma'il al Munqari al Tabudhaki ,Wada...</td>\n",
       "      <td>তিনি বলেন, 'আল্লাহর বাণী বর্ণনা করে ইবনে আব্বা...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14961</th>\n",
       "      <td>36</td>\n",
       "      <td>57</td>\n",
       "      <td>Sahih Muslim</td>\n",
       "      <td>57</td>\n",
       "      <td>7744</td>\n",
       "      <td>The Book of Commentary</td>\n",
       "      <td>30201, 20345, 11404, 11052, 18</td>\n",
       "      <td>حدثنا أبو بكر بن أبي شيبة، حدثنا علي بن مسهر، ...</td>\n",
       "      <td>Ibn Umar reported that Umar delivered a sermo...</td>\n",
       "      <td>Abu Bakr bin Abi Shayba ,'Ali bin Ms'hr ,Yahya...</td>\n",
       "      <td>ইবনে উমর (রাঃ) বলেন, উমর আল্লার রসূল (সাঃ) এর ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14962</th>\n",
       "      <td>37</td>\n",
       "      <td>57</td>\n",
       "      <td>Sahih Muslim</td>\n",
       "      <td>57</td>\n",
       "      <td>7745</td>\n",
       "      <td>The Book of Commentary</td>\n",
       "      <td>30216, 20102, 11404, 11052, 18</td>\n",
       "      <td>وحدثنا أبو كريب، أخبرنا ابن إدريس، حدثنا أبو ح...</td>\n",
       "      <td>Ibn Umar reported:                      I hea...</td>\n",
       "      <td>Abu Karayb, Muhammad bin al 'Ala' ,'Abdullah b...</td>\n",
       "      <td>ইবনে উমর রিপোর্ট করেছেনঃ আমি উমরকে শুনলাম খাত্...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14963</th>\n",
       "      <td>38</td>\n",
       "      <td>57</td>\n",
       "      <td>Sahih Muslim</td>\n",
       "      <td>57</td>\n",
       "      <td>7746</td>\n",
       "      <td>The Book of Commentary</td>\n",
       "      <td>30201, 20208, 30123, 20347</td>\n",
       "      <td>وحدثنا أبو بكر بن أبي شيبة، حدثنا إسماعيل ابن ...</td>\n",
       "      <td>This hadith has been transmitted on the same ...</td>\n",
       "      <td>Abu Bakr bin Abi Shayba ,Ishaq bin Ibrahim bin...</td>\n",
       "      <td>এই হাদিস একই সূত্র থেকে প্রেরিত হয়েছে কিন্তু ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14964</th>\n",
       "      <td>39</td>\n",
       "      <td>57</td>\n",
       "      <td>Sahih Muslim</td>\n",
       "      <td>57</td>\n",
       "      <td>7747</td>\n",
       "      <td>The Book of Commentary</td>\n",
       "      <td>30365, 20468, 11454, 11313, 11609</td>\n",
       "      <td>حدثنا عمرو بن زرارة، حدثنا هشيم، عن أبي هاشم، ...</td>\n",
       "      <td>Abu Dharr took an oath that this verse:      ...</td>\n",
       "      <td>'Amr bin Zararah bin Waqid ,Hashaym bin Bashay...</td>\n",
       "      <td>আবু ধারর শপথ নিলেন যে এই আয়াতটিঃ \" এই দুই শত্...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14965</th>\n",
       "      <td>40</td>\n",
       "      <td>57</td>\n",
       "      <td>Sahih Muslim</td>\n",
       "      <td>57</td>\n",
       "      <td>7748</td>\n",
       "      <td>The Book of Commentary</td>\n",
       "      <td>30201, 20032, 30170, 20005, 11454, 11313, 11609</td>\n",
       "      <td>حدثنا أبو بكر بن أبي شيبة، حدثنا وكيع، ح وحدثن...</td>\n",
       "      <td>This hadith has been narrated on the authorit...</td>\n",
       "      <td>Abu Bakr bin Abi Shayba ,Waki' bin al Jarrah ,...</td>\n",
       "      <td>এই হাদিস আবু যারর কর্তৃক অন্য একটি প্রেরক শৃঙ্...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14966 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  hadith_id           source  chapter_no  hadith_no  \\\n",
       "0       0          1   Sahih Bukhari            1          1   \n",
       "1       1          2   Sahih Bukhari            1          2   \n",
       "2       2          3   Sahih Bukhari            1          3   \n",
       "3       3          4   Sahih Bukhari            1          4   \n",
       "4       4          5   Sahih Bukhari            1          5   \n",
       "...    ..        ...              ...         ...        ...   \n",
       "14961  36         57    Sahih Muslim           57       7744   \n",
       "14962  37         57    Sahih Muslim           57       7745   \n",
       "14963  38         57    Sahih Muslim           57       7746   \n",
       "14964  39         57    Sahih Muslim           57       7747   \n",
       "14965  40         57    Sahih Muslim           57       7748   \n",
       "\n",
       "                       chapter  \\\n",
       "0                  Revelation    \n",
       "1                  Revelation    \n",
       "2                  Revelation    \n",
       "3                  Revelation    \n",
       "4                  Revelation    \n",
       "...                        ...   \n",
       "14961  The Book of Commentary    \n",
       "14962  The Book of Commentary    \n",
       "14963  The Book of Commentary    \n",
       "14964  The Book of Commentary    \n",
       "14965  The Book of Commentary    \n",
       "\n",
       "                                            chain_indx  \\\n",
       "0                 30418, 20005, 11062, 11213, 11042, 3   \n",
       "1                       30355, 20001, 11065, 10511, 53   \n",
       "2                30399, 20023, 11207, 11013, 10511, 53   \n",
       "3                                     11013, 10567, 34   \n",
       "4                       20040, 20469, 11399, 11050, 17   \n",
       "...                                                ...   \n",
       "14961                   30201, 20345, 11404, 11052, 18   \n",
       "14962                   30216, 20102, 11404, 11052, 18   \n",
       "14963                       30201, 20208, 30123, 20347   \n",
       "14964                30365, 20468, 11454, 11313, 11609   \n",
       "14965  30201, 20032, 30170, 20005, 11454, 11313, 11609   \n",
       "\n",
       "                                                 text_ar  \\\n",
       "0      حدثنا الحميدي عبد الله بن الزبير، قال حدثنا سف...   \n",
       "1      حدثنا عبد الله بن يوسف، قال أخبرنا مالك، عن هش...   \n",
       "2      حدثنا يحيى بن بكير، قال حدثنا الليث، عن عقيل، ...   \n",
       "3      قال ابن شهاب وأخبرني أبو سلمة بن عبد الرحمن، أ...   \n",
       "4      حدثنا موسى بن إسماعيل، قال حدثنا أبو عوانة، قا...   \n",
       "...                                                  ...   \n",
       "14961  حدثنا أبو بكر بن أبي شيبة، حدثنا علي بن مسهر، ...   \n",
       "14962  وحدثنا أبو كريب، أخبرنا ابن إدريس، حدثنا أبو ح...   \n",
       "14963  وحدثنا أبو بكر بن أبي شيبة، حدثنا إسماعيل ابن ...   \n",
       "14964  حدثنا عمرو بن زرارة، حدثنا هشيم، عن أبي هاشم، ...   \n",
       "14965  حدثنا أبو بكر بن أبي شيبة، حدثنا وكيع، ح وحدثن...   \n",
       "\n",
       "                                                 text_en  \\\n",
       "0            Narrated Umar bin Al-Khattab:           ...   \n",
       "1            Narrated Aisha:                         ...   \n",
       "2       Narrated Aisha:                       (the mo...   \n",
       "3       Narrated Jabir bin Abdullah Al-Ansari while t...   \n",
       "4            Narrated Said bin Jubair:               ...   \n",
       "...                                                  ...   \n",
       "14961   Ibn Umar reported that Umar delivered a sermo...   \n",
       "14962   Ibn Umar reported:                      I hea...   \n",
       "14963   This hadith has been transmitted on the same ...   \n",
       "14964   Abu Dharr took an oath that this verse:      ...   \n",
       "14965   This hadith has been narrated on the authorit...   \n",
       "\n",
       "                                               narrators  \\\n",
       "0      'Abdullah bin al Zubair bin 'Isa ,Sufyan bin '...   \n",
       "1      'Abdullah bin Yusuf al Tunaysi ,Imam Maalik ,H...   \n",
       "2      Yahya bin Yahya bin Bukayr ,al Laith bin Sa'd ...   \n",
       "3      al Zuhri , ,Abu Salama bin 'Abdur Rahman ,Jabi...   \n",
       "4      Musa bin Isma'il al Munqari al Tabudhaki ,Wada...   \n",
       "...                                                  ...   \n",
       "14961  Abu Bakr bin Abi Shayba ,'Ali bin Ms'hr ,Yahya...   \n",
       "14962  Abu Karayb, Muhammad bin al 'Ala' ,'Abdullah b...   \n",
       "14963  Abu Bakr bin Abi Shayba ,Ishaq bin Ibrahim bin...   \n",
       "14964  'Amr bin Zararah bin Waqid ,Hashaym bin Bashay...   \n",
       "14965  Abu Bakr bin Abi Shayba ,Waki' bin al Jarrah ,...   \n",
       "\n",
       "                                                 text_bn  \n",
       "0      উমর বিন আল-খাত্তাবের বর্ণনায় উমর বিন আল-খাত্ত...  \n",
       "1      আয়েশা (রাঃ) এর কাহিনী বর্ণিত হয়েছে: (বিশ্বাস...  \n",
       "2      আয়েশা (রাঃ) এর বানীঃ (বিশ্বাসী মুমিনদের মা) আ...  \n",
       "3      বর্ণিত জাবির বিন আবদুল্লাহ আল-আনসারি যখন প্রকা...  \n",
       "4      তিনি বলেন, 'আল্লাহর বাণী বর্ণনা করে ইবনে আব্বা...  \n",
       "...                                                  ...  \n",
       "14961  ইবনে উমর (রাঃ) বলেন, উমর আল্লার রসূল (সাঃ) এর ...  \n",
       "14962  ইবনে উমর রিপোর্ট করেছেনঃ আমি উমরকে শুনলাম খাত্...  \n",
       "14963  এই হাদিস একই সূত্র থেকে প্রেরিত হয়েছে কিন্তু ...  \n",
       "14964  আবু ধারর শপথ নিলেন যে এই আয়াতটিঃ \" এই দুই শত্...  \n",
       "14965  এই হাদিস আবু যারর কর্তৃক অন্য একটি প্রেরক শৃঙ্...  \n",
       "\n",
       "[14966 rows x 11 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_bn_bukhari_muslim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset link -> https://www.kaggle.com/datasets/mobassir/en-bn-sahih-bukhari-muslim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kaggle",
   "language": "python",
   "name": "kaggle"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}