play_with_summarizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from googlesearch import search\n",
    "import numpy as np\n",
    "import requests\n",
    "from transformers import GPT2Config, GPT2LMHeadModel\n",
    "import torch\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "\n",
    "from dataset import GPT21024Dataset \n",
    "from utils import add_special_tokens, beam_search, generate_beam_sample, generate_sample, sample_seq, set_seed, top_k_top_p_filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(device=device(type='cuda'), ids_file='./CNN/ids.json', model_dir='./weights', n_gpu=1, num_workers=4, output_dir='./output', root_dir='./CNN/gpt2_1024_data', seed=42)\n"
     ]
    }
   ],
   "source": [
    "#please change default arguments if needed\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "\n",
    "parser.add_argument(\"--seed\",default=42, type=int,  help=\"seed to replicate results\")\n",
    "parser.add_argument(\"--num_workers\",default=4, type=int,  help=\"num of cpus available\")\n",
    "parser.add_argument(\"--device\",default=torch.device('cuda'), help=\"torch.device object\")\n",
    "parser.add_argument(\"--output_dir\",default='./output', type=str,  help=\"path to save evaluation results\")\n",
    "parser.add_argument(\"--model_dir\",default='./weights', type=str,  help=\"path to save trained model\")\n",
    "parser.add_argument(\"--root_dir\",default='./CNN/gpt2_1024_data', type=str, help=\"location of json dataset.\")\n",
    "parser.add_argument(\"--ids_file\",default='./CNN/ids.json', type=str, help=\"location of train, valid and test file indexes\")\n",
    "args = parser.parse_args([])\n",
    "print(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using the same validation and training data as during training\n",
    "tokenizer = add_special_tokens()\n",
    "# train_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='train',length=3000)\n",
    "# valid_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='valid',length=500)\n",
    "test_data = GPT21024Dataset(args.root_dir,args.ids_file,mode='test',length=500)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GPT2LMHeadModel(\n",
       "  (transformer): GPT2Model(\n",
       "    (wte): Embedding(50259, 1024)\n",
       "    (wpe): Embedding(1024, 1024)\n",
       "    (drop): Dropout(p=0.1, inplace=False)\n",
       "    (h): ModuleList(\n",
       "      (0): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (1): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (2): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (3): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (4): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (5): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (6): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (7): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (8): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (9): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (10): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (11): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (12): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (13): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (14): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (15): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (16): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (17): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (18): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (19): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (20): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (21): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (22): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (23): Block(\n",
       "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (attn): Attention(\n",
       "          (c_attn): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (attn_dropout): Dropout(p=0.1, inplace=False)\n",
       "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): MLP(\n",
       "          (c_fc): Conv1D()\n",
       "          (c_proj): Conv1D()\n",
       "          (dropout): Dropout(p=0.1, inplace=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=1024, out_features=50259, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model_file and config_file are files used to load finetuned model, change these name as per your file names\n",
    "\n",
    "# model_file = os.path.join(args.model_dir, 'model_data{}_trained_after_{}_epochs_only_sum_loss_ignr_pad.bin'.format(len(train_data),args.num_train_epochs))\n",
    "# config_file = os.path.join(args.model_dir, 'config_data{}_trained_after_{}_epochs_only_sum_loss_ignr_pad.json'.format(len(train_data),args.num_train_epochs))\n",
    "\n",
    "# path to model and config files\n",
    "model_file = \"345-model_O0_data3000_trained_after_5_epochs_only_sum_loss_ignr_pad.bin\"\n",
    "config_file = \"345-config_O0_data3000_trained_after_5_epochs_only_sum_loss_ignr_pad.json\"\n",
    "\n",
    "config = GPT2Config.from_json_file(config_file)\n",
    "model = GPT2LMHeadModel(config)\n",
    "state_dict = torch.load(model_file)\n",
    "model.load_state_dict(state_dict)\n",
    "model.eval()\n",
    "model.to(args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e221cca0a184f9085ef380425607399",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "new_article\n",
      "\n",
      "Rome -LRB- CNN -RRB- -- A cruise ship of the Costa Cruises line is adrift off the coast of the Seychelles after a fire in its engine room, the Italian coast guard said Monday. The ship, the Allegra, is a sister of the Costa Concordia, which wrecked off the coast of Italy on January 13, killing at least 21 people. The fire left the Allegra without propulsion, although its communications equipment is intact, the authorities said. The Allegra's fire has been put out, and the passengers are all in good health, the authorities said. The Seychelles is sending a tug, and merchant ships in the area are steaming toward the Allegra, the coast guard said.\n",
      "\n",
      "generated_summary\n",
      "\n",
      " The ship is carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying cargo from the Seychelles . The ship was carrying\n",
      "\n",
      "actual_summary\n",
      "\n",
      "An engine room fire leaves the Costa Allegra without propulsion, authorities say. Its sister ship, the Costa Concordia, shipwrecked last month, killing at least 21. <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|>\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb23b7f487aa4008ae913f92a1c73fd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "new_article\n",
      "\n",
      "Islamabad, Pakistan -LRB- CNN -RRB- -- A Pakistani politician and his bodyguard were killed Monday in a suicide attack in northwest Pakistan, a police official told CNN. Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha when a bomber approached his car and detonated his explosives, police official Muhammad Ejaz Khan said. The attack took place in the Swabi district of Khyber Pakhtunkhwa province, about 80 kilometers -LRB- 50 miles -RRB- west of Islamabad. Jadoon was a member of the Awami National Party, a secular party often targeted by the Taliban. Nine others were injured in the attack. No one has claimed responsibility for the attack, police said.\n",
      "\n",
      "generated_summary\n",
      "\n",
      " Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha . The attack took place in Swabi district of Khyber Pakhtunkhwa province , about 80 kilometers -LRB- 50 miles -RRB- west of Islamabad . Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha . The attack took place in Swabi district of Khyber Pakhtunkhwa province , about 80 kilometers\n",
      "\n",
      "actual_summary\n",
      "\n",
      "Hanif Jadoon was a member of the Awami National Party. The secular party is often targeted by the Taliban. Police say no one has claimed responsibility for the attack. Nine others are injured. <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generate_sample(test_data, tokenizer, model, num=2, length=100, temperature=1, top_k=10, top_p=0.5, device=args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\IMP_REPOS\\Generating_Text_Summary_With_GPT2\\utils.py:103: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  next_token_probs = F.softmax(next_token_logits)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7462ecf2e68e4cbc8ebfdb01bf1f9314",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=99), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\IMP_REPOS\\Generating_Text_Summary_With_GPT2\\utils.py:114: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  next_token_probs = F.softmax(next_token_logits)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "new_article\n",
      "\n",
      "Rome -LRB- CNN -RRB- -- A cruise ship of the Costa Cruises line is adrift off the coast of the Seychelles after a fire in its engine room, the Italian coast guard said Monday. The ship, the Allegra, is a sister of the Costa Concordia, which wrecked off the coast of Italy on January 13, killing at least 21 people. The fire left the Allegra without propulsion, although its communications equipment is intact, the authorities said. The Allegra's fire has been put out, and the passengers are all in good health, the authorities said. The Seychelles is sending a tug, and merchant ships in the area are steaming toward the Allegra, the coast guard said\n",
      "\n",
      "actual_summary\n",
      "\n",
      "An engine room fire leaves the Costa Allegra without propulsion, authorities say. Its sister ship, the Costa Concordia, shipwrecked last month, killing at least 21. <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|>\n",
      "\n",
      "generated_summary-1 and Score is 4.5763957245981146e-07.\n",
      "\n",
      " The ship is a sister of the Costa Concordia , which crashed off the coast of Italy on January 13 . The ship is a sister of the Costa Concordia , which crashed off the coast of Italy on January 13 . The ship is a sister of the Costa Concordia , which crashed off the coast of Italy on January 13 . The ship is a sister of the Costa Concordia , which crashed off the coast of Italy on January 13 . The ship is a sister of the Costa Concordia , which\n",
      "\n",
      "generated_summary-2 and Score is 6.15412121263148e-09.\n",
      "\n",
      " A Alleg was carrying passengers ship Costa Concord Cruises . The was carrying passengers ship Costa Concord in January ship , Crew fire was put out , passengers are Cruises , whose was on Italyy coast Sicily in January ship , Ship Allegra sister ship to Costa Concord Cruises . crashed crash on January 13 ,es in engine room , Ship Allegra sister ship to Costa Concordises . . crashed crash on January 13 , ship in engine room , Ship Allegra sister ship to Costa Concordises . . crashed\n",
      "\n",
      "generated_summary-3 and Score is 2.09466621825527e-09.\n",
      "\n",
      " Costa Cruraises ship onwreck Cruia ship which the wrecked ship Italy on January 13 Cruia .wreck killing Allegra passengers on January 13 in allising . the wrecked in Se ship , , . December . , Crews was able to of Costa ship-wreck , whose was in Seychell killing . December . , A Allegra sister ship to Costa Cru , ,, who crash in Seychelles . December . , A Allegra sister ship to Costa Cru , crashes ship ship\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e666fe6284b143ffb3ff2ed80c6019c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=99), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "new_article\n",
      "\n",
      "Islamabad, Pakistan -LRB- CNN -RRB- -- A Pakistani politician and his bodyguard were killed Monday in a suicide attack in northwest Pakistan, a police official told CNN. Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha when a bomber approached his car and detonated his explosives, police official Muhammad Ejaz Khan said. The attack took place in the Swabi district of Khyber Pakhtunkhwa province, about 80 kilometers -LRB- 50 miles -RRB- west of Islamabad. Jadoon was a member of the Awami National Party, a secular party often targeted by the Taliban. Nine others were injured in the attack. No one has claimed responsibility for the attack, police said\n",
      "\n",
      "actual_summary\n",
      "\n",
      "Hanif Jadoon was a member of the Awami National Party. The secular party is often targeted by the Taliban. Police say no one has claimed responsibility for the attack. Nine others are injured. <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|> <|pad|>\n",
      "\n",
      "generated_summary-1 and Score is 1.4044410599767243e-11.\n",
      "\n",
      " Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha when a bomber approached his car . bodyadian Hanif Jadoon was a member of the Awami National Party , a secular party often targeted by the Taliban . Nine others were injured in the attack .ha one of the others othersguards was injured in the attack . Hanif Jadoon had just finished morning prayers on the Islamic holiday of Eid al-Adha when a bomber approached his\n",
      "\n",
      "generated_summary-2 and Score is 3.9127810149905407e-13.\n",
      "\n",
      " Theadha is of was a secular party often targeted Eid al-Adha . Hanif J . Jadoon of vehicle and Theguards were killed in a suicide attack had just finished morning prayers . Hanif J . Jadoon had just finished morning prayers on The Islamic holiday of Eid al-Ad Noqq politician nine nine body a a killed in a suicide attack .one has claimed responsibility was a member of the Aw Eid holiday party . Hanif Ad Adoon . Hanifoon ,\n",
      "\n",
      "generated_summary-3 and Score is 3.8339730695780005e-13.\n",
      "\n",
      " J attack took holiday a , of Eid al-Ad by holiday . . Han , The Pakistani politician , was a member was the , aguardons killed . the attack took , holiday by holiday . on The Islamic holidayad police official says was a member of the Aw Noone are . . Han Ad , The Pakistani person and his was was guard killed . The attack , No one claimed responsibility . for the attack , holiday ,ami Muslim holiday . The attack Jadoon , Jad Ad was\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generate_beam_sample(test_data, tokenizer, model, num=2, length=100, beam_size=3, device=args.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download An Article Given A Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentences_from_query(query):\n",
    "    # Get url\n",
    "    if query.startswith(\"http\"):\n",
    "        url = query\n",
    "    else:\n",
    "        url = search(query, num_results=1)[0]\n",
    "    print(url)\n",
    "    page = requests.get(url).text\n",
    "    soup = BeautifulSoup(page)\n",
    "    # Get text from all <p> tags.\n",
    "    p_tags = soup.find_all('p')\n",
    "    # Get the text from each of the \"p\" tags and strip surrounding whitespace.\n",
    "    p_tags_text = \" \".join([tag.get_text().strip() for tag in p_tags])\n",
    "    return p_tags_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://towardsdatascience.com/neural-network-embeddings-explained-4d028e6f0526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1957 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "article = sentences_from_query(\"neural embedding\")\n",
    "article = tokenizer.encode(article)[:900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7aa77e633e8b441f9baac6610ed379ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "generated_text = sample_seq(model, article, 50, args.device, temperature=1, top_k=10, top_p=0.5)\n",
    "generated_text = generated_text[0, len(article):].tolist()\n",
    "text = tokenizer.convert_ids_to_tokens(generated_text,skip_special_tokens=True)\n",
    "text = tokenizer.convert_tokens_to_string(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Article: \n",
      "\n",
      "Applications of neural networks have expanded significantly in recent years from image segmentation to natural language processing to time-series forecasting. One notably successful use of deep learning is embedding, a method used to represent discrete variables as continuous vectors. This technique has found practical applications with word embeddings for machine translation and entity embeddings for categorical variables. In this article, I’ll explain what neural network embeddings are, why we want to use them, and how they are learned. We’ll go through these concepts in the context of a real problem I’m working on: representing all the books on Wikipedia as vectors to create a book recommendation system. An embedding is a mapping of a discrete — categorical — variable to a vector of continuous numbers. In the context of neural networks, embeddings are low-dimensional, learned continuous vector representations of discrete variables. Neural network embeddings are useful because they can reduce the dimensionality of categorical variables and meaningfully represent categories in the transformed space. Neural network embeddings have 3 primary purposes: This means in terms of the book project, using neural network embeddings, we can take all 37,000 book articles on Wikipedia and represent each one using only 50 numbers in a vector. Moreover, because embeddings are learned, books that are more similar in the context of our learning problem are closer to one another in the embedding space. Neural network embeddings overcome the two limitations of a common method for representing categorical variables: one-hot encoding. The operation of one-hot encoding categorical variables is actually a simple embedding where each category is mapped to a different vector. This process takes discrete entities and maps each observation to a vector of 0s and a single 1 signaling the specific category. The one-hot encoding technique has two main drawbacks: The first problem is well-understood: for each additional category — referred to as an entity — we have to add another number to the one-hot encoded vector. If we have 37,000 books on Wikipedia, then representing these requires a 37,000-dimensional vector for each book, which makes training any machine learning model on this representation infeasible. The second problem is equally limiting: one-hot encoding does not place similar entities closer to one another in vector space. If we measure similarity between vectors using the cosine distance, then after one-hot encoding, the similarity is 0 for every comparison between entities. This means that entities such as War and Peace and Anna Karenina (both classic books by Leo Tolstoy) are no closer to one another than War and Peace is to The Hitchhiker’s Guide to the Galaxy if we use one-hot encoding. Considering these two problems, the ideal solution for representing categorical variables would require fewer numbers than the number of unique categories and would place similar categories closer to one another. To construct a better representation of categorical entities, we can use an embedding neural network and a supervised task to learn embeddings. The main issue with one-hot encoding is that the transformation does not rely on any supervision. We can greatly improve embeddings by learning them using a neural network on a supervised task. The embeddings form the parameters — weights — of the network which are adjusted to minimize loss on the task. The resulting embedded vectors are representations of categories where similar categories — relative to the task — are closer to one another. For example, if we have a vocabulary of 50,000 words used in a collection of movie reviews, we could learn 100-dimensional embeddings for each word using an embedding neural network trained to predict the sentimentality of the reviews. (For exactly this application see this Google Colab Notebook). Words in the vocabulary that are associated with positive reviews such as “brilliant” or “excellent” will come out closer in the embedding space because the network has learned these are both associated with positive reviews. In the book example given above, our supervised task could be “identify whether or not a book was written by Leo Tolstoy” and the resulting embeddings would place books written by Tolstoy closer to each other. Figuring out how to create the supervised task to produce relevant representations is the toughest part of making embeddings. In the Wikipedia book project (complete notebook here), the supervised learning task is set as predicting\n",
      "------------------------------------------------------------ \n",
      "\n",
      "Generated Summary: \n",
      "\n",
      " the sentiment of each book based on the<|sep|>sentimentality of the book.<|sep|>Learning neural networks<|sep|>Learning neural networks<|sep|>Learning neural networks to predict sentimentality of books is a<|sep|>problem which requires a neural network trained on a supervised task. This\n"
     ]
    }
   ],
   "source": [
    "print(\"Article: \\n\")\n",
    "print(tokenizer.decode(article))\n",
    "print(\"------------------------------------------------------------ \\n\")\n",
    "print(\"Generated Summary: \\n\")\n",
    "print(text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}