nlp-peft-lora-dialogsum-dialogue-summarize.ipynb

{"metadata":{"availableInstances":[{"_defaultOrder":0,"_isFastLaunch":true,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":4,"name":"ml.t3.medium","vcpuNum":2},{"_defaultOrder":1,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":8,"name":"ml.t3.large","vcpuNum":2},{"_defaultOrder":2,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.t3.xlarge","vcpuNum":4},{"_defaultOrder":3,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.t3.2xlarge","vcpuNum":8},{"_defaultOrder":4,"_isFastLaunch":true,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":8,"name":"ml.m5.large","vcpuNum":2},{"_defaultOrder":5,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.m5.xlarge","vcpuNum":4},{"_defaultOrder":6,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.m5.2xlarge","vcpuNum":8},{"_defaultOrder":7,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":64,"name":"ml.m5.4xlarge","vcpuNum":16},{"_defaultOrder":8,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":128,"name":"ml.m5.8xlarge","vcpuNum":32},{"_defaultOrder":9,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":192,"name":"ml.m5.12xlarge","vcpuNum":48},{"_defaultOrder":10,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":256,"name":"ml.m5.16xlarge","vcpuNum":64},{"_defaultOrder":11,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":384,"name":"ml.m5.24xlarge","vcpuNum":96},{"_defaultOrder":12,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":8,"name":"ml.m5d.large","vcpuNum":2},{"_defaultOrder":13,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.m5d.xlarge","vcpuNum":4},{"_defaultOrder":14,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.m5d.2xlarge","vcpuNum":8},{"_defaultOrder":15,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":64,"name":"ml.m5d.4xlarge","vcpuNum":16},{"_defaultOrder":16,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":128,"name":"ml.m5d.8xlarge","vcpuNum":32},{"_defaultOrder":17,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":192,"name":"ml.m5d.12xlarge","vcpuNum":48},{"_defaultOrder":18,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":256,"name":"ml.m5d.16xlarge","vcpuNum":64},{"_defaultOrder":19,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":384,"name":"ml.m5d.24xlarge","vcpuNum":96},{"_defaultOrder":20,"_isFastLaunch":false,"category":"General purpose","gpuNum":0,"hideHardwareSpecs":true,"memoryGiB":0,"name":"ml.geospatial.interactive","supportedImageNames":["sagemaker-geospatial-v1-0"],"vcpuNum":0},{"_defaultOrder":21,"_isFastLaunch":true,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":4,"name":"ml.c5.large","vcpuNum":2},{"_defaultOrder":22,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":8,"name":"ml.c5.xlarge","vcpuNum":4},{"_defaultOrder":23,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.c5.2xlarge","vcpuNum":8},{"_defaultOrder":24,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.c5.4xlarge","vcpuNum":16},{"_defaultOrder":25,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":72,"name":"ml.c5.9xlarge","vcpuNum":36},{"_defaultOrder":26,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":96,"name":"ml.c5.12xlarge","vcpuNum":48},{"_defaultOrder":27,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":144,"name":"ml.c5.18xlarge","vcpuNum":72},{"_defaultOrder":28,"_isFastLaunch":false,"category":"Compute optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":192,"name":"ml.c5.24xlarge","vcpuNum":96},{"_defaultOrder":29,"_isFastLaunch":true,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.g4dn.xlarge","vcpuNum":4},{"_defaultOrder":30,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.g4dn.2xlarge","vcpuNum":8},{"_defaultOrder":31,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":64,"name":"ml.g4dn.4xlarge","vcpuNum":16},{"_defaultOrder":32,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":128,"name":"ml.g4dn.8xlarge","vcpuNum":32},{"_defaultOrder":33,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":4,"hideHardwareSpecs":false,"memoryGiB":192,"name":"ml.g4dn.12xlarge","vcpuNum":48},{"_defaultOrder":34,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":256,"name":"ml.g4dn.16xlarge","vcpuNum":64},{"_defaultOrder":35,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":61,"name":"ml.p3.2xlarge","vcpuNum":8},{"_defaultOrder":36,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":4,"hideHardwareSpecs":false,"memoryGiB":244,"name":"ml.p3.8xlarge","vcpuNum":32},{"_defaultOrder":37,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":8,"hideHardwareSpecs":false,"memoryGiB":488,"name":"ml.p3.16xlarge","vcpuNum":64},{"_defaultOrder":38,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":8,"hideHardwareSpecs":false,"memoryGiB":768,"name":"ml.p3dn.24xlarge","vcpuNum":96},{"_defaultOrder":39,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.r5.large","vcpuNum":2},{"_defaultOrder":40,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.r5.xlarge","vcpuNum":4},{"_defaultOrder":41,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":64,"name":"ml.r5.2xlarge","vcpuNum":8},{"_defaultOrder":42,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":128,"name":"ml.r5.4xlarge","vcpuNum":16},{"_defaultOrder":43,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":256,"name":"ml.r5.8xlarge","vcpuNum":32},{"_defaultOrder":44,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":384,"name":"ml.r5.12xlarge","vcpuNum":48},{"_defaultOrder":45,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":512,"name":"ml.r5.16xlarge","vcpuNum":64},{"_defaultOrder":46,"_isFastLaunch":false,"category":"Memory Optimized","gpuNum":0,"hideHardwareSpecs":false,"memoryGiB":768,"name":"ml.r5.24xlarge","vcpuNum":96},{"_defaultOrder":47,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":16,"name":"ml.g5.xlarge","vcpuNum":4},{"_defaultOrder":48,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":32,"name":"ml.g5.2xlarge","vcpuNum":8},{"_defaultOrder":49,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":64,"name":"ml.g5.4xlarge","vcpuNum":16},{"_defaultOrder":50,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":128,"name":"ml.g5.8xlarge","vcpuNum":32},{"_defaultOrder":51,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":1,"hideHardwareSpecs":false,"memoryGiB":256,"name":"ml.g5.16xlarge","vcpuNum":64},{"_defaultOrder":52,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":4,"hideHardwareSpecs":false,"memoryGiB":192,"name":"ml.g5.12xlarge","vcpuNum":48},{"_defaultOrder":53,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":4,"hideHardwareSpecs":false,"memoryGiB":384,"name":"ml.g5.24xlarge","vcpuNum":96},{"_defaultOrder":54,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":8,"hideHardwareSpecs":false,"memoryGiB":768,"name":"ml.g5.48xlarge","vcpuNum":192},{"_defaultOrder":55,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":8,"hideHardwareSpecs":false,"memoryGiB":1152,"name":"ml.p4d.24xlarge","vcpuNum":96},{"_defaultOrder":56,"_isFastLaunch":false,"category":"Accelerated computing","gpuNum":8,"hideHardwareSpecs":false,"memoryGiB":1152,"name":"ml.p4de.24xlarge","vcpuNum":96}],"colab":{"name":"Fine-tune a language model","provenance":[]},"instance_type":"ml.m5.2xlarge","kernelspec":{"display_name":"Python 3 (Data Science)","language":"python","name":"python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.10"},"vscode":{"interpreter":{"hash":"aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"}}},"nbformat_minor":4,"nbformat":4,"cells":[{"source":"<a href=\"https://www.kaggle.com/code/yannicksteph/nlp-peft-lora-dialogsum-dialog-summarize?scriptVersionId=145588245\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown"},{"cell_type":"markdown","source":"# | NLP | PEFT/LoRA | DialogSum | Dialog Summarize |\n\n## NLP (Natural Language Processing) with PEFT (Parameter Efficient Fine-Tuning) and LoRA (Low-Rank Adaptation) for Dialogue Summarization\n\n# <b>1 <span style='color:#78D118'>|</span> Introduction</b>\n\nThis project delves into the capabilities of LLM (Language Model) with a specific focus on leveraging Parameter Efficient Fine-Tuning (PEFT) for enhancing dialogue summarization using the FLAN-T5 model.\n\nOur goal is to enhance the quality of dialogue summarization by employing a comprehensive fine-tuning approach and evaluating the results using ROUGE metrics. Additionally, we will explore the advantages of Parameter Efficient Fine-Tuning (PEFT), demonstrating that its benefits outweigh any potential minor performance trade-offs.\n\n - NOTE: This is an example and we not using the entirety of the data used for PERF / LoRA.\n \n## Objectives :\n - Train LLM for Dialogue Summarization.\n \n \n ## The DialogSum Dataset:\nThe [DialogSum Dataset](https://huggingface.co/datasets/knkarthick/dialogsum) DialogSum is a large-scale dialogue summarization dataset, consisting of 13,460 (Plus 100 holdout data for topic generation) dialogues with corresponding manually labeled summaries and topics.\n\n## Project Workflow:\n\n- **Setup**: Import necessary libraries and define project parameters.\n- **Dataset Exploration**: Discovering DialogSum Dataset.\n- **Test Model Zero Shot Inferencing**: Initially, test the FLAN-T5 model for zero-shot inferencing on dialogue summarization tasks to establish a baseline performance.\n- **Dataset Preprocess Dialog and Summary**: Preprocess the dialog and its corresponding summary from the dataset to prepare for the train.\n-  **Perform Parameter Efficient Fine-Tuning (PEFT)**: Implement Parameter Efficient Fine-Tuning (PEFT), a more efficient fine-tuning approach that can significantly reduce training time while maintaining performance.\n-  **Evaluation**:\n    - Perform human evaluation to gauge the model's output in terms of readability and coherence. This can involve annotators ranking generated summaries for quality.\n    - Utilize ROUGE metrics to assess the quality of the generated summaries. ROUGE measures the overlap between generated summaries and human-written references.\n\n# <b>2<span style='color:#78D118'>|</span> Setup</b>\n## <b>2.1 <span style='color:#78D118'>|</span> Imports</b>","metadata":{}},{"cell_type":"code","source":"%pip install --upgrade pip\n%pip install --disable-pip-version-check \\\n    torch==1.13.1 \\\n    torchdata==0.5.1 --quiet\n\n%pip install \\\n    transformers==4.27.2 \\\n    datasets==2.11.0 \\\n    evaluate==0.4.0 \\\n    rouge_score==0.1.2 \\\n    loralib==0.1.1 \\\n    peft==0.3.0 --quiet","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":2,"outputs":[{"name":"stdout","output_type":"stream","text":"Requirement already satisfied: pip in /opt/conda/lib/python3.7/site-packages (23.2.1)\n\n\u001b[33mDEPRECATION: pyodbc 4.0.0-unsupported has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pyodbc or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n\n\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n\n\u001b[33mDEPRECATION: pyodbc 4.0.0-unsupported has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pyodbc or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n\n\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n\n\u001b[33mDEPRECATION: pyodbc 4.0.0-unsupported has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pyodbc or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n\n\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n\npytest-astropy 0.8.0 requires pytest-cov>=2.0, which is not installed.\n\npytest-astropy 0.8.0 requires pytest-filter-subpackage>=0.1, which is not installed.\n\nspyder 4.0.1 requires pyqt5<5.13; python_version >= \"3\", which is not installed.\n\nspyder 4.0.1 requires pyqtwebengine<5.13; python_version >= \"3\", which is not installed.\n\nnotebook 6.5.5 requires pyzmq<25,>=17, but you have pyzmq 25.1.1 which is incompatible.\n\npathos 0.3.1 requires dill>=0.3.7, but you have dill 0.3.6 which is incompatible.\n\npathos 0.3.1 requires multiprocess>=0.70.15, but you have multiprocess 0.70.14 which is incompatible.\n\nsparkmagic 0.20.4 requires nest-asyncio==1.5.5, but you have nest-asyncio 1.5.7 which is incompatible.\n\nspyder 4.0.1 requires jedi==0.14.1, but you have jedi 0.19.0 which is incompatible.\u001b[0m\u001b[31m\n\n\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"}]},{"cell_type":"code","source":"from datasets import load_dataset\nfrom transformers import AutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig, TrainingArguments, Trainer\nimport torch\nimport time\nimport evaluate\nimport pandas as pd\nimport numpy as np\nfrom peft import LoraConfig, get_peft_model, TaskType\nfrom peft import PeftModel, PeftConfig","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"rouge = evaluate.load('rouge')\ndash_line = '-'.join('' for x in range(100))","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":43,"outputs":[]},{"cell_type":"markdown","source":"Load the dataset","metadata":{}},{"cell_type":"code","source":"huggingface_dataset_name = \"knkarthick/dialogsum\"\ndataset = load_dataset(huggingface_dataset_name)","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":37,"outputs":[{"name":"stderr","output_type":"stream","text":"WARNING:datasets.builder:Found cached dataset csv (/root/.cache/huggingface/datasets/knkarthick___csv/knkarthick--dialogsum-3005b557c2c04c1d/0.0.0/6954658bab30a358235fa864b05cf819af0e179325c740e4bc853bcc7ec513e1)\n"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1d5f50859901406fb6934d823fedb0bd","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/3 [00:00<?, ?it/s]"]},"metadata":{}}]},{"cell_type":"markdown","source":"Load the pre-trained [FLAN-T5 model](https://huggingface.co/google/flan-t5-base) and its tokenizer directly from Hugging Face. We'll be using the smaller version of FLAN-T5 for this project.\n\nTo optimize memory usage, set `torch_dtype=torch.bfloat16` to specify the memory type used by this model.","metadata":{"tags":[]}},{"cell_type":"code","source":"model_name='google/flan-t5-base'\noriginal_model = AutoModelForSeq2SeqLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)\ntokenizer = AutoTokenizer.from_pretrained(model_name)","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":5,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"50bd2cc15f2b407e81567b874b0cf37e","version_major":2,"version_minor":0},"text/plain":["Downloading (…)lve/main/config.json:   0%|          | 0.00/1.40k [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"4b10a81535714f2f90fad98ab1ba803c","version_major":2,"version_minor":0},"text/plain":["Downloading pytorch_model.bin:   0%|          | 0.00/990M [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"1c65096204244cdca1bc832e5b69bd4a","version_major":2,"version_minor":0},"text/plain":["Downloading (…)neration_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"e38c167340cf4dd38aab0b5d3400c219","version_major":2,"version_minor":0},"text/plain":["Downloading (…)okenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"af7e11fd22564f1eaf6833e8f568478b","version_major":2,"version_minor":0},"text/plain":["Downloading spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"f1552aa2da0c4546b03a749adcaa88ef","version_major":2,"version_minor":0},"text/plain":["Downloading (…)/main/tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"6ea97c8c50cd401991a58e1c7edcef0c","version_major":2,"version_minor":0},"text/plain":["Downloading (…)cial_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"]},"metadata":{}}]},{"cell_type":"markdown","source":"## <b>2.2 <span style='color:#78D118'>|</span> Methods</b>","metadata":{}},{"cell_type":"code","source":"def print_number_of_trainable_model_parameters(model):\n    trainable_model_params = 0\n    all_model_params = 0\n    for _, param in model.named_parameters():\n        all_model_params += param.numel()\n        if param.requires_grad:\n            trainable_model_params += param.numel()\n    return f\"trainable model parameters: {trainable_model_params}\\nall model parameters: {all_model_params}\\npercentage of trainable model parameters: {100 * trainable_model_params / all_model_params:.2f}%\"\n\ndef tokenize_function(example):\n    start_prompt = 'Summarize the following conversation.\\n\\n'\n    end_prompt = '\\n\\nSummary: '\n    prompt = [start_prompt + dialogue + end_prompt for dialogue in example[\"dialogue\"]]\n    example['input_ids'] = tokenizer(prompt, padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n    example['labels'] = tokenizer(example[\"summary\"], padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n    \n    return example","metadata":{"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <b>3<span style='color:#78D118'>|</span> Data Exploration</b>","metadata":{}},{"cell_type":"code","source":"print(dash_line)\nprint(print_number_of_trainable_model_parameters(original_model))\nprint(dash_line)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":41,"outputs":[{"name":"stdout","output_type":"stream","text":"---------------------------------------------------------------------------------------------------\n\ntrainable model parameters: 3538944\n\nall model parameters: 251116800\n\npercentage of trainable model parameters: 1.41%\n\n---------------------------------------------------------------------------------------------------\n"}]},{"cell_type":"code","source":"print(\n    \"\"\"\n---------------------------------------------------------------------------------------------------\n\nPROMPT:\n\nSummarize the following conversation.\n\n\n#Person1#: Have you considered upgrading your system?\n\n#Person2#: Yes, but I'm not sure what exactly I would need.\n\n#Person1#: You could consider adding a painting program to your software. It would allow you to make up your own flyers and banners for advertising.\n\n#Person2#: That would be a definite bonus.\n\n#Person1#: You might also want to upgrade your hardware because it is pretty outdated now.\n\n#Person2#: How can we do that?\n\n#Person1#: You'd probably need a faster processor, to begin with. And you also need a more powerful hard disc, more memory and a faster modem. Do you have a CD-ROM drive?\n\n#Person2#: No.\n\n#Person1#: Then you might want to add a CD-ROM drive too, because most new software programs are coming out on Cds.\n\n#Person2#: That sounds great. Thanks.\n\n\nSummary:\n\n---------------------------------------------------------------------------------------------------\n\nHUMAN SUMMARY:\n\n#Person1# teaches #Person2# how to upgrade software and hardware in #Person2#'s system.\n\n---------------------------------------------------------------------------------------------------\n    \"\"\"\n)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-10-07T12:33:35.813739Z","iopub.execute_input":"2023-10-07T12:33:35.814073Z","iopub.status.idle":"2023-10-07T12:33:35.823464Z","shell.execute_reply.started":"2023-10-07T12:33:35.814047Z","shell.execute_reply":"2023-10-07T12:33:35.822521Z"},"trusted":true},"execution_count":28,"outputs":[{"name":"stdout","text":"\n---------------------------------------------------------------------------------------------------\n\nPROMPT:\n\nSummarize the following conversation.\n\n\n#Person1#: Have you considered upgrading your system?\n\n#Person2#: Yes, but I'm not sure what exactly I would need.\n\n#Person1#: You could consider adding a painting program to your software. It would allow you to make up your own flyers and banners for advertising.\n\n#Person2#: That would be a definite bonus.\n\n#Person1#: You might also want to upgrade your hardware because it is pretty outdated now.\n\n#Person2#: How can we do that?\n\n#Person1#: You'd probably need a faster processor, to begin with. And you also need a more powerful hard disc, more memory and a faster modem. Do you have a CD-ROM drive?\n\n#Person2#: No.\n\n#Person1#: Then you might want to add a CD-ROM drive too, because most new software programs are coming out on Cds.\n\n#Person2#: That sounds great. Thanks.\n\n\nSummary:\n\n---------------------------------------------------------------------------------------------------\n\nHUMAN SUMMARY:\n\n#Person1# teaches #Person2# how to upgrade software and hardware in #Person2#'s system.\n\n---------------------------------------------------------------------------------------------------\n    \n","output_type":"stream"}]},{"cell_type":"markdown","source":"# <b>4<span style='color:#78D118'>|</span> Test Model Zero Shot Inferencing</b>\n\nTest the model using zero-shot inference. It's evident that the model faces challenges in summarizing the dialogue when compared to the baseline summary. However, it manages to extract some crucial information from the text, suggesting that fine-tuning.","metadata":{"tags":[]}},{"cell_type":"code","source":"index = 200\n\ndialogue = dataset['test'][index]['dialogue']\nsummary = dataset['test'][index]['summary']\n\nprompt = f\"\"\"\nSummarize the following conversation.\n\n{dialogue}\n\nSummary:\n\"\"\"\n\ninputs = tokenizer(prompt, return_tensors='pt')\noutput = tokenizer.decode(\n    original_model.generate(\n        inputs[\"input_ids\"], \n        max_new_tokens=200,\n    )[0], \n    skip_special_tokens=True\n)\nprint(dash_line)\nprint(\"ZERO SHOT\")\nprint(dash_line)\nprint(f'PROMPT:\\n{prompt}')\nprint(dash_line)\nprint(f'HUMAN SUMMARY:\\n{summary}\\n')\nprint(dash_line)\nprint(f'ORIGINAL MODEL SUMMARY:\\n{output}')\nprint(dash_line)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":50,"outputs":[{"name":"stdout","output_type":"stream","text":"---------------------------------------------------------------------------------------------------\n\nZERO SHOT\n\n---------------------------------------------------------------------------------------------------\n\nPROMPT:\n\n\n\nSummarize the following conversation.\n\n\n\n#Person1#: Have you considered upgrading your system?\n\n#Person2#: Yes, but I'm not sure what exactly I would need.\n\n#Person1#: You could consider adding a painting program to your software. It would allow you to make up your own flyers and banners for advertising.\n\n#Person2#: That would be a definite bonus.\n\n#Person1#: You might also want to upgrade your hardware because it is pretty outdated now.\n\n#Person2#: How can we do that?\n\n#Person1#: You'd probably need a faster processor, to begin with. And you also need a more powerful hard disc, more memory and a faster modem. Do you have a CD-ROM drive?\n\n#Person2#: No.\n\n#Person1#: Then you might want to add a CD-ROM drive too, because most new software programs are coming out on Cds.\n\n#Person2#: That sounds great. Thanks.\n\n\n\nSummary:\n\n\n\n---------------------------------------------------------------------------------------------------\n\nHUMAN SUMMARY:\n\n#Person1# teaches #Person2# how to upgrade software and hardware in #Person2#'s system.\n\n\n\n---------------------------------------------------------------------------------------------------\n\nORIGINAL MODEL SUMMARY:\n\n#Person1#: Have you considered upgrading your system?\n\n---------------------------------------------------------------------------------------------------\n"}]},{"cell_type":"markdown","source":"# <b>5<span style='color:#78D118'>|</span> Dataset Preprocess Dialog and Summary</b>","metadata":{}},{"cell_type":"markdown","source":"Transform the dialog-summary (prompt-response) pairs by adding specific instructions for the Language Model (LLM). Add the instruction \"Summarize the following conversation\" at the beginning of the dialog and \"Summary\" at the beginning of the summary like this:\n\nTraining prompt (dialogue):\n```\nSummarize the following conversation.\n\n    Chris: This is his part of the conversation.\n    Antje: This is her part of the conversation.\n    \nSummary: \n```\n\nTraining response (summary):\n```\nBoth Chris and Antje participated in the conversation.\n```\n\nNow we preprocess the prompt-response dataset by tokenizing the text and extracting their input_ids, with one input_id assigned per token.","metadata":{"tags":[]}},{"cell_type":"code","source":"tokenized_datasets = dataset.map(tokenize_function, batched=True)\ntokenized_datasets = tokenized_datasets.remove_columns(['id', 'topic', 'dialogue', 'summary',])","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Map:   0%|          | 0/12460 [00:00<?, ? examples/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Map:   0%|          | 0/500 [00:00<?, ? examples/s]"]},"metadata":{}}]},{"cell_type":"code","source":"tokenized_datasets = tokenized_datasets.filter(lambda example, index: index % 100 == 0, with_indices=True)","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":9,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Filter:   0%|          | 0/12460 [00:00<?, ? examples/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Filter:   0%|          | 0/1500 [00:00<?, ? examples/s]"]},"metadata":{}},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"","version_major":2,"version_minor":0},"text/plain":["Filter:   0%|          | 0/500 [00:00<?, ? examples/s]"]},"metadata":{}}]},{"cell_type":"markdown","source":" - NOTE: This is an example and we not using the entirety of the data used for PERF / LoRA.","metadata":{}},{"cell_type":"code","source":"print(dash_line)\nprint(f\"Shapes of the datasets:\")\nprint(f\"Training: {tokenized_datasets['train'].shape}\")\nprint(f\"Validation: {tokenized_datasets['validation'].shape}\")\nprint(f\"Test: {tokenized_datasets['test'].shape}\")\nprint(tokenized_datasets)\nprint(dash_line)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":42,"outputs":[{"name":"stdout","output_type":"stream","text":"---------------------------------------------------------------------------------------------------\n\nShapes of the datasets:\n\nTraining: (125, 2)\n\nValidation: (5, 2)\n\nTest: (15, 2)\n\nDatasetDict({\n\n    train: Dataset({\n\n        features: ['input_ids', 'labels'],\n\n        num_rows: 125\n\n    })\n\n    test: Dataset({\n\n        features: ['input_ids', 'labels'],\n\n        num_rows: 15\n\n    })\n\n    validation: Dataset({\n\n        features: ['input_ids', 'labels'],\n\n        num_rows: 5\n\n    })\n\n})\n\n---------------------------------------------------------------------------------------------------\n"}]},{"cell_type":"markdown","source":"Check the shapes of all three parts of the dataset:","metadata":{"tags":[]}},{"cell_type":"markdown","source":"# <b>6 <span style='color:#78D118'>|</span> Dataset Preprocess Dialog and Summary</b>\n\nLet's delve into the process of Parameter Efficient Fine-Tuning (PEFT), which offers a more efficient alternative to full fine-tuning. PEFT encompasses various techniques, including Low-Rank Adaptation (LoRA) and prompt tuning (distinct from prompt engineering).\n\nPEFT, it typically involves Low-Rank Adaptation (LoRA).\n\nLoRA, in essence, enables fine-tuning of your model with significantly fewer computational resources, sometimes even just a single GPU. After fine-tuning for a specific task, use case, or tenant using LoRA, the original Language Model (LLM) remains unchanged, while a newly-trained \"LoRA adapter\" emerges. This LoRA adapter is substantially smaller than the original LLM, often only a fraction of its size (in megabytes rather than gigabytes).\n\nHowever, during inference, the LoRA adapter needs to be reintegrated and combined with its original LLM to fulfill the inference request. The advantage lies in the fact that multiple LoRA adapters can reuse the same original LLM, reducing overall memory requirements when serving multiple tasks and use cases.","metadata":{}},{"cell_type":"markdown","source":"## <b>6.1 <span style='color:#78D118'>|</span> PEFT/LoRA model for Fine-Tuning</b>\n\nTo configure the PEFT/LoRA model for fine-tuning with a new parameter adapter, we follow these steps:\n\n1. **PEFT/LoRA Setup**: \n   - We are using PEFT/LoRA, which means we freeze the underlying Language Model (LLM) and focus on training only the adapter.\n\n2. **Adapter Configuration**:\n   - LoRA configuration below, the `rank (r)` hyper-parameter. This hyper-parameter determines the rank or dimensionality of the adapter that will be trained.\n\nBy employing PEFT/LoRA, we ensure that the core LLM remains unchanged while adapting a separate parameterized layer for our specific task or use case. The `rank (r)` hyper-parameter plays a critical role in determining the adapter's complexity and capacity for the target task.","metadata":{}},{"cell_type":"code","source":"lora_config = LoraConfig(\n    r=32, # Rank\n    lora_alpha=32,\n    target_modules=[\"q\", \"v\"],\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=TaskType.SEQ_2_SEQ_LM # FLAN-T5\n)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":23,"outputs":[]},{"cell_type":"markdown","source":"Incorporate LoRA adapter layers and parameters into the original Language Model (LLM) for training.","metadata":{"tags":[]}},{"cell_type":"code","source":"peft_model = get_peft_model(original_model, \n                            lora_config)\nprint(print_number_of_trainable_model_parameters(peft_model))","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":24,"outputs":[{"name":"stdout","output_type":"stream","text":"trainable model parameters: 3538944\n\nall model parameters: 251116800\n\npercentage of trainable model parameters: 1.41%\n"}]},{"cell_type":"markdown","source":"## <b>6.2 <span style='color:#78D118'>|</span> Train PEFT/LoRA Adapter</b>","metadata":{"tags":[]}},{"cell_type":"code","source":"output_dir = f'./peft-dialogue-summary-training'\n\npeft_training_args = TrainingArguments(\n    output_dir=output_dir,\n    auto_find_batch_size=True,\n    learning_rate=1e-3, # Higher learning rate than full fine-tuning.\n    num_train_epochs=1,\n    logging_steps=1,\n    max_steps=1    \n)\n    \npeft_trainer = Trainer(\n    model=peft_model,\n    args=peft_training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":47,"outputs":[]},{"cell_type":"code","source":"peft_trainer.train()\n\npeft_model_path=\"./peft-dialogue-summary-checkpoint-local\"\n\npeft_trainer.model.save_pretrained(peft_model_path)\ntokenizer.save_pretrained(peft_model_path)","metadata":{"tags":[],"_kg_hide-input":true,"_kg_hide-output":true},"execution_count":26,"outputs":[{"name":"stderr","output_type":"stream","text":"/opt/conda/lib/python3.7/site-packages/transformers/optimization.py:395: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n\n  FutureWarning,\n"},{"output_type":"display_data","data":{"text/html":["\n","    <div>\n","      \n","      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [1/1 00:00, Epoch 0/1]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n"," <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>1</td>\n","      <td>51.000000</td>\n","    </tr>\n","  </tbody>\n","</table><p>"],"text/plain":["<IPython.core.display.HTML object>"]},"metadata":{}},{"execution_count":26,"output_type":"execute_result","data":{"text/plain":["('./peft-dialogue-summary-checkpoint-local/tokenizer_config.json',\n"," './peft-dialogue-summary-checkpoint-local/special_tokens_map.json',\n"," './peft-dialogue-summary-checkpoint-local/tokenizer.json')"]},"metadata":{}}]},{"cell_type":"code","source":"peft_model_base = AutoModelForSeq2SeqLM.from_pretrained(\"google/flan-t5-base\", torch_dtype=torch.bfloat16)\ntokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-base\")\n\npeft_model = PeftModel.from_pretrained(peft_model_base, \n                                       peft_model_path, \n                                       torch_dtype=torch.bfloat16,\n                                       is_trainable=False)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":29,"outputs":[]},{"cell_type":"code","source":"print(print_number_of_trainable_model_parameters(peft_model))","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":30,"outputs":[{"name":"stdout","output_type":"stream","text":"trainable model parameters: 0\n\nall model parameters: 251116800\n\npercentage of trainable model parameters: 0.00%\n"}]},{"cell_type":"markdown","source":"# <b>7 <span style='color:#78D118'>|</span> Evaluation</b>\n\n## <b>7.1 <span style='color:#78D118'>|</span> Evaluate the Model Qualitatively (Human Evaluation)</b>","metadata":{}},{"cell_type":"code","source":"index = 200\ndialogue = dataset['test'][index]['dialogue']\nbaseline_human_summary = dataset['test'][index]['summary']\n\nprompt = f\"\"\"\nSummarize the following conversation.\n\n{dialogue}\n\nSummary: \"\"\"\n\ninput_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n\noriginal_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\noriginal_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n\npeft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\npeft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n\nprint(dash_line)\nprint(f'HUMAN SUMMARY:\\n{human_baseline_summary}')\nprint(dash_line)\nprint(f'ORIGINAL MODEL:\\n{original_model_text_output}')\nprint(dash_line)\nprint(f'PEFT MODEL: {peft_model_text_output}')\nprint(dash_line)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":48,"outputs":[{"name":"stdout","output_type":"stream","text":"---------------------------------------------------------------------------------------------------\n\nHUMAN SUMMARY:\n\n#Person1# teaches #Person2# how to upgrade software and hardware in #Person2#'s system.\n\n---------------------------------------------------------------------------------------------------\n\nORIGINAL MODEL:\n\n#Person1#: I'm looking for a computer for painting. #Person2#: I'm not sure what I'd need. #Person1#: How about a computer for a painting program? #Person2#: I'm not sure. #Person2#: I'm not sure. #Person1#: I'd need a computer for painting. #Person2#: I'm not sure. #Person1#: I'm not sure. #Person2#: I'm not sure. #Person1#: I'm not sure. #Person2#: I'm not sure.\n\n---------------------------------------------------------------------------------------------------\n\nPEFT MODEL: #Person1# recommends adding a painting program to #Person2#'s software and upgrading hardware. #Person2# also wants to upgrade the hardware because it's outdated now.\n\n---------------------------------------------------------------------------------------------------\n"}]},{"cell_type":"code","source":"dialogues = dataset['test'][0:10]['dialogue']\nhuman_baseline_summaries = dataset['test'][0:10]['summary']\n\noriginal_model_summaries = []\ninstruct_model_summaries = []\npeft_model_summaries = []\n\nfor idx, dialogue in enumerate(dialogues):\n    prompt = f\"\"\"\nSummarize the following conversation.\n\n{dialogue}\n\nSummary: \"\"\"\n    \n    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n\n    human_baseline_text_output = human_baseline_summaries[idx]\n    \n    original_model_outputs = original_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n\n    peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n    peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n\n    original_model_summaries.append(original_model_text_output)\n    peft_model_summaries.append(peft_model_text_output)\n\nzipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, peft_model_summaries))\n \ndf = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'peft_model_summaries'])\ndf","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":52,"outputs":[{"execution_count":52,"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>human_baseline_summaries</th>\n","      <th>original_model_summaries</th>\n","      <th>peft_model_summaries</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Ms. Dawson helps #Person1# to write a memo to ...</td>\n","      <td>#Person1#: Thank you for your help. #Person2#:...</td>\n","      <td>#Person1# asks Ms. Dawson to take a dictation ...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>In order to prevent employees from wasting tim...</td>\n","      <td>#Person1#: This memo should be distributed to ...</td>\n","      <td>#Person1# asks Ms. Dawson to take a dictation ...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Ms. Dawson takes a dictation for #Person1# abo...</td>\n","      <td>This memo is to go out to all employees by thi...</td>\n","      <td>#Person1# asks Ms. Dawson to take a dictation ...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>#Person2# arrives late because of traffic jam....</td>\n","      <td>The traffic is terrible.</td>\n","      <td>#Person2# got stuck in traffic and #Person1# s...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>#Person2# decides to follow #Person1#'s sugges...</td>\n","      <td>The traffic jam at Carrefour is a real problem.</td>\n","      <td>#Person2# got stuck in traffic and #Person1# s...</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>#Person2# complains to #Person1# about the tra...</td>\n","      <td>#Person1#: I'm finally here. #Person2#: I'm st...</td>\n","      <td>#Person2# got stuck in traffic and #Person1# s...</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>#Person1# tells Kate that Masha and Hero get d...</td>\n","      <td>#Person2#: They are having a separation for 2 ...</td>\n","      <td>Kate tells #Person2# Masha and Hero are gettin...</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>#Person1# tells Kate that Masha and Hero are g...</td>\n","      <td>Masha and Hero are getting divorced.</td>\n","      <td>Kate tells #Person2# Masha and Hero are gettin...</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>#Person1# and Kate talk about the divorce betw...</td>\n","      <td>Masha and Hero are having a separation for 2 m...</td>\n","      <td>Kate tells #Person2# Masha and Hero are gettin...</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>#Person1# and Brian are at the birthday party ...</td>\n","      <td>People are at the party.</td>\n","      <td>Brian remembers his birthday and invites #Pers...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                            human_baseline_summaries  \\\n","0  Ms. Dawson helps #Person1# to write a memo to ...   \n","1  In order to prevent employees from wasting tim...   \n","2  Ms. Dawson takes a dictation for #Person1# abo...   \n","3  #Person2# arrives late because of traffic jam....   \n","4  #Person2# decides to follow #Person1#'s sugges...   \n","5  #Person2# complains to #Person1# about the tra...   \n","6  #Person1# tells Kate that Masha and Hero get d...   \n","7  #Person1# tells Kate that Masha and Hero are g...   \n","8  #Person1# and Kate talk about the divorce betw...   \n","9  #Person1# and Brian are at the birthday party ...   \n","\n","                            original_model_summaries  \\\n","0  #Person1#: Thank you for your help. #Person2#:...   \n","1  #Person1#: This memo should be distributed to ...   \n","2  This memo is to go out to all employees by thi...   \n","3                           The traffic is terrible.   \n","4    The traffic jam at Carrefour is a real problem.   \n","5  #Person1#: I'm finally here. #Person2#: I'm st...   \n","6  #Person2#: They are having a separation for 2 ...   \n","7               Masha and Hero are getting divorced.   \n","8  Masha and Hero are having a separation for 2 m...   \n","9                           People are at the party.   \n","\n","                                peft_model_summaries  \n","0  #Person1# asks Ms. Dawson to take a dictation ...  \n","1  #Person1# asks Ms. Dawson to take a dictation ...  \n","2  #Person1# asks Ms. Dawson to take a dictation ...  \n","3  #Person2# got stuck in traffic and #Person1# s...  \n","4  #Person2# got stuck in traffic and #Person1# s...  \n","5  #Person2# got stuck in traffic and #Person1# s...  \n","6  Kate tells #Person2# Masha and Hero are gettin...  \n","7  Kate tells #Person2# Masha and Hero are gettin...  \n","8  Kate tells #Person2# Masha and Hero are gettin...  \n","9  Brian remembers his birthday and invites #Pers...  "]},"metadata":{}}]},{"cell_type":"markdown","source":"## <b>7.2 <span style='color:#78D118'>|</span> Evaluate the Model Quantitatively (ROUGE Metric)</b>\n\nThe [ROUGE metric](https://en.wikipedia.org/wiki/ROUGE_(metric)) is a valuable tool for assessing the quality of summaries generated by models. It evaluates these summaries by comparing them to a \"baseline\" summary, typically crafted by a human. Although not flawless, the ROUGE metric provides insights into the improvement in the overall effectiveness of summarization achieved through fine-tuning.","metadata":{}},{"cell_type":"code","source":"human_baseline_summaries = results['human_baseline_summaries'].values\noriginal_model_summaries = results['original_model_summaries'].values\npeft_model_summaries     = results['peft_model_summaries'].values\n\noriginal_model_results = rouge.compute(\n    predictions=original_model_summaries,\n    references=human_baseline_summaries[0:len(original_model_summaries)],\n    use_aggregator=True,\n    use_stemmer=True,\n)\n\npeft_model_results = rouge.compute(\n    predictions=peft_model_summaries,\n    references=human_baseline_summaries[0:len(peft_model_summaries)],\n    use_aggregator=True,\n    use_stemmer=True,\n)\n\nprint(dash_line)\nprint('ORIGINAL MODEL:')\nprint(original_model_results)\nprint(dash_line)\nprint('PEFT MODEL:')\nprint(peft_model_results)\nprint(dash_line)","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":53,"outputs":[{"name":"stdout","output_type":"stream","text":"---------------------------------------------------------------------------------------------------\n\nORIGINAL MODEL:\n\n{'rouge1': 0.2334158581572823, 'rouge2': 0.07603964187010573, 'rougeL': 0.20145520923859048, 'rougeLsum': 0.20145899339006135}\n\n---------------------------------------------------------------------------------------------------\n\nPEFT MODEL:\n\n{'rouge1': 0.40810631575616746, 'rouge2': 0.1633255794568712, 'rougeL': 0.32507074586565354, 'rougeLsum': 0.3248950182867091}\n\n---------------------------------------------------------------------------------------------------\n"}]},{"cell_type":"code","source":"print(\"Absolute percentage improvement of PEFT MODEL over ORIGINAL MODEL\")\n\nimprovement = (np.array(list(peft_model_results.values())) - np.array(list(original_model_results.values())))\nfor key, value in zip(peft_model_results.keys(), improvement):\n    print(f'{key}: {value*100:.2f}%')","metadata":{"tags":[],"_kg_hide-input":true},"execution_count":54,"outputs":[{"name":"stdout","output_type":"stream","text":"Absolute percentage improvement of PEFT MODEL over ORIGINAL MODEL\n\nrouge1: 17.47%\n\nrouge2: 8.73%\n\nrougeL: 12.36%\n\nrougeLsum: 12.34%\n"}]},{"cell_type":"markdown","source":"## References\n\nThe creation of this document was greatly influenced by the following key sources of information:\n\n1. [DialogSum Dataset](https://huggingface.co/datasets/knkarthick/dialogsum) DialogSum is a large-scale dialogue summarization dataset, consisting of 13,460 (Plus 100 holdout data for topic generation) dialogues with corresponding manually labeled summaries and topics.\n2. [Generative AI with Large Language Models | Coursera](https://www.coursera.org/learn/generative-ai-with-llms?utm_medium=sem&utm_source=gg&utm_campaign=B2C_NAMER_generative-ai-with-llms_deeplearning-ai_FTCOF_learn_country-US-country-CA&campaignid=20534248984&adgroupid=160068579824&device=c&keyword=&matchtype=&network=g&devicemodel=&adposition=&creativeid=673251286004&hide_mobile_promo&gclid=CjwKCAjwg4SpBhAKEiwAdyLwvEW_WnNyptOwzHtsGmn5-OxT5BKsQeUXHPahO-opBJ0JjsSynHkPAxoCaoAQAvD_BwE) - An informative guide that provides in-depth explanations and examples on various LLMs.","metadata":{}}]}