llama2_13b_notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyN0el5i7lid6x1mXU+UAe1T",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c6fbb43c7d4942c7b78843a54b000646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d07b48b402bc44f4a5f7847c6f586f2d",
              "IPY_MODEL_4c304d098aba4bf5960f81a9bec76101",
              "IPY_MODEL_715a80bd6d754352b31e8f487c5cef36"
            ],
            "layout": "IPY_MODEL_88d5e91fa2dc4413a7ae8d6c19cbf271"
          }
        },
        "275db051f7c04830979d0d0eb22e4df3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3d7f8807816d4c34bae08beb4d7ac003",
            "placeholder": "​",
            "style": "IPY_MODEL_8a3b7cc6a68f4617951aa4e744710a5d",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "8823b50bc07147e198c8f952b90d06d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_fce64b4b1e3749c785b8000da49b938f",
            "placeholder": "​",
            "style": "IPY_MODEL_a883fd6feeed467385c72d108c8b8d97",
            "value": ""
          }
        },
        "ebb5b4c1587a4e789316a4faf7ba1659": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_bc15ab4918ce4d44827dbc28c2a8feb0",
            "style": "IPY_MODEL_2bf4d30b58ad4faf89cc232f16c9f32b",
            "value": false
          }
        },
        "d8c86b17d5774d5c994151fd750ec1fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_efa59a226c08401b84c4c4803c77030d",
            "style": "IPY_MODEL_2abb7bb92112472e82a5f837034fa225",
            "tooltip": ""
          }
        },
        "c80966fd806c4b2d851b74a6a3008a98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52fbaa6f32a94d1485520cf7444283fb",
            "placeholder": "​",
            "style": "IPY_MODEL_27b1e03704d442f0bfb7f365e351dc2f",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "88d5e91fa2dc4413a7ae8d6c19cbf271": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "3d7f8807816d4c34bae08beb4d7ac003": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a3b7cc6a68f4617951aa4e744710a5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fce64b4b1e3749c785b8000da49b938f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a883fd6feeed467385c72d108c8b8d97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bc15ab4918ce4d44827dbc28c2a8feb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2bf4d30b58ad4faf89cc232f16c9f32b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "efa59a226c08401b84c4c4803c77030d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2abb7bb92112472e82a5f837034fa225": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "52fbaa6f32a94d1485520cf7444283fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27b1e03704d442f0bfb7f365e351dc2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20d07d6aea5d407e9250c5d6119451b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a5cb1d234354f02883e86c3a50cc7e7",
            "placeholder": "​",
            "style": "IPY_MODEL_afd66be2ccc1466db4aac278f37b6fa5",
            "value": "Connecting..."
          }
        },
        "1a5cb1d234354f02883e86c3a50cc7e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afd66be2ccc1466db4aac278f37b6fa5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d07b48b402bc44f4a5f7847c6f586f2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ac19394b168452997eea31a0114d32a",
            "placeholder": "​",
            "style": "IPY_MODEL_acdb104e98d24ce48b99b5cd22f02f91",
            "value": "Token is valid (permission: read)."
          }
        },
        "4c304d098aba4bf5960f81a9bec76101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd6ebb0289624e0a86ec46d5c777c4df",
            "placeholder": "​",
            "style": "IPY_MODEL_5242cc498d3a4342939d3c1bea32a6df",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "715a80bd6d754352b31e8f487c5cef36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9c64cd4df144ef29c1fca402f53468e",
            "placeholder": "​",
            "style": "IPY_MODEL_205fa6293a3f40f282cd395d5a4c6508",
            "value": "Login successful"
          }
        },
        "6ac19394b168452997eea31a0114d32a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acdb104e98d24ce48b99b5cd22f02f91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd6ebb0289624e0a86ec46d5c777c4df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5242cc498d3a4342939d3c1bea32a6df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d9c64cd4df144ef29c1fca402f53468e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "205fa6293a3f40f282cd395d5a4c6508": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f4e3d352518d428b9d34517de7ac40fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f7aaae3fe56240c48ccb9b0887b6908c",
              "IPY_MODEL_2a65bb7a8c3742869b5b36e7101595eb",
              "IPY_MODEL_b5b130785c1b4ed99b5068618e328e46"
            ],
            "layout": "IPY_MODEL_b09aa7b5f7ba4e3f9caeb0dcc32bc268"
          }
        },
        "f7aaae3fe56240c48ccb9b0887b6908c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db9e85a2f0ec437c94eaf1972c05878e",
            "placeholder": "​",
            "style": "IPY_MODEL_c0ecb2ac013845e79ac945feb102b0f1",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "2a65bb7a8c3742869b5b36e7101595eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8756d9f9876415bb019af6ceecf1f8c",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_91da6344233a4b468adbd402bce9b4a9",
            "value": 3
          }
        },
        "b5b130785c1b4ed99b5068618e328e46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f495e0f275d148b5a250a9200be07137",
            "placeholder": "​",
            "style": "IPY_MODEL_891d29a3da8e49a28eb358dd5a772e01",
            "value": " 3/3 [02:22&lt;00:00, 44.49s/it]"
          }
        },
        "b09aa7b5f7ba4e3f9caeb0dcc32bc268": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db9e85a2f0ec437c94eaf1972c05878e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0ecb2ac013845e79ac945feb102b0f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e8756d9f9876415bb019af6ceecf1f8c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91da6344233a4b468adbd402bce9b4a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f495e0f275d148b5a250a9200be07137": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "891d29a3da8e49a28eb358dd5a772e01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/americanthinker/vectorsearch-applications/blob/main/llama2_13b_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Overview - READ THIS FIRST"
      ],
      "metadata": {
        "id": "g3F8TUH0NOAn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The purpose of this notebook is to familiarize the student with Llama-2 model setup, prompt creation, and trying out RAG methods using an LLM.  This notebook is primarily intended for practice and getting used to making calls with Llama-2.  Students are NOT expected to run their Streamlit app in this environment, rather a HuggingFace endpoint will be creatd for you later in the course, when you are ready to integrate Llama-2 calls into your overall RAG system**\n",
        "1. The raw data for the Impact Theory is downloaded, but it's up to you on how (or if) you use it when practicing with making Llama-2 calls.\n",
        "2. The Weaviate Client code is also downloaded, and this is the recommended way of getting context data into the model prompts.  Environment variables can easily be configured in the Secrets section of Colab on the left of the notebook."
      ],
      "metadata": {
        "id": "PZR3YFTYMdQj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install bitsandbytes --quiet\n",
        "!pip install accelerate --quiet\n",
        "!pip install einops --quiet\n",
        "!pip install tqdm --quiet"
      ],
      "metadata": {
        "id": "H0cxxXW4r_uv"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "nfht0rZBrdap"
      },
      "outputs": [],
      "source": [
        "from huggingface_hub import notebook_login\n",
        "from transformers.pipelines.text_generation import TextGenerationPipeline\n",
        "from torch import cuda, bfloat16\n",
        "import bitsandbytes\n",
        "import transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import userdata\n",
        "from transformers import AutoConfig"
      ],
      "metadata": {
        "id": "BfJE8VdryXVp"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pass in environement variables using the `userdata.get('Name of your secret')` method."
      ],
      "metadata": {
        "id": "Js7Bchg4ZZKu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!curl -o impact_theory_data.json https://raw.githubusercontent.com/americanthinker/vectorsearch-applications/main/data/impact_theory_data.json\n",
        "!curl -o weaviate_interface.py https://raw.githubusercontent.com/americanthinker/vectorsearch-applications/main/weaviate_interface.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bHOiw3YE3Qw8",
        "outputId": "3e837630-1802-4cde-9259-6011c3724916"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 25.6M  100 25.6M    0     0  36.0M      0 --:--:-- --:--:-- --:--:-- 36.0M\n",
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 17590  100 17590    0     0  83333      0 --:--:-- --:--:-- --:--:-- 83364\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Using the HuggingFace `notebook_login` method is a streamlined way of authenticating with HF that lasts throughout the entirety of the Colab session.  Simply copy and paste your HF Token when prompted."
      ],
      "metadata": {
        "id": "uBafSazTHlkY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "notebook_login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "c6fbb43c7d4942c7b78843a54b000646",
            "275db051f7c04830979d0d0eb22e4df3",
            "8823b50bc07147e198c8f952b90d06d4",
            "ebb5b4c1587a4e789316a4faf7ba1659",
            "d8c86b17d5774d5c994151fd750ec1fd",
            "c80966fd806c4b2d851b74a6a3008a98",
            "88d5e91fa2dc4413a7ae8d6c19cbf271",
            "3d7f8807816d4c34bae08beb4d7ac003",
            "8a3b7cc6a68f4617951aa4e744710a5d",
            "fce64b4b1e3749c785b8000da49b938f",
            "a883fd6feeed467385c72d108c8b8d97",
            "bc15ab4918ce4d44827dbc28c2a8feb0",
            "2bf4d30b58ad4faf89cc232f16c9f32b",
            "efa59a226c08401b84c4c4803c77030d",
            "2abb7bb92112472e82a5f837034fa225",
            "52fbaa6f32a94d1485520cf7444283fb",
            "27b1e03704d442f0bfb7f365e351dc2f",
            "20d07d6aea5d407e9250c5d6119451b8",
            "1a5cb1d234354f02883e86c3a50cc7e7",
            "afd66be2ccc1466db4aac278f37b6fa5",
            "d07b48b402bc44f4a5f7847c6f586f2d",
            "4c304d098aba4bf5960f81a9bec76101",
            "715a80bd6d754352b31e8f487c5cef36",
            "6ac19394b168452997eea31a0114d32a",
            "acdb104e98d24ce48b99b5cd22f02f91",
            "bd6ebb0289624e0a86ec46d5c777c4df",
            "5242cc498d3a4342939d3c1bea32a6df",
            "d9c64cd4df144ef29c1fca402f53468e",
            "205fa6293a3f40f282cd395d5a4c6508"
          ]
        },
        "id": "p_E7BQrcrmGS",
        "outputId": "886e9add-4a0f-4b69-80e1-a6e649875a8b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c6fbb43c7d4942c7b78843a54b000646"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#load data for later user\n",
        "import json\n",
        "with open('/content/impact_theory_data.json') as f:\n",
        "  data = json.load(f)"
      ],
      "metadata": {
        "id": "0yRjdrya30p4"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_id = 'meta-llama/Llama-2-13b-chat-hf'\n",
        "\n",
        "# 4-bit Quanityzation to load Llama 2 with less GPU memory\n",
        "bnb_config = transformers.BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_quant_type='nf4',\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_compute_dtype=bfloat16\n",
        ")"
      ],
      "metadata": {
        "id": "8NnzYCXErtGL"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download the model tokenizer and the Llama-model itself...it's big!  Pass in the 4-bit quantization configuration set up in the previous cell, and set the model status to `eval`"
      ],
      "metadata": {
        "id": "xyEfKDtRITwW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Llama 2 Tokenizer\n",
        "tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)\n",
        "\n",
        "# Llama 2 Model\n",
        "model = transformers.AutoModelForCausalLM.from_pretrained(\n",
        "    model_id,\n",
        "    trust_remote_code=True,\n",
        "    quantization_config=bnb_config,\n",
        "    device_map='auto'\n",
        ")\n",
        "model.eval()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527,
          "referenced_widgets": [
            "f4e3d352518d428b9d34517de7ac40fb",
            "f7aaae3fe56240c48ccb9b0887b6908c",
            "2a65bb7a8c3742869b5b36e7101595eb",
            "b5b130785c1b4ed99b5068618e328e46",
            "b09aa7b5f7ba4e3f9caeb0dcc32bc268",
            "db9e85a2f0ec437c94eaf1972c05878e",
            "c0ecb2ac013845e79ac945feb102b0f1",
            "e8756d9f9876415bb019af6ceecf1f8c",
            "91da6344233a4b468adbd402bce9b4a9",
            "f495e0f275d148b5a250a9200be07137",
            "891d29a3da8e49a28eb358dd5a772e01"
          ]
        },
        "id": "YCLf_-IIr4xx",
        "outputId": "393a1e14-d183-457d-9024-48254f76c12a"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f4e3d352518d428b9d34517de7ac40fb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LlamaForCausalLM(\n",
              "  (model): LlamaModel(\n",
              "    (embed_tokens): Embedding(32000, 5120)\n",
              "    (layers): ModuleList(\n",
              "      (0-39): 40 x LlamaDecoderLayer(\n",
              "        (self_attn): LlamaAttention(\n",
              "          (q_proj): Linear4bit(in_features=5120, out_features=5120, bias=False)\n",
              "          (k_proj): Linear4bit(in_features=5120, out_features=5120, bias=False)\n",
              "          (v_proj): Linear4bit(in_features=5120, out_features=5120, bias=False)\n",
              "          (o_proj): Linear4bit(in_features=5120, out_features=5120, bias=False)\n",
              "          (rotary_emb): LlamaRotaryEmbedding()\n",
              "        )\n",
              "        (mlp): LlamaMLP(\n",
              "          (gate_proj): Linear4bit(in_features=5120, out_features=13824, bias=False)\n",
              "          (up_proj): Linear4bit(in_features=5120, out_features=13824, bias=False)\n",
              "          (down_proj): Linear4bit(in_features=13824, out_features=5120, bias=False)\n",
              "          (act_fn): SiLUActivation()\n",
              "        )\n",
              "        (input_layernorm): LlamaRMSNorm()\n",
              "        (post_attention_layernorm): LlamaRMSNorm()\n",
              "      )\n",
              "    )\n",
              "    (norm): LlamaRMSNorm()\n",
              "  )\n",
              "  (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "HuggingFace has a series of pipelines that abstract away a lot of the details for various model tasks (Named Entity Recognition, Question Answering, Masked Language Modeling, etc.).  For our use case, we want to go with `text-generation` given that is the primary task that the Llama-2 model series was trained to do.  "
      ],
      "metadata": {
        "id": "I_SknJtcJviw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llama = transformers.pipeline(model=model,\n",
        "                              tokenizer=tokenizer,\n",
        "                              task='text-generation',\n",
        "                              temperature=0.15,\n",
        "                              max_new_tokens=250,\n",
        "                              repetition_penalty=1.1)"
      ],
      "metadata": {
        "id": "591qi0EWtx4i"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Given our objective of Retrieval Augmented Generation, we'll want to setup our prompt in a specific way that balances feeding highly relevant context, but not too much, along with instructing the model to stick to the context. I've included some text chunks for you to practice with getting used to creating prompts for the Llama model.  This text chunks were generated by using this query:\n",
        "**\"What is Ian Bremmer's opinion on AI threats?\"**"
      ],
      "metadata": {
        "id": "UBMqjkhJKLxf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_chunks = [\n",
        "\"And that idea of, okay, there are things that we could rally around that take us out of our smaller narrative into a larger narrative, hence the title of the book, The Power of Crisis, there is a thing that that can bring us together and give us that shared narrative. But what scares me is if you plug in AI bias into this equation, you can't get now I Yeah, now I'm like, whoa, like, one, who gets to decide what the AI's value system is what the AI's belief system is how the AI interprets truth, what the AI reinforces. And then if there are a lot of AI, which which is probably the thing that protects us from an authoritarian answer, but at the same time, then you have all this competing reinforcement that again, just brings us back to fragmentation. So as you look at that suite of unnerving potential problems, what do you see is our path to the other side of this to doing it well? Yeah. So President Biden just two weeks ago, had a group of seven AI founders slash CEOs, the most powerful companies in this space, as of right now, that will not be true in a year or two, there'll be vastly more.\",\n",
        "\"So I do think the motivation to get this right is going to be there. I just, I hope we're up for it. And, you know, again, I'm an optimist. I'm hopeful. I mean, at the end of the day, I mean, the fact that we're here and we're talking about it means that we're capable of doing something. My only fear is that with global warming, you can't win global warming and get a leg up over China or Russia, but you can win AI and get a leg up and be better. And I think that one thing that people aren't talking about enough for sure is that AI is going to be an adversarial system, meaning bad guys are going to have AI and they're going to try to do things to hurt me with that AI. And then others are going to build AI that is protective and try to stop the bad guys. And so you will have, just like with normal hacking, you'll have an ever escalating arms race of AI. And so even if only with the best of intentions, we will end up getting to AI super intelligence because we're trying to stop somebody from doing a bad thing.\",\n",
        "\"But I'm sure that criminal malware developers are saying, I can't imagine developing criminal malware or spear phishing without using these new AI tools, because I mean, it's just going to allow them to target in such an extraordinary and pinpoint way, and also to send out so much more, you know, sort of capable malware that will elicit so much more engagement, and therefore, you know, bring so much more money to them or shut down so many more servers and give them so much more illicit data and so much of the illicit data that they've already collected from the hacks on, you know, all of these companies that you've heard about, Target, for example, other firms. I mean, so much of that so far is just, oh, we're just selling that for people that want to like use the credit cards. No, now you're going to sell it to people that are empowered with AI that can generate malware against that data. And that again, and that's, that's like, we're going to develop all these new vaccines and new pharmaceuticals that will deal with Alzheimer's and deal with cancers. And it's going to be an incredible time for medicine.\"\n",
        "]"
      ],
      "metadata": {
        "id": "Fufw3xGQO1p0"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(text_chunks)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KM1aMI_9PEVc",
        "outputId": "197798ad-5e9e-4b63-bc81-f246a42fb7df"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "When crafting your prompt, make note of the way the special tags are used:\n",
        "\n",
        "*   `<s>` - start prompt tag\n",
        "*   `[INST], [/INST]` - Opening and closing model instruction tags\n",
        "*   `<<<SYS>>>, <</SYS>>` - Opening and closing system prompt tags\n",
        "\n",
        "Below is \"an\" example of a question answering prompt for the model."
      ],
      "metadata": {
        "id": "1BNy0iBZPKw0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "question_answering_prompt = '''\n",
        "<s>[INST] <<SYS>>\n",
        "\n",
        "You are an expert at creating high quality, context-based answers to questions when given contextual information and part of a podcast episode transcript.\n",
        "\n",
        "<</SYS>>\n",
        "Your task is to synthesize and reason over a transcript of a snippet of an interview between Tom Bilyeu and his guest(s).\n",
        "User the contextual information that is provided to answer the question, which includes the show summary, guest, and the \\\n",
        "transcript itself. After your synthesis, use the transcript to answer the below question.\\n\n",
        "\n",
        "```\n",
        "Show Summary: {summary}\n",
        "Show Guest: {guest}\n",
        "Transcript: {transcript}\n",
        "```\\n\\n\n",
        "Question: {question}\\n\n",
        "Answer the question and provide reasoning if necessary to explain the answer.\\n\n",
        "If the context does not provide enough information to answer the question, then \\n\n",
        "state that you cannot answer the question with the provided context. [/INST]\n",
        "\n",
        "Answer:\n",
        "'''"
      ],
      "metadata": {
        "id": "DtQmfoerKweV"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#set constants\n",
        "summary = '''\n",
        "\"In this episode, Ian Bremmer discusses the rise of big tech as a third superpower and the potential dangers and opportunities it presents. He highlights the immense power held by tech companies in shaping society, the economy, and national security, emphasizing their sovereignty over the digital world. Bremmer expresses concerns about the growing influence of AI and its potential to outstrip government regulation, leading to a reality where tech companies wield significant power over individuals. He also delves into the risks associated with AI proliferation, including the potential for non-governments to control and misuse the technology, exacerbating social inequalities and disinformation. Bremmer emphasizes the need to address negative externalities and regulate AI to mitigate its adverse impacts. Additionally, he discusses the implications of AI on job displacement and social discontent, particularly for marginalized communities. The conversation delves into the breakdown of truth in the digital age, driven by algorithmic sorting and micro-targeting, leading to fragmented echo chambers and the erosion of consensus on facts. Both Bremmer and the host explore the challenges of navigating truth in a polarized and algorithmically driven information landscape, highlighting the need for critical thinking and a focus on human flourishing as a guiding principle in the face of AI's transformative impact.\"\n",
        "'''\n",
        "guest = 'Ian Bremmer'\n",
        "title = \"THE BIG AI RESET: The Next Global SuperPower Isn't Who You Think | Ian Bremmer\""
      ],
      "metadata": {
        "id": "obJFuNA1QcNA"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#define your query\n",
        "query = \"What is Ian Bremmer's opinion on AI threats?\""
      ],
      "metadata": {
        "id": "XGLcH_EvQ8NR"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prompt = question_answering_prompt.format(summary=summary, guest=guest, transcript=text_chunks[0], question=query)\n",
        "print(prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uB798METQKh7",
        "outputId": "bb083230-12ba-4958-b83e-411946bc0530"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "<s>[INST] <<SYS>>\n",
            "\n",
            "You are an expert at creating high quality, context-based answers to questions when given contextual information and part of a podcast episode transcript.\n",
            "\n",
            "<</SYS>>\n",
            "Your task is to synthesize and reason over a transcript of a snippet of an interview between Tom Bilyeu and his guest(s).\n",
            "User the contextual information that is provided to answer the question, which includes the show summary, guest, and the transcript itself. After your synthesis, use the transcript to answer the below question.\n",
            "\n",
            "\n",
            "```\n",
            "Show Summary: \n",
            "\"In this episode, Ian Bremmer discusses the rise of big tech as a third superpower and the potential dangers and opportunities it presents. He highlights the immense power held by tech companies in shaping society, the economy, and national security, emphasizing their sovereignty over the digital world. Bremmer expresses concerns about the growing influence of AI and its potential to outstrip government regulation, leading to a reality where tech companies wield significant power over individuals. He also delves into the risks associated with AI proliferation, including the potential for non-governments to control and misuse the technology, exacerbating social inequalities and disinformation. Bremmer emphasizes the need to address negative externalities and regulate AI to mitigate its adverse impacts. Additionally, he discusses the implications of AI on job displacement and social discontent, particularly for marginalized communities. The conversation delves into the breakdown of truth in the digital age, driven by algorithmic sorting and micro-targeting, leading to fragmented echo chambers and the erosion of consensus on facts. Both Bremmer and the host explore the challenges of navigating truth in a polarized and algorithmically driven information landscape, highlighting the need for critical thinking and a focus on human flourishing as a guiding principle in the face of AI's transformative impact.\"\n",
            "\n",
            "Show Guest: Ian Bremmer\n",
            "Transcript: And that idea of, okay, there are things that we could rally around that take us out of our smaller narrative into a larger narrative, hence the title of the book, The Power of Crisis, there is a thing that that can bring us together and give us that shared narrative. But what scares me is if you plug in AI bias into this equation, you can't get now I Yeah, now I'm like, whoa, like, one, who gets to decide what the AI's value system is what the AI's belief system is how the AI interprets truth, what the AI reinforces. And then if there are a lot of AI, which which is probably the thing that protects us from an authoritarian answer, but at the same time, then you have all this competing reinforcement that again, just brings us back to fragmentation. So as you look at that suite of unnerving potential problems, what do you see is our path to the other side of this to doing it well? Yeah. So President Biden just two weeks ago, had a group of seven AI founders slash CEOs, the most powerful companies in this space, as of right now, that will not be true in a year or two, there'll be vastly more.\n",
            "```\n",
            "\n",
            "\n",
            "Question: What is Ian Bremmer's opinion on AI threats?\n",
            "\n",
            "Answer the question and provide reasoning if necessary to explain the answer.\n",
            "\n",
            "If the context does not provide enough information to answer the question, then \n",
            "\n",
            "state that you cannot answer the question with the provided context. [/INST]\n",
            "\n",
            "Answer:\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### **As a final check, we can make sure that our prompt is below the model's max context window length of 4,096 tokens.**"
      ],
      "metadata": {
        "id": "cb6aH1qbRarU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(f'Total Prompt Tokens: {len(tokenizer.encode(prompt))}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9oYDjqiGRnou",
        "outputId": "f6f99580-f777-40b0-c292-cfa961122659"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Prompt Tokens: 818\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pass in the prompt as an arg to the model's `call` method and make sure to set the `return_full_text` param to False, otherwise you'll get the entire prompt back as part of the models' answer.  Also, be aware that the `temperature` setting and the `max_new_tokens` settings are initialized when the model was passed into the HuggingFace pipeline. Final note, if the model is taking too long to process the prompt, you can try switching out for the faster, but less performant, version: `meta-llama/Llama-2-7b-chat-hf`"
      ],
      "metadata": {
        "id": "V086AnfQT0zv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = llama(prompt, return_full_text=False)"
      ],
      "metadata": {
        "id": "bIBtKOZRT0B-"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response[0]['generated_text']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "id": "YohM5ShcU1_S",
        "outputId": "75567c94-8a3d-42e5-c5b2-31af26894b6d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\nBased on the transcript, Ian Bremmer expresses concerns about the potential dangers of AI, specifically the risk of AI bias and the potential for non-government entities to control and misuse the technology. He highlights the need to address negative externalities and regulate AI to mitigate its adverse impacts. Additionally, he notes the potential for AI to exacerbate social inequalities and disinformation, and emphasizes the importance of critical thinking and a focus on human flourishing in the face of AI's transformative impact.\\n\\nReasoning:\\n\\nFrom the transcript, it is clear that Ian Bremmer has a nuanced view of the potential threats posed by AI. On one hand, he acknowledges the immense power held by tech companies in shaping society, the economy, and national security, and expresses concerns about the growing influence of AI and its potential to outstrip government regulation. However, he also recognizes the potential benefits of AI and emphasizes the need to address negative externalities and regulate AI to mitigate its adverse impacts. Furthermore, he highlight\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Practice, Practice, Practice\n",
        "\n",
        "Learning how to use the Llama-2 model simply comes down to practice.  Try changing the temperature setting, the repeat_penalty setting, and mix up the model prompts.  You can even set up your weaviate client - `weaviate_interface` should already be available if you ran the download code at the top of the notebook - to start practicing setting up your RAG system.  I've created a simple function for you below that allows you to generate query - context pairs as an example of something you can do with the model, that in this case, allows you to create these pairs for embedding model fine-tuning."
      ],
      "metadata": {
        "id": "ZuU15H9vX1SR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_query_context_pairs(llm: TextGenerationPipeline,\n",
        "                                 summary: str,\n",
        "                                 guest: str,\n",
        "                                 transcript: str,\n",
        "                                 prompt: str,\n",
        "                                 num_questions: int,\n",
        "                                 ) -> str:\n",
        "  prompt = prompt.format(summary=summary, guest=guest, transcript=transcript, num_questions_per_chunk=num_questions)\n",
        "  response = llm(prompt, return_full_text=False)\n",
        "  return response[0]['generated_text']"
      ],
      "metadata": {
        "id": "L0Uj1WSFwfni"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qa_prompt = '''\n",
        "<s>[INST] <<SYS>>\n",
        "\n",
        "You are an expert at creating high quality questions when given text from a show episode.\n",
        "\n",
        "<</SYS>>\n",
        "Impact Theory episode summary and episode guest are below:\n",
        "\n",
        "---------------------\n",
        "Summary: {summary}\n",
        "---------------------\n",
        "Guest: {guest}\n",
        "---------------------\n",
        "\n",
        "Given the Summary and Guest of the episode as context use the following randomly selected transcript section \\\n",
        "of the episode and not prior knowledge, generate questions that can be answered by the transcript section:\n",
        "\n",
        "---------------------\n",
        "Transcript: {transcript}\n",
        "---------------------\n",
        "\n",
        "Your task is to create {num_questions_per_chunk} questions that can only be answered given the previous context and transcript details.\\\n",
        "The questions should randomly start with How, Why, or What. Only respond with the question, do not answer the question or provide reasoning. \\\n",
        "Do not provide an intro to the questions, just respond with the generated questions. [/INST]\n",
        "'''"
      ],
      "metadata": {
        "id": "4Gn2_L-rCPvi"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response = generate_query_context_pairs(llama, summary, guest, text_chunks[0], prompt=qa_prompt, num_questions=2)"
      ],
      "metadata": {
        "id": "udPNE9nVVMdW"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for question in response.split('\\n\\n')[1:]:\n",
        "  print(question)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zry6SPd7W3yL",
        "outputId": "fb2f2d84-4e99-4240-832b-7aa065709224"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "What specific AI biases might reinforce existing societal inequalities and fragmentation, according to Ian Bremmer?\n",
            "Why does Ian Bremmer believe that having multiple AI systems with different value systems and beliefs may actually bring us back to fragmentation, rather than providing a shared narrative?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_V3D-zOUXf98"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}