openhermes-functions-with-vllm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## install vllm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! git clone https://github.com/Tostino/vllm.git\n",
    "! cd vllm\n",
    "! pip install -e ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Starting services"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! python -m vllm.entrypoints.openai.api_server --model OpenHermes-2.5-Mistral-7B/ --max-model-len 32768 --chat-template ./examples/template_chatml.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests  # \n",
    "\n",
    "# function for generating API requests\n",
    "def generate_api_request(query, functions=[]):\n",
    "    func_string = \"\\n\\n\".join([json.dumps(fn) for fn in functions])\n",
    "    system_message = {\n",
    "        \"role\": \"system\",\n",
    "        \"content\": f\"\"\"You are a helpful assistant with access to the following functions:\n",
    "\n",
    "{func_string}\n",
    "\n",
    "To use these functions respond with:\n",
    "<functioncall> {{\"name\": \"function_name\", \"arguments\": {{\"arg_1\": \"value_1\", \"arg_2\": \"value_2\", ...}}}} </functioncall>\n",
    "\n",
    "Edge cases you must handle:\n",
    "- If there are no functions that match the user request, you will respond politely that you cannot help.\"\"\"\n",
    "    }\n",
    "    user_message = {\"role\": \"user\", \"content\": query}\n",
    "\n",
    "    return {\n",
    "        \"model\": \"gpt-3.5-turbo-16k-0613\",\n",
    "        \"messages\": [system_message, user_message],\n",
    "        \"temperature\": 0.7,\n",
    "        \"max_tokens\": 150\n",
    "    }\n",
    "\n",
    "# function to send an API request and get the response\n",
    "def send_request(request_payload):\n",
    "    response = requests.post('localhost:8000/v1/chat/completions', json=request_payload)\n",
    "    return response.json()\n",
    "\n",
    "# \n",
    "functions = [\n",
    "    {\n",
    "        \"name\": \"call_uber\",\n",
    "        \"description\": \"Find suitable ride for customers given the location, type of ride, and the amount of time the customer is willing to wait as parameters\",\n",
    "        \"parameters\":  [\n",
    "            {\"name\": \"loc\", \"type\": \"string\", \"description\": \"location of the starting place of the uber ride\"},\n",
    "            {\"name\":\"type\", \"type\": \"string\", \"enum\": [\"plus\", \"comfort\", \"black\"], \"description\": \"types of uber ride user is ordering\"},\n",
    "            {\"name\": \"time\", \"type\": \"number\", \"description\": \"the amount of time in minutes the customer is willing to wait\"}\n",
    "            ]\n",
    "    },\n",
    "    {\n",
    "        \"name\": \"get_current_weather\",\n",
    "        \"description\": \"Gets the current weather for a given location\",\n",
    "        \"parameters\": {\n",
    "            \"type\": \"object\",\n",
    "            \"properties\": {\n",
    "                \"location\": {\n",
    "                    \"type\": \"string\",\n",
    "                    \"description\": \"The city and state, e.g. San Francisco, CA\",\n",
    "                },\n",
    "                \"format\": {\n",
    "                    \"type\": \"string\",\n",
    "                    \"enum\": [\"celsius\", \"fahrenheit\"],\n",
    "                    \"description\": \"The temperature unit to use. Infer this from the users location.\",\n",
    "                },\n",
    "            },\n",
    "            \"required\": [\"location\", \"format\"],\n",
    "        },\n",
    "    },\n",
    "]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 'cmpl-672fbcb175144b41992236d8d0e9dd1c', 'object': 'chat.completion', 'created': 19974236, 'model': 'gpt-3.5-turbo-16k-0613', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': '<functioncall>\\n  {\\n    \"name\": \"get_current_weather\",\\n    \"arguments\": {\\n      \"location\": \"New York, NY\",\\n      \"format\": \"celsius\"\\n    }\\n  }\\n</functioncall>'}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 374, 'total_tokens': 433, 'completion_tokens': 59}}\n"
     ]
    }
   ],
   "source": [
    "query = \"What is the weather in New York?\"\n",
    "request_payload = generate_api_request(query, functions)\n",
    "response = send_request(request_payload)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 'cmpl-b0a7fc061d3342ea887472aef9192d5e', 'object': 'chat.completion', 'created': 19975695, 'model': 'gpt-3.5-turbo-16k-0613', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': '<functioncall> {\"name\": \"call_uber\", \"arguments\": {\"loc\": \"Berkeley, 94704\", \"type\": \"plus\", \"time\": 10}} </functioncall>'}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 388, 'total_tokens': 435, 'completion_tokens': 47}}\n",
      "{'id': 'cmpl-860658086e8b49fc80d943fbb0752887', 'object': 'chat.completion', 'created': 19975696, 'model': 'gpt-3.5-turbo-16k-0613', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': '<functioncall>\\n  {\\n    \"name\": \"call_uber\",\\n    \"arguments\": {\\n      \"loc\": \"Berkeley 94704\",\\n      \"type\": \"plus\",\\n      \"time\": 10\\n    }\\n  }\\n</functioncall>'}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 388, 'total_tokens': 454, 'completion_tokens': 66}}\n",
      "{'id': 'cmpl-ca2ae780ce41455db7e9d910e53c7b4f', 'object': 'chat.completion', 'created': 19975697, 'model': 'gpt-3.5-turbo-16k-0613', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': '<functioncall>\\n  {\"name\": \"call_uber\",\\n  \"arguments\": {\\n    \"loc\": \"Berkeley, CA 94704\",\\n    \"type\": \"plus\",\\n    \"time\": 10\\n  }}\\n</functioncall>'}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 388, 'total_tokens': 450, 'completion_tokens': 62}}\n"
     ]
    }
   ],
   "source": [
    "query = \"Call me an plus Uber in Berkeley at zipcode 94704 in 10 minutes\"\n",
    "# call_uber first function in list\n",
    "prompt = generate_api_request(query,functions=functions)\n",
    "response = send_request(prompt)\n",
    "print(response)\n",
    "# call_uber last function in list\n",
    "prompt = generate_api_request(query,functions=functions)\n",
    "response = send_request(prompt)\n",
    "print(response)\n",
    "# Only call_uber in list\n",
    "prompt = generate_api_request(query,functions=functions)\n",
    "response = send_request(prompt)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}