feat: add cost logging.

cost_viz.ipynb

@@ -0,0 +1,117 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: openai-cost-logger==0.3.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (0.3.0)\n",
+      "Requirement already satisfied: openai in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai-cost-logger==0.3.0) (1.13.3)\n",
+      "Requirement already satisfied: pandas in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai-cost-logger==0.3.0) (2.0.3)\n",
+      "Requirement already satisfied: matplotlib in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai-cost-logger==0.3.0) (3.6.3)\n",
+      "Requirement already satisfied: contourpy>=1.0.1 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (1.2.1)\n",
+      "Requirement already satisfied: cycler>=0.10 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (0.11.0)\n",
+      "Requirement already satisfied: fonttools>=4.22.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (4.51.0)\n",
+      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (1.4.4)\n",
+      "Requirement already satisfied: numpy>=1.19 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (1.25.2)\n",
+      "Requirement already satisfied: packaging>=20.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (23.0)\n",
+      "Requirement already satisfied: pillow>=6.2.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (8.4.0)\n",
+      "Requirement already satisfied: pyparsing>=2.2.1 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (3.0.6)\n",
+      "Requirement already satisfied: python-dateutil>=2.7 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from matplotlib->openai-cost-logger==0.3.0) (2.8.2)\n",
+      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (4.3.0)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (1.9.0)\n",
+      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (0.27.0)\n",
+      "Requirement already satisfied: pydantic<3,>=1.9.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (2.7.1)\n",
+      "Requirement already satisfied: sniffio in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (1.3.0)\n",
+      "Requirement already satisfied: tqdm>4 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (4.62.3)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from openai->openai-cost-logger==0.3.0) (4.11.0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from pandas->openai-cost-logger==0.3.0) (2021.3)\n",
+      "Requirement already satisfied: tzdata>=2022.1 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from pandas->openai-cost-logger==0.3.0) (2024.1)\n",
+      "Requirement already satisfied: idna>=2.8 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from anyio<5,>=3.5.0->openai->openai-cost-logger==0.3.0) (3.3)\n",
+      "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from anyio<5,>=3.5.0->openai->openai-cost-logger==0.3.0) (1.1.0)\n",
+      "Requirement already satisfied: certifi in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from httpx<1,>=0.23.0->openai->openai-cost-logger==0.3.0) (2021.10.8)\n",
+      "Requirement already satisfied: httpcore==1.* in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from httpx<1,>=0.23.0->openai->openai-cost-logger==0.3.0) (1.0.5)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai->openai-cost-logger==0.3.0) (0.14.0)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from pydantic<3,>=1.9.0->openai->openai-cost-logger==0.3.0) (0.6.0)\n",
+      "Requirement already satisfied: pydantic-core==2.18.2 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from pydantic<3,>=1.9.0->openai->openai-cost-logger==0.3.0) (2.18.2)\n",
+      "Requirement already satisfied: six>=1.5 in /home/drudao/anaconda3/envs/simulacra/lib/python3.9/site-packages (from python-dateutil>=2.7->matplotlib->openai-cost-logger==0.3.0) (1.16.0)\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%pip install openai-cost-logger==0.3.0"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai_cost_logger import DEFAULT_LOG_PATH\n",
+    "from openai_cost_logger import OpenAICostLoggerViz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "LOG_PATH = DEFAULT_LOG_PATH"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "OpenAICostLoggerViz.print_total_cost(path=LOG_PATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "simulacra",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

reverie/backend_server/persona/prompt_template/gpt_structure.py

@@ -5,13 +5,38 @@
 Description: Wrapper functions for calling OpenAI APIs.
 """
 import json
-import random
 import openai
 import time 
 
 from utils import *
 
+from openai_cost_logger import OpenAICostLogger
+from openai_cost_logger import DEFAULT_LOG_PATH
+import threading
+
 openai.api_key = openai_api_key
+exp_name = "simulacra-test-1"
+cost_upperbound = 10
+
+cost_logger = OpenAICostLogger(
+    experiment_name = exp_name,
+    log_folder = DEFAULT_LOG_PATH,
+    cost_upperbound = cost_upperbound
+)
+lock = threading.Lock()  # Lock to ensure thread safety when updating the cost logger.
+
+def update_cost_logger(response: dict, input_cost: float, output_cost: float = 0) -> None:
+  """
+  Update the cost logger with the cost of the response.
+
+  Args:
+    response (dict): The response from the OpenAI API.
+    input_cost (float): The cost of the input to the API.
+    output_cost (float. Optional): The cost of the output from the API. Defaults to 0.
+  """ 
+  # Need to acquire the lock since the cost_logger is shared among threads.
+  with lock:
+    cost_logger.update_cost(response=response, input_cost=input_cost, output_cost=output_cost)
 
 def temp_sleep(seconds=0.1):
   time.sleep(seconds)
@@ -23,9 +48,9 @@ def ChatGPT_single_request(prompt):
     model="gpt-3.5-turbo", 
     messages=[{"role": "user", "content": prompt}]
   )
+  update_cost_logger(completion, input_cost=0.5, output_cost=1.5)
   return completion["choices"][0]["message"]["content"]
 
-
 # ============================================================================
 # #####################[SECTION 1: CHATGPT-3 STRUCTURE] ######################
 # ============================================================================
@@ -49,6 +74,7 @@ def GPT4_request(prompt):
     model="gpt-4", 
     messages=[{"role": "user", "content": prompt}]
     )
+    update_cost_logger(completion, input_cost=30.0, output_cost=60.0)
     return completion["choices"][0]["message"]["content"]
 
   except: 
@@ -74,6 +100,7 @@ def ChatGPT_request(prompt):
     model="gpt-3.5-turbo", 
     messages=[{"role": "user", "content": prompt}]
     )
+    update_cost_logger(completion, input_cost=0.5, output_cost=1.5)
     return completion["choices"][0]["message"]["content"]
 
   except: 
@@ -145,10 +172,6 @@ def ChatGPT_safe_generate_response(prompt,
       end_index = curr_gpt_response.rfind('}') + 1
       curr_gpt_response = curr_gpt_response[:end_index]
       curr_gpt_response = json.loads(curr_gpt_response)["output"]
-
-      # print ("---ashdfaf")
-      # print (curr_gpt_response)
-      # print ("000asdfhia")
 
       if func_validate(curr_gpt_response, prompt=prompt): 
         return func_clean_up(curr_gpt_response, prompt=prompt)
@@ -218,6 +241,7 @@ def GPT_request(prompt, gpt_parameter):
                 presence_penalty=gpt_parameter["presence_penalty"],
                 stream=gpt_parameter["stream"],
                 stop=gpt_parameter["stop"],)
+    update_cost_logger(response=response, input_cost=0.5, output_cost=1.5)
     return response.choices[0].text
   except: 
     print ("TOKEN LIMIT EXCEEDED")
@@ -277,8 +301,9 @@ def get_embedding(text, model="text-embedding-ada-002"):
   text = text.replace("\n", " ")
   if not text: 
     text = "this is blank"
-  return openai.Embedding.create(
-          input=[text], model=model)['data'][0]['embedding']
+  response = openai.Embedding.create(input=[text], model=model)
+  update_cost_logger(response=response, input_cost=0.1, output_cost=0.0)
+  return response['data'][0]['embedding']
 
 
 if __name__ == '__main__':
@@ -327,5 +352,3 @@ def __func_clean_up(gpt_response):
 
 
 
-
-