assess_ethics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0427d40c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 1000\n",
      " running accuracy=1.0\n",
      " running accuracy=0.5\n",
      " running accuracy=0.6666666666666666\n",
      " running accuracy=0.5\n",
      " running accuracy=0.6\n",
      "5 / 1000\n",
      " running accuracy=0.6666666666666666\n",
      " running accuracy=0.7142857142857143\n",
      " running accuracy=0.625\n",
      " running accuracy=0.6666666666666666\n",
      " running accuracy=0.7\n",
      "10 / 1000\n",
      " running accuracy=0.6363636363636364\n",
      " running accuracy=0.5833333333333334\n",
      " running accuracy=0.6153846153846154\n",
      " running accuracy=0.6428571428571429\n",
      " running accuracy=0.6\n",
      "15 / 1000\n",
      " running accuracy=0.5625\n",
      " running accuracy=0.5882352941176471\n",
      " running accuracy=0.6111111111111112\n",
      " running accuracy=0.5789473684210527\n",
      " running accuracy=0.55\n",
      "20 / 1000\n",
      " running accuracy=0.5714285714285714\n",
      " running accuracy=0.5909090909090909\n",
      " running accuracy=0.5652173913043478\n",
      " running accuracy=0.5416666666666666\n",
      " running accuracy=0.56\n",
      "25 / 1000\n",
      " running accuracy=0.5769230769230769\n",
      " running accuracy=0.5555555555555556\n",
      " running accuracy=0.5357142857142857\n",
      " running accuracy=0.5517241379310345\n",
      " running accuracy=0.5333333333333333\n",
      "30 / 1000\n",
      " running accuracy=0.5483870967741935\n",
      " running accuracy=0.53125\n"
     ]
    },
    {
     "ename": "CancelledError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mSSLWantReadError\u001b[0m                          Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/anyio/streams/tls.py:108\u001b[0m, in \u001b[0;36mTLSStream._call_sslobject_method\u001b[0;34m(self, func, *args)\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 108\u001b[0m     result \u001b[39m=\u001b[39m func(\u001b[39m*\u001b[39;49margs)\n\u001b[1;32m    109\u001b[0m \u001b[39mexcept\u001b[39;00m ssl\u001b[39m.\u001b[39mSSLWantReadError:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/ssl.py:889\u001b[0m, in \u001b[0;36mSSLObject.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m    888\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 889\u001b[0m     v \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sslobj\u001b[39m.\u001b[39;49mread(\u001b[39mlen\u001b[39;49m)\n\u001b[1;32m    890\u001b[0m \u001b[39mreturn\u001b[39;00m v\n",
      "\u001b[0;31mSSLWantReadError\u001b[0m: The operation did not complete (read) (_ssl.c:2633)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mCancelledError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 108\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[39mfor\u001b[39;00m task \u001b[39min\u001b[39;00m benchmark_files:\n\u001b[1;32m    106\u001b[0m     \u001b[39m# try:\u001b[39;00m\n\u001b[1;32m    107\u001b[0m     model \u001b[39m=\u001b[39m RecipeModel(agent_name\u001b[39m=\u001b[39magent)\n\u001b[0;32m--> 108\u001b[0m     accuracy, answers \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m evaluate_task(task, model, agent, \u001b[39m1000\u001b[39m, amplification\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m    109\u001b[0m     accuracy, answers \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m evaluate_task(task, model, agent, \u001b[39m1000\u001b[39m, amplification\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n",
      "Cell \u001b[0;32mIn[1], line 57\u001b[0m, in \u001b[0;36mevaluate_task\u001b[0;34m(task_name, model, agent_name, task_count, amplification)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[39mif\u001b[39;00m amplification:\n\u001b[1;32m     56\u001b[0m     prompt \u001b[39m=\u001b[39m make_deontology_prompt(row[\u001b[39m'\u001b[39m\u001b[39mscenario\u001b[39m\u001b[39m'\u001b[39m], row[\u001b[39m'\u001b[39m\u001b[39mexcuse\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[0;32m---> 57\u001b[0m     model_rough_answer \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m answer_by_amplification(prompt, agent_name)\n\u001b[1;32m     58\u001b[0m \u001b[39melse\u001b[39;00m: \n\u001b[1;32m     59\u001b[0m     prompt \u001b[39m=\u001b[39m make_deontology_prompt(row[\u001b[39m'\u001b[39m\u001b[39mscenario\u001b[39m\u001b[39m'\u001b[39m], row[\u001b[39m'\u001b[39m\u001b[39mexcuse\u001b[39m\u001b[39m'\u001b[39m])\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/amplification.py:90\u001b[0m, in \u001b[0;36manswer_by_amplification\u001b[0;34m(prompt, engine)\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39manswer_by_amplification\u001b[39m(\n\u001b[1;32m     88\u001b[0m     prompt: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mIs it ethical to clone humans?\u001b[39m\u001b[39m\"\u001b[39m, engine: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mchatgpt\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     89\u001b[0m ):\n\u001b[0;32m---> 90\u001b[0m     subs \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m get_subs(prompt)\n\u001b[1;32m     91\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m answer(prompt, subs\u001b[39m=\u001b[39msubs, engine\u001b[39m=\u001b[39mengine)\n\u001b[1;32m     92\u001b[0m     subs \u001b[39m=\u001b[39m [(q, a) \u001b[39mfor\u001b[39;00m q, a \u001b[39min\u001b[39;00m subs]\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/amplification.py:67\u001b[0m, in \u001b[0;36mget_subs\u001b[0;34m(question)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mget_subs\u001b[39m(\n\u001b[1;32m     65\u001b[0m     question: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mWhat is the effect of creatine on cognition?\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m     66\u001b[0m ):\n\u001b[0;32m---> 67\u001b[0m     subquestions \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m ask_subquestions(question\u001b[39m=\u001b[39mquestion)\n\u001b[1;32m     68\u001b[0m     subs_answer \u001b[39m=\u001b[39m partial(sub_answer, question\u001b[39m=\u001b[39mquestion)\n\u001b[1;32m     69\u001b[0m     subanswers \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m map_async(subquestions, subs_answer)\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/ice/recipes/primer/subquestions.py:20\u001b[0m, in \u001b[0;36mask_subquestions\u001b[0;34m(question)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mask_subquestions\u001b[39m(\n\u001b[1;32m     17\u001b[0m     question: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mWhat is the effect of creatine on cognition?\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m     18\u001b[0m ):\n\u001b[1;32m     19\u001b[0m     prompt \u001b[39m=\u001b[39m make_subquestion_prompt(question)\n\u001b[0;32m---> 20\u001b[0m     subquestions_text \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m recipe\u001b[39m.\u001b[39magent()\u001b[39m.\u001b[39mcomplete(prompt\u001b[39m=\u001b[39mprompt)\n\u001b[1;32m     21\u001b[0m     subquestions \u001b[39m=\u001b[39m [line\u001b[39m.\u001b[39mstrip(\u001b[39m\"\u001b[39m\u001b[39m- \u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m subquestions_text\u001b[39m.\u001b[39msplit(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m)]\n\u001b[1;32m     22\u001b[0m     \u001b[39mreturn\u001b[39;00m subquestions\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/ice/agents/openai.py:45\u001b[0m, in \u001b[0;36mOpenAIAgent.complete\u001b[0;34m(self, prompt, stop, verbose, default, max_tokens)\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[39mif\u001b[39;00m verbose:\n\u001b[1;32m     44\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_print_markdown(prompt)\n\u001b[0;32m---> 45\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_complete(prompt, stop\u001b[39m=\u001b[39mstop, max_tokens\u001b[39m=\u001b[39mmax_tokens)\n\u001b[1;32m     46\u001b[0m completion \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_extract_completion(response)\n\u001b[1;32m     47\u001b[0m \u001b[39mif\u001b[39;00m verbose:\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/ice/agents/openai.py:102\u001b[0m, in \u001b[0;36mOpenAIAgent._complete\u001b[0;34m(self, prompt, **kwargs)\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Send a completion request to the OpenAI API with the given prompt and parameters.\"\"\"\u001b[39;00m\n\u001b[1;32m     94\u001b[0m kwargs\u001b[39m.\u001b[39mupdate(\n\u001b[1;32m     95\u001b[0m     {\n\u001b[1;32m     96\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mmodel\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    100\u001b[0m     }\n\u001b[1;32m    101\u001b[0m )\n\u001b[0;32m--> 102\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m openai_complete(prompt, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    103\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mchoices\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m response:\n\u001b[1;32m    104\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mNo choices in response: \u001b[39m\u001b[39m{\u001b[39;00mresponse\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/werk/ice/ice/trace.py:229\u001b[0m, in \u001b[0;36mtrace.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    227\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    228\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m trace_enabled():\n\u001b[0;32m--> 229\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    231\u001b[0m     \u001b[39m@wraps\u001b[39m(fn)\n\u001b[1;32m    232\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39minner_wrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    233\u001b[0m         \u001b[39mid\u001b[39m \u001b[39m=\u001b[39m make_id()\n",
      "File \u001b[0;32m~/werk/ice/ice/apis/openai.py:166\u001b[0m, in \u001b[0;36mopenai_complete\u001b[0;34m(prompt, stop, top_p, temperature, model, max_tokens, logprobs, logit_bias, n, echo, cache_id)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[39mif\u001b[39;00m logit_bias:\n\u001b[1;32m    165\u001b[0m     params[\u001b[39m\"\u001b[39m\u001b[39mlogit_bias\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m logit_bias  \u001b[39m# type: ignore[assignment]\u001b[39;00m\n\u001b[0;32m--> 166\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m _post(\u001b[39m\"\u001b[39m\u001b[39mcompletions\u001b[39m\u001b[39m\"\u001b[39m, json\u001b[39m=\u001b[39mparams, cache_id\u001b[39m=\u001b[39mcache_id)\n\u001b[1;32m    167\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(response, TooLongRequestError):\n\u001b[1;32m    168\u001b[0m     \u001b[39mraise\u001b[39;00m response\n",
      "File \u001b[0;32m~/werk/ice/ice/cache.py:41\u001b[0m, in \u001b[0;36mdiskcache.<locals>.decorator.<locals>.async_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     38\u001b[0m cache\u001b[39m.\u001b[39mclose()\n\u001b[1;32m     39\u001b[0m \u001b[39m# close connection while waiting for async function so we don't have a bunch of connections open simultaneously\u001b[39;00m\n\u001b[0;32m---> 41\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m     43\u001b[0m cache \u001b[39m=\u001b[39m get_cache(func)\n\u001b[1;32m     44\u001b[0m cache[key] \u001b[39m=\u001b[39m result\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/tenacity/_asyncio.py:88\u001b[0m, in \u001b[0;36mAsyncRetrying.wraps.<locals>.async_wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[39m@functools\u001b[39m\u001b[39m.\u001b[39mwraps(fn)\n\u001b[1;32m     87\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39masync_wrapped\u001b[39m(\u001b[39m*\u001b[39margs: t\u001b[39m.\u001b[39mAny, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: t\u001b[39m.\u001b[39mAny) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m t\u001b[39m.\u001b[39mAny:\n\u001b[0;32m---> 88\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/tenacity/_asyncio.py:47\u001b[0m, in \u001b[0;36mAsyncRetrying.__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m     45\u001b[0m retry_state \u001b[39m=\u001b[39m RetryCallState(retry_object\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m, fn\u001b[39m=\u001b[39mfn, args\u001b[39m=\u001b[39margs, kwargs\u001b[39m=\u001b[39mkwargs)\n\u001b[1;32m     46\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m---> 47\u001b[0m     do \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49miter(retry_state\u001b[39m=\u001b[39;49mretry_state)\n\u001b[1;32m     48\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(do, DoAttempt):\n\u001b[1;32m     49\u001b[0m         \u001b[39mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/tenacity/__init__.py:314\u001b[0m, in \u001b[0;36mBaseRetrying.iter\u001b[0;34m(self, retry_state)\u001b[0m\n\u001b[1;32m    312\u001b[0m is_explicit_retry \u001b[39m=\u001b[39m fut\u001b[39m.\u001b[39mfailed \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(fut\u001b[39m.\u001b[39mexception(), TryAgain)\n\u001b[1;32m    313\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (is_explicit_retry \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mretry(retry_state)):\n\u001b[0;32m--> 314\u001b[0m     \u001b[39mreturn\u001b[39;00m fut\u001b[39m.\u001b[39;49mresult()\n\u001b[1;32m    316\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mafter \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    317\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mafter(retry_state)\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/concurrent/futures/_base.py:439\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    437\u001b[0m     \u001b[39mraise\u001b[39;00m CancelledError()\n\u001b[1;32m    438\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_state \u001b[39m==\u001b[39m FINISHED:\n\u001b[0;32m--> 439\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__get_result()\n\u001b[1;32m    441\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_condition\u001b[39m.\u001b[39mwait(timeout)\n\u001b[1;32m    443\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_state \u001b[39min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/concurrent/futures/_base.py:391\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    389\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception:\n\u001b[1;32m    390\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 391\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception\n\u001b[1;32m    392\u001b[0m     \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m    393\u001b[0m         \u001b[39m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m    394\u001b[0m         \u001b[39mself\u001b[39m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/tenacity/_asyncio.py:50\u001b[0m, in \u001b[0;36mAsyncRetrying.__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(do, DoAttempt):\n\u001b[1;32m     49\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 50\u001b[0m         result \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m fn(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m     51\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m:  \u001b[39m# noqa: B902\u001b[39;00m\n\u001b[1;32m     52\u001b[0m         retry_state\u001b[39m.\u001b[39mset_exception(sys\u001b[39m.\u001b[39mexc_info())  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n",
      "File \u001b[0;32m~/werk/ice/ice/apis/openai.py:114\u001b[0m, in \u001b[0;36m_post\u001b[0;34m(endpoint, json, timeout, cache_id)\u001b[0m\n\u001b[1;32m    111\u001b[0m cache_id  \u001b[39m# unused\u001b[39;00m\n\u001b[1;32m    113\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mwith\u001b[39;00m httpx\u001b[39m.\u001b[39mAsyncClient() \u001b[39mas\u001b[39;00m client:\n\u001b[0;32m--> 114\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m client\u001b[39m.\u001b[39mpost(\n\u001b[1;32m    115\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mOPENAI_BASE_URL\u001b[39m}\u001b[39;00m\u001b[39m/\u001b[39m\u001b[39m{\u001b[39;00mendpoint\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m,\n\u001b[1;32m    116\u001b[0m         json\u001b[39m=\u001b[39mjson,\n\u001b[1;32m    117\u001b[0m         headers\u001b[39m=\u001b[39mmake_headers(),\n\u001b[1;32m    118\u001b[0m         timeout\u001b[39m=\u001b[39mtimeout \u001b[39mor\u001b[39;00m \u001b[39m60\u001b[39m,\n\u001b[1;32m    119\u001b[0m     )\n\u001b[1;32m    120\u001b[0m     \u001b[39mif\u001b[39;00m response\u001b[39m.\u001b[39mstatus_code \u001b[39m==\u001b[39m \u001b[39m429\u001b[39m:\n\u001b[1;32m    121\u001b[0m         \u001b[39mraise\u001b[39;00m RateLimitError(response)\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1848\u001b[0m, in \u001b[0;36mAsyncClient.post\u001b[0;34m(self, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[1;32m   1827\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mpost\u001b[39m(\n\u001b[1;32m   1828\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m   1829\u001b[0m     url: URLTypes,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1841\u001b[0m     extensions: typing\u001b[39m.\u001b[39mOptional[RequestExtensions] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m   1842\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Response:\n\u001b[1;32m   1843\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   1844\u001b[0m \u001b[39m    Send a `POST` request.\u001b[39;00m\n\u001b[1;32m   1845\u001b[0m \n\u001b[1;32m   1846\u001b[0m \u001b[39m    **Parameters**: See `httpx.request`.\u001b[39;00m\n\u001b[1;32m   1847\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1848\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrequest(\n\u001b[1;32m   1849\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mPOST\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m   1850\u001b[0m         url,\n\u001b[1;32m   1851\u001b[0m         content\u001b[39m=\u001b[39mcontent,\n\u001b[1;32m   1852\u001b[0m         data\u001b[39m=\u001b[39mdata,\n\u001b[1;32m   1853\u001b[0m         files\u001b[39m=\u001b[39mfiles,\n\u001b[1;32m   1854\u001b[0m         json\u001b[39m=\u001b[39mjson,\n\u001b[1;32m   1855\u001b[0m         params\u001b[39m=\u001b[39mparams,\n\u001b[1;32m   1856\u001b[0m         headers\u001b[39m=\u001b[39mheaders,\n\u001b[1;32m   1857\u001b[0m         cookies\u001b[39m=\u001b[39mcookies,\n\u001b[1;32m   1858\u001b[0m         auth\u001b[39m=\u001b[39mauth,\n\u001b[1;32m   1859\u001b[0m         follow_redirects\u001b[39m=\u001b[39mfollow_redirects,\n\u001b[1;32m   1860\u001b[0m         timeout\u001b[39m=\u001b[39mtimeout,\n\u001b[1;32m   1861\u001b[0m         extensions\u001b[39m=\u001b[39mextensions,\n\u001b[1;32m   1862\u001b[0m     )\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1530\u001b[0m, in \u001b[0;36mAsyncClient.request\u001b[0;34m(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[1;32m   1501\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   1502\u001b[0m \u001b[39mBuild and send a request.\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[39m[0]: /advanced/#merging-of-configuration\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuild_request(\n\u001b[1;32m   1518\u001b[0m     method\u001b[39m=\u001b[39mmethod,\n\u001b[1;32m   1519\u001b[0m     url\u001b[39m=\u001b[39murl,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1528\u001b[0m     extensions\u001b[39m=\u001b[39mextensions,\n\u001b[1;32m   1529\u001b[0m )\n\u001b[0;32m-> 1530\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msend(request, auth\u001b[39m=\u001b[39mauth, follow_redirects\u001b[39m=\u001b[39mfollow_redirects)\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1617\u001b[0m, in \u001b[0;36mAsyncClient.send\u001b[0;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[1;32m   1609\u001b[0m follow_redirects \u001b[39m=\u001b[39m (\n\u001b[1;32m   1610\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfollow_redirects\n\u001b[1;32m   1611\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(follow_redirects, UseClientDefault)\n\u001b[1;32m   1612\u001b[0m     \u001b[39melse\u001b[39;00m follow_redirects\n\u001b[1;32m   1613\u001b[0m )\n\u001b[1;32m   1615\u001b[0m auth \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_request_auth(request, auth)\n\u001b[0;32m-> 1617\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_send_handling_auth(\n\u001b[1;32m   1618\u001b[0m     request,\n\u001b[1;32m   1619\u001b[0m     auth\u001b[39m=\u001b[39mauth,\n\u001b[1;32m   1620\u001b[0m     follow_redirects\u001b[39m=\u001b[39mfollow_redirects,\n\u001b[1;32m   1621\u001b[0m     history\u001b[39m=\u001b[39m[],\n\u001b[1;32m   1622\u001b[0m )\n\u001b[1;32m   1623\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1624\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m stream:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1645\u001b[0m, in \u001b[0;36mAsyncClient._send_handling_auth\u001b[0;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[1;32m   1642\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m auth_flow\u001b[39m.\u001b[39m\u001b[39m__anext__\u001b[39m()\n\u001b[1;32m   1644\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m-> 1645\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_send_handling_redirects(\n\u001b[1;32m   1646\u001b[0m         request,\n\u001b[1;32m   1647\u001b[0m         follow_redirects\u001b[39m=\u001b[39mfollow_redirects,\n\u001b[1;32m   1648\u001b[0m         history\u001b[39m=\u001b[39mhistory,\n\u001b[1;32m   1649\u001b[0m     )\n\u001b[1;32m   1650\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1651\u001b[0m         \u001b[39mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1682\u001b[0m, in \u001b[0;36mAsyncClient._send_handling_redirects\u001b[0;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[1;32m   1679\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mrequest\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[1;32m   1680\u001b[0m     \u001b[39mawait\u001b[39;00m hook(request)\n\u001b[0;32m-> 1682\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_send_single_request(request)\n\u001b[1;32m   1683\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m   1684\u001b[0m     \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mresponse\u001b[39m\u001b[39m\"\u001b[39m]:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_client.py:1719\u001b[0m, in \u001b[0;36mAsyncClient._send_single_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m   1714\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[1;32m   1715\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mAttempted to send an sync request with an AsyncClient instance.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   1716\u001b[0m     )\n\u001b[1;32m   1718\u001b[0m \u001b[39mwith\u001b[39;00m request_context(request\u001b[39m=\u001b[39mrequest):\n\u001b[0;32m-> 1719\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m transport\u001b[39m.\u001b[39mhandle_async_request(request)\n\u001b[1;32m   1721\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(response\u001b[39m.\u001b[39mstream, AsyncByteStream)\n\u001b[1;32m   1722\u001b[0m response\u001b[39m.\u001b[39mrequest \u001b[39m=\u001b[39m request\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpx/_transports/default.py:353\u001b[0m, in \u001b[0;36mAsyncHTTPTransport.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    340\u001b[0m req \u001b[39m=\u001b[39m httpcore\u001b[39m.\u001b[39mRequest(\n\u001b[1;32m    341\u001b[0m     method\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mmethod,\n\u001b[1;32m    342\u001b[0m     url\u001b[39m=\u001b[39mhttpcore\u001b[39m.\u001b[39mURL(\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    350\u001b[0m     extensions\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mextensions,\n\u001b[1;32m    351\u001b[0m )\n\u001b[1;32m    352\u001b[0m \u001b[39mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m--> 353\u001b[0m     resp \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_pool\u001b[39m.\u001b[39mhandle_async_request(req)\n\u001b[1;32m    355\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(resp\u001b[39m.\u001b[39mstream, typing\u001b[39m.\u001b[39mAsyncIterable)\n\u001b[1;32m    357\u001b[0m \u001b[39mreturn\u001b[39;00m Response(\n\u001b[1;32m    358\u001b[0m     status_code\u001b[39m=\u001b[39mresp\u001b[39m.\u001b[39mstatus,\n\u001b[1;32m    359\u001b[0m     headers\u001b[39m=\u001b[39mresp\u001b[39m.\u001b[39mheaders,\n\u001b[1;32m    360\u001b[0m     stream\u001b[39m=\u001b[39mAsyncResponseStream(resp\u001b[39m.\u001b[39mstream),\n\u001b[1;32m    361\u001b[0m     extensions\u001b[39m=\u001b[39mresp\u001b[39m.\u001b[39mextensions,\n\u001b[1;32m    362\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/connection_pool.py:262\u001b[0m, in \u001b[0;36mAsyncConnectionPool.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    260\u001b[0m     \u001b[39mwith\u001b[39;00m AsyncShieldCancellation():\n\u001b[1;32m    261\u001b[0m         \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresponse_closed(status)\n\u001b[0;32m--> 262\u001b[0m     \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m    263\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    264\u001b[0m     \u001b[39mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/connection_pool.py:245\u001b[0m, in \u001b[0;36mAsyncConnectionPool.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    242\u001b[0m         \u001b[39mraise\u001b[39;00m exc\n\u001b[1;32m    244\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 245\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m connection\u001b[39m.\u001b[39mhandle_async_request(request)\n\u001b[1;32m    246\u001b[0m \u001b[39mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[1;32m    247\u001b[0m     \u001b[39m# The ConnectionNotAvailable exception is a special case, that\u001b[39;00m\n\u001b[1;32m    248\u001b[0m     \u001b[39m# indicates we need to retry the request on a new connection.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    252\u001b[0m     \u001b[39m# might end up as an HTTP/2 connection, but which actually ends\u001b[39;00m\n\u001b[1;32m    253\u001b[0m     \u001b[39m# up as HTTP/1.1.\u001b[39;00m\n\u001b[1;32m    254\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_pool_lock:\n\u001b[1;32m    255\u001b[0m         \u001b[39m# Maintain our position in the request queue, but reset the\u001b[39;00m\n\u001b[1;32m    256\u001b[0m         \u001b[39m# status so that the request becomes queued again.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/connection.py:96\u001b[0m, in \u001b[0;36mAsyncHTTPConnection.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[39melif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_connection\u001b[39m.\u001b[39mis_available():\n\u001b[1;32m     94\u001b[0m         \u001b[39mraise\u001b[39;00m ConnectionNotAvailable()\n\u001b[0;32m---> 96\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_connection\u001b[39m.\u001b[39mhandle_async_request(request)\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/http11.py:121\u001b[0m, in \u001b[0;36mAsyncHTTP11Connection.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[39masync\u001b[39;00m \u001b[39mwith\u001b[39;00m Trace(\u001b[39m\"\u001b[39m\u001b[39mresponse_closed\u001b[39m\u001b[39m\"\u001b[39m, logger, request) \u001b[39mas\u001b[39;00m trace:\n\u001b[1;32m    120\u001b[0m         \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_response_closed()\n\u001b[0;32m--> 121\u001b[0m \u001b[39mraise\u001b[39;00m exc\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/http11.py:99\u001b[0m, in \u001b[0;36mAsyncHTTP11Connection.handle_async_request\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_send_request_body(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m     91\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mwith\u001b[39;00m Trace(\n\u001b[1;32m     92\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mreceive_response_headers\u001b[39m\u001b[39m\"\u001b[39m, logger, request, kwargs\n\u001b[1;32m     93\u001b[0m ) \u001b[39mas\u001b[39;00m trace:\n\u001b[1;32m     94\u001b[0m     (\n\u001b[1;32m     95\u001b[0m         http_version,\n\u001b[1;32m     96\u001b[0m         status,\n\u001b[1;32m     97\u001b[0m         reason_phrase,\n\u001b[1;32m     98\u001b[0m         headers,\n\u001b[0;32m---> 99\u001b[0m     ) \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_receive_response_headers(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    100\u001b[0m     trace\u001b[39m.\u001b[39mreturn_value \u001b[39m=\u001b[39m (\n\u001b[1;32m    101\u001b[0m         http_version,\n\u001b[1;32m    102\u001b[0m         status,\n\u001b[1;32m    103\u001b[0m         reason_phrase,\n\u001b[1;32m    104\u001b[0m         headers,\n\u001b[1;32m    105\u001b[0m     )\n\u001b[1;32m    107\u001b[0m \u001b[39mreturn\u001b[39;00m Response(\n\u001b[1;32m    108\u001b[0m     status\u001b[39m=\u001b[39mstatus,\n\u001b[1;32m    109\u001b[0m     headers\u001b[39m=\u001b[39mheaders,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    115\u001b[0m     },\n\u001b[1;32m    116\u001b[0m )\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/http11.py:164\u001b[0m, in \u001b[0;36mAsyncHTTP11Connection._receive_response_headers\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m    161\u001b[0m timeout \u001b[39m=\u001b[39m timeouts\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mread\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m    163\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m--> 164\u001b[0m     event \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_receive_event(timeout\u001b[39m=\u001b[39mtimeout)\n\u001b[1;32m    165\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(event, h11\u001b[39m.\u001b[39mResponse):\n\u001b[1;32m    166\u001b[0m         \u001b[39mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_async/http11.py:200\u001b[0m, in \u001b[0;36mAsyncHTTP11Connection._receive_event\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    197\u001b[0m     event \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_h11_state\u001b[39m.\u001b[39mnext_event()\n\u001b[1;32m    199\u001b[0m \u001b[39mif\u001b[39;00m event \u001b[39mis\u001b[39;00m h11\u001b[39m.\u001b[39mNEED_DATA:\n\u001b[0;32m--> 200\u001b[0m     data \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_network_stream\u001b[39m.\u001b[39mread(\n\u001b[1;32m    201\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mREAD_NUM_BYTES, timeout\u001b[39m=\u001b[39mtimeout\n\u001b[1;32m    202\u001b[0m     )\n\u001b[1;32m    204\u001b[0m     \u001b[39m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[1;32m    205\u001b[0m     \u001b[39m#\u001b[39;00m\n\u001b[1;32m    206\u001b[0m     \u001b[39m#     httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    210\u001b[0m     \u001b[39m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[1;32m    211\u001b[0m     \u001b[39m# it as a ConnectError.\u001b[39;00m\n\u001b[1;32m    212\u001b[0m     \u001b[39mif\u001b[39;00m data \u001b[39m==\u001b[39m \u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_h11_state\u001b[39m.\u001b[39mtheir_state \u001b[39m==\u001b[39m h11\u001b[39m.\u001b[39mSEND_RESPONSE:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/httpcore/_backends/anyio.py:34\u001b[0m, in \u001b[0;36mAnyIOStream.read\u001b[0;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[39mwith\u001b[39;00m anyio\u001b[39m.\u001b[39mfail_after(timeout):\n\u001b[1;32m     33\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 34\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_stream\u001b[39m.\u001b[39mreceive(max_bytes\u001b[39m=\u001b[39mmax_bytes)\n\u001b[1;32m     35\u001b[0m     \u001b[39mexcept\u001b[39;00m anyio\u001b[39m.\u001b[39mEndOfStream:  \u001b[39m# pragma: nocover\u001b[39;00m\n\u001b[1;32m     36\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mb\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/anyio/streams/tls.py:171\u001b[0m, in \u001b[0;36mTLSStream.receive\u001b[0;34m(self, max_bytes)\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39mreceive\u001b[39m(\u001b[39mself\u001b[39m, max_bytes: \u001b[39mint\u001b[39m \u001b[39m=\u001b[39m \u001b[39m65536\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mbytes\u001b[39m:\n\u001b[0;32m--> 171\u001b[0m     data \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call_sslobject_method(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ssl_object\u001b[39m.\u001b[39mread, max_bytes)\n\u001b[1;32m    172\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m data:\n\u001b[1;32m    173\u001b[0m         \u001b[39mraise\u001b[39;00m EndOfStream\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/anyio/streams/tls.py:115\u001b[0m, in \u001b[0;36mTLSStream._call_sslobject_method\u001b[0;34m(self, func, *args)\u001b[0m\n\u001b[1;32m    112\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_write_bio\u001b[39m.\u001b[39mpending:\n\u001b[1;32m    113\u001b[0m         \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtransport_stream\u001b[39m.\u001b[39msend(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_write_bio\u001b[39m.\u001b[39mread())\n\u001b[0;32m--> 115\u001b[0m     data \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtransport_stream\u001b[39m.\u001b[39mreceive()\n\u001b[1;32m    116\u001b[0m \u001b[39mexcept\u001b[39;00m EndOfStream:\n\u001b[1;32m    117\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_read_bio\u001b[39m.\u001b[39mwrite_eof()\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/site-packages/anyio/_backends/_asyncio.py:1105\u001b[0m, in \u001b[0;36mSocketStream.receive\u001b[0;34m(self, max_bytes)\u001b[0m\n\u001b[1;32m   1103\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_protocol\u001b[39m.\u001b[39mread_event\u001b[39m.\u001b[39mis_set() \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_transport\u001b[39m.\u001b[39mis_closing():\n\u001b[1;32m   1104\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_transport\u001b[39m.\u001b[39mresume_reading()\n\u001b[0;32m-> 1105\u001b[0m     \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_protocol\u001b[39m.\u001b[39mread_event\u001b[39m.\u001b[39mwait()\n\u001b[1;32m   1106\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_transport\u001b[39m.\u001b[39mpause_reading()\n\u001b[1;32m   1108\u001b[0m \u001b[39mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/asyncio/locks.py:226\u001b[0m, in \u001b[0;36mEvent.wait\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    224\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_waiters\u001b[39m.\u001b[39mappend(fut)\n\u001b[1;32m    225\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 226\u001b[0m     \u001b[39mawait\u001b[39;00m fut\n\u001b[1;32m    227\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m    228\u001b[0m \u001b[39mfinally\u001b[39;00m:\n",
      "File \u001b[0;32m~/miniconda3/envs/ice_env/lib/python3.9/asyncio/futures.py:284\u001b[0m, in \u001b[0;36mFuture.__await__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    282\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdone():\n\u001b[1;32m    283\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_asyncio_future_blocking \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m--> 284\u001b[0m     \u001b[39myield\u001b[39;00m \u001b[39mself\u001b[39m  \u001b[39m# This tells Task to wait for completion.\u001b[39;00m\n\u001b[1;32m    285\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdone():\n\u001b[1;32m    286\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mawait wasn\u001b[39m\u001b[39m'\u001b[39m\u001b[39mt used with future\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[0;31mCancelledError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import json\n",
    "from fvalues import F\n",
    "from ice.recipe import recipe\n",
    "import pandas as pd\n",
    "from amplification import answer_by_amplification\n",
    "\n",
    "from prompt_generator import make_deontology_prompt , extract_prompt, make_justice_prompt\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class RecipeModel():\n",
    "    def __init__(self, agent_name):\n",
    "        self.agent_name = agent_name\n",
    "\n",
    "    def generate_text(self, inputs, max_length=250):\n",
    "        response = recipe.agent(agent_name=self.agent_name).complete(prompt=inputs, stop='\"', max_tokens=max_length)\n",
    "        return response\n",
    "\n",
    "\n",
    "\n",
    "async def evaluate_task(task_name, model, agent_name, task_count, amplification=False):\n",
    "    task= \"\"\n",
    "    if \"deontology\" in task_name:\n",
    "        task = \"deontology\"\n",
    "    elif \"justice\" in task_name:\n",
    "        task = \"justice\"\n",
    "    else:\n",
    "        raise Exception(\"Unsupported task\")\n",
    "        \n",
    "    task_path = os.path.join(f'/Users/bensturgeon/werk/ice/ethics/{task}/', task_name)\n",
    "    task_data = pd.read_csv(task_path)\n",
    "\n",
    "    correct_answers = 0\n",
    "    total_questions = 0\n",
    "    answers = []\n",
    "    result = {}\n",
    "\n",
    "    \n",
    "\n",
    "    for index, row in task_data.iterrows():\n",
    "        try:\n",
    "            if index % 5 == 0:\n",
    "                print(f\"{index} / {task_count}\")\n",
    "            if task == \"deontology\":\n",
    "                if amplification:\n",
    "                    prompt = make_deontology_prompt(row['scenario'], row['excuse'])\n",
    "                    model_rough_answer = await answer_by_amplification(prompt, agent_name)\n",
    "                else: \n",
    "                    prompt = make_deontology_prompt(row['scenario'], row['excuse'])\n",
    "                    model_rough_answer = await model.generate_text(prompt)\n",
    "            if task == \"justice\":\n",
    "                if amplification:\n",
    "                    prompt = make_justice_prompt(row['scenario'])\n",
    "                    model_rough_answer = await answer_by_amplification(prompt, agent_name)\n",
    "                else: \n",
    "                    prompt = make_justice_prompt(row['scenario'])\n",
    "                    model_rough_answer = await model.generate_text(prompt)\n",
    "\n",
    "\n",
    "            \n",
    "            extraction_prompt = extract_prompt(model_rough_answer)\n",
    "            model_extracted_answer = await model.generate_text(extraction_prompt)\n",
    "            correct_answer = row['label']\n",
    "\n",
    "            if str(model_extracted_answer) == str(correct_answer):\n",
    "                correct_answers += 1\n",
    "            total_questions += 1\n",
    "            answers.append( {'index': index +1,'input': row['scenario'], 'model_answer': str(model_extracted_answer), 'correct_answer': correct_answer})\n",
    "            accuracy = correct_answers / total_questions\n",
    "\n",
    "            result['answers'] = answers\n",
    "            result['running_accuracy'] = accuracy\n",
    "\n",
    "            print(f\" running {accuracy=}\")\n",
    "            task_name_no_extension = task_name.split(\".csv\")[0]\n",
    "\n",
    "            amplified = \"base\"\n",
    "            if amplification:\n",
    "                amplified = \"amplified\"\n",
    "            with open(f'benchmarks/results/{task_name_no_extension}_{agent_name}_{amplified}.json', 'w') as f:\n",
    "                json.dump(result, f,indent=4)\n",
    "            if index > task_count:\n",
    "                break\n",
    "        except Exception as e:\n",
    "            print(f\"There was an exception: {e}\")\n",
    "    del accuracy, answers\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    agents = [\"chatgpt\", \"gpt-4\"]\n",
    "    benchmark_files = ['deontology_test_hard.csv', 'justice_test_hard.csv']\n",
    "    for agent in agents:\n",
    "        for task in benchmark_files:\n",
    "            # try:\n",
    "            model = RecipeModel(agent_name=agent)\n",
    "            accuracy, answers = await evaluate_task(task, model, agent, 1000, amplification=True)\n",
    "            accuracy, answers = await evaluate_task(task, model, agent, 1000, amplification=False)\n",
    "            # except Exception as e:\n",
    "            #     print(\"There was an error: \", e)\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "91b71e87",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_answer = await model.generate_text(\"What is the sky?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eb89d421",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(model_answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89297aed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_questions_from_json(json_path: str) -> list[str]:\n",
    "    with open(json_path, 'r') as file:\n",
    "        data = json.load(file)\n",
    "        # Assume the questions are stored in a list under the key \"questions\"\n",
    "        return data['questions']\n",
    "\n",
    "async def answer_by_amplification(json_path: str):\n",
    "    questions = load_questions_from_json(json_path)\n",
    "    answers = []\n",
    "    for question in questions:\n",
    "        answer = await answer(question)\n",
    "        answers.append((question, answer))\n",
    "    return answers"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ice",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}