Skip to content

Commit

Permalink
Remove OpenAI API key requirement from custom evals (#665)
Browse files Browse the repository at this point in the history 
* Remvoe OpenAI API key requirement from custom evals

* Fix import in docs

* Reorgnize imports
  • Loading branch information
@Dominastorm
Dominastorm authored Mar 22, 2024
1 parent f249582 commit fe72823
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 60 deletions.
24 changes: 5 additions & 19 deletions docs/predefined-evaluations/custom-evals/custom-eval.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ We will use UpTrain to check for these custom evaluations over the following cas
First, let's import the required dependencies

```python
from uptrain.framework.base import Settings
from uptrain.operators.base import TransformOp, register_custom_op, TYPE_TABLE_OUTPUT
from uptrain import EvalLLM, Settings
from uptrain.operators.base import ColumnOp, register_custom_op, TYPE_TABLE_OUTPUT
import polars as pl
```

Expand Down Expand Up @@ -95,21 +95,12 @@ data = [
]
```

First, let's import the required dependencies


```python
from uptrain import EvalLLM, Settings
import os
```

All done! Now let's run these evaluations


```python
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")

eval_llm = EvalLLM(openai_api_key=OPENAI_API_KEY)
eval_llm = EvalLLM(Settings())

results = eval_llm.evaluate(
project_name = 'UpTrain',
Expand Down Expand Up @@ -146,14 +137,9 @@ We have already defined some prebuilt evaluations that you can use without the h


```python
from uptrain.operators.language.text import WordCount
from uptrain.operators.language.text import KeywordDetector
from uptrain import EvalLLM

OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")

from uptrain.operators.language.text import WordCount, KeywordDetector

eval_llm = EvalLLM(openai_api_key=OPENAI_API_KEY)
eval_llm = EvalLLM(Settings())

results = eval_llm.evaluate(
project_name = 'UpTrain',
Expand Down
111 changes: 70 additions & 41 deletions examples/checks/custom/custom_evals.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"#!pip install uptrain"
"%pip install uptrain"
]
},
{
Expand All @@ -75,13 +75,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from uptrain.framework.base import Settings\n",
"from uptrain import EvalLLM, Settings\n",
"from uptrain.operators.base import ColumnOp, register_custom_op, TYPE_TABLE_OUTPUT\n",
"import polars as pl"
]
Expand All @@ -102,7 +102,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -129,7 +129,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -162,7 +162,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -182,23 +182,6 @@
"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First, let's import the required dependencies"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from uptrain import EvalLLM\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -208,13 +191,11 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\n",
"\n",
"eval_llm = EvalLLM(openai_api_key=OPENAI_API_KEY)\n",
"eval_llm = EvalLLM(Settings())\n",
"\n",
"results = eval_llm.evaluate(\n",
" project_name = 'UpTrain',\n",
Expand All @@ -235,9 +216,31 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"[{'question': 'What are the primary components of a cell?',\n",
" 'response': 'A cell comprises a cell membrane, cytoplasm, and nucleus. The cell membrane regulates substance passage, the cytoplasm contains organelles, and the nucleus houses genetic material.',\n",
" 'score_diverse_vocabulary': 0.84,\n",
" 'score_average_word_length': 6.24},\n",
" {'question': 'How does photosynthesis work?',\n",
" 'response': 'Photosynthesis converts light energy into chemical energy in plants, algae, and some bacteria. Chlorophyll absorbs sunlight, synthesizing glucose from carbon dioxide and water, with oxygen released as a byproduct.',\n",
" 'score_diverse_vocabulary': 0.93,\n",
" 'score_average_word_length': 6.38},\n",
" {'question': 'What are the key features of the Python programming language?',\n",
" 'response': 'Python is a high-level, interpreted language known for readability. It supports object-oriented, imperative, and functional programming with a large standard library, dynamic typing, and automatic memory management.',\n",
" 'score_diverse_vocabulary': 0.93,\n",
" 'score_average_word_length': 7.0}]"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"results"
]
Expand Down Expand Up @@ -290,18 +293,22 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2024-03-21 16:36:59.739\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36muptrain.framework.base\u001b[0m:\u001b[36mrun\u001b[0m:\u001b[36m217\u001b[0m - \u001b[34m\u001b[1mExecuting node: operator_0 for operator DAG: dummy\u001b[0m\n",
"\u001b[32m2024-03-21 16:36:59.740\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36muptrain.framework.base\u001b[0m:\u001b[36mrun\u001b[0m:\u001b[36m217\u001b[0m - \u001b[34m\u001b[1mExecuting node: operator_0 for operator DAG: dummy\u001b[0m\n"
]
}
],
"source": [
"from uptrain.operators.language.text import WordCount\n",
"from uptrain.operators.language.text import KeywordDetector\n",
"from uptrain import EvalLLM\n",
"\n",
"OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")\n",
"\n",
"from uptrain.operators.language.text import WordCount, KeywordDetector\n",
"\n",
"eval_llm = EvalLLM(openai_api_key=OPENAI_API_KEY)\n",
"eval_llm = EvalLLM(Settings())\n",
"\n",
"results = eval_llm.evaluate(\n",
" project_name = 'UpTrain',\n",
Expand All @@ -315,9 +322,31 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"[{'question': 'What are the primary components of a cell?',\n",
" 'response': 'A cell comprises a cell membrane, cytoplasm, and nucleus. The cell membrane regulates substance passage, the cytoplasm contains organelles, and the nucleus houses genetic material.',\n",
" 'word_count': 25,\n",
" 'keyword_detector': False},\n",
" {'question': 'How does photosynthesis work?',\n",
" 'response': 'Photosynthesis converts light energy into chemical energy in plants, algae, and some bacteria. Chlorophyll absorbs sunlight, synthesizing glucose from carbon dioxide and water, with oxygen released as a byproduct.',\n",
" 'word_count': 29,\n",
" 'keyword_detector': False},\n",
" {'question': 'What are the key features of the Python programming language?',\n",
" 'response': 'Python is a high-level, interpreted language known for readability. It supports object-oriented, imperative, and functional programming with a large standard library, dynamic typing, and automatic memory management.',\n",
" 'word_count': 27,\n",
" 'keyword_detector': True}]"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"results"
]
Expand Down

0 comments on commit fe72823

Please sign in to comment.