feat(groq): integrate native Groq SDK and update documentation (promptfoo#1479)

Author: mldangelo

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "editor.formatOnSave": true,
-  "cSpell.words": ["openai", "promptfoo", "promptfooconfig", "redteam"]
+  "cSpell.words": ["Groq", "openai", "promptfoo", "promptfooconfig", "redteam"]
 }

examples/groq/README.md

@@ -1,4 +1,4 @@
-To get started, set your GROQ_API_KEY environment variable.
+To get started, set your `GROQ_API_KEY` environment variable.
 
 Next, edit promptfooconfig.yaml.
 

examples/groq/promptfooconfig.yaml

@@ -1,19 +1,27 @@
 prompts:
-  - 'Answer this question as concisely as possible: {{question}}'
+  - 'Answer this question as concisely as possible: {{calculation}}'
+  - 'Calculate the following and provide a brief explanation: {{calculation}}'
 
 providers:
-  - id: openai:chat:mixtral-8x7b-32768
+  - id: groq:gemma2-9b-it
+  - id: groq:llama3-groq-8b-8192-tool-use-preview
     config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
-  - id: openai:chat:llama3-8b-8192
-    config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
+      tools:
+        - type: function
+          function:
+            name: calculate
+            description: 'Evaluate a mathematical expression'
+            parameters:
+              type: object
+              properties:
+                expression:
+                  type: string
+                  description: 'The mathematical expression to evaluate'
+              required: ['expression']
+      tool_choice: auto
 
 tests:
   - vars:
-      question: What is the capital of France?
-    assert:
-      - type: icontains
-        value: Paris
+      calculation: 25 * 4 + 10
+  - vars:
+      calculation: (8 + 2) * 5

examples/tool-use/external_tools.yaml

@@ -0,0 +1,17 @@
+- type: function
+  function:
+    name: get_weather
+    description: Get the current weather in a given location
+    input_schema:
+      type: object
+      properties:
+        location:
+          type: string
+          description: The city and state, e.g. San Francisco, CA
+        unit:
+          type: string
+          enum:
+            - celsius
+            - fahrenheit
+      required:
+        - location

examples/tool-use/promptfooconfig.yaml

@@ -40,9 +40,29 @@ providers:
   - id: openai:chat:gpt-4o-mini
     config:
       tools: file://external_tools.yaml
+  - id: groq:llama3-groq-70b-8192-tool-use-preview
+    config:
+      tools:
+        - type: function
+          function:
+            name: get_weather
+            description: 'Get the current weather in a given location'
+            parameters:
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: 'The city and state, e.g. San Francisco, CA'
+                unit:
+                  type: string
+                  enum:
+                    - celsius
+                    - fahrenheit
+              required: ['location']
+      tool_choice: auto
 
 prompts:
-  - Tell me about the weather in {{city}}.
+  - Tell me about the weather in {{city}} in the default unit for the location.
 
 tests:
   - vars:

package-lock.json

@@ -147,6 +147,7 @@
     "fast-xml-parser": "^4.4.1",
     "fastest-levenshtein": "^1.0.16",
     "glob": "^10.4.3",
+    "groq-sdk": "^0.5.0",
     "inquirer": "^10.1.8",
     "js-rouge": "3.0.0",
     "js-yaml": "^4.1.0",

package.json

@@ -1,98 +1,124 @@
 # Groq
 
-The [Groq API](https://wow.groq.com) supports the OpenAI format, which makes it easy to integrate with promptfoo as a variation of the `openai` provider.
+The [Groq API](https://wow.groq.com) is integrated into promptfoo using the Groq SDK, providing a native experience for using Groq models in your evaluations. Groq offers high-performance inference for various large language models.
 
-## Prerequisites
+## Setup
 
-Before you begin, make sure you have a Groq API key. You can obtain one from the [Groq Console](https://console.groq.com/) and set it as the `GROQ_API_KEY` environment variable.
+To use Groq, you need to set up your API key:
 
-## Configuration
-
-To use the Groq API with Promptfoo, you need to configure the provider in your promptfoo configuration file.
+1. Create a Groq API key in the [Groq Console](https://console.groq.com/).
+2. Set the `GROQ_API_KEY` environment variable:
 
-Here's an example configuration:
-
-```yaml
-providers:
-  - id: openai:chat:mixtral-8x7b-32768
-    config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
-  - id: openai:chat:llama2-70b-4096
-    config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
+```sh
+export GROQ_API_KEY=your_api_key_here
 ```
 
-In this configuration:
+Alternatively, you can specify the `apiKey` in the provider configuration (see below).
 
-- `id` specifies the provider ID in the format `openai:chat:<model>`. Replace `<model>` with the desired Groq model.
-- `config.apiBaseUrl` points it to the Groq API.
-- `config.apiKeyEnvar` specifies the environment variable that holds your Groq API key.
+## Configuration
 
-You can also directly specify the API key in the configuration using the `apiKey` field instead of `apiKeyEnvar`:
+Configure the Groq provider in your promptfoo configuration file:
 
 ```yaml
 providers:
-  - id: openai:chat:mixtral-8x7b-32768
+  - id: groq:llama3-groq-70b-8192-tool-use-preview
     config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKey: gsk_abc123_your_groq_api_key
+      temperature: 0.7
+      max_tokens: 100
+      tools:
+        - type: function
+          function:
+            name: get_weather
+            description: 'Get the current weather in a given location'
+            parameters:
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: 'The city and state, e.g. San Francisco, CA'
+                unit:
+                  type: string
+                  enum: [celsius, fahrenheit]
+              required: [location]
+      tool_choice: auto
 ```
 
+Key configuration options:
+
+- `temperature`: Controls randomness in output (0.0 to 1.0)
+- `max_tokens`: Maximum number of tokens in the response
+- `tools`: Defines functions the model can use (for tool use/function calling)
+- `tool_choice`: Specifies how the model should choose tools ('auto', 'none', or a specific tool)
+
 ## Supported Models
 
-As of the time of writing, the Groq API supports the following models:
+Groq supports a variety of models, including:
 
-- `mixtral-8x7b-32768`
+- `gemma-7b-it`
+- `gemma2-9b-it`
+- `llama-3.1-405b-reasoning`
+- `llama-3.1-70b-versatile`
+- `llama-3.1-8b-instant`
 - `llama2-70b-4096`
+- `llama3-70b-8192`
+- `llama3-8b-8192`
+- `llama3-groq-70b-8192-tool-use-preview` (recommended for tool use)
+- `llama3-groq-8b-8192-tool-use-preview` (recommended for tool use)
+- `mixtral-8x7b-32768`
 
-You can find the latest list of supported models in the [Groq Console documentation](https://console.groq.com/docs/models).
+For the most up-to-date list and detailed information about each model, refer to the [Groq Console documentation](https://console.groq.com/docs/models).
 
 ## Using the Provider
 
-Once you have configured the Groq provider, you can use it in your Promptfoo tests just like any other OpenAI-compatible provider. Specify the provider ID in your test configuration, and Promptfoo will send the requests to the Groq API.
-
-Here's an example test configuration:
+Specify the Groq provider in your test configuration:
 
 ```yaml
-prompts:
-  - 'Answer this as concisely as possible: {{question}}'
-
 providers:
-  - id: openai:chat:mixtral-8x7b-32768
-    config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
-  - id: openai:chat:llama2-70b-4096
+  - id: groq:llama3-groq-70b-8192-tool-use-preview
     config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
+      temperature: 0.5
+      max_tokens: 150
+
+prompts:
+  - Tell me about the weather in {{city}} in the default unit for the location.
 
 tests:
   - vars:
-      question: What is the capital of France?
-    assert:
-      - type: equals
-        value: Paris
+      city: Boston
+  - vars:
+      city: New York
 ```
 
-In this example, the test will be run against both the `mixtral-8x7b-32768` and `llama2-70b-4096` models using the Groq API.
+## Tool Use (Function Calling)
 
-## Additional Configuration
-
-The Groq provider supports additional configuration options, such as `temperature`, `max_tokens`, etc. You can specify these options under the `config` field for each provider.
-
-For example:
+Groq supports tool use, allowing models to call predefined functions. Configure tools in your provider settings:
 
 ```yaml
 providers:
-  - id: openai:chat:mixtral-8x7b-32768
+  - id: groq:llama3-groq-70b-8192-tool-use-preview
     config:
-      apiBaseUrl: https://api.groq.com/openai/v1
-      apiKeyEnvar: GROQ_API_KEY
-      temperature: 0.7
-      max_tokens: 100
+      tools:
+        - type: function
+          function:
+            name: get_weather
+            description: 'Get the current weather in a given location'
+            parameters:
+              type: object
+              properties:
+                location:
+                  type: string
+                  description: 'The city and state, e.g. San Francisco, CA'
+                unit:
+                  type: string
+                  enum: [celsius, fahrenheit]
+              required: [location]
+      tool_choice: auto
 ```
 
-Refer to the [Groq OpenAI compatibility docs](https://console.groq.com/docs/openai) as well as the [OpenAI documentation](https://platform.openai.com/docs/api-reference/completions/create) for the full list of supported configuration options.
+For complex tools or ambiguous queries, use the `llama3-groq-70b-8192-tool-use-preview` model.
+
+## Additional Capabilities
+
+- **Caching**: Groq provider caches previous LLM requests by default for improved performance.
+- **Token Usage Tracking**: Provides detailed information on token usage for each request.
+- **Cost Calculation**: Automatically calculates the cost of each request based on token usage and the specific model used.

site/docs/providers/groq.md

@@ -15,6 +15,7 @@ import { BAMChatProvider, BAMEmbeddingProvider } from './providers/bam';
 import { AwsBedrockCompletionProvider, AwsBedrockEmbeddingProvider } from './providers/bedrock';
 import * as CloudflareAiProviders from './providers/cloudflare-ai';
 import { CohereChatCompletionProvider } from './providers/cohere';
+import { GroqProvider } from './providers/groq';
 import { HttpProvider } from './providers/http';
 import {
   HuggingfaceFeatureExtractionProvider,
@@ -363,6 +364,9 @@ export async function loadApiProvider(
     ret = new RedteamCrescendoProvider(providerOptions.config);
   } else if (providerPath === 'promptfoo:manual-input') {
     ret = new ManualInputProvider(providerOptions);
+  } else if (providerPath.startsWith('groq:')) {
+    const modelName = providerPath.split(':')[1];
+    ret = new GroqProvider(modelName, providerOptions);
   } else {
     if (providerPath.startsWith('file://')) {
       providerPath = providerPath.slice('file://'.length);
@@ -439,5 +443,6 @@ export default {
   LocalAiChatProvider,
   BAMChatProvider,
   BAMEmbeddingProvider,
+  GroqProvider,
   loadApiProvider,
 };