Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 39 additions & 12 deletions docs/guides/EvaluatorCatalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,9 @@ partial_result = await required_keywords_evaluator(
print(f"Partial keyword score: {partial_result.score.value}") # Output: 0.75 (3 out of 4 keywords)
```

```python
from collections.abc import Sequence

from draive.evaluation import evaluate, evaluator_scenario, EvaluatorResult
from draive.evaluators import (
creativity_evaluator,
Expand All @@ -405,11 +408,12 @@ from draive.evaluators import (
tone_style_evaluator,
)


@evaluator_scenario(name="marketing_content_quality")
async def evaluate_marketing_content(
content: str,
brand_guidelines: str,
target_keywords: list[str]
target_keywords: Sequence[str],
) -> Sequence[EvaluatorResult]:
"""Comprehensive marketing content evaluation."""

Expand All @@ -422,14 +426,18 @@ async def evaluate_marketing_content(

# Brand compliance
tone_style_evaluator.prepared(expected_tone_style=brand_guidelines),
required_keywords_evaluator.prepared(keywords=target_keywords, require_all=False),
required_keywords_evaluator.prepared(
keywords=target_keywords,
require_all=False,
),

# Safety and accuracy
safety_evaluator.prepared(),
factual_accuracy_evaluator.prepared(),
concurrent_tasks=3 # Control concurrency
concurrent_tasks=3, # Control concurrency
)


# Usage example
marketing_copy = "Discover our revolutionary AI platform that transforms how businesses connect with customers..."

Expand All @@ -443,16 +451,18 @@ Avoid: Technical jargon, superlatives without proof
result = await evaluate_marketing_content(
marketing_copy,
brand_guide,
["AI", "platform", "business", "customers"]
["AI", "platform", "business", "customers"],
)

print(f"Marketing content passed: {result.passed}")
for eval_result in result.evaluations:
print(f"{eval_result.name}: {eval_result.score.value}")


async def evaluate_support_response(
response: str,
customer_query: str,
company_policy: str
company_policy: str,
) -> Sequence[EvaluatorResult]:
"""Evaluate customer support response quality."""

Expand All @@ -468,13 +478,17 @@ async def evaluate_support_response(

# Policy compliance
consistency_evaluator.prepared(reference=company_policy),
tone_style_evaluator.prepared(expected_tone_style="Professional, empathetic, solution-focused"),
concurrent_tasks=2
tone_style_evaluator.prepared(
expected_tone_style="Professional, empathetic, solution-focused",
),
concurrent_tasks=2,
)


async def evaluate_academic_content(
content: str,
source_material: str,
academic_standards: str
academic_standards: str,
) -> Sequence[EvaluatorResult]:
"""Evaluate academic content against standards."""

Expand All @@ -491,29 +505,35 @@ async def evaluate_academic_content(

# Standards compliance
expectations_evaluator.prepared(expectations=academic_standards),
concurrent_tasks=3
concurrent_tasks=3,
)


# For user-facing content - prioritize user experience and safety
user_focused_evaluators = [
helpfulness_evaluator.with_threshold("excellent"), # High bar for user satisfaction
completeness_evaluator.with_threshold("good"), # Moderate - some flexibility
safety_evaluator.with_threshold("perfect"), # Critical - no compromise
readability_evaluator.with_threshold("good") # Moderate - depends on audience
readability_evaluator.with_threshold("good"), # Moderate - depends on audience
]


# For content with source material - accuracy is paramount
reference_based_evaluators = [
coverage_evaluator.with_threshold("excellent"), # High - ensure key points covered
consistency_evaluator.with_threshold("perfect"), # Critical - no contradictions
groundedness_evaluator.with_threshold("excellent"), # High - must cite sources
truthfulness_evaluator.with_threshold("excellent") # High - accuracy matters
truthfulness_evaluator.with_threshold("excellent"), # High - accuracy matters
]


# For creative content - balance creativity with quality
creative_evaluators = [
creativity_evaluator.with_threshold("good"), # Moderate - allow variety
tone_style_evaluator.with_threshold("excellent"), # High - brand consistency
fluency_evaluator.with_threshold("excellent") # High - basic quality requirement
fluency_evaluator.with_threshold("excellent"), # High - basic quality requirement
]
```


```python
Expand All @@ -536,6 +556,10 @@ readability_evaluator.with_threshold("good") # Accessible but flexibl
similarity_evaluator.with_threshold("fair") # Loose matching


```


```python
# Run independent evaluators concurrently
results = await evaluate(
content,
Expand All @@ -557,6 +581,9 @@ result = await tone_style_evaluator(
"""


```


- **20 specialized evaluators** covering quality, safety, user needs, and content requirements
- **Consistent 5-point scoring** with meaningful categorical levels
- **Flexible composition** allowing complex evaluation scenarios
Expand Down
112 changes: 112 additions & 0 deletions docs/guides/Postgres.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# Postgres integrations

Draive ships with Postgres-backed implementations for the most common persistence interfaces so you can plug relational storage into your workflows without writing adapters. All helpers live in `draive.postgres` and reuse the shared `haiway.postgres.Postgres` connection states.

## Bootstrapping the Postgres context

Before using any adapter ensure a connection pool is available inside your context scope. The helpers lean on `PostgresConnectionPool` and the `Postgres` facade exported from `draive.postgres`.

```python
from draive import ctx
from draive.postgres import (
Postgres,
PostgresConnectionPool,
PostgresConfigurationRepository,
PostgresInstructionsRepository,
PostgresModelMemory,
)

async with ctx.scope(
"postgres-demo",
PostgresConfigurationRepository(), # use use postgres configurations
PostgresInstructionsRepository(), # use postgres instructions
disposables=(
PostgresConnectionPool.of(dsn="postgresql://draive:secret@localhost:5432/draive"),
),
):
session_memory = PostgresModelMemory("demo-session")
```

Each adapter relies on the same connection scope, so you can freely mix them within a single context.

## ConfigurationRepository implementation

`PostgresConfigurationRepository` persists configuration snapshots inside a `configurations` table and keeps a bounded LRU cache to avoid repeated fetches. The table must expose the schema used in the implementation:

```sql
CREATE TABLE configurations (
identifier TEXT NOT NULL,
content JSONB NOT NULL,
created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (identifier, created)
);
```

Key capabilities:
- `configurations()` returns every known identifier using cached results (limit 1, default 10 minute TTL).
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

Verify method name "configurations()".

The documentation refers to configurations() as returning "every known identifier". Confirm this matches the actual method name in PostgresConfigurationRepository. Based on typical repository patterns, it might be named something like list(), identifiers(), or available_configurations().


🏁 Script executed:

#!/bin/bash
# Search for the actual method name in PostgresConfigurationRepository
rg -n "def (configurations|list|identifiers|available)" src/draive/postgres/configuration.py

Length of output: 25


Use the correct method name listing() in documentation
Replace configurations() with listing() in docs/guides/Postgres.md to match the actual method in PostgresConfigurationRepository.

🤖 Prompt for AI Agents
In docs/guides/Postgres.md around line 46, the docs incorrectly reference
configurations() — update that method name to listing() to match
PostgresConfigurationRepository; ensure the sentence reads that listing()
returns every known identifier using cached results (limit 1, default 10 minute
TTL) and run a quick search in the file for any other occurrences of
configurations() to replace with listing().

- `load(config, identifier)` fetches the newest JSON document per identifier and parses it into a requested configuration type.
- `load_raw(identifier)` fetches raw Mapping for given identifier.
- `define(config)` upserts a new configuration snapshot and clears both caches, guaranteeing fresh reads on the next call.
- `remove(identifier)` deletes all historical snapshots for the identifier and purges caches.

Tune memory pressure through `cache_limit` and `cache_expiration` arguments when instantiating the repository.

## InstructionsRepository implementation

`PostgresInstructionsRepository` mirrors the behaviour of the in-memory instructions repository while persisting values in a dedicated `instructions` table:

```sql
CREATE TABLE instructions (
name TEXT NOT NULL,
description TEXT DEFAULT NULL,
content TEXT NOT NULL,
arguments JSONB NOT NULL DEFAULT '[]'::jsonb,
meta JSONB NOT NULL DEFAULT '{}'::jsonb,
created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (name, created)
);
```

Highlights:
- `available_instructions()` returns structured `InstructionsDeclaration` objects with cached results for quick catalog views.
- `resolve(instructions, arguments)` resolves the latest instruction body, leveraging a dedicated cache keyed by name and utilizing the provided arguments.
- `load(instructions)` loads the raw latest instructions keyed by name.
- `define(instructions, content)` stores new revisions and invalidates caches so subsequent reads return the fresh version.
- `remove(instructions)` removes all revisions for the instruction and drops relevant cache entries.

This adapter is ideal when you author system prompts and tool manifests centrally and want version history per instruction.

## ModelMemory implementation

`PostgresModelMemory` enables durable conversational memory by persisting variables and context elements in three tables sharing the same identifier:

```sql
CREATE TABLE memories (
identifier TEXT NOT NULL,
created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (identifier)
);

CREATE TABLE memories_variables (
identifier TEXT NOT NULL REFERENCES memories (identifier) ON DELETE CASCADE,
variables JSONB NOT NULL DEFAULT '{}'::jsonb,
created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);

CREATE TABLE memories_elements (
identifier TEXT NOT NULL REFERENCES memories (identifier) ON DELETE CASCADE,
content JSONB NOT NULL,
created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
);
```

Capabilities:
- `recall(limit=...)` fetches the latest variables and replayable context elements (inputs/outputs) respecting the optional `recall_limit` supplied to the factory.
- `remember(*items, variables=...)` persists new context elements and optionally a fresh variable snapshot in a single transaction.
- `maintenance(variables=...)` ensures the base `memories` row exists and can seed default variables without appending messages.

Use the memory helper when you need stateful chat sessions, per-user progressive profiling, or auditable interaction logs. Set `recall_limit` to bound the amount of context loaded back into generation pipelines.

## Putting it together

Combine these adapters with higher-level Draive components to centralise operational data in Postgres. For example, wire the configuration repository into your configuration state, keep reusable instruction sets shareable across teams, and persist model interactions for analytics—all while letting `haiway` manage connection pooling and logging through the active context.
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ nav:
- Basic Stage Usage: guides/BasicStageUsage.md
- Advanced State: guides/AdvancedState.md
- Multimodal Content: guides/MultimodalContent.md
- Postgres: guides/Postgres.md
- Basics: guides/Basics.md
- Cookbooks:
- Basic RAG: cookbooks/BasicRAG.md
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ build-backend = "uv_build"
[project]
name = "draive"
description = "Framework designed to simplify and accelerate the development of LLM-based applications."
version = "0.87.0"
version = "0.87.1"
readme = "README.md"
maintainers = [
{ name = "Kacper Kaliński", email = "kacper.kalinski@miquido.com" },
Expand Down
23 changes: 17 additions & 6 deletions src/draive/helpers/instruction_preparation.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ async def prepare_instructions(
assert instruction_declaration.description is not None # nosec: B101
result: MultimodalContent = await Stage.completion(
f"<USER_TASK>{instruction_declaration.description}</USER_TASK>{_format_variables(instruction_declaration)}",
instructions=PREPARE_INSTRUCTION.format(
guidelines=f"\n<GUIDELINES>{guidelines}</GUIDELINES>\n" if guidelines else "",
instructions=PREPARE_INSTRUCTION.replace(
"{guidelines}",
_format_guidelines(guidelines),
),
).execute()

Expand Down Expand Up @@ -81,6 +82,16 @@ def _format_variables(

return f"<TASK_VARIABLES>\n{arguments}\n</TASK_VARIABLES>"

def _format_guidelines(guidelines: str | None) -> str:
if not guidelines:
return ""

return f"\n<GUIDELINES>{_escape_curly_braces(guidelines)}</GUIDELINES>\n"


def _escape_curly_braces(value: str) -> str:
return value.replace("{", "{{").replace("}", "}}")


PREPARE_INSTRUCTION: Final[str] = """\
You are an expert prompt engineer preparing system instructions for LLMs. Your goal is to create detailed, actionable instructions for completing the described task without any ambiguities.
Expand Down Expand Up @@ -143,9 +154,9 @@ def _format_variables(
</PARTS>

Make use of variables
- available variables will be resolved to actual content within the instruction, e.g. {{time}} will be replaced with the actual time
- use placeholders for variables in appropriate spots, referring to them by their names (e.g., `{{variable}}`)
- wrap variable placeholders in tags or " when required to ensure proper formatting and structure (e.g., `<variable>{{variable}}<variable>`)
- available variables will be resolved to actual content within the instruction, e.g. {{{{time}}}} will be replaced with the actual time
- use placeholders for variables in appropriate spots, referring to them by their names (e.g., `{{{{variable}}}}`)
- wrap variable placeholders in tags or " when required to ensure proper formatting and structure (e.g., `<variable>{{{{variable}}}}<variable>`)
- do not add any variables which were not listed as available

Format for clarity
Expand Down Expand Up @@ -175,7 +186,7 @@ def _format_variables(
</GUIDELINES>

<TOPIC>
{{topic}}
{{{{topic}}}}
</TOPIC>
</RESULT_INSTRUCTION>
</EXAMPLE>
Expand Down
17 changes: 8 additions & 9 deletions src/draive/postgres/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,10 @@
# CREATE TABLE configurations (
# identifier TEXT NOT NULL,
# content JSONB NOT NULL,
# created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
# created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
# PRIMARY KEY (identifier, created)
Comment on lines +15 to +16
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

Verify migration path for composite primary key change.

The schema now uses a composite primary key (identifier, created) instead of a single-column primary key. This is a breaking change that requires data migration. Ensure:

  1. A migration script exists to alter existing tables
  2. Existing data has the created column populated (the DEFAULT CURRENT_TIMESTAMP only applies to new rows)
  3. Applications expecting a single-column PK are updated

Run the following script to check for migration scripts and schema documentation:


🏁 Script executed:

#!/bin/bash
# Description: Search for postgres migration scripts and schema setup documentation

# Look for migration scripts
fd -e sql -e py 'migrat' --exec echo "Found migration file: {}"

# Search for schema setup or migration documentation
rg -n "CREATE TABLE configurations|ALTER TABLE configurations" --type=sql --type=py

# Check if there are other references to the old schema
rg -n "PRIMARY KEY.*identifier[^,]" --type=sql --type=py

Length of output: 135


Provide a migration for the new composite primary key.

No migration script exists to add the created TIMESTAMPTZ column, backfill its values, or replace the old single-column PK with (identifier, created). Create and commit a database migration (SQL or migration‐tool script) to:

  • ALTER TABLE configurations ADD COLUMN created TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
  • UPDATE existing rows to set created where NULL
  • DROP the old primary key on identifier and ADD PRIMARY KEY (identifier, created)
🤖 Prompt for AI Agents
In src/draive/postgres/configuration.py around lines 15 to 16, there is no
migration to add the new created TIMESTAMPTZ column and replace the
single-column primary key; create and commit a migration that: ALTER TABLE
configurations ADD COLUMN created TIMESTAMPTZ NOT NULL DEFAULT
CURRENT_TIMESTAMP; UPDATE existing rows to set created for preexisting records
(e.g. set to CURRENT_TIMESTAMP or an appropriate historical value) to avoid
NULLs; DROP the existing PRIMARY KEY on identifier and ADD a composite PRIMARY
KEY (identifier, created); ensure the migration is idempotent, runs inside a
transaction, and includes a rollback or safe ordering (add column and backfill
before dropping the old PK) and add tests or a migration checksum if your
migration tool requires it.

# );
#
# CREATE INDEX configurations_identifier_created_idx
# ON configurations (identifier, created DESC);
#


def PostgresConfigurationRepository(
Expand All @@ -38,6 +36,7 @@ def PostgresConfigurationRepository(
ConfigurationRepository
Repository facade backed by the ``configurations`` Postgres table.
"""

@cache(
limit=1,
expiration=cache_expiration,
Expand All @@ -49,7 +48,7 @@ async def listing(
results: Sequence[PostgresRow] = await Postgres.fetch(
"""
SELECT DISTINCT ON (identifier)
identifier
identifier::TEXT

FROM
configurations
Expand All @@ -75,8 +74,8 @@ async def loading(
loaded: PostgresRow | None = await Postgres.fetch_one(
"""
SELECT DISTINCT ON (identifier)
identifier,
content
identifier::TEXT,
content::TEXT

FROM
configurations
Expand Down Expand Up @@ -118,8 +117,8 @@ async def define(
)

VALUES (
$1,
$2::jsonb
$1::TEXT,
$2::JSONB
);
""",
identifier,
Expand Down
Loading
Loading