Merge Main into Prod (prod-2.6.7)

.github/workflows/ai_explain.py

@@ -3,6 +3,7 @@
 
 from openai import OpenAI
 from github import Github
+from tiktoken import encoding_for_model
 
 # Initialize GitHub and OpenAI clients
 github_token = os.getenv("GITHUB_TOKEN")
@@ -69,6 +70,16 @@
 # Prepare OpenAI prompt
 prompt = prompt_template.format(diff=diff)
 
+# check number of tokens
+encoding = encoding_for_model("gpt-4o-mini")
+tokens = encoding.encode(prompt)
+print(f"Number of tokens: {len(tokens)}")
+# 128k is max tokens for gpt-4o-mini
+num_output_tokens = 1000
+if len(tokens) > 128000 - num_output_tokens:
+    tokens = tokens[: 128000 - num_output_tokens]
+    prompt = encoding.decode(tokens)
+
 # Call OpenAI API
 client = OpenAI()
 response = client.chat.completions.create(

.github/workflows/ai_explain.yaml

@@ -18,12 +18,13 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.x"
+          python-version: '3.x'
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install openai
+          pip install tiktoken
           pip install PyGithub
 
       - name: Explain PR

.github/workflows/docs.yaml

@@ -1,14 +1,15 @@
 # This workflow will install Python dependencies and generate Markdown
 # documentation from docstrings using Sphinx. We generate the HTML
 # documentation to keep it up to date with the Markdown files
-
-name: Framework API docs
+name: Python Library API docs
 
 on:
   push:
     branches:
       - main
       - release-v1
+    paths-ignore:
+      - 'docs/_build/**'
   workflow_dispatch:
     inputs:
       note:
@@ -21,7 +22,8 @@ permissions:
 
 jobs:
   docs:
-    runs-on: ubuntu-latest
+    runs-on:
+      group: ubuntu-vm-large
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/integration.yaml

@@ -9,15 +9,16 @@ on:
       - main
       - prod
       - release-v1
+    paths-ignore:
+      - 'docs/_build/**'
 
 permissions:
   contents: read
 
 jobs:
   integration:
-    # Don't rerun the integration tests after docs were generated
-    if: "!contains(github.event.head_commit.message, 'Generate docs')"
-    runs-on: ubuntu-latest
+    runs-on:
+      group: ubuntu-vm-large
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/python.yaml

@@ -6,8 +6,16 @@ name: Code Quality and Unit Testing
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - 'docs/_build/**'
+      - 'notebooks/**'
+      - 'scripts/**'
   pull_request:
-    branches: ["*"]
+    branches: ['*']
+    paths-ignore:
+      - 'docs/_build/**'
+      - 'notebooks/**'
+      - 'scripts/**'
 
 permissions:
   contents: read
@@ -17,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      # TODO figure out why dev-framework requires so much disk space and fix that
+      # TODO figure out why VM Library requires so much disk space and fix that
       # - name: Free Disk Space (Ubuntu)
       #   uses: jlumbroso/free-disk-space@main
       #   with:
@@ -42,8 +50,8 @@ jobs:
       - name: Set up Python 3.9
         uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
-          cache: "poetry"
+          python-version: '3.9'
+          cache: 'poetry'
 
       - name: Install Dependencies
         run: |

Makefile

@@ -30,15 +30,26 @@ else
 	poetry run python -m unittest discover tests
 endif
 
+test-unit:
+	poetry run python -m unittest "tests.test_unit_tests"
+
 test-integration:
 	poetry run python scripts/run_e2e_notebooks.py
 
 docs:
 	rm -rf docs/_build
-	poetry run pdoc validmind -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/vm-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico -o docs/_build
+ifeq ($(shell uname),Darwin)
+	poetry run pdoc validmind !validmind.tests.data_validation.Protected* -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico -o docs/_build
+else
+	poetry run pdoc validmind -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico -o docs/_build
+endif
 
 docs-serve:
-	poetry run pdoc validmind -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/vm-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico
+ifeq ($(shell uname),Darwin)
+	poetry run pdoc validmind !validmind.tests.data_validation.Protected* -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico
+else
+	poetry run pdoc validmind -d google -t docs/templates --no-show-source --logo https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg --favicon https://vmai.s3.us-west-1.amazonaws.com/favicon.ico
+endif
 
 version:
 	@:$(call check_defined, tag, new semver version tag to use on pyproject.toml)

README.md

@@ -1,30 +1,30 @@
 # ValidMind Library
 
 <!-- TODO: put back in when workflows are working properly -->
-<!-- [![Code Quality](https://github.com/validmind/developer-framework/actions/workflows/python.yaml/badge.svg)](https://github.com/validmind/developer-framework/actions/workflows/python.yaml)
-[![Integration Tests](https://github.com/validmind/developer-framework/actions/workflows/integration.yaml/badge.svg)](https://github.com/validmind/developer-framework/actions/workflows/integration.yaml) -->
+<!-- [![Code Quality](https://github.com/validmind/validmind-library/actions/workflows/python.yaml/badge.svg)](https://github.com/validmind/validmind-library/actions/workflows/python.yaml)
+[![Integration Tests](https://github.com/validmind/validmind-library/actions/workflows/integration.yaml/badge.svg)](https://github.com/validmind/validmind-library/actions/workflows/integration.yaml) -->
 
-![ValidMind logo](images/ValidMind-logo-color.svg "ValidMind logo")
+![ValidMind logo](https://vmai.s3.us-west-1.amazonaws.com/validmind-logo.svg "ValidMind logo")
 
-ValidMind's Python library automates the documentation and validation of your models through a comprehensive suite of developer tools and methods.
+The ValidMind Library is a suite of developer tools and methods designed to run validation tests and automate the documentation of your models.
 
-Built to be model agnostic, it works seamlessly with any Python model without requiring developers to rewrite existing code.
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
 
-It includes a suite of rich documentation and model testing capabilities - from dataset descriptions to identifying model weak spots and overfit areas. Through this library, you can automate documentation generation by feeding artifacts and test results to the ValidMind platform.
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
 
 ## Contributing to the ValidMind Library
 
 We believe in the power of collaboration and welcome contributions to the ValidMind Library. If you've noticed a bug, have a feature request, or want to contribute a test, please create a pull request or submit an issue and refer to the [contributing guide](README.md#how-to-contribute) below.
 
 - Interested in connecting with fellow AI model risk practitioners? Join our [Community Slack](https://docs.validmind.ai/about/contributing/join-community.html)!
 
-- For more information about ValidMind's open source tests and Jupyter notebooks, read the [Library docs](https://docs.validmind.ai/developer/get-started-developer-framework.html).
+- For more information about ValidMind's open-source tests and Jupyter Notebooks, read the [ValidMind Library docs](https://docs.validmind.ai/developer/get-started-validmind-library.html).
 
 ## Getting started
 
 ### Install from PyPI
 
-To install the library and all optional dependencies, run:
+To install the ValidMind Library and all optional dependencies, run:
 
 ```bash
 pip install validmind[all]
@@ -80,7 +80,7 @@ This will install the dependencies and git hooks for the project.
 a new kernel with Jupyter:
 
 ```bash
-poetry run python -m ipykernel install --user --name dev-framework --display-name "Library"
+poetry run python -m ipykernel install --user --name validmind --display-name "ValidMind Library"
 ```
 
 ### Installing LLM validation dependencies
@@ -143,7 +143,7 @@ Single file:
 poetry run python scripts/add_test_description.py review validmind/tests/ongoing_monitoring/FeatureDrift.py
 ```
 
-## Adding a Copyright Header
+## Adding a copyright header
 
 When adding new files to the project, you can add the ValidMind copyright header to any files that
 are missing it by running:
@@ -152,7 +152,7 @@ are missing it by running:
 make copyright
 ```
 
-## Known Issues
+## Known issues
 
 ### ValidMind wheel errors
 

README.pypi.md

@@ -1,48 +1,71 @@
 # ValidMind Library
 
-ValidMind's Python library automates the documentation and validation of your models through a comprehensive suite of developer tools and methods.
+The ValidMind Library is a suite of developer tools and methods designed to automate the documentation and validation of your models.
 
-Built to be model agnostic, it works seamlessly with any Python model without requiring developers to rewrite existing code.
+Designed to be model agnostic, the ValidMind Library provides all the standard functionality without requiring you to rewrite any functions as long as your model is built in Python.
 
-It includes a suite of rich documentation and model testing capabilities - from dataset descriptions to identifying model weak spots and overfit areas. Through this library, you can automate documentation generation by feeding artifacts and test results to the ValidMind platform.
+With a rich array of documentation tools and test suites, from documenting descriptions of your datasets to testing your models for weak spots and overfit areas, the ValidMind Library helps you automate model documentation by feeding the ValidMind Platform with documentation artifacts and test results.
 
-## Getting started
+## What is ValidMind?
 
-### Install from PyPI
+ValidMind helps developers, data scientists and risk and compliance stakeholders identify potential risks in their AI and large language models, and generate robust, high-quality model documentation that meets regulatory requirements.
 
-To install the library and all optional dependencies, run:
+[The ValidMind AI risk platform](https://docs.validmind.ai/about/overview.html) consists of two intertwined product offerings:
+
+- **The ValidMind Library** — Designed to be incorporated into your existing model development environment, you use the ValidMind Library to run tests and log documentation to the ValidMind Platform. Driven by the power of open-source, the ValidMind Library welcomes contributions to our code and developer samples: [`validmind-library` @ GitHub](https://github.com/validmind/validmind-library)
+- **The ValidMind Platform** — A cloud-hosted user interface allowing you to comprehensively track your model inventory throughout the entire model lifecycle according to the unique requirements of your organization. You use the ValidMind Platform to oversee your model risk management process via the customizable model inventory.
+
+### What do I need to get started with ValidMind?
+
+> **All you need to get started with ValidMind is an account with us.**
+>
+> Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
+
+That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and [**Get started with the ValidMind Library**](https://docs.validmind.ai/developer/get-started-validmind-library.html)!
+
+### How do I do more with the ValidMind Library?
+
+**[Explore our code samples!](https://docs.validmind.ai/developer/samples-jupyter-notebooks.html)**
+
+Our selection of Jupyter Notebooks showcase the capabilities and features of the ValidMind Library, while also providing you with useful examples that you can build on and adapt for your own use cases.
+
+## Installation
+
+To install the ValidMind Library and all optional dependencies, run:
 
 ```bash
 pip install validmind[all]
 ```
 
-To just install the core functionality without optional dependencies (some tests and models may not work), run:
+To install the ValidMind Library without optional dependencies (core functionality only), run:
 
 ```bash
 pip install validmind
 ```
 
-#### Extra dependencies
+### Extra dependencies
 
-- **Install with LLM Support**
+The ValidMind Library has optional dependencies that can be installed separately to support additional model types and tests.
+
+- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
 
     ```bash
     pip install validmind[llm]
     ```
 
-- **Install with Hugging Face `transformers` support**
+- **PyTorch Models**: To use pytorch models with the ValidMind Library, install the `torch` extra:
 
     ```bash
-    pip install validmind[transformers]
+    pip install validmind[torch]
     ```
 
-- **Install with PyTorch support**
+- **Hugging Face Transformers**: To use Hugging Face Transformers models with the ValidMind Library, install the `transformers` extra:
 
     ```bash
-    pip install validmind[pytorch]
+    pip install validmind[transformers]
     ```
 
-- **Install with R support (requires R to be installed)**
+- **R Models**: To use R models with the ValidMind Library, install the `r` extra:
 
     ```bash
     pip install validmind[r-support]