Added Features: Prompt Creation and Opro + Workflows

.flake8

@@ -1,2 +1,3 @@
 [flake8]
 max-line-length = 120
+ignore = F401, W503

.gitattributes

@@ -0,0 +1 @@
+*.ipynb linguist-documentation

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+* @finitearth
+* @timo282
+* @mo374z

.github/actions/python-poetry/action.yml

@@ -0,0 +1,39 @@
+name: Setup-Python-Poetry
+description: 'Setup Python and Poetry'
+
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: true
+    default: '3.11'
+  poetry-version:
+    description: 'Poetry version to use'
+    required: true
+    default: 'latest'
+  groups:
+    required: false
+    default: 'main'
+    type: string
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Setup Poetry
+      uses: snok/install-poetry@v1.4.1
+      with:
+        python-version: ${{ inputs.python-version }}
+        virtualenvs-in-project: true
+        virtualenvs-create: true
+    - uses: actions/cache@v3
+      id: cached-poetry-dependencies
+      with:
+        path: .venv
+        key: ${{ runner.os }}-poetry-${{ inputs.poetry-version }}-${{ hashFiles('**/poetry.lock') }}
+    - name: Install dependencies
+      run: poetry install --no-interaction --no-root --only ${{ inputs.groups }}
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      shell: bash

.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    types: [opened, synchronize]
+    branches:
+      - main
+      - dev
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Set up Python and Poetry
+        uses: ./.github/actions/python-poetry
+        with:
+          groups: dev
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+      # - name: Run tests
+      #   run: poetry run python -m pytest
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python and Poetry
+        uses: ./.github/actions/python-poetry
+
+      - name: Build wheel
+        run: poetry build --format wheel
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.event.repository.name }}
+          path: dist/

.github/workflows/cicd.yml

@@ -0,0 +1,102 @@
+name: CICD
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Run Tests, Pre-Commits, and Build Python package
+    uses: ./.github/workflows/ci.yml
+
+  check-version:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      version: ${{ steps.get_version.outputs.VERSION }}
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python and Poetry
+      uses: ./.github/actions/python-poetry
+      with:
+        cache: true
+
+    - name: Get version from pyproject.toml
+      id: get_version
+      run: |
+        echo "VERSION=$(poetry version -s)" >> $GITHUB_OUTPUT
+
+    - name: Get latest release version
+      id: get_latest_release
+      run: |
+        latest_release=$(curl -s https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .tag_name)
+        echo "LATEST_VERSION=${latest_release#v}" >> $GITHUB_OUTPUT
+
+    - name: Compare versions
+      run: |
+        current_version="${{ steps.get_version.outputs.VERSION }}"
+        latest_version="${{ steps.get_latest_release.outputs.LATEST_VERSION }}"
+        if [ "$(printf '%s\n' "$latest_version" "$current_version" | sort -V | tail -n1)" != "$current_version" ]; then
+          echo "Error: Current version ($current_version) is not higher than the latest release ($latest_version)"
+          exit 1
+        fi
+
+  create-release:
+    needs: check-version
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Create and push tag
+      run: |
+        git config --local user.email "action@github.com"
+        git config --local user.name "GitHub Action"
+        git tag -a v${{ needs.check-version.outputs.version }} -m "Release v${{ needs.check-version.outputs.version }}"
+        git push origin v${{ needs.check-version.outputs.version }}
+
+    - name: Create GitHub Release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: v${{ needs.check-version.outputs.version }}
+        release_name: Release v${{ needs.check-version.outputs.version }}
+        draft: false
+        prerelease: false
+
+  publish-to-pypi:
+    needs: [build, check-version, create-release]
+    name: Publish Python package to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python and Poetry
+      uses: ./.github/actions/python-poetry
+      with:
+        cache: true  # Enable caching if your custom action supports it
+
+    - name: Download wheel
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ github.event.repository.name }}
+        path: dist/
+
+    - name: Publish to PyPI
+      env:
+        PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      run: |
+        poetry config pypi-token.pypi $PYPI_TOKEN
+        poetry publish

.github/workflows/docs.yml

@@ -0,0 +1,27 @@
+name: Docs
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-deploy:
+    concurrency: ci-${{ github.ref }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup python and poetry
+        uses: ./.github/actions/python-poetry
+        with:
+          groups: "docs"
+
+      - name: Deploy docs
+        run: |
+          poetry run mkdocs gh-deploy --force

.gitignore

@@ -5,3 +5,4 @@ groqtoken.txt
 rsync_exclude.txt
 __pycache__/
 temp/
+dist/

.pre-commit-config.yaml

@@ -1,15 +1,49 @@
+fail_fast: true
+exclude: '^(?!promptolution/).*$'
 repos:
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.18.2
+    hooks:
+      - id: gitleaks
   - repo: https://github.com/psf/black
-    rev: 23.3.0  # Use the latest version
+    rev: 23.7.0
     hooks:
       - id: black
-
+      - id: black-jupyter
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0  # Use the latest version
+    rev: 6.0.0
     hooks:
       - id: flake8
-
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1  # Use the latest version
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
       - id: isort
+  - repo: https://github.com/pycqa/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+  - repo: https://github.com/python-poetry/poetry
+    rev: "1.6.0"
+    hooks:
+      - id: poetry-check
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: check-builtin-literals
+      - id: check-json
+      - id: check-toml
+      - id: check-xml
+      - id: debug-statements
+      - id: detect-private-key
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v3.4.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+        args: []

README.md

@@ -1,5 +1,5 @@
+![promptolution](https://github.com/user-attachments/assets/84c050bd-61a1-4f2e-bc4e-874d9b4a69af)
 # Promptolution
-
 Promptolution is a library that provides a modular and extensible framework for implementing prompt tuning experiments. It offers a user-friendly interface to assemble the core components for various prompt optimization tasks.
 
 In addition, this repository contains our experiments for the paper "Towards Cost-Effective Prompt Tuning: Evaluating the Effects of Model Size, Model Family and Task Descriptions in EvoPrompt".

configs/experiment_eval.ini

@@ -12,4 +12,4 @@ subsample_seed = 42
 n_samples = 200
 
 [downstream_llm]
-name = meta-llama/Meta-Llama-3-70B-Instruct
+name = meta-llama/Meta-Llama-3-70B-Instruct

configs/experiment_eval_task_descr.ini

@@ -10,4 +10,4 @@ subsample_seed = 42
 n_samples = 200
 
 [downstream_llm]
-name = meta-llama/Meta-Llama-3-70B-Instruct
+name = meta-llama/Meta-Llama-3-70B-Instruct

configs/experiment_gpt.ini

@@ -12,4 +12,4 @@ subsample_seed = 42
 n_samples = 200
 
 [downstream_llm]
-name = gpt-4o-2024-05-13
+name = gpt-4o-2024-05-13

configs/experiment_initial_prompts.ini

@@ -8,4 +8,4 @@ n_prompts = 3
 n_samples = 200
 
 [downstream_llm]
-name = meta-llama/Meta-Llama-3-70B-Instruct
+name = meta-llama/Meta-Llama-3-70B-Instruct

configs/experiment_task_descr.ini

@@ -18,4 +18,4 @@ name = meta-llama/Meta-Llama-3-8B-Instruct
 name = meta-llama/Meta-Llama-3-8B-Instruct
 
 [downstream_llm]
-name = meta-llama/Meta-Llama-3-70B-Instruct
+name = meta-llama/Meta-Llama-3-70B-Instruct

data_sets/cls/agnews/description.json

@@ -9,4 +9,4 @@
         "Business",
         "Tech"
     ]
-}
+}

data_sets/cls/agnews/prompts.txt

@@ -1,15 +1,15 @@
 Give the main topic of the news article and then choose from World, Sports, Tech and Business.
-Determine the the topic of the item and then choose from World, Sports, Business and Tech. 
+Determine the the topic of the item and then choose from World, Sports, Business and Tech.
 Your task is to classify the news item as "World", "Sports", "Tech" or "Business".
-Based on the main theme of given the news article, categorize it into World, Sports, Business, or Tech. 
+Based on the main theme of given the news article, categorize it into World, Sports, Business, or Tech.
 Determine the theme of the news item. Choose from World, Sports, Business and Tech.
-Your objective is to classify a news article into one of four themes: World, Sports, Business and Tech. 
-Your task is to identify the primary topic of the news artical and choose from World, Sports, Business and Tech. 
+Your objective is to classify a news article into one of four themes: World, Sports, Business and Tech.
+Your task is to identify the primary topic of the news artical and choose from World, Sports, Business and Tech.
 Your job is to determine whether a news article belongs to the World, Sports, Business, or Tech category based on its primary theme.
 You will be given a news article and asked to classify it as World, Sports, Business and Tech, depending on its main topic.
 Your responsibility is to assign a news article to one of four categories: World, Sports, Business, or Tech, based on its main idea.
 Categorize the news article into one of four categories: World, Sports, Business, or Tech, based on its content.
 Choose a word from World, Sports, Business and Tech to categorize the given text.
 You will be required to classify a news article as World, Sports, Business, or Tech based on its primary topic.
 In this task, you are given a news article. Your task is to classify the article to one out of the four topics "World", "Sports", "Business", "Tech" if the article"s main topic is relevant to the world, sports, business, and technology, correspondingly. If you are not sure about the topic, choose the closest option.
-Classify the topic of the following news as "World", "Sports", "Tech" or "Business".
+Classify the topic of the following news as "World", "Sports", "Tech" or "Business".

data_sets/cls/agnews/prompts_auto.txt

@@ -17,4 +17,4 @@ classify each input-output pair according to the topic.
 identify the topic of the text.
 determine the topic of the given text.
 identify the topic of a given article.
-identify the category of the article.
+identify the category of the article.

data_sets/cls/cr/description.json

@@ -7,4 +7,4 @@
         "negative",
         "positive"
     ]
-}
+}

data_sets/cls/cr/prompts.txt

@@ -10,7 +10,7 @@ Determine the sentiment of the text provided, positve or negative.
 Classify a sentence as expressing optimism or pessimism.
 In this task, you are given sentences from product reviews. The task is to classify a sentence as  positive or as negative.
 When given an English sentence, your task is to analyze it and generate a sentiment word to describe the original text, such as positive or negative.
-Now you are a classifier, your task is to determine the sentiment polarity of the given text, whether positive or negative. 
-You are a sentiment classifier. To do this, you must first understand the meaning of the sentence and any relevant context. And then you should classify it as positive or negative. 
-Please label the sentiment towards the product of the given product review. The sentiment label should be "positive" or "negative". 
-Given text, classify whether it is positive or negative.
+Now you are a classifier, your task is to determine the sentiment polarity of the given text, whether positive or negative.
+You are a sentiment classifier. To do this, you must first understand the meaning of the sentence and any relevant context. And then you should classify it as positive or negative.
+Please label the sentiment towards the product of the given product review. The sentiment label should be "positive" or "negative".
+Given text, classify whether it is positive or negative.

data_sets/cls/cr/seed5/dev.txt

@@ -197,4 +197,4 @@ the biggest problem is the installation .	0
 i 've always liked symantec products , except for this version 2004 .	0
 another thing you should know , the metal on the flip side of the ipod is very tacky , and scratches very easily .	0
 if you experience problems with installation , you can visit symantec 's website , click the support tab , click the 'home or home office support ' and there is an automated support assistant that can scan the norton program files on your computer and it will tell you what is not working right and how to fix it .	0
-norton internet security 2004 is one of the more buggy implementations .	0
+norton internet security 2004 is one of the more buggy implementations .	0