Update LlamaIndex to version 0.10

The new version introduces changes in the package interface, so an important refactor was needed. Additionally, PDM is introduced instead of poetry due to its better performance.
GPTStonks · Feb 27, 2024 · 38b15bf · 38b15bf
1 parent aefb4d3
commit 38b15bf
Show file tree

Hide file tree

Showing 15 changed files with 4,805 additions and 4,790 deletions.
diff --git a/.github/workflows/docker-pr-build.yaml b/.github/workflows/docker-pr-build.yaml
@@ -0,0 +1,71 @@
+name: "Docker build and push"
+on:
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  docker-build-pr:
+    name: "Build Docker image for PR tests"
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      # Reference: https://github.com/jlumbroso/free-disk-space
+      - name: Free disk space
+        run: |
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          docker rmi $(docker image ls -aq)
+          sudo rm -rf /usr/local/lib/android || true
+          sudo rm -rf /usr/share/dotnet || true
+          sudo rm -rf /opt/ghc || true
+          sudo rm -rf /usr/local/.ghcup || true
+          sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^llvm-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..."
+          sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..."
+          sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..."
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
+          df -h
+
+      - name: Checkout repository to build machine
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -1,6 +1,5 @@
 name: "Docker build and push"
 on:
-  pull_request:
   push:
     branches:
       - main

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -15,7 +15,6 @@ jobs:
       matrix:
         os: ["ubuntu-latest"]
         python-version: ["3.10"]
-        poetry-version: ["1.5.1"]
 
     timeout-minutes: 20
 
@@ -28,20 +27,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-
       - name: Install dependencies
         run: |
-          poetry install
-          poetry add pytest
-          poetry add sh
+          pip install pdm
+          pdm install --no-editable --no-self -G :all
 
       - name: Run pytest
         run: |
-          poetry run pytest -v
+          .venv/bin/pytest -v
 
   run_tests_windows:
     runs-on: ${{ matrix.os }}
@@ -50,8 +43,6 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest"]
-        python-version: ["3.10"]
-        poetry-version: ["1.5.1"]
 
     timeout-minutes: 20
 
@@ -64,29 +55,19 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-
       - name: Install dependencies
         run: |
-          poetry install
-          poetry add pytest
-          poetry add sh
+          pip install pdm
+          pdm install --no-editable --no-self -G :all
 
       - name: Run pytest
         run: |
-          poetry run pytest -v
+          .\.venv\Scripts\pytest
 
   # upload code coverage report
   code-coverage:
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        poetry-version: ["1.5.1"]
-
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -96,22 +77,13 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-
       - name: Install dependencies
         run: |
-          poetry install
-          poetry add pytest
-          poetry add sh
-          poetry add pytest-cov[toml]
+          pip install pdm
+          pdm install --no-editable --no-self -G :all
 
       - name: Run tests and collect coverage
-        run: poetry run pytest --cov openbb_chat
+        run: .venv/bin/pytest --cov openbb_chat
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -16,7 +16,6 @@ repos:
       - id: check-toml
       - id: check-case-conflict
       - id: check-added-large-files
-        exclude: poetry.lock
 
   # remove unused imports
   - repo: https://github.com/hadialqattan/pycln
@@ -106,7 +105,7 @@ repos:
     hooks:
       - id: codespell
         args:
-          - --skip=logs/**,data/**,*.ipynb,poetry.lock
+          - --skip=logs/**,data/**,*.ipynb,*.lock
           # - --ignore-words-list=abc,def
 
   # jupyter notebook cell output clearing

diff --git a/Dockerfile b/Dockerfile
@@ -3,29 +3,15 @@ FROM nvidia/cuda:11.7.1-devel-ubuntu22.04
 # Update system
 RUN apt update && apt install -y python3-pip && apt install -y python3-venv
 
-# Configure Poetry
-ENV POETRY_VERSION=1.5.1
-ENV POETRY_HOME=/opt/poetry
-ENV POETRY_VENV=/opt/poetry-venv
-ENV POETRY_CACHE_DIR=/opt/.cache
-
-# Install poetry separated from system interpreter
-RUN python3 -m venv $POETRY_VENV \
-    && $POETRY_VENV/bin/pip install -U pip setuptools \
-    && $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
-
-# Add `poetry` to PATH
-ENV PATH="${PATH}:${POETRY_VENV}/bin"
-
 WORKDIR /openbb-chat/
 
 # Add dependencies
-ADD poetry.lock /openbb-chat/poetry.lock
+ADD pdm.lock /openbb-chat/pdm.lock
 ADD pyproject.toml /openbb-chat/pyproject.toml
-RUN poetry install --no-root
 ADD openbb_chat /openbb-chat/openbb_chat
 ADD README.md /openbb-chat/README.md
-RUN poetry install --only-root
+RUN pip install --no-cache-dir setuptools==68.2.2 wheel==0.41.3 pdm==2.12.3 && \
+    pdm install --prod
 
 # Add project root
 ADD .project-root /openbb-chat/.project-root

diff --git a/README.md b/README.md
@@ -27,30 +27,18 @@ OpenBB Chat provides chat capabilities to [OpenBB](https://github.com/OpenBB-fin
 
 ## Installation
 
-#### Poetry
+#### PDM
 
 ```bash
 # clone project
 git clone https://github.com/Dedalo314/openbb-chat
 cd openbb-chat
 
-# [OPTIONAL] create conda environment
-conda create -n myenv python=3.10
-conda activate myenv
+# install pdm
+pip install pdm
 
-# install poetry (change paths as needed)
-POETRY_VERSION=1.5.1
-POETRY_HOME=/opt/poetry
-POETRY_VENV=/opt/poetry-venv
-POETRY_CACHE_DIR=/opt/.cache
-python3 -m venv $POETRY_VENV \
-&& $POETRY_VENV/bin/pip install -U pip setuptools \
-&& $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
-
-# add poetry to PATH
-PATH="${PATH}:${POETRY_VENV}/bin"
-
-poetry install
+# install package
+pdm install
 ```
 
 ## How to run
@@ -59,22 +47,22 @@ Train model with default configuration
 
 ```bash
 # train demo on CPU
-poetry run python openbb_chat/train.py trainer=cpu
+python openbb_chat/train.py trainer=cpu
 
 # train demo on GPU
-poetry run python openbb_chat/train.py trainer=gpu
+python openbb_chat/train.py trainer=gpu
 ```
 
 Train model with chosen experiment configuration from [configs/experiment/](configs/experiment/)
 
 ```bash
-poetry run python openbb_chat/train.py experiment=experiment_name.yaml
+python openbb_chat/train.py experiment=experiment_name.yaml
 ```
 
 You can override any parameter from command line like this
 
 ```bash
-poetry run python openbb_chat/train.py trainer.max_epochs=20 data.batch_size=64
+python openbb_chat/train.py trainer.max_epochs=20 data.batch_size=64
 ```
 
 ## Released models

diff --git a/docs/index.md b/docs/index.md
@@ -11,7 +11,7 @@ OpenBB Chat provides chat capabilities to [OpenBB](https://github.com/OpenBB-fin
 
 ## Installation
 
-#### Poetry
+#### PDM
 
 ```bash
 # clone project
@@ -22,19 +22,10 @@ cd openbb-chat
 conda create -n myenv python=3.10
 conda activate myenv
 
-# install poetry (change paths as needed)
-POETRY_VERSION=1.5.1
-POETRY_HOME=/opt/poetry
-POETRY_VENV=/opt/poetry-venv
-POETRY_CACHE_DIR=/opt/.cache
-python3 -m venv $POETRY_VENV \
-&& $POETRY_VENV/bin/pip install -U pip setuptools \
-&& $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
+# install pdm (change paths as needed)
+pip install pdm
 
-# add poetry to PATH
-PATH="${PATH}:${POETRY_VENV}/bin"
-
-poetry install
+pdm install
 ```
 
 ## How to run
@@ -43,22 +34,22 @@ Train model with default configuration
 
 ```bash
 # train demo on CPU
-poetry run python openbb_chat/train.py trainer=cpu
+python openbb_chat/train.py trainer=cpu
 
 # train demo on GPU
-poetry run python openbb_chat/train.py trainer=gpu
+python openbb_chat/train.py trainer=gpu
 ```
 
 Train model with chosen experiment configuration from [configs/experiment/](configs/experiment/)
 
 ```bash
-poetry run python openbb_chat/train.py experiment=experiment_name.yaml
+python openbb_chat/train.py experiment=experiment_name.yaml
 ```
 
 You can override any parameter from command line like this
 
 ```bash
-poetry run python openbb_chat/train.py trainer.max_epochs=20 data.batch_size=64
+python openbb_chat/train.py trainer.max_epochs=20 data.batch_size=64
 ```
 
 ## Released models