Move to Poetry to manage deps (#1158)

.github/workflows/tests.yml

@@ -17,30 +17,25 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
       - name: Install dependencies
         run: |
           make install-dev
       - name: Lint
         run: |
           make lint
 
-  tests:
+  mlserver:
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         # NOTE: There's no pre-built `grpcio` wheel for Python 3.11 yet
         # https://github.com/grpc/grpc/issues/32454
         python-version: ["3.8", "3.9", "3.10"]
-        tox-environment:
-          - mlserver
-          - sklearn
-          - xgboost
-          - lightgbm
-          - mlflow
-          - huggingface
-          - alibi-explain
-          - alibi-detect
         is-pr:
           - ${{ github.event_name == 'pull_request' }}
         exclude:
@@ -70,12 +65,59 @@ jobs:
         with:
           auto-update-conda: true
           auto-activate-base: false
-      - name: Install dependencies
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install Dependencies
+        run: |
+          poetry install --sync --only dev
+      - name: Test
+        run: |
+          tox -e mlserver
+
+  runtimes:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.8", "3.9", "3.10"]
+        tox-environment:
+          - sklearn
+          - xgboost
+          - lightgbm
+          - mlflow
+          - huggingface
+          - alibi-explain
+          - alibi-detect
+        is-pr:
+          - ${{ github.event_name == 'pull_request' }}
+        exclude:
+          # MacOS tests take a lot of time, so we will run them only on merge
+          # From https://github.com/orgs/community/discussions/26253
+          - is-pr: true
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          auto-activate-base: false
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install Dependencies
         run: |
-          pip install -r requirements/dev.txt
+          poetry install --sync --only dev
       - name: Test
         run: |
-          tox -e ${{ matrix.tox-environment }}
+          tox -c ./runtimes/${{ matrix.tox-environment }}
 
   all-runtimes:
     if: github.event_name == 'push'
@@ -115,9 +157,13 @@ jobs:
         with:
           auto-update-conda: true
           auto-activate-base: false
-      - name: Install dependencies
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install Dependencies
         run: |
-          pip install -r requirements/dev.txt
+          poetry install --sync --only dev
       - name: Test
         run: |
           tox -e all-runtimes

.readthedocs.yaml

@@ -13,16 +13,19 @@ sphinx:
 formats:
   - pdf
 
-# Optionally set the version of Python and requirements required to build your
-# docs
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - requirements: requirements/rtd.txt
-    - requirements: runtimes/alibi-detect/requirements/dev.txt
-    - requirements: requirements/docker.txt
-
 build:
   os: ubuntu-22.04
   tools:
     python: "3.8"
+  jobs:
+    # Set up Poetry
+    # From https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-poetry
+    post_create_environment:
+      # Install poetry
+      # https://python-poetry.org/docs/#installing-manually
+      - pip install poetry==1.4.2
+      # Tell poetry to not use a virtual environment
+      - poetry config virtualenvs.create false
+      # Install dependencies with 'docs' dependency group
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      - poetry install -vvv --without dev --with docs --with all-runtimes --with all-runtimes-dev

Dockerfile

@@ -1,19 +1,23 @@
 FROM python:3.10-slim AS wheel-builder
 SHELL ["/bin/bash", "-l", "-c"]
 
+ARG POETRY_VERSION="1.4.2"
+
 COPY ./hack/build-wheels.sh ./hack/build-wheels.sh
 COPY ./mlserver ./mlserver
-COPY ./openapi ./openapi
 COPY ./runtimes ./runtimes
 COPY \
-    setup.py \
-    MANIFEST.in \
+    pyproject.toml \
+    poetry.lock \
     README.md \
     .
 
-# This will build the wheels and place will place them in the
-# /opt/mlserver/dist folder
-RUN ./hack/build-wheels.sh /opt/mlserver/dist
+# Install Poetry, build wheels and export constraints
+RUN pip install poetry==$POETRY_VERSION && \
+    ./hack/build-wheels.sh /opt/mlserver/dist && \
+    poetry export --with all-runtimes \
+        --format constraints.txt \
+        -o /opt/mlserver/dist/constraints.txt
 
 FROM registry.access.redhat.com/ubi9/ubi-minimal
 SHELL ["/bin/bash", "-c"]
@@ -74,7 +78,6 @@ RUN useradd -u 1000 -s /bin/bash mlserver -d $MLSERVER_PATH && \
     chmod -R 776 $MLSERVER_PATH
 
 COPY --from=wheel-builder /opt/mlserver/dist ./dist
-COPY ./requirements/docker.txt ./requirements/docker.txt
 # NOTE: if runtime is "all" we install mlserver-<version>-py3-none-any.whl
 # we have to use this syntax to return the correct file: $(ls ./dist/mlserver-*.whl)
 # NOTE: Temporarily excluding mllib from the main image due to:
@@ -88,19 +91,18 @@ RUN . $CONDA_PATH/etc/profile.d/conda.sh && \
         for _wheel in "./dist/mlserver_"*.whl; do \
             if [[ ! $_wheel == *"mllib"* ]]; then \
                 echo "--> Installing $_wheel..."; \
-                pip install $_wheel; \
+                pip install $_wheel --constraints ./dist/constraints.txt; \
             fi \
         done \
     else \
         for _runtime in $RUNTIMES; do \
             _wheelName=$(echo $_runtime | tr '-' '_'); \
             _wheel="./dist/$_wheelName-"*.whl; \
             echo "--> Installing $_wheel..."; \
-            pip install $_wheel; \
+            pip install $_wheel --constraints ./dist/constraints.txt; \
         done \
     fi && \
-    pip install $(ls "./dist/mlserver-"*.whl) && \
-    pip install -r ./requirements/docker.txt && \
+    pip install $(ls "./dist/mlserver-"*.whl) --constraints ./dist/constraints.txt && \
     rm -f /opt/conda/lib/python3.10/site-packages/spacy/tests/package/requirements.txt && \
     rm -rf /root/.cache/pip
 

Makefile

@@ -6,17 +6,7 @@ IMAGE_NAME := seldonio/mlserver
 	push-test push test lint fmt version clean licenses
 
 install-dev:
-	pip install -r requirements/dev.txt
-	for _runtime in ./runtimes/*; \
-	do \
-		pip install --editable $$_runtime; \
-		if [[ -f $$_runtime/requirements/dev.txt ]]; \
-		then \
-			pip install -r $$_runtime/requirements/dev.txt; \
-		fi \
-	done
-	pip install --editable .
-	pip install -r ./requirements/docker.txt
+	poetry install --sync --with all-runtimes --with all-runtimes-dev
 
 _generate: # "private" target to call `fmt` after `generate`
 	./hack/generate-types.sh
@@ -32,31 +22,44 @@ build: clean
 	./hack/build-wheels.sh ./dist
 
 clean:
-	rm -rf ./dist ./build *.egg-info
+	rm -rf ./dist ./build *.egg-info .tox
 	for _runtime in ./runtimes/*; \
 	do \
-		rm -rf $$_runtime/dist; \
-		rm -rf $$_runtime/build; \
-		rm -rf $$_runtime/*.egg-info; \
+		rm -rf \
+			$$_runtime/dist \
+			$$_runtime/build \
+			$$_runtime/*.egg-info \
+			$$_runtime/.tox; \
 	done
 
 push-test:
-	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+	poetry config repositories.pypi-test https://test.pypi.org/legacy/
+	poetry publish -r pypi-test
+	for _runtime in ./runtimes/*; \
+	do \
+		poetry publish -C $$_runtime -r pypi-test; \
+	done
 
 push:
-	twine upload dist/*
+	poetry publish
 	docker push ${IMAGE_NAME}:${VERSION}
 	docker push ${IMAGE_NAME}:${VERSION}-slim
 	for _runtime in ./runtimes/*; \
 	do \
 	  _runtimeName=$$(basename $$_runtime); \
+		poetry publish -C $$_runtime; \
 		docker push ${IMAGE_NAME}:${VERSION}-$$_runtimeName; \
 	done
 
 test:
 	tox
+	for _runtime in ./runtimes/*; \
+	do \
+		tox -c $$_runtime; \
+	done
 
 lint: generate
+	black --check .
 	flake8 .
 	mypy ./mlserver
 	for _runtime in ./runtimes/*; \
@@ -77,8 +80,7 @@ licenses:
 		> ./licenses/license_info.no_versions.csv
 
 fmt:
-	black . \
-		--exclude "/(\.tox.*|mlserver/grpc/dataplane_pb2.*|venv.*)/"
+	black .
 
 version:
 	@echo ${VERSION}

docs/Makefile

@@ -23,4 +23,4 @@ start:
 	sphinx-autobuild . ./_build/html
 
 install-dev:
-	pip install -r ./requirements.txt
+	poetry install -C .. --with docs --with all-runtimes --with all-runtimes-dev

hack/build-wheels.sh

@@ -17,12 +17,12 @@ _buildWheel() {
   local _outputPath=$2
   local _currentDir=$PWD
 
-  # Python really expects the `setup.py` to be on the current folder, so we'll
-  # move into the source folder and then go back again.
+  # Poetry doesn't let us send the output to a separate folder so we'll `cd`
+  # into the folder and them move the wheels out
+  # https://github.com/python-poetry/poetry/issues/3586
   pushd $_srcPath
-  python setup.py \
-    sdist -d $_outputPath \
-    bdist_wheel -d $_outputPath
+  poetry build
+  cp ./dist/* $_outputPath
   popd
 }
 
@@ -35,7 +35,7 @@ _main() {
 
   # Build MLServer
   echo "---> Building MLServer wheel"
-  _buildWheel $ROOT_FOLDER $_outputPath
+  poetry build
 
   for _runtime in "$ROOT_FOLDER/runtimes/"*; do
     echo "---> Building MLServer runtime: '$_runtime'"

hack/generate-types.sh

@@ -21,6 +21,7 @@ _generatePydantic() {
 
   datamodel-codegen \
     --input "${ROOT_FOLDER}/openapi/$apiName.yaml" \
+    --input-file-type openapi \
     --output "${ROOT_FOLDER}/mlserver/types/$apiName.py" \
     --custom-template-dir "${ROOT_FOLDER}/hack/templates" \
     --base-class ".base.BaseModel" \

hack/update-version.sh

@@ -11,6 +11,15 @@ if [ "$#" -ne 1 ]; then
   exit 1
 fi
 
+_updatePyproject() {
+  local _newVersion=$1
+  local _pyproject=$2
+
+  sed \
+    -i "s/^version = \"\(.*\)\"$/version = \"$_newVersion\"/" \
+    "$_pyproject"
+}
+
 _updateVersion() {
   local _newVersion=$1
   local _versionPy=$2
@@ -33,6 +42,7 @@ _main() {
 
   # To call within `-exec`
   export -f _updateVersion
+  export -f _updatePyproject
 
   find $ROOT_FOLDER \
     -type f -name version.py \
@@ -42,6 +52,14 @@ _main() {
     \) \
     -exec bash -c "_updateVersion $_newVersion {}" \;
 
+  find $ROOT_FOLDER \
+    -type f -name pyproject.toml \
+    \( \
+    -path "$ROOT_FOLDER/*" -or \
+    -path "$ROOT_FOLDER/runtimes/*" \
+    \) \
+    -exec bash -c "_updatePyproject $_newVersion {}" \;
+
   _updateDocs $_newVersion
 }