Optional requirements related changes

Author: pranavrth

DEVELOPER.md

@@ -17,7 +17,7 @@ If librdkafka is installed in a non-standard location provide the include and li
 
 Install sphinx and sphinx_rtd_theme packages:
 
-    $ pip install sphinx sphinx_rtd_theme
+    $ pip install .[docs]
 
 Build HTML docs:
 

docs/conf.py

@@ -17,18 +17,13 @@
 import os
 from glob import glob
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path[:0] = [os.path.abspath(x) for x in glob('../build/lib.*')]
-
 # -- General configuration ------------------------------------------------
 
 
 ######################################################################
 # General information about the project.
 project = u'confluent-kafka'
-copyright = u'2016-2023, Confluent Inc.'
+copyright = u'2016-2024, Confluent Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

examples/README.md

@@ -30,8 +30,8 @@ To setup a venv with the latest release version of confluent-kafka and dependenc
 ```
 $ python3 -m venv venv_examples
 $ source venv_examples/bin/activate
-$ cd examples
-$ pip install -r requirements.txt
+$ pip install confluent_kafka
+$ pip install -r requirements/requirements-examples.txt
 ```
 
 To setup a venv that uses the current source tree version of confluent_kafka, you
@@ -42,11 +42,7 @@ need to have a C compiler and librdkafka installed
 ```
 $ python3 -m venv venv_examples
 $ source venv_examples/bin/activate
-$ rm -rf dist
-$ python -m build
-$ pip install -e dist/confluent_kafka*whl
-$ cd examples
-$ pip install -r requirements.txt
+$ pip install .[examples]
 ```
 
 When you're finished with the venv:

pyproject.toml

@@ -13,10 +13,10 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Topic :: Software Development :: Libraries :: Python Modules"]
-dependencies = []
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.7"
+dynamic = ["optional-dependencies","dependencies"]
 
 [[project.authors]]
 name = "Confluent Inc."
@@ -25,13 +25,17 @@ email = "support@confluent.io"
 [project.urls]
 Homepage = "https://github.com/confluentinc/confluent-kafka-python"
 
-[project.optional-dependencies]
-schema-registry = [ "requests",]
-avro = [ "fastavro>=1.0", "avro>=1.11.1,<2", "requests",]
-json = [ "pyrsistent", "jsonschema", "requests",]
-protobuf = [ "protobuf", "requests",]
-dev = [ "pytest", "pytest-timeout", "flake8", "fastavro>=1.0", "avro>=1.11.1,<2", "requests",]
-doc = [ "sphinx", "sphinx-rtd-theme", "fastavro>=1.0", "avro>=1.11.1,<2", "requests",]
-
 [tool.setuptools]
 include-package-data = false
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements/requirements.txt"]}
+optional-dependencies.schemaregistry = { file = ["requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.avro = { file = ["requirements/requirements-avro.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.json = { file = ["requirements/requirements-json.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.protobuf = { file = ["requirements/requirements-protobuf.txt", "requirements/requirements-schemaregistry.txt"] }
+optional-dependencies.dev = { file = ["requirements/requirements-tests.txt", "requirements/requirements-docs.txt", "requirements/examples.txt"] }
+optional-dependencies.docs = { file = ["requirements/requirements-docs.txt"] }
+optional-dependencies.tests = { file = ["requirements/requirements-tests.txt"] }
+optional-dependencies.examples = { file = ["requirements/examples.txt"] }
+optional-dependencies.soaktest = { file = ["requirements/soaktest.txt"] }

requirements/requirements-all.txt

@@ -0,0 +1,9 @@
+-r requirements.txt
+-r requirements-schemaregistry.txt
+-r requirements-avro.txt
+-r requirements-protobuf.txt
+-r requirements-json.txt
+-r requirements-examples.txt
+-r requirements-tests.txt
+-r requirements-docs.txt
+-r requirements-soaktest.txt

src/confluent_kafka/avro/requirements.txt renamed to requirements/requirements-avro.txt

@@ -1,3 +1,3 @@
 fastavro>=0.23.0
 requests
-avro>=1.11.1,<2
+avro>=1.11.1,<2

docs/requirements.txt renamed to requirements/requirements-docs.txt

@@ -0,0 +1,2 @@
+pyrsistent
+jsonschema

examples/requirements.txt renamed to requirements/requirements-examples.txt

@@ -0,0 +1 @@
+protobuf

requirements/requirements-json.txt

@@ -0,0 +1 @@
+requests

requirements/requirements-protobuf.txt

@@ -0,0 +1,8 @@
+urllib3<2.0.0;python_version<="3.7"
+urllib3
+flake8
+pytest
+pytest-timeout
+requests-mock
+trivup>=0.8.3
+build

requirements/requirements-schemaregistry.txt

@@ -18,7 +18,7 @@ A python3 env suitable for running tests:
 
     $ python3 -m venv venv_test
     $ source venv_test/bin/activate
-    $ pip install -r tests/requirements.txt
+    $ pip install -r requirements/requirements-tests.txt
     $ python3 -m pip install .
 
 When you're finished with it:

tests/soak/requirements.txt renamed to requirements/requirements-soaktest.txt

@@ -46,9 +46,7 @@ source $venv/bin/activate
 
 pip install -U pip
 
-pip install -v .
-
-pip install -r tests/soak/requirements.txt
+pip install -v .[soaktest]
 
 popd # ..python
 

requirements/requirements-tests.txt

@@ -92,7 +92,7 @@ for PYBIN in /opt/python/cp*/bin; do
             "${PYBIN}/pip" -V
             "${PYBIN}/pip" install --no-index -f /io/wheelhouse confluent_kafka 
             "${PYBIN}/python" -c 'import confluent_kafka; print(confluent_kafka.libversion())'
-            "${PYBIN}/pip" install -r /io/tests/requirements.txt
+            "${PYBIN}/pip" install -r /io/requirements/requirements-tests.txt
             "${PYBIN}/pytest" /io/tests/test_Producer.py
             echo "## Uninstalling $PYBIN"
             "${PYBIN}/pip" uninstall -y confluent_kafka

requirements/requirements.txt

@@ -60,7 +60,7 @@ for py in 3.8 ; do
         hash -r
 
         pip install -U pip pkginfo
-        pip install -r tests/requirements.txt
+        pip install -r requirements/requirements-tests.txt
 
         # Get the packages version so we can pin the install
         # command to this version (which hopefully loads it from the wheeldir

src/confluent_kafka/requirements.txt

@@ -5,9 +5,9 @@
 #
 set -e
 
-pip install -r docs/requirements.txt
+pip install -r requirements/requirements-docs.txt
 pip install -U protobuf
-pip install -r tests/requirements.txt
+pip install -r requirements/requirements-tests.txt
 pip install -U build
 
 lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
@@ -17,9 +17,7 @@ export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
 export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/$lib_dir"
 
-rm -rf dist
-python3 -m build
-pip install dist/confluent_kafka*.whl
+python3 -m pip install .
 
 if [[ $OS_NAME == linux && $ARCH == x64 ]]; then
     flake8 --exclude ./_venv,*_pb2.py

src/confluent_kafka/schema_registry/requirements.txt

@@ -15,7 +15,7 @@ echo on
 
 set CIBW_BUILD=cp37-%BW_ARCH% cp38-%BW_ARCH% cp39-%BW_ARCH% cp310-%BW_ARCH% cp311-%BW_ARCH% cp312-%BW_ARCH%
 set CIBW_BEFORE_BUILD=python -m pip install delvewheel==1.1.4
-set CIBW_TEST_REQUIRES=-r tests/requirements.txt
+set CIBW_TEST_REQUIRES=-r requirements/requirements-tests.txt
 set CIBW_TEST_COMMAND=pytest {project}\tests\test_Producer.py
 rem set CIBW_BUILD_VERBOSITY=3
 set include=%cd%\%DEST%\build\native\include

tests/README.md

@@ -10,7 +10,7 @@ this_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Skip PyPy, Python2, old Python3 versions, musl, and x86 builds.
 export CIBW_SKIP="pp* cp27-* cp35-* cp36-* *i686 *musllinux* $CIBW_SKIP"
 # Run a simple test suite
-export CIBW_TEST_REQUIRES="-r tests/requirements.txt"
+export CIBW_TEST_REQUIRES="-r requirements/requirements-tests.txt"
 export CIBW_TEST_COMMAND="pytest {project}/tests/test_Producer.py"
 
 

tests/requirements.txt

@@ -10,7 +10,7 @@ set
 rem Download and install librdkafka from NuGet.
 call tools\windows-install-librdkafka.bat %LIBRDKAFKA_NUGET_VERSION% dest || exit /b 1
 
-pip install -U -r tests/requirements.txt -r src/confluent_kafka/avro/requirements.txt
+pip install -U -r requirements/requirements-tests.txt -r requirements/requirements-avro.txt
 pip install cibuildwheel==0.12.0 || exit /b 1
 
 rem Build wheels (without tests)
@@ -51,8 +51,8 @@ for %%W in (wheelhouse\confluent_kafka-*cp%PYTHON_SHORTVER%*win*%PYTHON_ARCH%.wh
   python -c "import struct; print(struct.calcsize('P') * 8)"
   7z l %%~W
   pip install %%~W || exit /b 1
-  pip install -r src\confluent_kafka\requirements.txt
-  pip install -r src\confluent_kafka\avro\requirements.txt
+  pip install -r requirements\requirements.txt
+  pip install -r requirements\requirements-avro.txt
 
   python -c "from confluent_kafka import libversion ; print(libversion())" || exit /b 1
 

tests/soak/ubuntu-bootstrap.sh

@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py37,py38,py39,py310
+envlist = flake8,py37,py38,py39,py310,py311,py312
 
 [testenv]
 setenv =
@@ -22,7 +22,7 @@ commands =
 
 deps =
     # https://docs.pytest.org/en/latest/changelog.html#id53
-    -rtests/requirements.txt
+    -rrequirements/requirement-dev.txt
 
 [testenv:flake8]
 deps = flake8