Merge upstream 2.3.0 to main

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,11 @@ jobs:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -51,7 +51,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make bootstrap
     #   make release
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v4

.github/workflows/python-package.yml

@@ -1,198 +1,92 @@
-name: CI/CD
+# Derived from https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml
+#
+name: Python Package
 
 on:
   push:
     branches: ["master"]
   pull_request:
     branches: ["master"]
-  release:
-    types: [created]
-    branches:
-      - 'master'
-  workflow_dispatch:
 
 env:
   FORCE_COLOR: "1"  # Make tools pretty.
   PIP_DISABLE_PIP_VERSION_CHECK: "1"
   PIP_NO_PYTHON_VERSION_WARNING: "1"
-  PYTHON_LATEST: "3.12"
-  KAFKA_LATEST: "2.6.0"
-
-  # For re-actors/checkout-python-sdist
-  sdist-artifact: python-package-distributions
 
 jobs:
+  build:
 
-  build-sdist:
-    name: 📦 Build the source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout project
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_LATEST }}
-          cache: pip
-      - run: python -m pip install build
-        name: Install core libraries for build and install
-      - name: Build artifacts
-        run: python -m build
-      - name: Upload built artifacts for testing
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.sdist-artifact }}
-          # NOTE: Exact expected file names are specified here
-          # NOTE: as a safety measure — if anything weird ends
-          # NOTE: up being in this dir or not all dists will be
-          # NOTE: produced, this will fail the workflow.
-          path: dist/${{ env.sdist-name }}
-          retention-days: 15
-
-  test-python:
-    name: Tests on ${{ matrix.python-version }}
-    needs:
-      - build-sdist
     runs-on: ubuntu-latest
-    continue-on-error: ${{ matrix.experimental }}
+    name: "Test: python ${{ matrix.python }} / kafka ${{ matrix.kafka }}"
+    continue-on-error: ${{ matrix.experimental || false }}
     strategy:
       fail-fast: false
       matrix:
-        python-version:
-          - "3.8"
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        experimental: [ false ]
-        include:
-          - python-version: "pypy3.9"
-            experimental: true
-          - python-version: "~3.13.0-0"
-            experimental: true
-    steps:
-      - name: Checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Setup java
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 11
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: |
-            requirements-dev.txt
-      - name: Check Java installation
-        run: source travis_java_install.sh
-      - name: Pull Kafka releases
-        run: ./build_integration.sh
-        env:
-          PLATFORM: ${{ matrix.platform }}
-          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
-        # TODO: Cache releases to expedite testing
-      - name: Install dependencies
-        run: |
-          sudo apt install -y libsnappy-dev libzstd-dev
-          python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
-          pip install .
-          pip install -r requirements-dev.txt
-      - name: Test with tox
-        run: tox
-        env:
-          PLATFORM: ${{ matrix.platform }}
-          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
-
-  test-kafka:
-    name: Tests for Kafka ${{ matrix.kafka-version }}
-    needs:
-      - build-sdist
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        kafka-version:
+        kafka:
           - "0.8.2.2"
           - "0.9.0.1"
           - "0.10.2.2"
-          - "0.11.0.2"
           - "0.11.0.3"
           - "1.1.1"
           - "2.4.0"
-          - "2.5.0"
-          - "2.6.0"
+          - "2.8.2"
+          - "3.0.2"
+          - "3.5.2"
+          - "3.9.0"
+          - "4.0.0"
+        python:
+          - "3.13"
+        include:
+          #- python: "pypy3.9"
+          #  kafka: "2.6.0"
+          #  experimental: true
+          - python: "3.8"
+            kafka: "4.0.0"
+          - python: "3.9"
+            kafka: "4.0.0"
+          - python: "3.10"
+            kafka: "4.0.0"
+          - python: "3.11"
+            kafka: "4.0.0"
+          - python: "3.12"
+            kafka: "4.0.0"
+
     steps:
-      - name: Checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Setup java
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v6
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v6
         with:
-          python-version: ${{ env.PYTHON_LATEST }}
+          python-version: ${{ matrix.python }}
           cache: pip
           cache-dependency-path: |
             requirements-dev.txt
-      - name: Pull Kafka releases
-        run: ./build_integration.sh
-        env:
-          # This is fast enough as long as you pull only one release at a time,
-          # no need to worry about caching
-          PLATFORM: ${{ matrix.platform }}
-          KAFKA_VERSION: ${{ matrix.kafka-version }}
       - name: Install dependencies
         run: |
           sudo apt install -y libsnappy-dev libzstd-dev
           python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
-          pip install .
           pip install -r requirements-dev.txt
-      - name: Test with tox
-        run: tox
-        env:
-          PLATFORM: ${{ matrix.platform }}
-          KAFKA_VERSION: ${{ matrix.kafka-version }}
-
-  check:  # This job does nothing and is only used for the branch protection
-    name: ✅ Ensure the required checks passing
-    if: always()
-    needs:
-      - build-sdist
-      - test-python
-      - test-kafka
-    runs-on: ubuntu-latest
-    steps:
-    - name: Decide whether the needed jobs succeeded or failed
-      uses: re-actors/alls-green@release/v1
-      with:
-        jobs: ${{ toJSON(needs) }}
-  publish:
-    name: 📦 Publish to PyPI
-    runs-on: ubuntu-latest
-    needs: [build-sdist]
-    permissions:
-      id-token: write
-    environment: pypi
-    if: github.event_name == 'release' && github.event.action == 'created'
-    steps:
-      - name: Download the sdist artifact
-        uses: actions/download-artifact@v3
+      - name: Pylint
+        run: pylint --recursive=y --errors-only --exit-zero kafka test
+      - name: Setup java
+        uses: actions/setup-java@v5
         with:
-          name: artifact
-          path: dist
-      - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+          distribution: temurin
+          java-version: 23
+      - name: Restore cached kafka releases
+        id: cache-servers-dist-restore
+        uses: actions/cache/restore@v4
         with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+          path: servers/dist
+          key: servers-dist-${{ matrix.kafka }}
+      - name: Install Kafka release
+        run: make servers/${{ matrix.kafka }}/kafka-bin
+      - name: Update kafka release cache
+        id: cache-servers-dist-save
+        uses: actions/cache/save@v4
+        with:
+          path: servers/dist
+          key: ${{ steps.cache-servers-dist-restore.outputs.cache-primary-key }}
+      - name: Pytest
+        run: make test
+        env:
+          KAFKA_VERSION: ${{ matrix.kafka }}

.gitignore

@@ -15,3 +15,5 @@ docs/_build
 integration-test/
 tests-env/
 .pytest_cache/
+.envrc
+shell.nix

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt