Modernize Python tooling

Move all configs into `pyproject.toml`, switch test/packaging to
use Hatch, and do all the linting with Ruff.

Fix issues that Ruff turned up, and annotate/ignore the pieces that
it got wrong.

Fixes: Issue NicolasLM#33

.github/workflows/python-publish.yml

@@ -18,11 +18,11 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install hatch twine
     - name: Build and publish
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist bdist_wheel
+        hatch build
         twine upload dist/*

.github/workflows/tests.yml

@@ -28,15 +28,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e .[tests]
+        pip install hatch
 
     - name: Lint
       run: |
-        pycodestyle --ignore=E252,W503,W504 spinach tests
+        hatch run pep8
 
     - name: Test with pytest
       run: |
-        pytest -v --cov=spinach tests/
+        hatch run ci
 
     - name: Coveralls
       uses: AndreMiras/coveralls-python-action@develop

.gitignore

@@ -1,3 +1,7 @@
+# _version is generated by setuptools_scm and requests not to be under a VCS
+spinach/_version.py
+
+.hatch
 .pyc
 __pycache__
 

doc/conf.py

@@ -22,6 +22,7 @@
 sys.path.insert(0, os.path.abspath('..'))
 
 import spinach
+from spinach import _version as spinach_version
 
 # -- General configuration ------------------------------------------------
 
@@ -66,9 +67,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = spinach.__version__
+version = spinach_version.__version__
 # The full version, including alpha/beta/rc tags.
-release = spinach.__version__
+release = spinach_version.__version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

pyproject.toml

@@ -0,0 +1,114 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "spinach"
+dynamic = ["version"]
+description = "Modern Redis task queue for Python 3"
+readme = "README.rst"
+license-files = { paths = ["LICENSE"] }
+requires-python = ">=3.8.0,<4.0.0"
+authors = [
+    { name = "Nicolas Le Manchet", email = "nicolas@lemanchet.fr" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: System :: Distributed Computing",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["task", "queue", "jobs", "redis"]
+urls.Source = "https://github.com/NicolasLM/spinach"
+
+dependencies = [
+    "redis",
+    "blinker"
+]
+
+[project.optional-dependencies]
+tests = [
+    "pytest",
+    "pytest-cov",
+    "pytest-threadleak",
+    "ruff",
+    "flask",
+    "django"
+]
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "spinach/_version.py"
+
+[tool.hatch.build.targets.sdist]
+ignore-vcs = true
+include = ["/spinach", "/tests"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["spinach"]
+
+[tool.hatch.build.targets.wheel.package-data]
+"spinach.brokers.redis_scripts" = [
+    "deregister.lua",
+    "enqueue_job.lua",
+    "enqueue_jobs_from_dead_broker.lua",
+    "flush.lua",
+    "get_jobs_from_queue.lua",
+    "move_future_jobs.lua",
+    "register_periodic_tasks.lua",
+    "remove_job_from_running.lua",
+    "set_concurrency_keys.lua",
+]
+
+[tool.hatch.envs.default]
+path = ".hatch"
+features = ["tests"]
+
+scripts.pep8 = ["ruff spinach tests"]
+
+scripts.py3 = [
+    "docker-compose -f tests/docker-compose.yml up -d",
+    "pytest tests {args}",
+    "docker-compose -f tests/docker-compose.yml down",
+]
+
+# A minimalist pytest runner for Github to use
+scripts.ci = ["pytest -v --cov=spinach tests {args}"]
+
+[tool.ruff.lint]
+select = ["B", "C9", "D", "E", "F", "S", "W"]
+ignore = [
+    "B018",  # Useless expression (seems to have false-positive bugs)
+    "S101",  # Use of `assert` detected (conflicts with pytest)
+    "D10",  # Missing docstring in public function / module / etc.
+    "D203",  # 1 blank line required before class docstring
+    "D205",  # 1 blank line required between summary line and description
+    "D213",  # Multi-line docstring summary should start at the second line
+    "D40",  # First line of docstring complaints
+    "D415",  # First line should end with a period, question mark, or exclamation point
+]
+mccabe.max-complexity = 13
+
+[tool.ruff.lint.per-file-ignores]
+# Aggregating symbols into a convenient location for import is good practice
+# for libraries, we don't care if they look unused
+"__init__.py" = ["F401"]
+
+[tool.coverage.run]
+relative_files = true
+include = ["spinach/*"]
+omit = ["spinach/_version.py"]
+
+[tool.pytest.ini_options]
+threadleak = true

spinach/__init__.py

@@ -1,8 +1,5 @@
 from .brokers.memory import MemoryBroker
 from .brokers.redis import RedisBroker
-from .const import VERSION
 from .engine import Engine
 from .task import Tasks, Batch, RetryException, AbortException
 from .worker import ThreadWorkers, AsyncioWorkers
-
-__version__ = VERSION

spinach/brokers/base.py

@@ -50,7 +50,7 @@ def wait_for_event(self):
         if self._something_happened.wait(timeout=timeout):
             self._something_happened.clear()
 
-    def start(self):
+    def start(self):  # noqa: B027
         """Start the broker.
 
         Only needed by arbiter.

spinach/const.py

@@ -1,5 +1,3 @@
-VERSION = '0.0.20'
-
 DEFAULT_QUEUE = 'spinach'
 DEFAULT_NAMESPACE = 'spinach'
 DEFAULT_MAX_RETRIES = 0

spinach/contrib/flask_spinach.py

@@ -82,14 +82,14 @@ def spin(self):
             return flask.current_app.extensions['spinach']
         except (AttributeError, TypeError, KeyError):
             raise RuntimeError('Spinach extension not initialized. '
-                               'Did you forget to call init_app?')
+                               'Did you forget to call init_app?') from None
 
     def register_tasks(self, app, tasks):
         try:
             app.extensions['spinach'].attach_tasks(tasks)
         except KeyError:
             raise RuntimeError('Spinach extension not initialized. '
-                               'Did you forget to call init_app?')
+                               'Did you forget to call init_app?') from None
 
     # Convenience access to common Engine attributes and methods
 

spinach/contrib/spinachd/mail.py

@@ -38,6 +38,6 @@ def serialize_email_messages(messages: List[EmailMessage]):
 def deserialize_email_messages(messages: List[str]):
     """Deserialize EmailMessages passed as task argument."""
     return [
-        pickle.loads(zlib.decompress(base64.b64decode(m)))
+        pickle.loads(zlib.decompress(base64.b64decode(m)))  # noqa: S301
         for m in messages
     ]

spinach/job.py

@@ -154,7 +154,7 @@ def check_signature(self):
             msg = 'Arguments of job not compatible with task {}: {}'.format(
                 self.task_name, e
             )
-            raise InvalidJobSignatureError(msg)
+            raise InvalidJobSignatureError(msg) from None
         except ValueError:
             logger.info('Cannot verify job signature, assuming it is correct')
 

spinach/task.py

@@ -81,6 +81,7 @@ class Tasks:
     :arg max_concurrency: maximum number of simultaneous Jobs that can be
         started for this Task. Requires max_retries to be also set.
     """
+
     # This class is not thread-safe because it doesn't need to be used
     # concurrently.
 

spinach/utils.py

@@ -75,7 +75,7 @@ def exponential_backoff(attempt: int, cap: int=1200) -> timedelta:
     """
     base = 3
     temp = min(base * 2 ** attempt, cap) // 2
-    return timedelta(seconds=temp + random.randint(0, temp))
+    return timedelta(seconds=temp + random.randint(0, temp))  # noqa: S311
 
 
 @contextlib.contextmanager

spinach/worker.py

@@ -1,4 +1,3 @@
-from abc import ABC
 import asyncio
 from logging import getLogger
 try:
@@ -18,7 +17,7 @@
 logger = getLogger(__name__)
 
 
-class BaseWorkers(ABC):
+class BaseWorkers:
     """Base class for Spinach workers.
 
     The Workers class receives jobs from the Engine via the `submit_job`

tests/functional/test_concurrency.py

@@ -1,5 +1,4 @@
 import pytest
-import time
 
 from spinach.brokers.memory import MemoryBroker
 from spinach.brokers.redis import RedisBroker

tests/test_memory_brokers.py

@@ -2,7 +2,7 @@
 import pytest
 
 from spinach.brokers.memory import MemoryBroker
-from spinach.job import Job, JobStatus
+from spinach.job import Job
 from spinach.task import Task
 
 

tests/test_redis_brokers.py

@@ -199,7 +199,7 @@ def test_get_jobs_from_queue_returns_all_requested(broker):
 
 
 def test_set_concurrency_keys_sets_new_keys_on_idempotent_tasks(broker):
-    max_c = random.randint(1, 10)
+    max_c = random.randint(1, 10)  # noqa: S311
     tasks = [
         Task(
             print, 'foo', 'q1', 10, timedelta(seconds=5),
@@ -229,7 +229,7 @@ def test_set_concurrency_keys_ignores_tasks_without_concurrency(broker):
 
 
 def test_set_concurrency_keys_doesnt_overwrite_existing_concurrency(broker):
-    current_c = random.randint(1, 10)
+    current_c = random.randint(1, 10)  # noqa: S311
     set_concurrency_keys(broker, current_concurrency=current_c)
     tasks = [
         Task(