cicd: add Ruff linting and formatting (#376)

Co-authored-by: Kori Kuzma <korikuzma@gmail.com>

.github/workflows/checks.yaml

@@ -1,65 +1,64 @@
 name: checks
 on: [push, pull_request]
 jobs:
-    test:
-        name: test py${{ matrix.python-version }}
-        runs-on: ubuntu-latest
-        env:
-            AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
-            AWS_DEFAULT_REGION: us-east-2
-            AWS_DEFAULT_OUTPUT: text
-            DISEASE_NORM_DB_URL: http://localhost:8002
-            THERAPY_NORM_DB_URL: http://localhost:8002
-            THERAPY_TEST: true
-        strategy:
-            matrix:
-              python-version: ['3.8', '3.9', '3.10', '3.11']
-        steps:
-            - uses: actions/checkout@v3
+  test:
+    name: test py${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      AWS_DEFAULT_OUTPUT: text
+      DISEASE_NORM_DB_URL: http://localhost:8002
+      THERAPY_NORM_DB_URL: http://localhost:8002
+      THERAPY_TEST: true
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
 
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: ${{ matrix.python-version }}
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
 
-            - name: Install dependencies
-              run: |
-                  python3 -m pip install ".[dev,test]"
+      - name: Install dependencies
+        run: |
+          python3 -m pip install ".[dev,test]"
 
-            - name: Build local DynamoDB
-              run: |
-                  chmod +x ./tests/scripts/dynamodb_run.sh
-                  ./tests/scripts/dynamodb_run.sh
+      - name: Build local DynamoDB
+        run: |
+          chmod +x ./tests/scripts/dynamodb_run.sh
+          ./tests/scripts/dynamodb_run.sh
 
-            - name: Run tests
-              run: python3 -m pytest
-    lint:
-        name: lint py${{ matrix.python-version }}
-        runs-on: ubuntu-latest
-        env:
-            AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
-            AWS_DEFAULT_REGION: us-east-2
-            AWS_DEFAULT_OUTPUT: text
-        strategy:
-            matrix:
-              python-version: ['3.8', '3.9', '3.10', '3.11']
-        steps:
-            - uses: actions/checkout@v3
+      - name: Run tests
+        run: python3 -m pytest
+  lint:
+    name: lint py${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.DUMMY_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.DUMMY_AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-east-2
+      AWS_DEFAULT_OUTPUT: text
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
 
-            - name: Setup Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: ${{ matrix.python-version }}
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
 
-            - name: Install dependencies
-              run: python3 -m pip install ".[dev,test]"
+      - name: Install dependencies
+        run: python3 -m pip install ".[dev,test]"
 
-            - name: Check style
-              run: python3 -m flake8 therapy/ tests/ setup.py
-
-            - name: Check type correctness
-              if: ${{ always() }}
-              run: python3 -m mypy --ignore-missing-imports therapy/
+      - name: Check style
+        run: python3 -m ruff check . && ruff format --check .
 
+      - name: Check type correctness
+        if: ${{ always() }}
+        run: python3 -m mypy --ignore-missing-imports therapy/

.pre-commit-config.yaml

@@ -1,11 +1,13 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v1.4.0
     hooks:
-    - id: flake8
-      additional_dependencies: [flake8-docstrings, flake8-quotes, flake8-import-order, flake8-annotations]
-    - id: check-added-large-files
-      args: ['--maxkb=2500']
-    - id: detect-private-key
+      - id: check-added-large-files
+      - id: detect-private-key
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.2
+    hooks:
+      - id: ruff
+      - id: ruff-format

Pipfile

@@ -23,11 +23,7 @@ ipykernel = "*"
 pytest = "*"
 pytest-cov = "*"
 pre-commit = "*"
-flake8 = "*"
-flake8-docstrings = "*"
-flake8-quotes = "*"
-flake8-import-order = "*"
-flake8-annotations = "*"
+ruff = ">=0.1.2"
 ipython = ">=8.10.0"
 jupyterlab = "*"
 civicpy = "*"

README.md

@@ -21,7 +21,7 @@ We use Amazon DynamoDB for data storage. To deploy locally, follow [these instru
 
 ### Initialize development environment
 
-Code style is managed by [flake8](https://github.com/PyCQA/flake8) and checked prior to commit.
+Code style is managed by [Ruff](https://docs.astral.sh/ruff/) and checked prior to commit.
 
 We use [pre-commit](https://pre-commit.com/#usage) to run conformance tests.
 

pyproject.toml

@@ -1,3 +1,43 @@
 [build-system]
 requires = ["setuptools", "wheel"]
 build-backend = "setuptools.build_meta:__legacy__"
+
+[tool.ruff]
+# pycodestyle (E, W)
+# Pyflakes (F)
+# flake8-annotations (ANN)
+# pydocstyle (D)
+# pep8-naming (N)
+# isort (I)
+select = ["E", "W", "F", "ANN", "D", "N", "I"]
+
+fixable = ["I", "F401"]
+
+# D203 - one-blank-line-before-class
+# D205 - blank-line-after-summary
+# D206 - indent-with-spaces*
+# D213 - multi-line-summary-second-line
+# D400 - ends-in-period
+# D415 - ends-in-punctuation
+# ANN101 - missing-type-self
+# ANN003 - missing-type-kwargs
+# E501 - line-too-long
+# W191 - tab-indentation*
+# *ignored for compatibility with formatter
+ignore = ["D203", "D205", "D206", "D213", "D400", "D415", "ANN101", "ANN003", "E501", "Q", "W191"]
+
+[tool.ruff.per-file-ignores]
+# ANN001 - missing-type-function-argument
+# ANN2 - missing-return-type
+# ANN201 - Missing type annotation
+# ANN102 - missing-type-cls
+# D103 - Missing docstring in public function
+# F821 - undefined-name
+# F401 - unused-import
+# I001 - Import block unsorted or unformatted
+# N805 - invalid-first-argument-name-for-method
+"tests/*" = ["ANN001", "ANN102", "ANN2"]
+"setup.py" = ["F821"]
+"*__init__.py" = ["F401"]
+"therapy/schemas.py" = ["ANN001", "ANN201", "N805"]
+"docs/source/conf.py" = ["D100", "I001", "D103", "ANN201", "ANN001"]

setup.cfg

@@ -42,16 +42,12 @@ dev =
     ipython >= 8.10
     pre-commit
     tox
-    flake8
-    flake8-docstrings
-    flake8-quotes
-    flake8-import-order
-    flake8-annotations
     mypy
     types-requests
     types-pyyaml
     lxml
     xmlformatter
+    ruff >= 0.1.2
 
 [tool:pytest]
 addopts = --ignore setup.py --ignore=analysis/ --cov-report term-missing --cov=therapy

tests/conftest.py

@@ -1,22 +1,22 @@
 """Pytest test config tools."""
-import os
-from typing import Optional, List, Callable
 import json
+import os
 from pathlib import Path
+from typing import Callable, List, Optional
 
 import pytest
 
+from therapy.database import AWS_ENV_VAR_NAME, Database
 from therapy.etl.base import Base
 from therapy.query import QueryHandler
-from therapy.schemas import Drug, MatchType, MatchesKeyed
-from therapy.database import AWS_ENV_VAR_NAME, Database
+from therapy.schemas import Drug, MatchesKeyed, MatchType
 
 
 def pytest_collection_modifyitems(items):
     """Modify test items in place to ensure test modules run in a given order.
     When creating new test modules, be sure to add them here.
     """
-    MODULE_ORDER = [
+    MODULE_ORDER = [  # noqa: N806
         "test_chembl",
         "test_chemidplus",
         "test_drugbank",
@@ -30,7 +30,7 @@ def pytest_collection_modifyitems(items):
         "test_database",
         "test_query",
         "test_emit_warnings",
-        "test_disease_indication"
+        "test_disease_indication",
     ]
     items.sort(key=lambda i: MODULE_ORDER.index(i.module.__name__))
 
@@ -51,9 +51,7 @@ def db():
     database = Database()
     if os.environ.get("THERAPY_TEST", "").lower() == "true":
         if os.environ.get(AWS_ENV_VAR_NAME):
-            assert False, (
-                f"Cannot have both THERAPY_TEST and {AWS_ENV_VAR_NAME} set."
-            )
+            assert False, f"Cannot have both THERAPY_TEST and {AWS_ENV_VAR_NAME} set."
         existing_tables = database.dynamodb_client.list_tables()["TableNames"]
         if "therapy_concepts" in existing_tables:
             database.dynamodb_client.delete_table(TableName="therapy_concepts")
@@ -78,23 +76,21 @@ def _normalize_disease(query: str):
 
 
 @pytest.fixture(scope="session")
-def test_source(
-        db: Database, test_data: Path, disease_normalizer: Callable
-):
+def test_source(db: Database, test_data: Path, disease_normalizer: Callable):
     """Provide query endpoint for testing sources. If THERAPY_TEST is set, will try to
     load DB from test data.
     :return: factory function that takes an ETL class instance and returns a query
     endpoint.
     """
-    def test_source_factory(EtlClass: Base):
+
+    def test_source_factory(EtlClass: Base):  # noqa: N803
         if os.environ.get("THERAPY_TEST", "").lower() == "true":
             test_class = EtlClass(db, test_data)  # type: ignore
             test_class._normalize_disease = disease_normalizer  # type: ignore
             test_class.perform_etl(use_existing=True)
             test_class.database.flush_batch()
 
         class QueryGetter:
-
             def __init__(self):
                 self.query_handler = QueryHandler()
                 self._src_name = EtlClass.__name__  # type: ignore
@@ -157,8 +153,11 @@ def compare_records():
 
 
 def _compare_response(
-    response: MatchesKeyed, match_type: MatchType, fixture: Optional[Drug] = None,
-    fixture_list: Optional[List[Drug]] = None, num_records: int = 0
+    response: MatchesKeyed,
+    match_type: MatchType,
+    fixture: Optional[Drug] = None,
+    fixture_list: Optional[List[Drug]] = None,
+    num_records: int = 0,
 ):
     """Check that test response is correct. Only 1 of {fixture, fixture_list}
     should be passed as arguments. num_records should only be passed with fixture_list.
@@ -176,8 +175,7 @@ def _compare_response(
     elif not fixture and not fixture_list:
         raise Exception("Must pass 1 of {fixture, fixture_list}")
     if fixture and num_records:
-        raise Exception("`num_records` should only be given with "
-                        "`fixture_list`.")
+        raise Exception("`num_records` should only be given with " "`fixture_list`.")
 
     assert response.match_type == match_type
     if fixture:

tests/scripts/build_chembl_data.py

@@ -1,6 +1,6 @@
 """Construct test data for ChEMBL source."""
-from pathlib import Path
 import sqlite3
+from pathlib import Path
 
 from therapy.database import Database
 from therapy.etl import ChEMBL