refactor: use pyproject.toml with ruff

.pre-commit-config.yaml

@@ -1,40 +1,34 @@
 # unless otherwise specified, hooks run on push only
 default_stages: [push]
 repos:
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.16.23
+    hooks:
+      - id: typos
   # formatters and linters are available in the virtualenv so they can be run from the makefile & vscode
   - repo: local
     hooks:
+      - id: ruff
+        name: ruff
+        entry: .venv/bin/ruff
+        language: system
+        types: [python]
       - id: black
         name: black
         entry: .venv/bin/black
         language: system
         types: [python]
         require_serial: true
-      - id: isort
-        name: isort
-        entry: .venv/bin/isort
-        language: system
-        types: [python]
-  - repo: local
-    hooks:
-      - id: flake8
-        name: flake8
-        entry: .venv/bin/flake8
-        language: system
-        types: [python]
-  # these hooks require the project's virtualenv
-  - repo: local
-    hooks:
       - id: pyright
         name: pyright
-        entry: make pyright
-        # run on all files
+        entry: node_modules/.bin/pyright
+        # run on all files to catch type errors that affect unchanged files
         pass_filenames: false
         language: system
         types: [python]
       - id: test
         name: test
-        entry: make test
+        entry: .venv/bin/pytest
         # run on all files
         pass_filenames: false
         language: system

CONTRIBUTING.md

@@ -10,7 +10,7 @@
 
 `. .venv/bin/activate` activates the virtualenv.
 
-The make targets will update the virtualenv when _setup.py_ changes.
+The make targets will update the virtualenv when _pyproject.toml_ changes.
 
 ## Usage
 

Makefile-common.mk

@@ -1,7 +1,7 @@
 MAKEFLAGS += --warn-undefined-variables
 SHELL = /bin/bash -o pipefail
 .DEFAULT_GOAL := help
-.PHONY: help clean install format check lint pyright test hooks install-hooks
+.PHONY: help clean install format check pyright test dist hooks install-hooks
 
 ## display help message
 help:
@@ -10,57 +10,48 @@ help:
 venv ?= .venv
 pip := $(venv)/bin/pip
 
-$(pip):
+$(pip): $(if $(value CI),|,) .python-version
 # create venv using system python even when another venv is active
 	PATH=$${PATH#$${VIRTUAL_ENV}/bin:} python3 -m venv --clear $(venv)
 	$(venv)/bin/python --version
-	$(pip) install pip~=23.1 wheel~=0.40
+	$(pip) install pip~=23.3 wheel~=0.40
 
-$(venv): setup.py $(pip)
+$(venv): $(if $(value CI),|,) pyproject.toml $(pip)
 	$(pip) install -e '.[dev]'
 	touch $(venv)
 
+node_modules: package.json
+	npm install --no-save
+	touch node_modules
+
 # delete the venv
 clean:
 	rm -rf $(venv)
 
 ## create venv and install this package and hooks
 install: $(venv) node_modules $(if $(value CI),,install-hooks)
 
-## format all code
-format: $(venv)
-	$(venv)/bin/autoflake --in-place --recursive --remove-all-unused-imports --exclude venv .
-	$(venv)/bin/black .
-	$(venv)/bin/isort .
-
-## lint code and run static type check
-check: lint pyright
+## format, lint and type check
+check: export SKIP=test
+check: hooks
 
-## lint using flake8
-lint: $(venv)
-	$(venv)/bin/flake8
+## format and lint
+format: export SKIP=pyright,test
+format: hooks
 
-node_modules: package.json
-	npm install --no-save
-	touch node_modules
-
-## pyright
+## pyright type check
 pyright: node_modules $(venv)
-# activate venv so pyright can find dependencies
-	PATH="$(venv)/bin:$$PATH" node_modules/.bin/pyright
+	node_modules/.bin/pyright
 
 ## run tests
 test: $(venv)
 	$(venv)/bin/pytest
 
 ## run pre-commit git hooks on all files
-hooks: $(venv)
-	$(venv)/bin/pre-commit run --show-diff-on-failure --color=always --all-files --hook-stage push
-
-install-hooks: .git/hooks/pre-commit .git/hooks/pre-push
+hooks: node_modules $(venv)
+	$(venv)/bin/pre-commit run --color=always --all-files --hook-stage push
 
-.git/hooks/pre-commit: $(venv)
-	$(venv)/bin/pre-commit install -t pre-commit
+install-hooks: .git/hooks/pre-push
 
 .git/hooks/pre-push: $(venv)
-	$(venv)/bin/pre-commit install -t pre-push
+	$(venv)/bin/pre-commit install --install-hooks -t pre-push

docs/blocks.md

@@ -77,7 +77,7 @@ select * from block_document;
 
 ```
 id|created|updated|name|data|block_schema_id|is_default_storage_block_document|block_type_id|is_anonymous
-b61967e0-f665-41a4-84d1-cf06f5514f2c|2022-07-09 06:41:41.242948|2022-07-09 06:41:41.243058|anonymous:6e5336ad1712905d5ca107b432a50e99|"gAAAAABiySMl_NR-DYaDKhF-5Xeqhp_-sawtFJwyB3NE4oHxGrNwRo1JpWWS4dHAUtUn-9sLzFQtgGESZI-YcTyT0pYwGT_kSBBgE7Q67GiupwRLmhpZRaqPFzfF0h0-ozwL0vZPqS3OMqpMoWqN9XxOgMsHfwiwRxMhBqtZsRovAVusn8IdxsMt-SzaBASlE2O1tbrZuKbLvHCDFTuhvIGEZ2hEO8Uar6hOPcsw7w7GaOsydft10GaXVDlmYMPk9LwgyijxxnsLhRRMyCV3z0oOoQ8rwmgR-jhDAn0A1eVezeLMDlkuBCOzM5Ky-ngjLa2iFCzgaR_rcaRi7Jg9umLg-L7ZA8oUlKu-9YG0AyWrFgHZm20jZjNYzXLdcn6hzVJQ1kLikt4TyXuvo1Pzff2YAFdaxZfWI1bWADGAmTrJYVbV1uhFpBBEPO0E-iSIruHWhaXf7w9rETdmf5dQ1Nz1YjdB-3GKdwKatCiCpP2gVbBst9hVoFAFeptSZuriItwqbhtAGPhwyawUNRL40LcxMmSRbcxYosFuYcuNIVtmNB_YC3wSTFa--uB6weTYtZ_fWSc3_7Ll22fCKlm_qcbZ-TfxkQOcoBdfTOa42ARBi64wVH9OH-MTMawU955t2CJnCpd_Ou3HDy-fYccr30qfTTfeVKC_d4ODh6itQ8M0DYogTJz6H2U09S2PjcvYVsA9jn1qXMXr70RnGYs0YtACR19Sst8n_ZzMGT8qnHlCpY4fYQkXWQLNANtlY-PoZSJjruin8Lc7J9rXSVhN9k6ty1DfsfGmq5JT3v5U9S7eTTKj1bfflc4_1tJj1wW6Cx5eIWdT_aKGQMjO8ucbzKeQIZ6EDxdGxVaD_7_IILG-S1lmjZuj1eH5ExbVVJtDIv2azIqF4QaSjJrBsN1SoPUvLdjXgrs3bG7hzRtqntlzHxID0sRw7vA7oHYts_Y5cuRCYA-hzXaKNrQzaO-b9J7b5jwGlKcYJnSmBEB5Q6Xv_-dQBJDra48j3G4w5ILhAnPlciOQFiBt_CrHaE02dhxsRXkGDgD-b4HsRSADxt3lPkNtr94-1nyFJsnb-DECnyPNqmNJAQDaW9Bt60w-n-QqOwTAF87ZtXOTwFCjBfIswR4D6FYbEo4RDPgA3_YmHjVHI1QTSzLa5EFSKyHNystLh_WuKHSvs5LQ9XDAbfal437QSFWaFF7uK3D1afkqGeZSV3ooM9zPx8yN5DpV3lLout8gUVFQAzn2fIOoDBbdclVvIo4lzSw="|02afbc00-fc1e-4dd5-8d42-57b165376620|0|4dfbd6a2-ba1b-4b44-bfb3-c2732f9fe5dd|1
+b61967e0-f665-41a4-84d1-cf06f5514f2c|2022-07-09 06:41:41.242948|2022-07-09 06:41:41.243058|anonymous:6e5336ad1712905d5ca107b432a50e99|"gAAAAABiySMl_NR-DYaDKhF-5Xeqhp_-sawtFJwyB3NE4oHxGrNwRo1JpWWS4dHAUtUn-9sLzFQtgGESZI-YcTyT0pYwGT_kSBBgE7Q67GiupwRLmhpZRaqPFzfF0h0-ozwL0vZPqS3OMqpMoWqN9XxOgMsHfwiwRxMhBqtZsRovAVusn8IdxsMt-SzaBYSlE2O1tbrZuKbLvHCDFTuhvIGEZ2hEO8Uar6hOPcsw7w7GaOsydft10GaXVDlmYMPk9LwgyijxxnsLhRRMyCV3z0oOoQ8rwmgR-jhDAn0A1eVezeLMDlkuBCOzM5Ky-ngjLa2iFCzgaR_rcaRi7Jg9umLg-L7ZA8oUlKu-9YG0AyWrFgHZm20jZjNYzXLdcn6hzVJQ1kLikt4TyXuvo1Pzff2YAFdaxZfWI1bWADGAmTrJYVbV1uhFpBBEPO0E-iSIruHWhaXf7w9rETdmf5dQ1Nz1YjdB-3GKdwKatCiCpP2gVbBst9hVoFAFeptSZuriItwqbhtAGPhwyawUNRL40LcxMmSRbcxYosFuYcuNIVtmNB_YC3wSTFa--uB6weTYtZ_fWSc3_7Ll22fCKlm_qcbZ-TfxkQOcoBdfTOa42ARBi64wVH9OH-MTMawU955t2CJnCpd_Ou3HDy-fYccr30qfTTfeVKC_d4ODh6itQ8M0DYogTJz6H2U09S2PjcvYVsA9jn1qXMXr70RnGYs0YtACR19Sst8n_ZzMGT8qnHlCpY4fYQkXWQLNANtlY-PoZSJjruin8Lc7J9rXSVhN9k6ty1DfsfGmq5JT3v5U9S7eTTKj1bfflc4_1tJj1wW6Cx5eIWdT_aKGQMjO8ucbzKeQIZ6EDxdGxVaD_7_IILG-S1lmjZuj1eH5ExbVVJtDIv2azIqF4QaSjJrBsN1SoPUvLdjXgrs3bG7hzRtqntlzHxID0sRw7vA7oHYts_Y5cuRCYA-hzXaKNrQzaO-b9J7b5jwGlKcYJnSmBEB5Q6Xv_-dQBJDra48j3G4w5ILhAnPlciOQFiBt_CrHaE02dhxsRXkGDgD-b4HsRSADxt3lPkNtr94-1nyFJsnb-DECnyPNqmNJAQDaW9Bt60w-n-QqOwTAF87ZtXOTwFCjBfIswR4D6FYbEo4RDPgA3_YmHjVHI1QTSzLa5EFSKyHNystLh_WuKHSvs5LQ9XDAbfal437QSFWaFF7uK3D1afkqGeZSV3ooM9zPx8yN5DpV3lLout8gUVFQAzn2fIOoDBbdclVvIo4lzSw="|02afbc00-fc1e-4dd5-8d42-57b165376620|0|4dfbd6a2-ba1b-4b44-bfb3-c2732f9fe5dd|1
 ```
 
 The encryption key is read from the [`ORION_ENCRYPTION_KEY` environment variable](https://github.com/PrefectHQ/prefect/blob/1d4dfa5055c46d7769c571b6a66aaec8e6cdfc13/src/prefect/orion/utilities/encryption.py#L15) if it exists, otherwise it's generated and stored in the `configuration` table in the database (alongside the same data that is being encrypted, which defeats the purpose!).

flows/context_flow.py

@@ -23,7 +23,7 @@ def log_context() -> None:
 
 
 @flow(log_prints=True)
-def context(i: int) -> None:
+def context() -> None:
     logger = get_run_logger()
     logger.info("Starting context flow")
     print("my name is", prefect.runtime.flow_run.name)  # type: ignore see https://github.com/PrefectHQ/prefect/issues/9027
@@ -32,4 +32,4 @@ def context(i: int) -> None:
 
 
 if __name__ == "__main__":
-    context(42)
+    context()

flows/dask_flow.py

@@ -1,5 +1,3 @@
-from typing import List
-
 from prefect import flow, get_run_logger, task
 from prefect_dask.task_runners import DaskTaskRunner
 
@@ -20,7 +18,7 @@ def say_goodbye(name: str) -> None:
 # TODO: can the task runner be parameterised so we don't duplicate the flow with dask_kubes_flow?
 # see https://github.com/PrefectHQ/prefect/issues/5560
 @flow(task_runner=DaskTaskRunner())
-def dask(names: List[str]) -> None:
+def dask(names: list[str]) -> None:
     for name in names:
         # tasks must be submitted to run on dask
         # if called without .submit() they are still tracked but

flows/dask_kubes_flow.py

@@ -1,5 +1,3 @@
-from typing import List
-
 import dask_kubernetes
 from dask_kubernetes import make_pod_spec
 from kubernetes.client import V1Pod
@@ -44,7 +42,7 @@ def dask_pod_spec() -> V1Pod:
         adapt_kwargs={"minimum": 1, "maximum": 2},
     )
 )
-def dask_kubes(names: List[str]) -> None:
+def dask_kubes(names: list[str]) -> None:
     for name in names:
         # tasks must be submitted to run on dask
         # if called without .submit() they are still tracked but

flows/failure_flow.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 from time import sleep
-from typing import List
 
 from prefect import allow_failure, flow, get_run_logger, task  # pyright: ignore[reportPrivateImportUsage]
 
@@ -23,13 +22,13 @@ def success() -> int:
 
 
 @task
-def the_end(scores: List[int]) -> None:
+def the_end(scores: list[int]) -> None:
     logger = get_run_logger()
     logger.info(f"Final score is {sum(scores)} 🏆")
 
 
 @task
-def the_end_handle_ex(scores: List[int | Exception]) -> None:
+def the_end_handle_ex(scores: list[int | Exception]) -> None:
     logger = get_run_logger()
     logger.info(f"Final score is {sum(s for s in scores if isinstance(s, int))} 🏆")
 

flows/flatten_flow.py

@@ -1,5 +1,3 @@
-from typing import List, Tuple
-
 from prefect import flow, get_run_logger, task
 
 
@@ -11,12 +9,12 @@ def setup() -> str:
 
 
 @task
-def fetch_batches() -> List[str]:
+def fetch_batches() -> list[str]:
     return [f"batch {i}" for i in range(10)]
 
 
 @task
-def repartition_batches(batch: str) -> List[List[str]]:
+def repartition_batches(batch: str) -> list[list[str]]:
     logger = get_run_logger()
     batch_size = 4
     materialised = [f"{batch}-{i}" for i in range(8)]
@@ -26,14 +24,14 @@ def repartition_batches(batch: str) -> List[List[str]]:
 
 
 @task
-def count_rows(batch: List[str]) -> int:
+def count_rows(batch: list[str]) -> int:
     logger = get_run_logger()
     logger.info(batch)
     return len(batch)
 
 
 @task
-def summary(count: List[int]) -> Tuple[int, int]:
+def summary(count: list[int]) -> tuple[int, int]:
     logger = get_run_logger()
     logger.info(f"{count} Num batches: {len(count)} Num rows: {sum(count)}")
     return len(count), sum(count)

flows/map_flow.py

@@ -1,5 +1,3 @@
-from typing import List, Tuple
-
 from prefect import flow, get_run_logger, task
 
 
@@ -11,7 +9,7 @@ def setup() -> str:
 
 
 @task
-def fetch_batches() -> List[str]:
+def fetch_batches() -> list[str]:
     return [f"batch {i}" for i in range(10)]
 
 
@@ -23,14 +21,14 @@ def count_rows(batch: str) -> int:
 
 
 @task
-def summary(count: List[int]) -> Tuple[int, int]:
+def summary(count: list[int]) -> tuple[int, int]:
     logger = get_run_logger()
     logger.info(f"Num batches: {len(count)} Num rows: {sum(count)}")
     return len(count), sum(count)
 
 
 # map is a bit more performant than a for loop as it gathers all upstream dependencies concurrently
-# but if you’re calling map without any upstream tasks as inputs, it’s probably going to be the same as a for loop
+# but if you're calling map without any upstream tasks as inputs, it's probably going to be the same as a for loop
 @flow
 def map_flow() -> None:
     logger = get_run_logger()

flows/ping_ray.py

@@ -7,11 +7,11 @@
 def ping() -> ray_client_pb2.ClusterInfoResponse:
     with grpc.insecure_channel("localhost:10001") as channel:
         stub = ray_client_pb2_grpc.RayletDriverStub(channel)
-        return stub.ClusterInfo(ray_client_pb2.ClusterInfoRequest(type="PING"))
+        return stub.ClusterInfo(ray_client_pb2.ClusterInfoRequest(type="PING"))  # type: ignore
 
 
 try:
     print(ping())
-except grpc._channel._InactiveRpcError as e:
+except grpc._channel._InactiveRpcError as e:  # noqa: SLF001
     print(f"{e.code()} {e.details()}")
     exit(42)

flows/ray_flow.py

@@ -1,5 +1,4 @@
 import sys
-from typing import List
 
 from prefect import flow, get_run_logger, task
 from prefect_ray.task_runners import RayTaskRunner
@@ -35,7 +34,7 @@ def say_goodbye(name: str) -> None:
         },
     )
 )
-def ray(names: List[str]) -> None:
+def ray(names: list[str]) -> None:
     # these tasks have no dependencies so will execute concurrently
 
     for name in names:

flows/storage.py

@@ -5,7 +5,7 @@ def minio_flows() -> S3:
     name = "minio-flows"
     try:
         fs: S3 = S3.load(name)  # type: ignore
-        print(f"Loaded S3 block '{fs.get_block_type_slug()}/{name}' (id {fs._block_document_id})")
+        print(f"Loaded S3 block '{fs.get_block_type_slug()}/{name}' (id {fs._block_document_id})")  # noqa: SLF001
         return fs
     except ValueError as e:
         if "Unable to find block document" in str(e):

flows/sub_flow.py

@@ -5,7 +5,7 @@
 def common(config: dict) -> int:
     # show in prefect ui
     logger = get_run_logger()
-    logger.info("I am a subgraph that shows up in lots of places!")
+    logger.info(f"I am a subgraph that shows up in lots of places! {config=}")
     intermediate_result = 42
     return intermediate_result
 

flows/submit_flow.py

@@ -70,7 +70,7 @@
 
 You can also mix within a flow ie: have some tasks without submit() to execute immediately and locally, and
 others that run via the task runner.
-"""
+"""  # noqa: E501
 
 
 @task

package.json

@@ -1,5 +1,5 @@
 {
     "devDependencies": {
-        "pyright": "1.1.296"
+        "pyright": "1.1.330"
     }
 }