From f696af6cb4a0a78f0a525327f2baac4e891e2003 Mon Sep 17 00:00:00 2001 From: Sakarias Johansson Date: Wed, 7 Feb 2024 17:07:19 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20ruff=20support=20for=20python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐶 🐕 🦮 🐕‍🦺 🐩 🌭 --- CHANGELOG.md | 10 ++ docs/src/python/components.md | 10 ++ docs/src/python/override.md | 12 +++ examples/hello/clients/hello/hello.nix | 1 + flake.lock | 8 +- python/default.nix | 10 +- python/hooks/check.bash | 132 +++++++++++++++++++------ python/hooks/config-merger.py | 9 +- python/hooks/config/ruff.toml | 1 + python/hooks/default.nix | 80 +++++++++++---- python/package.nix | 2 +- 11 files changed, 215 insertions(+), 60 deletions(-) create mode 100644 python/hooks/config/ruff.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f9f5d7..8cf81b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Ruff support for python components. Currently have to opt in by + setting `defaultCheckPhase = "ruffStandardTests";` on + `base.languages.python`. This will change the global default for the + python language. To set it for a single component set + `installCheckPhase` to `standardTests` or `ruffStandardTests` or any + combination of the two. To individually set the formatter set the + `formatter` on the component to either "standard" (black + isort) or + to "ruff" (ruff 🐕). If formatter is not set it will be detected from the value of `installCheckPhase`. + ## [3.0.0] - 2024-02-06 ### Added diff --git a/docs/src/python/components.md b/docs/src/python/components.md index 249c8a1..0acc818 100644 --- a/docs/src/python/components.md +++ b/docs/src/python/components.md @@ -62,6 +62,16 @@ to add python linting to other types of components. [^pylint]: ⚠️ Note that Nedryglot does not support pylint config in setup.cfg. +In case you want to use ruff instead of black, isort, pylint, and +flake8, you can set `installCheckPhase` to `ruffStandardTests` +instead. + +#### Formatting + +By default the formatter will follow the selected check variant (ruff +or black+isort). However, this can be overridden by setting the +`formatter` attribute to `ruff` or `standard`. + ### Api docs Nedryglot will output python API docs, by default using diff --git a/docs/src/python/override.md b/docs/src/python/override.md index 799c7fe..120b440 100644 --- a/docs/src/python/override.md +++ b/docs/src/python/override.md @@ -31,3 +31,15 @@ nedryland.mkProject ]; } ``` + + +# Changing the default checkphase + +```nix +base.languages.python.override { + # Change the default tests and formatter to use ruff + defaultCheckPhase = "ruffStandardTests"; +}; +``` + +The above example changes the default tests and formatter to use ruff. diff --git a/examples/hello/clients/hello/hello.nix b/examples/hello/clients/hello/hello.nix index 9e7eb86..b808224 100644 --- a/examples/hello/clients/hello/hello.nix +++ b/examples/hello/clients/hello/hello.nix @@ -9,4 +9,5 @@ base.languages.python.mkClient { # Here we just use numpyWrapper since it's our own # package and not part of the python version packages. propagatedBuildInputs = [ numpyWrapper ]; + installCheckPhase = "ruffStandardTests"; } diff --git a/flake.lock b/flake.lock index a8b0f03..81d5909 100644 --- a/flake.lock +++ b/flake.lock @@ -38,16 +38,16 @@ ] }, "locked": { - "lastModified": 1704793135, - "narHash": "sha256-aSnaRtIiC+9upOv5mLmdaQmQ9Lzd56FXWzfv2MRyEyQ=", + "lastModified": 1707233712, + "narHash": "sha256-h7PC08PmZeG2jaoUS4SvY3ZimBHaszKPXFxU8fJqSfY=", "owner": "goodbyekansas", "repo": "nedryland", - "rev": "3999f03f5087297c299f563ca064278b58a86703", + "rev": "cd760aeb2b5581bbb3c405eca9a440f6b85cabf0", "type": "github" }, "original": { "owner": "goodbyekansas", - "ref": "nixpkgs-23.11", + "ref": "10.0.0", "repo": "nedryland", "type": "github" } diff --git a/python/default.nix b/python/default.nix index 13e675f..d3a46d4 100644 --- a/python/default.nix +++ b/python/default.nix @@ -1,9 +1,15 @@ -{ base, callPackage, lib, python, pythonVersions ? { } }: +{ base +, callPackage +, lib +, python +, pythonVersions ? { } +, defaultCheckPhase ? "standardTests" +}: let pythons = pythonVersions // { inherit python; }; defaultPythonName = "python"; - hooks = callPackage ./hooks { }; + hooks = callPackage ./hooks { inherit defaultCheckPhase; }; mkPackage = callPackage ./package.nix { inherit base; checkHook = hooks.check; }; diff --git a/python/hooks/check.bash b/python/hooks/check.bash index aa50395..6bcf0b8 100644 --- a/python/hooks/check.bash +++ b/python/hooks/check.bash @@ -1,4 +1,5 @@ #! /usr/bin/env bash +# shellcheck disable=SC2030,SC2031 printStatus() { case "$1" in @@ -14,64 +15,133 @@ printStatus() { esac } +runTool() { + local tool + tool="$1" + shift 1 + + local displayTool + displayTool="$1" + shift 1 + + local disableVar + disableVar="dontRun${tool^}" + local statusVar + statusVar="${tool}Status" + + if [ -z "${!disableVar:-}" ] && [[ "$(command -v "$tool")" =~ ^/nix/store/.*$ ]]; then + echo -e "\n\x1b[1;36m${displayTool^}:\x1b[0m" + "$tool" "$@" 2>&1 | sed 's/^/ /' + declare -xg "$statusVar"=$? + else + echo "$tool is disabled." + fi +} + standardTests() ( # clean up after pip rm -rf build/ set +e - echo -e "\n\x1b[1;36mBlack:\x1b[0m" - if [[ "$(command -v black)" =~ ^/nix/store/.*$ ]]; then - # shellcheck disable=SC2086 - black ${blackArgs:-} --check . 2>&1 | sed 's/^/ /' - blackStatus=$? - else - echo " Black not supported on platform ${system:-unknown}." - fi + set -o pipefail + + # shellcheck disable=SC2086 + runTool black black ${blackArgs:-} --check . - echo -e "\n\x1b[1;36mIsort:\x1b[0m" # shellcheck disable=SC2086 - isort ${isortArgs:-} --check . 2>&1 | sed 's/^/ /' - isortStatus=$? + runTool isort isort ${isortArgs:-} --check . - echo -e "\n\x1b[1;36mPylint:\x1b[0m" # shellcheck disable=SC2046,SC2086 - HOME=$TMP pylint ${pylintArgs:-} --recursive=y . 2>&1 | sed 's/^/ /' - pylintStatus=$? + HOME=$TMP runTool pylint pylint ${pylintArgs:-} --recursive=y . - echo -e "\n\x1b[1;36mFlake8:\x1b[0m" # shellcheck disable=SC2086 - flake8 ${flake8Args:-} . 2>&1 | sed 's/^/ /' - flake8Status=$? + runTool flake8 flake8 ${flake8Args:-} . - echo -e "\n\x1b[1;36mMypy:\x1b[0m" # shellcheck disable=SC2086 - mypy ${mypyArgs:-} . 2>&1 | sed 's/^/ /' - mypyStatus=$? + runTool mypy mypy ${mypyArgs:-} . - echo -e "\n\x1b[1;36mPytest:\x1b[0m" # shellcheck disable=SC2086 - pytest ${pytestArgs:-} . 2>&1 | sed 's/^/ /' - pytestStatus=$? + runTool pytest pytest ${pytestArgs:-} . # no tests ran - if [ $pytestStatus -eq 5 ]; then + if [ "${pytestStatus:-0}" -eq 5 ]; then + local pytestStatus pytestStatus=0 fi echo -e "Summary: black: $(printStatus "${blackStatus:-skipped}") - isort: $(printStatus $isortStatus) - pylint: $(printStatus $pylintStatus) - flake8: $(printStatus $flake8Status) - mypy: $(printStatus $mypyStatus) - pytest: $(printStatus $pytestStatus)" + isort: $(printStatus "${isortStatus:-skipped}") + pylint: $(printStatus "${pylintStatus:-skipped}") + flake8: $(printStatus "${flake8Status:-skipped}") + mypy: $(printStatus "${mypyStatus:-skipped}") + pytest: $(printStatus "${pytestStatus:-skipped}")" - blackStatus=${blackStatus:-0} + : "${blackStatus:=0}" "${isortStatus:=0}" "${pylintStatus:=0}" + : "${flake8Status:=0}" "${mypyStatus:=0}" "${pytestStatus:=0}" exit $((blackStatus + isortStatus + pylintStatus + flake8Status + mypyStatus + pytestStatus)) ) +ruffStandardTests() ( + # clean up after pip + rm -rf build/ + + set +e + set -o pipefail + + # shellcheck disable=SC2086 + runTool ruff "Ruff Check" check ${ruffArgs:-} . + local ruffCheckStatus + ruffCheckStatus=${ruffStatus:-} + + # shellcheck disable=SC2086 + runTool ruff "Ruff Format" format --diff ${ruffArgs:-} . + local ruffFormatStatus + ruffFormatStatus=${ruffStatus:-} + + # shellcheck disable=SC2086 + runTool mypy mypy ${mypyArgs:-} . + + # shellcheck disable=SC2086 + runTool pytest pytest ${pytestArgs:-} . + + # no tests ran + if [ "${pytestStatus:-0}" -eq 5 ]; then + local pytestStatus + pytestStatus=0 + fi + + echo -e "Summary: + ruff check: $(printStatus "${ruffCheckStatus:-skipped}") + ruff format: $(printStatus "${ruffFormatStatus:-skipped}") + mypy: $(printStatus "${mypyStatus:-skipped}") + pytest: $(printStatus "${pytestStatus:-skipped}")" + + : "${ruffCheckStatus:=0}" "${ruffFormatStatus:=0}" + : "${mypyStatus:=0}" "${pytestStatus:=0}" + exit $((ruffCheckStatus + ruffFormatStatus + mypyStatus + pytestStatus)) +) + # If there is a checkPhase declared, mk-python-component in nixpkgs will put it in # installCheckPhase so we use that phase as well (since this is executed later). if [ -n "${doStandardTests-}" ] && [ -z "${installCheckPhase-}" ]; then - installCheckPhase=standardTests + installCheckPhase=@defaultCheckPhase@ fi + +runStandardFormat() { + black . && isort . +} + +ruffRunFormat() { + ruff format . +} + +runFormat() { + if [ "${formatter:-}" = "standard" ]; then + runStandardFormat + elif [ "${formatter:-}" = "ruff" ] || [[ "${installCheckPhase:-}" =~ "ruffStandardTests" ]]; then + ruffRunFormat + else + runStandardFormat + fi +} diff --git a/python/hooks/config-merger.py b/python/hooks/config-merger.py index c9d02d4..832705b 100644 --- a/python/hooks/config-merger.py +++ b/python/hooks/config-merger.py @@ -31,7 +31,12 @@ def change_header(config: dict, from_header: str, to_header: str) -> dict: else: return {} - return {to_header: sub_config} + header_sub_config = to_header.split(".") + + for key in reversed(header_sub_config): + sub_config = {key: sub_config} + + return sub_config def parse_toml(config_file: str) -> dict: @@ -192,7 +197,7 @@ def main(): with open(out_file, "w", encoding="utf-8") as output_file: if out_file.suffix == ".toml": - toml.dump({"tool": combined_config}, output_file) + toml.dump(combined_config, output_file) else: config_parser = ConfigParser() config_parser.read_dict(combined_config) diff --git a/python/hooks/config/ruff.toml b/python/hooks/config/ruff.toml new file mode 100644 index 0000000..146f834 --- /dev/null +++ b/python/hooks/config/ruff.toml @@ -0,0 +1 @@ +[tool.ruff] diff --git a/python/hooks/default.nix b/python/hooks/default.nix index ca6ed18..c7bcfbf 100644 --- a/python/hooks/default.nix +++ b/python/hooks/default.nix @@ -1,4 +1,12 @@ -{ makeSetupHook, writeTextFile, pkgs, bat, findutils, lib }: +{ makeSetupHook +, writeTextFile +, pkgs +, bat +, findutils +, lib +, defaultCheckPhase +, ruff +}: let generateConfigurationRunner = { toolDerivation @@ -8,6 +16,8 @@ let , configFlag ? "--config" , extraArgs ? "" , files ? [ ] + , customExecution ? null + , }: let remove = builtins.foldl' (acc: cur: "${acc} ${cur}") "" @@ -17,6 +27,11 @@ let files)); py = if lib.versionAtLeast lib.version "23.05pre-git" then pkgs.python3 else pkgs.python310; + execution = + if customExecution == null then + "${toolDerivation}/bin/${toolName} ${configFlag} \"$config_file\" \"$@\" ${extraArgs}" + else + customExecution; in writeTextFile { name = "${toolName}-with-nedryglot-cfg"; @@ -55,33 +70,55 @@ let exit 0 fi - ${toolDerivation}/bin/${toolName} ${configFlag} "$config_file" "$@" ${extraArgs} + ${execution} ''; destination = "/bin/${toolName}"; }; blackWithConfig = toolDerivation: generateConfigurationRunner { inherit toolDerivation; - key = "black"; + key = "tool.black"; config = "black.toml"; files = [ - { path = "pyproject.toml"; key = "tool.black"; } - "setup.cfg" - { path = ./config/black.toml; key = "tool.black"; } + "pyproject.toml" + { path = "setup.cfg"; key = "black"; } + ./config/black.toml + ]; + }; + + ruffWithConfig = toolDerivation: generateConfigurationRunner { + inherit toolDerivation; + # Ruff can only take the config argument for the sub commands + # check and format which is kind of annoying. + customExecution = '' + configArgs="" + if [[ " $* " =~ [[:blank:]]check[[:blank:]] ]] || [[ " $* " =~ [[:blank:]]format[[:blank:]] ]]; then + configArgs="--config $config_file" + fi + + ${toolDerivation}/bin/ruff "$@" $configArgs + ''; + key = ""; + config = "ruff.toml"; + files = [ + { path = "pyproject.toml"; key = "tool.ruff"; } + "ruff.toml" + ".ruff.toml" + { path = ./config/ruff.toml; key = "tool.ruff"; } ]; }; coverageWithConfig = toolDerivation: generateConfigurationRunner { inherit toolDerivation; - key = "coverage"; + key = "tool.coverage"; config = "coverage.toml"; configFlag = "--rcfile"; files = [ - ".coveragerc" + { path = ".coveragerc"; key = "coverage"; } { path = "setup.cfg"; key = "tool:coverage"; } { path = "tox.ini"; key = "tool:coverage"; } - { path = "pyproject.toml"; key = "tool.coverage"; } - { path = ./config/coverage.toml; key = "tool.coverage"; } + "pyproject.toml" + ./config/coverage.toml ]; }; @@ -133,19 +170,17 @@ let inherit toolDerivation; configFlag = "--rcfile"; config = "pylint.toml"; - key = "pylint"; + key = "tool.pylint"; files = [ { path = "pylintrc"; key = ""; } { path = ".pylintrc"; key = ""; } - { path = "pyproject.toml"; key = "tool.pylint"; } - { - path = - if lib.versionAtLeast toolDerivation.version "2.14" then - ./config/pylint2_14.toml - else - ./config/pylint.toml; - key = "tool.pylint"; - } + "pyproject.toml" + ( + if lib.versionAtLeast toolDerivation.version "2.14" then + ./config/pylint2_14.toml + else + ./config/pylint.toml + ) ]; }; @@ -171,6 +206,10 @@ in makeSetupHook { name = "check-hook"; + substitutions = { + inherit defaultCheckPhase; + }; + "${depsAttr}" = with pythonPkgs; [ findutils (coverageWithConfig coverage) @@ -179,6 +218,7 @@ in (mypyWithConfig mypy) (pylintWithConfig pylint) (pytestWithConfig pytest) + (ruffWithConfig ruff) # pytest is also useful as a module in PYTHONPATH for fixtures and such pytest ] diff --git a/python/package.nix b/python/package.nix index 0481181..e39597c 100644 --- a/python/package.nix +++ b/python/package.nix @@ -146,7 +146,7 @@ let show = attrs.doCheck or true; }; format = { - script = "black . && isort ."; + script = "runFormat"; description = "Format the code."; }; build = {