Ruff migrate (#149)

* build: Change lint settings to ruff

- Removes references to flake8 and isort
- Sets certain basic rules, including the default flake8 and isort ones.
- Several other proposed rules are included, but only whitespace/non-functional/non-refactoring are not commented out.

Co-Authored-By: Daniel Parizher <105245560+Arborym@users.noreply.github.com>

* build: Adjust ruff/black run logic

- In particular, black may need to enforce changes after ruff performs an auto fix. This places ruff ahead of black.

* style: Whitespace/non-functional linting+formatting

- Applies new linting and formatting rules.

* refactor: Apply more linting rules from ruff

style: Certain changes were also stylistic in nature.
- Enabled and Enforced `flake8-return` (RET), `flake8-simplify` (SIM), flake8-unused-arguments (ARG) and RUF100 (checks for unused `#noqa` tags)
- Certain changes, such as from `flake8-return` represent a stylistic decision on my part, and may not be agreeable to everyone.
- The high line count changes in `webui.py` are mostly consolidating context managers
- `model-stats.py` and `pop-stats.py` had raw `open()` calls, and regardless of rules ultimately chosen, should be fixed.

* build: Re-adjust proposed rules

style: Also enforces flake8-comprehensions ("C4") and flake8-raise ("RSE")

* build: Suggests flake8-blind-except

* build: Include pyflakes and pycodestyle linting rules.

* build: Show ruff diff on CI fail

* build: Upgrade actions versions

* build: Remove bash call, invoke python directly for lint

* build: style.py debug option,  Check if 'nataili' folder exists

- Adds the `--debug` option to style.py to help sort of CI linting issues.
- Checks to see if 'nataili' exists, which has the effect of confusing ruff.

* style: Post 'nataili' folder check import sort fixes

- These import sorts were not being included due to my local repo containing a folder named 'nataili'.

* Revert "build: Remove bash call, invoke python directly for lint"

This reverts commit a976c83.

* fix: allow to run with nataili folder

---------

Co-authored-by: Daniel Parizher <105245560+Arborym@users.noreply.github.com>
Co-authored-by: Divided by Zer0 <mail@dbzer0.com>

.github/workflows/lint.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
 

model-stats.py

@@ -53,12 +53,12 @@ def get_date(self):
         return adate
 
     def get_num_lines(self, file_path):
-        fp = open(file_path, "r+")
-        buf = mmap.mmap(fp.fileno(), 0)
-        lines = 0
-        while buf.readline():
-            lines += 1
-        return lines
+        with open(file_path, "r+") as fp:
+            buf = mmap.mmap(fp.fileno(), 0)
+            lines = 0
+            while buf.readline():
+                lines += 1
+            return lines
 
     def download_stats(self, period, model_type="img"):
         self.unused_models = []  # not relevant
@@ -80,10 +80,10 @@ def parse_log(self):
         if self.period == PERIOD_HORDE_DAY:
             self.download_stats("day")
             return
-        elif self.period == PERIOD_HORDE_MONTH:
+        if self.period == PERIOD_HORDE_MONTH:
             self.download_stats("month")
             return
-        elif self.period == PERIOD_TEXT_HORDE_MONTH:
+        if self.period == PERIOD_TEXT_HORDE_MONTH:
             self.download_stats("month", "text")
             return
 
@@ -158,7 +158,10 @@ def print_stats(self):
     parser.add_argument("-d", "--horde", help="Show statistics for the entire horde for the day", action="store_true")
     parser.add_argument("-k", "--kudos", help="Show statistics for the kudos per hour", action="store_true")
     parser.add_argument(
-        "-m", "--hordemonth", help="Show statistics for the entire horde for the month", action="store_true"
+        "-m",
+        "--hordemonth",
+        help="Show statistics for the entire horde for the month",
+        action="store_true",
     )
     parser.add_argument(
         "-x",

pop-stats.py

@@ -44,15 +44,14 @@ def get_date(self):
         return adate
 
     def get_num_lines(self, file_path):
-        fp = open(file_path, "r+")
-        buf = mmap.mmap(fp.fileno(), 0)
-        lines = 0
-        while buf.readline():
-            lines += 1
-        return lines
+        with open(file_path, "r+") as fp:
+            buf = mmap.mmap(fp.fileno(), 0)
+            lines = 0
+            while buf.readline():
+                lines += 1
+            return lines
 
     def parse_log(self):
-
         # Identify all log files and total number of log lines
         total_log_lines = 0
         for logfile in glob.glob(self.logfile):

pyproject.toml

@@ -1,17 +1,43 @@
 [tool.black]
 line-length = 119
 
-[tool.isort]
-py_version = 38
-line_length = 119
-profile = "black"
-known_third_party = ["creds", "gfpgan", "ray", "bridgeData", "nataili"]
+[tool.ruff]
+line-length = 119
+select = [
+    "A",    # flake8-builtins
+    "I",    # isort 
+    # "S",    # Bandit
+    "F",    # pyflakes
+    "E",    # pycodestyle errors
+    "W",    # pycodestyle warnings
 
-[tool.pylint.FORMAT]
-max-line-length=500
 
-[tool.ruff]
-line-length = 500
+    "YTT",  # flake8-2020
+    # "BLE",    # flake8-blind-except
+    # "B",    # flake8-bugbear
+    "COM",  # flake8-commas
+    "C4",   # flake8-comprehensions
+    # "G",    # flake8-logging-format
+    "INP",  # flake8-no-pep420
+    "PIE",  # flake8-pie
+    # "T20",  # flake8-print
+    # "UP",   # pyupgrade
+    "RSE",  # flake8-raise 
+    "RET",  # flake8-return
+    # "SLF",  # flake8-self
+    "SIM",  # flake8-simplify
+    "ARG",  # flake8-unused-arguments
+    # "TRY",  # tryceratops 
+    "RUF100"
+]
+ignore = [
+    "A002", # Argument `x` is shadowing a python builtin
+    "A003", # Class attribute `x` is shadowing a python builtin
+]
+
+[tool.ruff.isort]
+known-third-party = ["creds", "gfpgan", "ray", "bridgeData", "nataili"]
+
 
 [build-system]
 requires = ["setuptools>=64.0"]

requirements.dev.txt

@@ -1,3 +1,2 @@
-black==22.3.0
-flake8==3.8.1
-isort==5.10.1
+black==23.3.0
+ruff>=0.0.259

style.py

@@ -4,8 +4,29 @@
 import subprocess
 import sys
 
+arg_parser = argparse.ArgumentParser()
+arg_parser.add_argument(
+    "--fix",
+    action="store_true",
+    required=False,
+    help="Fix issues which can be fixed automatically",
+)
+arg_parser.add_argument(
+    "--debug",
+    action="store_true",
+    required=False,
+    help="Show some extra information for debugging purposes",
+)
+args = arg_parser.parse_args()
+
 # Set the working directory to where this script is located
-os.chdir(os.path.dirname(os.path.abspath(__file__)))
+thisFilePath = os.path.abspath(__file__)
+workingDirectory = os.path.dirname(thisFilePath)
+
+if args.debug:
+    print(f"working out of {workingDirectory}")
+    print(f"style.py located at {thisFilePath}")
+os.chdir(workingDirectory)
 
 src = [
     "worker",
@@ -22,43 +43,40 @@
 
 src = [item for item in src if item not in ignore_src]
 
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument(
-    "--fix",
-    action="store_true",
-    required=False,
-    help="Fix issues which can be fixed automatically",
-)
-args = arg_parser.parse_args()
 
 black_args = [
     "black",
     "--line-length=119",
 ]
-flake8_args = [
-    "flake8",
-]
-isort_args = [
-    "isort",
+ruff_args = [
+    "ruff",
 ]
 
 if args.fix:
     print("fix requested")
+    ruff_args.extend(("check", "--fix"))
 else:
     print("fix not requested")
 
     black_args.extend(("--check", "--diff"))
-    isort_args.extend(("--check-only", "--diff"))
+    ruff_args.append("--diff")
+
 lint_processes = [
+    ruff_args,
     black_args,
-    isort_args,
-    flake8_args,
 ]
 
+if args.debug:
+    for process in lint_processes:
+        VERSION_COMMAND = process[0] + " --version"
+        subprocess.run(VERSION_COMMAND, shell=True, check=True)
+        print()
+
 for process_args in lint_processes:
     process_args.extend(src)
 
-    COMMAND = " ".join(process_args)
+    COMMAND = "python -s -m "
+    COMMAND += " ".join(process_args)
     print(f"\nRunning {COMMAND}")
     try:
         subprocess.run(COMMAND, shell=True, check=True)