chore: set up ruff as a new linter/formatter (#28158)

.pre-commit-config.yaml

@@ -19,25 +19,6 @@ repos:
     rev: v0.2.2
     hooks:
       - id: auto-walrus
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
-    hooks:
-      - id: pyupgrade
-        exclude: scripts/change_detector.py
-        args:
-          - --py39-plus
-  - repo: https://github.com/hadialqattan/pycln
-    rev: v2.1.2
-    hooks:
-      - id: pycln
-        args:
-          - --disable-all-dunder-policy
-          - --exclude=superset/config.py
-          - --extend-exclude=tests/integration_tests/superset_test_config.*.py
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.3.0
     hooks:
@@ -73,11 +54,6 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
         args: ["--markdown-linebreak-ext=md"]
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
-    hooks:
-      - id: black
-        language_version: python3
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0 # Use the sha or tag you want to point at
     hooks:
@@ -95,3 +71,9 @@ repos:
     hooks:
       - id: helm-docs
         files: helm
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.0
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format

.pylintrc

@@ -77,6 +77,7 @@ disable=
     cyclic-import,  # re-enable once this no longer raises false positives
     missing-docstring,
     duplicate-code,
+    line-too-long,
     unspecified-encoding,
     too-many-instance-attributes  # re-enable once this no longer raises false positives
 
@@ -171,7 +172,7 @@ max-nested-blocks=5
 [FORMAT]
 
 # Maximum number of characters on a single line.
-max-line-length=90
+max-line-length=100
 
 # Regexp for a line that is allowed to be longer than the limit.
 ignore-long-lines=^\s*(# )?<?https?://\S+>?$

RELEASING/changelog.py

@@ -94,10 +94,10 @@ def _fetch_github_pr(self, pr_number: int) -> PullRequest:
             if not pull_request:
                 pull_request = github_repo.get_pull(pr_number)
                 self._github_prs[pr_number] = pull_request
-        except BadCredentialsException as ex:
+        except BadCredentialsException:
             print(
-                f"Bad credentials to github provided"
-                f" use access_token parameter or set GITHUB_TOKEN"
+                "Bad credentials to github provided"
+                " use access_token parameter or set GITHUB_TOKEN"
             )
             sys.exit(1)
 
@@ -167,8 +167,8 @@ def _is_risk_pull_request(self, labels: list[Any]) -> bool:
     def _get_changelog_version_head(self) -> str:
         if not len(self._logs):
             print(
-                f"No changes found between revisions. "
-                f"Make sure your branch is up to date."
+                "No changes found between revisions. "
+                "Make sure your branch is up to date."
             )
             sys.exit(1)
         return f"### {self._version} ({self._logs[0].time})"

pyproject.toml

@@ -191,6 +191,7 @@ development = [
     "pytest-cov",
     "pytest-mock",
     "python-ldap>=3.4.4",
+    "ruff",
     "sqloxide",
     "statsd",
     "tox",
@@ -267,7 +268,6 @@ usedevelop = true
 allowlist_externals =
     npm
     pkill
-    {toxinidir}/superset-frontend/cypress_build.sh
 
 [testenv:cypress]
 setenv =
@@ -393,3 +393,81 @@ envlist =
     pylint
 skipsdist = true
 """
+[tool.ruff]
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.8
+target-version = "py310"
+
+[tool.ruff.lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"

requirements/development.txt

@@ -10,6 +10,8 @@
     # via
     #   -r requirements/base.in
     #   -r requirements/development.in
+appnope==0.1.4
+    # via ipython
 astroid==3.1.0
     # via pylint
 asttokens==2.2.1
@@ -239,6 +241,8 @@ rfc3339-validator==0.1.4
     # via openapi-schema-validator
 rfc3986==2.0.0
     # via tableschema
+ruff==0.4.0
+    # via apache-superset
 s3transfer==0.6.1
     # via boto3
 sqlalchemy-bigquery==1.10.0

scripts/build_docker.py

@@ -82,7 +82,7 @@ def is_latest_release(release: str) -> bool:
     return "SKIP_TAG::false" in output
 
 
-def make_docker_tag(l: list[str]) -> str:
+def make_docker_tag(l: list[str]) -> str:  # noqa: E741
     return f"{REPO}:" + "-".join([o for o in l if o])
 
 
@@ -140,7 +140,7 @@ def get_docker_command(
     build_context_ref: str,
     force_latest: bool = False,
 ) -> str:
-    tag = ""
+    tag = ""  # noqa: F841
     build_target = ""
     py_ver = BASE_PY_IMAGE
     docker_context = "."
@@ -284,7 +284,7 @@ def main(
         script = script + docker_build_command
         if verbose:
             run_cmd("cat Dockerfile")
-        stdout = run_cmd(script)
+        stdout = run_cmd(script)  # noqa: F841
     else:
         print("Dry Run - Docker Build Command:")
         print(docker_build_command)

scripts/cancel_github_workflows.py

@@ -32,6 +32,7 @@
   # cancel all jobs of a PR, including the latest runs
   ./cancel_github_workflows.py 1024 --include-last
 """
+
 import os
 from collections.abc import Iterable, Iterator
 from typing import Any, Literal, Optional, Union

scripts/change_detector.py

@@ -21,7 +21,6 @@
 import re
 import subprocess
 from typing import List
-from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
 # Define patterns for each group of files you're interested in
@@ -102,12 +101,12 @@ def main(event_type: str, sha: str, repo: str) -> None:
     if event_type == "pull_request":
         pr_number = os.getenv("GITHUB_REF", "").split("/")[-2]
         files = fetch_changed_files_pr(repo, pr_number)
-        print(f"PR files:")
+        print("PR files:")
         print_files(files)
 
     elif event_type == "push":
         files = fetch_changed_files_push(repo, sha)
-        print(f"Files touched since previous commit:")
+        print("Files touched since previous commit:")
         print_files(files)
     else:
         raise ValueError("Unsupported event type")

scripts/erd/erd.py

@@ -19,6 +19,7 @@
 Entity-Relationship Diagram (ERD) from SQLAlchemy
 and onto a plantuml file.
 """
+
 import json
 import os
 from collections import defaultdict
@@ -27,9 +28,8 @@
 
 import click
 import jinja2
-from flask.cli import FlaskGroup, with_appcontext
 
-from superset import app, db
+from superset import db
 
 GROUPINGS: dict[str, Iterable[str]] = {
     "Core": [

superset/__init__.py

@@ -19,19 +19,19 @@
 from flask import current_app, Flask
 from werkzeug.local import LocalProxy
 
-from superset.app import create_app
+from superset.app import create_app  # noqa: F401
 from superset.extensions import (
-    appbuilder,
+    appbuilder,  # noqa: F401
     cache_manager,
-    db,
-    event_logger,
+    db,  # noqa: F401
+    event_logger,  # noqa: F401
     feature_flag_manager,
     manifest_processor,
     results_backend_manager,
-    security_manager,
-    talisman,
+    security_manager,  # noqa: F401
+    talisman,  # noqa: F401
 )
-from superset.security import SupersetSecurityManager
+from superset.security import SupersetSecurityManager  # noqa: F401
 
 #  All of the fields located here should be considered legacy. The correct way
 #  to declare "global" dependencies is to define it in extensions.py,

superset/advanced_data_type/plugins/internet_port.py

@@ -88,9 +88,9 @@ def port_translation_func(req: AdvancedDataTypeRequest) -> AdvancedDataTypeRespo
                 else port_conversion_dict[string_value]
             )
         except (KeyError, ValueError):
-            resp[
-                "error_message"
-            ] = f"'{string_value}' does not appear to be a port name or number"
+            resp["error_message"] = (
+                f"'{string_value}' does not appear to be a port name or number"
+            )
             break
         else:
             resp["display_value"] = ", ".join(

superset/advanced_data_type/schemas.py

@@ -17,6 +17,7 @@
 """
 Schemas for advanced data types
 """
+
 from marshmallow import fields, Schema
 
 advanced_data_type_convert_schema = {

superset/annotation_layers/filters.py

@@ -24,9 +24,7 @@
 from superset.views.base import BaseFilter
 
 
-class AnnotationLayerAllTextFilter(
-    BaseFilter
-):  # pylint: disable=too-few-public-methods
+class AnnotationLayerAllTextFilter(BaseFilter):  # pylint: disable=too-few-public-methods
     name = _("All Text")
     arg_name = "annotation_layer_all_text"
 

superset/cachekeys/api.py

@@ -109,10 +109,8 @@ def invalidate(self) -> Response:
                 )
 
             try:
-                delete_stmt = (
-                    CacheKey.__table__.delete().where(  # pylint: disable=no-member
-                        CacheKey.cache_key.in_(cache_keys)
-                    )
+                delete_stmt = CacheKey.__table__.delete().where(  # pylint: disable=no-member
+                    CacheKey.cache_key.in_(cache_keys)
                 )
                 db.session.execute(delete_stmt)
                 db.session.commit()

superset/charts/filters.py

@@ -130,9 +130,7 @@ def apply(self, query: Query, value: Any) -> Query:
         )
 
 
-class ChartOwnedCreatedFavoredByMeFilter(
-    BaseFilter
-):  # pylint: disable=too-few-public-methods
+class ChartOwnedCreatedFavoredByMeFilter(BaseFilter):  # pylint: disable=too-few-public-methods
     """
     Custom filter for the GET chart that filters all charts the user
     owns, created, changed or favored.

superset/columns/models.py

@@ -23,6 +23,7 @@
 
 These models are not fully implemented, and shouldn't be used yet.
 """
+
 import sqlalchemy as sa
 from flask_appbuilder import Model
 

superset/commands/chart/export.py

@@ -78,9 +78,10 @@ def _file_content(model: Slice) -> str:
     def _export(
         model: Slice, export_related: bool = True
     ) -> Iterator[tuple[str, Callable[[], str]]]:
-        yield ExportChartsCommand._file_name(
-            model
-        ), lambda: ExportChartsCommand._file_content(model)
+        yield (
+            ExportChartsCommand._file_name(model),
+            lambda: ExportChartsCommand._file_content(model),
+        )
 
         if model.table and export_related:
             yield from ExportDatasetsCommand([model.table.id]).run()