Apply Flake8 type checking and Ruff-specific rules (#4038)

* Apply Flake8 type checking and Ruff-specific rules

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

* Change outdated import path for `BaseSessionStore`

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

* Remove parenthetical refs to `DataCatalog` in docs

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

* Make `DataCatalog` top level for ReadTheDocs build

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

* Update the patch path to `kedro.framework.context`

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

* Use `exclude_also`, and add `TYPE_CHECKING` blocks

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

---------

Signed-off-by: Deepyaman Datta <deepyaman.datta@utexas.edu>

docs/source/conf.py

@@ -476,7 +476,7 @@ def autolink_classes_and_methods(lines):
             lines[i] = re.sub(existing, rf"{replacement}", lines[i])
 
 
-def autodoc_process_docstring(app, what, name, obj, options, lines):  # noqa: PLR0913
+def autodoc_process_docstring(app, what, name, obj, options, lines):
     try:
         # guarded method to make sure build never fails
         log_suggestions(lines, name)
@@ -486,7 +486,7 @@ def autodoc_process_docstring(app, what, name, obj, options, lines):  # noqa: PL
             style(
                 "Failed to check for class name mentions that can be "
                 f"converted to reStructuredText links in docstring of {name}. "
-                f"Error is: \n{str(e)}",
+                f"Error is: \n{e!s}",
                 fg="red",
             )
         )
@@ -515,7 +515,7 @@ def setup(app):
         style(
             "Failed to create list of (regex, reStructuredText link "
             "replacement) for class names and method names in docstrings. "
-            f"Error is: \n{str(e)}",
+            f"Error is: \n{e!s}",
             fg="red",
         )
     )

features/environment.py

@@ -1,5 +1,4 @@
 """Behave environment setup commands."""
-# noqa: unused-argument
 from __future__ import annotations
 
 import os
@@ -65,7 +64,7 @@ def _setup_context_with_venv(context, venv_dir):
     path = context.env["PATH"].split(os.pathsep)
     path = [p for p in path if not (Path(p).parent / "pyvenv.cfg").is_file()]
     path = [p for p in path if not (Path(p).parent / "conda-meta").is_dir()]
-    path = [str(bin_dir)] + path
+    path = [str(bin_dir), *path]
     context.env["PATH"] = os.pathsep.join(path)
 
     # Create an empty pip.conf file and point pip to it

features/steps/cli_steps.py

@@ -342,7 +342,7 @@ def commit_changes_to_git(context):
 def exec_kedro_target(context, command):
     """Execute Kedro target."""
     split_command = command.split()
-    cmd = [context.kedro] + split_command
+    cmd = [context.kedro, *split_command]
     context.result = run(cmd, env=context.env, cwd=str(context.root_project_dir))
 
 
@@ -378,7 +378,7 @@ def get_kedro_version_python(context):
 def exec_notebook(context, command):
     """Execute Kedro Jupyter target."""
     split_command = command.split()
-    cmd = [context.kedro, "jupyter"] + split_command
+    cmd = [context.kedro, "jupyter", *split_command]
 
     # Jupyter notebook forks a child process from a parent process, and
     # only kills the parent process when it is terminated
@@ -711,7 +711,7 @@ def check_docs_generated(context: behave.runner.Context):
         context.root_project_dir / "docs" / "build" / "html" / "index.html"
     ).read_text("utf-8")
     project_repo = context.project_name.replace("-", "_")
-    assert f"Welcome to project {project_repo}’s API docs!" in index_html, index_html
+    assert f"Welcome to project {project_repo}'s API docs!" in index_html, index_html
 
 
 @then("requirements should be generated")

features/steps/sh_run.py

@@ -90,7 +90,7 @@ def terminate(self) -> None:
         """Terminate process and children."""
         try:
             proc = psutil.Process(self.pid)
-            procs = [proc] + proc.children(recursive=True)
+            procs = [proc, *proc.children(recursive=True)]
         except psutil.NoSuchProcess:
             pass
         else:

features/steps/util.py

@@ -5,9 +5,11 @@
 import os
 import re
 from contextlib import contextmanager
-from pathlib import Path
 from time import sleep, time
-from typing import Any, Callable, Iterator
+from typing import TYPE_CHECKING, Any, Callable, Iterator
+
+if TYPE_CHECKING:
+    from pathlib import Path
 
 
 @contextmanager

kedro/config/omegaconf_config.py

@@ -296,7 +296,6 @@ def load_and_merge_dir_config(  # noqa: PLR0913
             Resulting configuration dictionary.
 
         """
-        # noqa: too-many-locals
 
         if not self._fs.isdir(Path(conf_path).as_posix()):
             raise MissingConfigException(
@@ -306,7 +305,7 @@ def load_and_merge_dir_config(  # noqa: PLR0913
 
         paths = []
         for pattern in patterns:
-            for each in self._fs.glob(Path(f"{str(conf_path)}/{pattern}").as_posix()):
+            for each in self._fs.glob(Path(f"{conf_path!s}/{pattern}").as_posix()):
                 if not self._is_hidden(each):
                     paths.append(Path(each))
 

kedro/framework/cli/catalog.py

@@ -4,8 +4,7 @@
 import copy
 from collections import defaultdict
 from itertools import chain
-from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import click
 import yaml
@@ -14,10 +13,14 @@
 from kedro.framework.cli.utils import KedroCliError, env_option, split_string
 from kedro.framework.project import pipelines, settings
 from kedro.framework.session import KedroSession
-from kedro.framework.startup import ProjectMetadata
-from kedro.io import AbstractDataset
 from kedro.io.data_catalog import DataCatalog
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from kedro.framework.startup import ProjectMetadata
+    from kedro.io import AbstractDataset
+
 
 def _create_session(package_name: str, **kwargs: Any) -> KedroSession:
     kwargs.setdefault("save_on_close", False)

kedro/framework/cli/hooks/specs.py

@@ -4,10 +4,13 @@
 """
 from __future__ import annotations
 
-from kedro.framework.startup import ProjectMetadata
+from typing import TYPE_CHECKING
 
 from .markers import cli_hook_spec
 
+if TYPE_CHECKING:
+    from kedro.framework.startup import ProjectMetadata
+
 
 class CLICommandSpecs:
     """Namespace that defines all specifications for Kedro CLI's lifecycle hooks."""

kedro/framework/cli/jupyter.py

@@ -7,7 +7,7 @@
 import os
 import shutil
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import click
 
@@ -19,7 +19,9 @@
     python_call,
 )
 from kedro.framework.project import validate_settings
-from kedro.framework.startup import ProjectMetadata
+
+if TYPE_CHECKING:
+    from kedro.framework.startup import ProjectMetadata
 
 
 @click.group(name="Kedro")
@@ -34,7 +36,7 @@ def jupyter() -> None:
 
 @forward_command(jupyter, "setup", forward_help=True)
 @click.pass_obj  # this will pass the metadata as first argument
-def setup(metadata: ProjectMetadata, /, args: Any, **kwargs: Any) -> None:  # noqa: unused-argument
+def setup(metadata: ProjectMetadata, /, args: Any, **kwargs: Any) -> None:
     """Initialise the Jupyter Kernel for a kedro project."""
     _check_module_importable("ipykernel")
     validate_settings()
@@ -53,7 +55,7 @@ def jupyter_notebook(
     env: str,
     args: Any,
     **kwargs: Any,
-) -> None:  # noqa: unused-argument
+) -> None:
     """Open Jupyter Notebook with project specific variables loaded."""
     _check_module_importable("notebook")
     validate_settings()
@@ -66,8 +68,11 @@ def jupyter_notebook(
 
     python_call(
         "jupyter",
-        ["notebook", f"--MultiKernelManager.default_kernel_name={kernel_name}"]
-        + list(args),
+        [
+            "notebook",
+            f"--MultiKernelManager.default_kernel_name={kernel_name}",
+            *list(args),
+        ],
     )
 
 
@@ -80,7 +85,7 @@ def jupyter_lab(
     env: str,
     args: Any,
     **kwargs: Any,
-) -> None:  # noqa: unused-argument
+) -> None:
     """Open Jupyter Lab with project specific variables loaded."""
     _check_module_importable("jupyterlab")
     validate_settings()
@@ -93,7 +98,7 @@ def jupyter_lab(
 
     python_call(
         "jupyter",
-        ["lab", f"--MultiKernelManager.default_kernel_name={kernel_name}"] + list(args),
+        ["lab", f"--MultiKernelManager.default_kernel_name={kernel_name}", *list(args)],
     )
 
 

kedro/framework/cli/micropkg.py

@@ -11,10 +11,9 @@
 import toml
 from importlib import import_module
 from pathlib import Path
-from typing import Any, Iterable, Iterator
+from typing import Any, Iterable, Iterator, TYPE_CHECKING
 
 import click
-from importlib_metadata import PackageMetadata
 from omegaconf import OmegaConf
 from packaging.requirements import InvalidRequirement, Requirement
 from packaging.utils import canonicalize_name
@@ -38,7 +37,10 @@
     env_option,
     python_call,
 )
-from kedro.framework.startup import ProjectMetadata
+
+if TYPE_CHECKING:
+    from kedro.framework.startup import ProjectMetadata
+    from importlib_metadata import PackageMetadata
 
 _PYPROJECT_TOML_TEMPLATE = """
 [build-system]
@@ -107,7 +109,7 @@ def __eq__(self, other: Any) -> bool:
         )
 
 
-def _check_module_path(ctx: click.core.Context, param: Any, value: str) -> str:  # noqa: unused-argument
+def _check_module_path(ctx: click.core.Context, param: Any, value: str) -> str:
     if value and not re.match(r"^[\w.]+$", value):
         message = (
             "The micro-package location you provided is not a valid Python module path"
@@ -116,7 +118,6 @@ def _check_module_path(ctx: click.core.Context, param: Any, value: str) -> str:
     return value
 
 
-# noqa: missing-function-docstring
 @click.group(name="Kedro")
 def micropkg_cli() -> None:  # pragma: no cover
     pass
@@ -379,7 +380,6 @@ def package_micropkg(  # noqa: PLR0913
 
 
 def _get_fsspec_filesystem(location: str, fs_args: str | None) -> Any:
-    # noqa: import-outside-toplevel
     import fsspec
 
     from kedro.io.core import get_protocol_and_path
@@ -389,7 +389,7 @@ def _get_fsspec_filesystem(location: str, fs_args: str | None) -> Any:
 
     try:
         return fsspec.filesystem(protocol, **fs_args_config)
-    except Exception as exc:  # noqa: broad-except
+    except Exception as exc:
         # Specified protocol is not supported by `fsspec`
         # or requires extra dependencies
         click.secho(str(exc), fg="red")
@@ -408,7 +408,6 @@ def safe_extract(tar: tarfile.TarFile, path: Path) -> None:
     for member in tar.getmembers():
         member_path = path / member.name
         if not _is_within_directory(path, member_path):
-            # noqa: broad-exception-raised
             raise Exception("Failed to safely extract tar file.")
         safe_members.append(member)
     tar.extractall(path, members=safe_members)  # nosec B202

kedro/framework/cli/pipeline.py

@@ -5,7 +5,7 @@
 import shutil
 from pathlib import Path
 from textwrap import indent
-from typing import Any, NamedTuple
+from typing import TYPE_CHECKING, Any, NamedTuple
 
 import click
 
@@ -17,7 +17,9 @@
     env_option,
 )
 from kedro.framework.project import settings
-from kedro.framework.startup import ProjectMetadata
+
+if TYPE_CHECKING:
+    from kedro.framework.startup import ProjectMetadata
 
 _SETUP_PY_TEMPLATE = """# -*- coding: utf-8 -*-
 from setuptools import setup, find_packages
@@ -65,7 +67,7 @@ def _assert_pkg_name_ok(pkg_name: str) -> None:
         raise KedroCliError(message)
 
 
-def _check_pipeline_name(ctx: click.Context, param: Any, value: str) -> str:  # noqa: unused-argument
+def _check_pipeline_name(ctx: click.Context, param: Any, value: str) -> str:
     if value:
         _assert_pkg_name_ok(value)
     return value
@@ -105,7 +107,7 @@ def create_pipeline(
     skip_config: bool,
     env: str,
     **kwargs: Any,
-) -> None:  # noqa: unused-argument
+) -> None:
     """Create a new modular pipeline by providing a name."""
     package_dir = metadata.source_dir / metadata.package_name
     project_root = metadata.project_path / metadata.project_name
@@ -148,7 +150,7 @@ def create_pipeline(
 @click.pass_obj  # this will pass the metadata as first argument
 def delete_pipeline(
     metadata: ProjectMetadata, /, name: str, env: str, yes: bool, **kwargs: Any
-) -> None:  # noqa: unused-argument
+) -> None:
     """Delete a modular pipeline by providing a name."""
     package_dir = metadata.source_dir / metadata.package_name
     conf_source = settings.CONF_SOURCE

kedro/framework/cli/project.py

@@ -4,7 +4,7 @@
 import os
 import sys
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import click
 
@@ -21,9 +21,11 @@
 )
 from kedro.framework.project import settings
 from kedro.framework.session import KedroSession
-from kedro.framework.startup import ProjectMetadata
 from kedro.utils import load_obj
 
+if TYPE_CHECKING:
+    from kedro.framework.startup import ProjectMetadata
+
 NO_DEPENDENCY_MESSAGE = """{module} is not installed. Please make sure {module} is in
 requirements.txt and run 'pip install -r requirements.txt'."""
 LINT_CHECK_ONLY_HELP = """Check the files for style guide violations, unsorted /
@@ -69,13 +71,13 @@ def project_group() -> None:  # pragma: no cover
 @forward_command(project_group, forward_help=True)
 @env_option
 @click.pass_obj  # this will pass the metadata as first argument
-def ipython(metadata: ProjectMetadata, /, env: str, args: Any, **kwargs: Any) -> None:  # noqa: unused-argument
+def ipython(metadata: ProjectMetadata, /, env: str, args: Any, **kwargs: Any) -> None:
     """Open IPython with project specific variables loaded."""
     _check_module_importable("IPython")
 
     if env:
         os.environ["KEDRO_ENV"] = env
-    call(["ipython", "--ext", "kedro.ipython"] + list(args))
+    call(["ipython", "--ext", "kedro.ipython", *list(args)])
 
 
 @project_group.command()