Add path_report capability to help observe file and directory challenges

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
     -   id: check-yaml
     -   id: detect-private-key
 -   repo: https://github.com/tox-dev/pyproject-fmt
-    rev: "v2.5.1"
+    rev: "v2.6.0"
     hooks:
     -   id: pyproject-fmt
 -   repo: https://github.com/citation-file-format/cffconvert
@@ -40,12 +40,12 @@ repos:
         additional_dependencies:
         -   mdformat-gfm
 -   repo: https://github.com/adrienverge/yamllint
-    rev: v1.37.0
+    rev: v1.37.1
     hooks:
     -   id: yamllint
         exclude: pre-commit-config.yaml
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.11.8"
+    rev: "v0.11.10"
     hooks:
     -   id: ruff-format
     -   id: ruff
@@ -62,10 +62,6 @@ repos:
     rev: 'v2.14'
     hooks:
     -   id: vulture
--   repo: local
-    hooks:
-    -   id: code-cov-gen
-        name: Generate code coverage
         language: system
         # freeze the environment so it does not update
         # and only runs tests for coverage analysis

README.md

@@ -1,9 +1,9 @@
 # nViz
 
 [![Build Status](https://github.com/WayScience/nViz/actions/workflows/run-tests.yml/badge.svg?branch=main)](https://github.com/WayScience/nViz/actions/workflows/run-tests.yml?query=branch%3Amain)
-![Coverage Status](https://raw.githubusercontent.com/WayScience/nViz/main/docs/src/_static/coverage-badge.svg)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+![Coverage Status](https://raw.githubusercontent.com/WayScience/nViz/main/docs/src/_static/coverage-badge.svg)
 
 This project focuses on ingesting a set of [TIFF](https://en.wikipedia.org/wiki/TIFF) images as [OME-Zarr](https://pmc.ncbi.nlm.nih.gov/articles/PMC9980008/) or [OME-TIFF](https://genomebiology.biomedcentral.com/articles/10.1186/gb-2005-6-5-r47).
 Each input image set[<sup>1</sup>](#image_set_ref) are organized by channel and z-slices which form four dimensional (4D) microscopy data.

pyproject.toml

@@ -90,12 +90,10 @@ pythonpath = [ "." ]
 [tool.coverage]
 # omit testing utils for code coverage
 run.omit = [
-  "tests/utils.py",
-  "tests/conftest.py",
+  "tests/*",
 ]
 report.omit = [
-  "tests/utils.py",
-  "tests/conftest.py",
+  "tests/*",
 ]
 
 [tool.jupytext]

src/nviz/cli.py

@@ -2,11 +2,14 @@
 CLI for nviz
 """
 
+import json
+import sys
 from typing import Dict, List, Optional, Tuple, Union
 
 import fire
 
 from nviz.image import tiff_to_ometiff, tiff_to_zarr
+from nviz.report import path_report
 from nviz.view import view_ometiff_with_napari, view_zarr_with_napari
 
 
@@ -153,6 +156,57 @@ def view_ometiff(
             ometiff_path=ometiff_path, scaling_values=scaling_values, headless=headless
         )
 
+    def path_report(
+        self,
+        base_path: str,
+        ignore: Optional[List[str]] = None,
+        print_report: bool = True,
+    ) -> Dict[str, Union[Dict[str, int], List[str], List[tuple]]]:
+        """
+        CLI interface for generating a report of the local file paths.
+
+        Args:
+            base_path (str):
+                The base path to analyze.
+            ignore (Optional[List[str]]):
+                A list of patterns to ignore which is sent to
+                the get_path_info function. (e.g.,
+                hidden files, specific file types).
+            print_report (bool):
+                Whether to print the report to the screen
+                in a human-readable format. If we don't
+                print the report we instead return the
+                JSON dump of the report.
+                Defaults to True.
+
+        Returns:
+            dict:
+                A dictionary containing the
+                path information and empty directories.
+        """
+
+        # gather the report
+        report = path_report(
+            base_path=base_path, ignore=ignore, print_report=print_report
+        )
+
+        if not print_report:
+            # if we're not printing the report, dump the json to the output
+            print(json.dumps(report))
+
+        if (
+            len(report["empty_directories"]) >= 1
+            and None not in report["empty_directories"]
+        ) or (
+            len(report["similarly_named_directories"]) >= 1
+            and None not in report["similarly_named_directories"]
+        ):
+            # if we find errors in the report, we want to exit with a non-zero code
+            sys.exit(1)
+
+        # otherwise, we can exit with a zero code
+        sys.exit(0)
+
 
 def trigger() -> None:
     """

src/nviz/report.py

@@ -0,0 +1,183 @@
+"""
+Module to analyze local file paths and return structured information.
+"""
+
+import pathlib
+from collections import Counter
+from difflib import SequenceMatcher
+from typing import Dict, List, Optional, Union
+
+paths_type = Dict[str, Union[str, bool, int, float, None]]
+
+
+def get_path_info(base_path: str, ignore: Optional[List[str]] = None) -> paths_type:
+    """
+    Takes a local path and returns a data structure with
+    details about the path.
+
+    Args:
+        base_path (str):
+            The local path to analyze.
+        ignore (Optional[List[str]]):
+            A list of patterns to ignore
+            (e.g., hidden files, specific file types).
+
+    Returns:
+        paths_type:
+            A dictionary containing the filepath,
+            filename, type (file/dir), extension (if file),
+            hidden status, and filesize.
+    """
+
+    if not (base_path := pathlib.Path(base_path)).exists():
+        raise FileNotFoundError(f"The path '{base_path}' does not exist.")
+
+    path_info = [
+        {
+            "filepath": str(path.resolve()),
+            "filename": path.name,
+            "filepath_parent": str(path.parent.resolve()),
+            "type": "directory" if path.is_dir() else "file",
+            "extension": path.suffix if path.is_file() else None,
+            "hidden": path.name.startswith("."),
+            "filesize": path.stat().st_size if path.is_file() else None,
+        }
+        for path in base_path.rglob("*")
+        if ignore is None
+        or (
+            path not in ignore
+            and not any(str(path).startswith(ignore_path) for ignore_path in ignore)
+            and not any(str(path).endswith(ignore_path) for ignore_path in ignore)
+        )
+    ]
+
+    return path_info
+
+
+def find_empty_directories(paths: paths_type) -> paths_type:
+    """
+    Recursively finds empty directories starting
+    from the given base path.
+
+    Args:
+        paths (paths_type):
+            The base path to start searching
+            for empty directories.
+
+    Returns:
+        list:
+            A list of paths to empty directories.
+    """
+
+    directory_children = Counter(path["filepath_parent"] for path in paths)
+
+    return [
+        path["filepath"]
+        for path in paths
+        if path["type"] == "directory" and directory_children[path["filepath"]] == 0
+    ]
+
+
+def count_file_extensions(paths: paths_type) -> Dict[str, int]:
+    """
+    Counts all file extensions in the given paths
+    and returns the count per file extension.
+
+    Args:
+        paths (paths_type):
+            The paths data structure containing file information.
+
+    Returns:
+        Dict[str, int]:
+            A dictionary with file extensions
+            as keys and their counts as values.
+    """
+    return Counter(
+        path["extension"]
+        for path in paths
+        if path["type"] == "file" and path["extension"]
+    )
+
+
+def find_similar_directories(
+    paths: paths_type, similarity_threshold: float = 0.97
+) -> List[tuple]:
+    """
+    Finds pairs of directory names that
+    are extremely similar to each other.
+
+    Args:
+        paths (paths_type):
+            The paths data structure containing
+            file and directory information.
+        similarity_threshold (float):
+            The threshold above which two
+            directory names are considered similar.
+
+    Returns:
+        List[tuple]:
+            A list of tuples containing pairs of
+            similar directory names.
+    """
+
+    directories = [path["filepath"] for path in paths if path["type"] == "directory"]
+    similar_pairs = []
+
+    for i, dir1 in enumerate(directories):
+        for dir2 in directories[i + 1 :]:
+            similarity = SequenceMatcher(None, dir1, dir2).ratio()
+            if similarity >= similarity_threshold:
+                similar_pairs.append((dir1, dir2))
+
+    return similar_pairs
+
+
+def path_report(
+    base_path: str, ignore: Optional[List[str]] = None, print_report: bool = False
+) -> Dict[str, Union[Dict[str, int], List[str], List[tuple]]]:
+    """
+    Generates a report of the local file paths.
+
+    Args:
+        base_path (str):
+            The base path to analyze.
+        ignore (Optional[List[str]]):
+            A list of patterns to ignore which is sent to
+            the get_path_info function. (e.g.,
+            hidden files, specific file types).
+        print_report (bool):
+            Whether to print the report to the screen.
+
+    Returns:
+        dict:
+            A dictionary containing the
+            path information and empty directories.
+    """
+
+    paths = get_path_info(base_path=base_path, ignore=ignore)
+
+    empty_dirs = find_empty_directories(paths=paths)
+    similar_dirs = find_similar_directories(paths=paths)
+
+    report = {
+        "file_extensions": count_file_extensions(paths=paths),
+        "empty_directories": empty_dirs if empty_dirs else [None],
+        "similarly_named_directories": similar_dirs if similar_dirs else [None],
+    }
+
+    if print_report:
+        print("\nnViz Path Report:\n--------------------")
+        print("File Extensions:")
+        for ext, count in report["file_extensions"].items():
+            print(f"  {ext}: {count}")
+        print("\nEmpty Directories:")
+        for directory in report["empty_directories"]:
+            print(f"  {directory}")
+        print("\nSimilarly Named Directories:")
+        if report["similarly_named_directories"] == [None]:
+            print("  None")
+        else:
+            for dir1, dir2 in report["similarly_named_directories"]:
+                print(f"  {dir1} <-> {dir2}")
+
+    return report

tests/linux_test_configuration.sh

@@ -0,0 +1,55 @@
+#! /bin/bash
+
+# This script is used to test the configuration of the Linux system.
+# We referenced the following to help with configuration:
+# https://github.com/pyvista/setup-headless-display-action/blob/v3/action.yml
+
+# One way to run this script through a non-Linux or "clean" environment
+# is through Docker. For example:
+# docker run --rm -it --platform linux/amd64 -w /app -v $PWD:/app python:3.11 /bin/bash
+
+# install dependencies
+apt update
+apt-get install -y \
+          libglx-mesa0 \
+          libgl1 \
+          xvfb \
+          x11-xserver-utils \
+          herbstluftwm \
+          libdbus-1-3 \
+          libegl1 \
+          libopengl0 \
+          libosmesa6 \
+          libxcb-cursor0 \
+          libxcb-icccm4 \
+          libxcb-image0 \
+          libxcb-keysyms1 \
+          libxcb-randr0 \
+          libxcb-render-util0 \
+          libxcb-shape0 \
+          libxcb-xfixes0 \
+          libxcb-xinerama0 \
+          libxcb-xinput0 \
+          libxkbcommon-x11-0 \
+          mesa-utils \
+          x11-utils
+
+# Set up the display
+export DISPLAY=:99.0
+Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+sleep 3
+
+# Start the window manager
+herbstluftwm &
+sleep 3
+
+# install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# shellcheck disable=SC1091
+source "$HOME/.local/bin/env"
+
+# install pre-commit
+pip install pre-commit
+
+# install pre-commit hooks
+pre-commit install-hooks