More cleanup for update_quick_index.py + helpers (pythonarcade#2113)

* Deduplicate logic in API module processing

* Move shared member iteration into a generator function

* Rename member special rules declarations dict to be general instead of class-specific

* Fix indentation for class rendering

* Add comments and top-level docstring to explain file structure

* Clean up unused items removed in earlier commits

* Explain the decision to keep NotExcludedBy

* Move Vfs and SharedPaths into util/doc_helpers

* Move reusable items into doc_helpers

* Move NotExcludedBy, get_module_path, and EMPTY_TUPLE into doc_helpers

* Update __all__

* Clean up imports

* Rename rules for tab-completion friendliness

* Fix docstring fro VirtualFile

* Type-annotate VirtualFile.write

* Remove extra whitespace from Vfs.py

* Annotate Vfs.write

* Use pathlib.Path instead of strings inside vfs objects

* Use EMPTY_TUPLE instead of recreating new list

* Line ending consistency

Author: pushfoo

util/create_resources_listing.py

@@ -12,7 +12,7 @@
 sys.path.insert(0, str(Path(__file__).parent.parent.resolve()))
 
 import arcade
-from vfs import Vfs
+from doc_helpers.vfs import Vfs
 
 MODULE_DIR = Path(__file__).parent.resolve()
 ARCADE_ROOT = MODULE_DIR.parent

util/doc_helpers/__init__.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Iterable
+
+from .vfs import VirtualFile, Vfs, F
+
+
+__all__ = (
+    'get_module_path',
+    'EMPTY_TUPLE',
+    'F',
+    'SharedPaths',
+    'NotExcludedBy',
+    'VirtualFile',
+    'Vfs'
+)
+
+
+EMPTY_TUPLE = tuple()
+
+
+class SharedPaths:
+    """These are often used to set up a Vfs and open files."""
+    REPO_UTILS_DIR = Path(__file__).parent.parent.resolve()
+    REPO_ROOT = REPO_UTILS_DIR.parent
+    ARCADE_ROOT = REPO_ROOT / "arcade"
+    DOC_ROOT = REPO_ROOT / "doc"
+    API_DOC_ROOT = DOC_ROOT / "api_docs"
+
+
+class NotExcludedBy:
+    """Helper predicate for exclusion.
+
+    This is here because we may eventually define excludes at per-module
+    level in our config below instead of a single list.
+    """
+    def __init__(self, collection: Iterable):
+        self.items = set(collection)
+
+    def __call__(self, item) -> bool:
+        return item not in self.items
+
+
+_VALID_MODULE_SEGMENT = re.compile(r"[_a-zA-Z][_a-z0-9]*")
+
+
+def get_module_path(module: str, root = SharedPaths.REPO_ROOT) -> Path:
+    """Quick-n-dirty module path estimation relative to the repo root.
+
+    :param module: A module path in the project.
+    :raises ValueError: When a can't be computed.
+    :return: A
+    """
+    # Convert module.name.here to module/name/here
+    current = root
+    for index, part in enumerate(module.split('.')):
+        if not _VALID_MODULE_SEGMENT.fullmatch(part):
+            raise ValueError(
+                f'Invalid module segment at index {index}: {part!r}')
+        # else:
+        #   print(current, part)
+        current /= part
+
+    # Account for the two kinds of modules:
+    # 1. arcade/module.py
+    # 2. arcade/module/__init__.py
+    as_package = current / "__init__.py"
+    have_package = as_package.is_file()
+    as_file = current.with_suffix('.py')
+    have_file = as_file.is_file()
+
+    # TODO: When 3.10 becomes our min Python, make this a match-case?
+    if have_package and have_file:
+        raise ValueError(
+            f"Module conflict between {as_package} and {as_file}")
+    elif have_package:
+        current = as_package
+    elif have_file:
+        current = as_file
+    else:
+        raise ValueError(
+            f"No folder package or file module detected for "
+            f"{module}")
+
+    return current

util/vfs.py renamed to util/doc_helpers/vfs.py

@@ -35,30 +35,28 @@
 from typing import Generator, Type, TypeVar, Generic
 
 
-class SharedPaths:
-    """These are often used to set up a Vfs and open files."""
-    REPO_UTILS_DIR = Path(__file__).parent.resolve()
-    REPO_ROOT = REPO_UTILS_DIR.parent
-    ARCADE_ROOT = REPO_ROOT / "arcade"
-    DOC_ROOT = REPO_ROOT / "doc"
-    API_DOC_ROOT = DOC_ROOT / "api_docs"
-
-
 class VirtualFile:
-    """Subclass these to add some magic.
+    """An abstraction over an in-memory stream.
 
+    This might later change to be an abstraction to over a StringIO
+    inside the Vfs class to better reflect file system behavior. What
+    don't change is the write method, which means you can safely do the
+    following:
+
+    1. Subclass this to add helper functions
+    2. Pass it to a Vfs at creation
     """
 
-    def __init__(self, path: str):
-        self.path = path
+    def __init__(self, path: Path | str):
+        self.path = Path(path)
         self._content = StringIO()
 
     def include_file(self, path: Path | str) -> int:
         """Copy the path's contents into this file."""
         contents = Path(path).read_text()
         return self.write(contents)
 
-    def write(self, str: str):
+    def write(self, str: str) -> int:
         return self._content.write(str)
 
     def close(self):
@@ -96,7 +94,7 @@ class Vfs(Generic[F]):
 
     def __init__(self, file_type: Type[F] = VirtualFile):
         self.file_type: Type[F] = file_type
-        self.files: dict[str, F] = dict()
+        self.files: dict[Path, F] = dict()
         self.files_to_delete: set[Path] = set()
 
     def request_culling_unwritten(self, directory: str | Path, glob: str):
@@ -109,11 +107,11 @@ def request_culling_unwritten(self, directory: str | Path, glob: str):
         Doing it this way allows us to leave the files untouched on disk if
         this build would emit an identical file.
         """
-        path = Path(str(directory))
+        path = Path(directory)
         for p in path.glob(glob):
             self.files_to_delete.add(p)
 
-    def write(self):
+    def write(self) -> None:
         """Sync all files of this Vfs to the real filesystem.
 
         This performs the following actions:
@@ -129,16 +127,16 @@ def write(self):
         for file in self.files.values():
             file._write_to_disk()
         for path in self.files_to_delete:
-            if not str(path) in file_paths:
+            if not path in file_paths:
                 print(f"Deleting {path}")
-                os.remove(path)
+                path.unlink()
 
     def exists(self, path: str | Path) -> bool:
         """Return True if the file has been opened in this Vfs."""
         return str(path) in self.files
 
     def open(self, path: str | Path, mode: str) -> F:
-        path = str(path)
+        path = Path(path).resolve()
         modes = set(mode)
 
         # Modes which are blatantly unsupported
@@ -158,7 +156,6 @@ def open(self, path: str | Path, mode: str) -> F:
 
         raise ValueError(f"Unsupported mode {mode!r}")
 
-
     # This is less nasty than dynamically generating a subclass
     # which then attaches instances to a specific Vfs on creation
     # and assigns itself as the .open value for that Vfs