Simplify the Assets API (#1653)

betty/app.py

@@ -181,8 +181,7 @@ def assets(self) -> AssetRepository:
         """
         if self._assets is None:
             self._assert_bootstrapped()
-            self._assets = AssetRepository()
-            self._assets.prepend(fs.ASSETS_DIRECTORY_PATH, "utf-8")
+            self._assets = AssetRepository(fs.ASSETS_DIRECTORY_PATH)
         return self._assets
 
     @property

betty/assets.py

@@ -4,48 +4,12 @@
 
 from __future__ import annotations
 
-import asyncio
-from collections import deque
-
-from contextlib import suppress
+from os import walk
+from typing import Sequence, TYPE_CHECKING
 from pathlib import Path
-from shutil import copy2
-from typing import AsyncContextManager, Sequence, AsyncIterable, TYPE_CHECKING
-
-import aiofiles
-from aiofiles.os import makedirs
-
-from betty.fs import iterfiles
 
 if TYPE_CHECKING:
-    from aiofiles.threadpool.text import AsyncTextIOWrapper
-    from types import TracebackType
-
-
-class _Open:
-    def __init__(self, fs: AssetRepository, file_paths: tuple[Path, ...]):
-        self._fs = fs
-        self._file_paths = file_paths
-        self._file: AsyncContextManager[AsyncTextIOWrapper] | None = None
-
-    async def __aenter__(self) -> AsyncTextIOWrapper:
-        for file_path in map(Path, self._file_paths):
-            for fs_path, fs_encoding in self._fs._paths:
-                with suppress(FileNotFoundError):
-                    self._file = aiofiles.open(
-                        fs_path / file_path, encoding=fs_encoding
-                    )
-                    return await self._file.__aenter__()
-        raise FileNotFoundError
-
-    async def __aexit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_val: BaseException | None,
-        exc_tb: TracebackType | None,
-    ) -> None:
-        if self._file is not None:
-            await self._file.__aexit__(None, None, None)
+    from collections.abc import Iterator
 
 
 class AssetRepository:
@@ -56,69 +20,42 @@ class AssetRepository:
     each other. Paths added later act as fallbacks, e.g. earlier paths have priority.
     """
 
-    def __init__(self, *paths: tuple[Path, str | None]):
-        self._paths = deque(paths)
-
-    def __len__(self) -> int:
-        return len(self._paths)
+    def __init__(self, *assets_directory_paths: Path):
+        self._assets_directory_paths = assets_directory_paths
+        self._assets = {}
+        for assets_directory_path in reversed(assets_directory_paths):
+            for directory_path, _, file_names in walk(assets_directory_path):
+                for file_name in file_names:
+                    file_path = Path(directory_path) / file_name
+                    self._assets[file_path.relative_to(assets_directory_path)] = (
+                        file_path
+                    )
 
     @property
-    def paths(self) -> Sequence[tuple[Path, str | None]]:
+    def assets_directory_paths(self) -> Sequence[Path]:
         """
-        The paths to the individual layers.
+        The paths to the individual virtual layers.
         """
-        return list(self._paths)
+        return self._assets_directory_paths
 
-    def prepend(self, path: Path, fs_encoding: str | None = None) -> None:
+    def walk(self, asset_directory_path: Path | None = None) -> Iterator[Path]:
         """
-        Prepend a layer path, e.g. override existing layers with the given one.
-        """
-        self._paths.appendleft((path, fs_encoding))
-
-    def clear(self) -> None:
-        """
-        Clear all layers from the file system.
-        """
-        self._paths.clear()
-
-    def open(self, *file_paths: Path) -> _Open:
-        """
-        Open a file.
-
-        :param file_paths: One or more file paths within the file system. The first file path to exist
-        will cause this function to return. Previously missing file paths will not cause errors.
+        Get virtual paths to available assets.
 
-        :raise FileNotFoundError: Raised when none of the provided paths matches an existing file.
+        :param asset_directory_path: If given, only asses under the directory are returned.
         """
-        return _Open(self, file_paths)
+        for asset_path in self._assets:
+            if (
+                asset_directory_path is None
+                or asset_directory_path in asset_path.parents
+            ):
+                yield asset_path
 
-    async def copy2(self, source_path: Path, destination_path: Path) -> Path:
-        """
-        Copy a file to a destination using :py:func:`shutil.copy2`.
+    def __getitem__(self, path: Path) -> Path:
         """
-        for fs_path, _ in self._paths:
-            with suppress(FileNotFoundError):
-                await asyncio.to_thread(copy2, fs_path / source_path, destination_path)
-                return destination_path
-        tried_paths = [str(fs_path / source_path) for fs_path, _ in self._paths]
-        raise FileNotFoundError("Could not find any of %s." % ", ".join(tried_paths))
+        Get the path to a single asset file.
 
-    async def copytree(
-        self, source_path: Path, destination_path: Path
-    ) -> AsyncIterable[Path]:
+        :param path: The virtual asset path.
+        :return: The path to the actual file on disk.
         """
-        Recursively copy the files in a directory tree to another directory.
-        """
-        file_destination_paths = set()
-        for fs_path, _ in self._paths:
-            async for file_source_path in iterfiles(fs_path / source_path):
-                file_destination_path = destination_path / file_source_path.relative_to(
-                    fs_path / source_path
-                )
-                if file_destination_path not in file_destination_paths:
-                    file_destination_paths.add(file_destination_path)
-                    await makedirs(file_destination_path.parent, exist_ok=True)
-                    await asyncio.to_thread(
-                        copy2, file_source_path, file_destination_path
-                    )
-                    yield file_destination_path
+        return self._assets[path]

betty/generate.py

@@ -9,7 +9,15 @@
 import logging
 import os
 import shutil
-from asyncio import create_task, Task, as_completed, Semaphore, CancelledError, sleep
+from asyncio import (
+    create_task,
+    Task,
+    as_completed,
+    Semaphore,
+    CancelledError,
+    sleep,
+    to_thread,
+)
 from contextlib import suppress
 from pathlib import Path
 from typing import (
@@ -27,6 +35,7 @@
 from aiofiles.threadpool.text import AsyncTextIOWrapper
 from math import floor
 
+from betty.asyncio import gather
 from betty.job import Context
 from betty.json.schema import Schema
 from betty.locale import get_display_name
@@ -243,6 +252,26 @@ async def _generate_dispatch(
     await project.dispatcher.dispatch(Generator)(job_context)
 
 
+async def _generate_public_asset(
+    asset_path: Path, project: Project, job_context: GenerationContext, locale: str
+) -> None:
+    www_directory_path = project.configuration.localize_www_directory_path(locale)
+    file_destination_path = www_directory_path / asset_path.relative_to(
+        Path("public") / "localized"
+    )
+    await makedirs(file_destination_path.parent, exist_ok=True)
+    await to_thread(
+        shutil.copy2,
+        project.assets[asset_path],
+        file_destination_path,
+    )
+    await project.renderer.render_file(
+        file_destination_path,
+        job_context=job_context,
+        localizer=await project.app.localizers.get(locale),
+    )
+
+
 async def _generate_public(
     job_context: GenerationContext,
     locale: str,
@@ -254,17 +283,31 @@ async def _generate_public(
             "Generating localized public files in {locale}..."
         ).format(
             locale=locale_label,
+            localizer=await project.app.localizers.get(locale),
         )
     )
-    async for file_path in project.assets.copytree(
-        Path("public") / "localized",
-        project.configuration.localize_www_directory_path(locale),
-    ):
-        await project.renderer.render_file(
-            file_path,
-            job_context=job_context,
-            localizer=await project.app.localizers.get(locale),
+    await gather(
+        *(
+            _generate_public_asset(asset_path, project, job_context, locale)
+            for asset_path in project.assets.walk(Path("public") / "localized")
         )
+    )
+
+
+async def _generate_static_public_asset(
+    asset_path: Path, project: Project, job_context: GenerationContext
+) -> None:
+    file_destination_path = (
+        project.configuration.www_directory_path
+        / asset_path.relative_to(Path("public") / "static")
+    )
+    await makedirs(file_destination_path.parent, exist_ok=True)
+    await to_thread(
+        shutil.copy2,
+        project.assets[asset_path],
+        file_destination_path,
+    )
+    await project.renderer.render_file(file_destination_path, job_context=job_context)
 
 
 async def _generate_static_public(
@@ -275,21 +318,21 @@ async def _generate_static_public(
     logging.getLogger(__name__).info(
         app.localizer._("Generating static public files...")
     )
-    async for file_path in project.assets.copytree(
-        Path("public") / "static", project.configuration.www_directory_path
-    ):
-        await project.renderer.render_file(
-            file_path,
-            job_context=job_context,
+    await gather(
+        *(
+            _generate_static_public_asset(asset_path, project, job_context)
+            for asset_path in project.assets.walk(Path("public") / "static")
         )
+    )
 
-        # Ensure favicon.ico exists, otherwise servers of Betty sites would log
-        # many a 404 Not Found for it, because some clients eagerly try to see
-        # if it exists.
-        await project.assets.copy2(
-            Path("public") / "static" / "betty.ico",
-            project.configuration.www_directory_path / "favicon.ico",
-        )
+    # Ensure favicon.ico exists, otherwise servers of Betty sites would log
+    # many a 404 Not Found for it, because some clients eagerly try to see
+    # if it exists.
+    await to_thread(
+        shutil.copy2,
+        project.assets[Path("public") / "static" / "betty.ico"],
+        project.configuration.www_directory_path / "favicon.ico",
+    )
 
 
 async def _generate_entity_type_list_html(

betty/jinja2/__init__.py

@@ -217,7 +217,7 @@ class Environment(Jinja2Environment):
 
     def __init__(self, project: Project):
         template_directory_paths = [
-            str(path / "templates") for path, _ in project.assets.paths
+            str(path / "templates") for path in project.assets.assets_directory_paths
         ]
         super().__init__(
             loader=FileSystemLoader(template_directory_paths),

betty/locale/__init__.py

@@ -950,7 +950,7 @@ def locales(self) -> Iterator[str]:
         if self._locales is None:
             self._locales = set()
             self._locales.add(DEFAULT_LOCALE)
-            for assets_directory_path, __ in reversed(self._assets.paths):
+            for assets_directory_path in reversed(self._assets.assets_directory_paths):
                 for po_file_path in assets_directory_path.glob("locale/*/betty.po"):
                     self._locales.add(po_file_path.parent.name)
         yield from self._locales
@@ -979,7 +979,7 @@ async def get_negotiated(self, *preferred_locales: str) -> Localizer:
 
     async def _build_translation(self, locale: str) -> Localizer:
         translations = gettext.NullTranslations()
-        for assets_directory_path, __ in reversed(self._assets.paths):
+        for assets_directory_path in reversed(self._assets.assets_directory_paths):
             opened_translations = await self._open_translations(
                 locale, assets_directory_path
             )
@@ -1041,7 +1041,7 @@ async def coverage(self, locale: Localey) -> tuple[int, int]:
         return len(translations), len(translatables)
 
     async def _get_translatables(self) -> AsyncIterator[str]:
-        for assets_directory_path, __ in self._assets.paths:
+        for assets_directory_path in self._assets.assets_directory_paths:
             with suppress(FileNotFoundError):
                 async with aiofiles.open(
                     assets_directory_path / "betty.pot"
@@ -1051,7 +1051,7 @@ async def _get_translatables(self) -> AsyncIterator[str]:
                         yield entry.msgid_with_context
 
     async def _get_translations(self, locale: str) -> AsyncIterator[str]:
-        for assets_directory_path, __ in reversed(self._assets.paths):
+        for assets_directory_path in reversed(self._assets.assets_directory_paths):
             with suppress(FileNotFoundError):
                 async with aiofiles.open(
                     assets_directory_path / "locale" / locale / "betty.po",

betty/project/__init__.py

@@ -1129,14 +1129,14 @@ def assets(self) -> AssetRepository:
         """
         if self._assets is None:
             self._assert_bootstrapped()
-            self._assets = AssetRepository()
-            # Mimic :py:attr:`betty.app.App.assets`.
-            self._assets.prepend(fs.ASSETS_DIRECTORY_PATH, "utf-8")
+            asset_paths = [self.configuration.assets_directory_path]
             for extension in self.extensions.flatten():
                 extension_assets_directory_path = extension.assets_directory_path()
                 if extension_assets_directory_path is not None:
-                    self._assets.prepend(extension_assets_directory_path, "utf-8")
-            self._assets.prepend(self.configuration.assets_directory_path)
+                    asset_paths.append(extension_assets_directory_path)
+            # Mimic :py:attr:`betty.app.App.assets`.
+            asset_paths.append(fs.ASSETS_DIRECTORY_PATH)
+            self._assets = AssetRepository(*asset_paths)
         return self._assets
 
     @property

betty/tests/locale/test___init__.py

@@ -623,7 +623,7 @@ async def test_get(self) -> None:
         locale = "nl-NL"
         async with TemporaryDirectory() as assets_directory_path_str:
             assets_directory_path = Path(assets_directory_path_str)
-            fs = AssetRepository((assets_directory_path, None))
+            assets = AssetRepository(assets_directory_path)
             lc_messages_directory_path = assets_directory_path / "locale" / locale
             lc_messages_directory_path.mkdir(parents=True)
             po = """
@@ -653,6 +653,6 @@ async def test_get(self) -> None:
 """
             async with aiofiles.open(lc_messages_directory_path / "betty.po", "w") as f:
                 await f.write(po)
-            sut = LocalizerRepository(fs)
+            sut = LocalizerRepository(assets)
             actual = (await sut.get(locale))._("Subject")
             assert actual == "Onderwerp"

betty/tests/project/test___init__.py

@@ -1365,7 +1365,7 @@ async def test_app(self, new_temporary_app: App) -> None:
     async def test_assets(self, new_temporary_app: App) -> None:
         sut = Project(new_temporary_app)
         async with sut:
-            assert len(sut.assets.paths) > 0
+            assert len(sut.assets.assets_directory_paths) > 0
 
     async def test_discover_extension_types(self, new_temporary_app: App) -> None:
         sut = Project(new_temporary_app)