diff --git a/betty/app.py b/betty/app.py index 300550ec1..2ee251290 100644 --- a/betty/app.py +++ b/betty/app.py @@ -181,8 +181,7 @@ def assets(self) -> AssetRepository: """ if self._assets is None: self._assert_bootstrapped() - self._assets = AssetRepository() - self._assets.prepend(fs.ASSETS_DIRECTORY_PATH, "utf-8") + self._assets = AssetRepository(fs.ASSETS_DIRECTORY_PATH) return self._assets @property diff --git a/betty/assets.py b/betty/assets.py index 5ae02ad7d..ecf308097 100644 --- a/betty/assets.py +++ b/betty/assets.py @@ -4,48 +4,12 @@ from __future__ import annotations -import asyncio -from collections import deque - -from contextlib import suppress +from os import walk +from typing import Sequence, TYPE_CHECKING from pathlib import Path -from shutil import copy2 -from typing import AsyncContextManager, Sequence, AsyncIterable, TYPE_CHECKING - -import aiofiles -from aiofiles.os import makedirs - -from betty.fs import iterfiles if TYPE_CHECKING: - from aiofiles.threadpool.text import AsyncTextIOWrapper - from types import TracebackType - - -class _Open: - def __init__(self, fs: AssetRepository, file_paths: tuple[Path, ...]): - self._fs = fs - self._file_paths = file_paths - self._file: AsyncContextManager[AsyncTextIOWrapper] | None = None - - async def __aenter__(self) -> AsyncTextIOWrapper: - for file_path in map(Path, self._file_paths): - for fs_path, fs_encoding in self._fs._paths: - with suppress(FileNotFoundError): - self._file = aiofiles.open( - fs_path / file_path, encoding=fs_encoding - ) - return await self._file.__aenter__() - raise FileNotFoundError - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - if self._file is not None: - await self._file.__aexit__(None, None, None) + from collections.abc import Iterator class AssetRepository: @@ -56,69 +20,42 @@ class AssetRepository: each other. Paths added later act as fallbacks, e.g. earlier paths have priority. """ - def __init__(self, *paths: tuple[Path, str | None]): - self._paths = deque(paths) - - def __len__(self) -> int: - return len(self._paths) + def __init__(self, *assets_directory_paths: Path): + self._assets_directory_paths = assets_directory_paths + self._assets = {} + for assets_directory_path in reversed(assets_directory_paths): + for directory_path, _, file_names in walk(assets_directory_path): + for file_name in file_names: + file_path = Path(directory_path) / file_name + self._assets[file_path.relative_to(assets_directory_path)] = ( + file_path + ) @property - def paths(self) -> Sequence[tuple[Path, str | None]]: + def assets_directory_paths(self) -> Sequence[Path]: """ - The paths to the individual layers. + The paths to the individual virtual layers. """ - return list(self._paths) + return self._assets_directory_paths - def prepend(self, path: Path, fs_encoding: str | None = None) -> None: + def walk(self, asset_directory_path: Path | None = None) -> Iterator[Path]: """ - Prepend a layer path, e.g. override existing layers with the given one. - """ - self._paths.appendleft((path, fs_encoding)) - - def clear(self) -> None: - """ - Clear all layers from the file system. - """ - self._paths.clear() - - def open(self, *file_paths: Path) -> _Open: - """ - Open a file. - - :param file_paths: One or more file paths within the file system. The first file path to exist - will cause this function to return. Previously missing file paths will not cause errors. + Get virtual paths to available assets. - :raise FileNotFoundError: Raised when none of the provided paths matches an existing file. + :param asset_directory_path: If given, only asses under the directory are returned. """ - return _Open(self, file_paths) + for asset_path in self._assets: + if ( + asset_directory_path is None + or asset_directory_path in asset_path.parents + ): + yield asset_path - async def copy2(self, source_path: Path, destination_path: Path) -> Path: - """ - Copy a file to a destination using :py:func:`shutil.copy2`. + def __getitem__(self, path: Path) -> Path: """ - for fs_path, _ in self._paths: - with suppress(FileNotFoundError): - await asyncio.to_thread(copy2, fs_path / source_path, destination_path) - return destination_path - tried_paths = [str(fs_path / source_path) for fs_path, _ in self._paths] - raise FileNotFoundError("Could not find any of %s." % ", ".join(tried_paths)) + Get the path to a single asset file. - async def copytree( - self, source_path: Path, destination_path: Path - ) -> AsyncIterable[Path]: + :param path: The virtual asset path. + :return: The path to the actual file on disk. """ - Recursively copy the files in a directory tree to another directory. - """ - file_destination_paths = set() - for fs_path, _ in self._paths: - async for file_source_path in iterfiles(fs_path / source_path): - file_destination_path = destination_path / file_source_path.relative_to( - fs_path / source_path - ) - if file_destination_path not in file_destination_paths: - file_destination_paths.add(file_destination_path) - await makedirs(file_destination_path.parent, exist_ok=True) - await asyncio.to_thread( - copy2, file_source_path, file_destination_path - ) - yield file_destination_path + return self._assets[path] diff --git a/betty/generate.py b/betty/generate.py index c1d3ee253..da2b2e231 100644 --- a/betty/generate.py +++ b/betty/generate.py @@ -9,7 +9,15 @@ import logging import os import shutil -from asyncio import create_task, Task, as_completed, Semaphore, CancelledError, sleep +from asyncio import ( + create_task, + Task, + as_completed, + Semaphore, + CancelledError, + sleep, + to_thread, +) from contextlib import suppress from pathlib import Path from typing import ( @@ -27,6 +35,7 @@ from aiofiles.threadpool.text import AsyncTextIOWrapper from math import floor +from betty.asyncio import gather from betty.job import Context from betty.json.schema import Schema from betty.locale import get_display_name @@ -243,6 +252,26 @@ async def _generate_dispatch( await project.dispatcher.dispatch(Generator)(job_context) +async def _generate_public_asset( + asset_path: Path, project: Project, job_context: GenerationContext, locale: str +) -> None: + www_directory_path = project.configuration.localize_www_directory_path(locale) + file_destination_path = www_directory_path / asset_path.relative_to( + Path("public") / "localized" + ) + await makedirs(file_destination_path.parent, exist_ok=True) + await to_thread( + shutil.copy2, + project.assets[asset_path], + file_destination_path, + ) + await project.renderer.render_file( + file_destination_path, + job_context=job_context, + localizer=await project.app.localizers.get(locale), + ) + + async def _generate_public( job_context: GenerationContext, locale: str, @@ -254,17 +283,31 @@ async def _generate_public( "Generating localized public files in {locale}..." ).format( locale=locale_label, + localizer=await project.app.localizers.get(locale), ) ) - async for file_path in project.assets.copytree( - Path("public") / "localized", - project.configuration.localize_www_directory_path(locale), - ): - await project.renderer.render_file( - file_path, - job_context=job_context, - localizer=await project.app.localizers.get(locale), + await gather( + *( + _generate_public_asset(asset_path, project, job_context, locale) + for asset_path in project.assets.walk(Path("public") / "localized") ) + ) + + +async def _generate_static_public_asset( + asset_path: Path, project: Project, job_context: GenerationContext +) -> None: + file_destination_path = ( + project.configuration.www_directory_path + / asset_path.relative_to(Path("public") / "static") + ) + await makedirs(file_destination_path.parent, exist_ok=True) + await to_thread( + shutil.copy2, + project.assets[asset_path], + file_destination_path, + ) + await project.renderer.render_file(file_destination_path, job_context=job_context) async def _generate_static_public( @@ -275,21 +318,21 @@ async def _generate_static_public( logging.getLogger(__name__).info( app.localizer._("Generating static public files...") ) - async for file_path in project.assets.copytree( - Path("public") / "static", project.configuration.www_directory_path - ): - await project.renderer.render_file( - file_path, - job_context=job_context, + await gather( + *( + _generate_static_public_asset(asset_path, project, job_context) + for asset_path in project.assets.walk(Path("public") / "static") ) + ) - # Ensure favicon.ico exists, otherwise servers of Betty sites would log - # many a 404 Not Found for it, because some clients eagerly try to see - # if it exists. - await project.assets.copy2( - Path("public") / "static" / "betty.ico", - project.configuration.www_directory_path / "favicon.ico", - ) + # Ensure favicon.ico exists, otherwise servers of Betty sites would log + # many a 404 Not Found for it, because some clients eagerly try to see + # if it exists. + await to_thread( + shutil.copy2, + project.assets[Path("public") / "static" / "betty.ico"], + project.configuration.www_directory_path / "favicon.ico", + ) async def _generate_entity_type_list_html( diff --git a/betty/jinja2/__init__.py b/betty/jinja2/__init__.py index 535adc329..2ebee7a18 100644 --- a/betty/jinja2/__init__.py +++ b/betty/jinja2/__init__.py @@ -217,7 +217,7 @@ class Environment(Jinja2Environment): def __init__(self, project: Project): template_directory_paths = [ - str(path / "templates") for path, _ in project.assets.paths + str(path / "templates") for path in project.assets.assets_directory_paths ] super().__init__( loader=FileSystemLoader(template_directory_paths), diff --git a/betty/locale/__init__.py b/betty/locale/__init__.py index c2272cf49..873db5ad5 100644 --- a/betty/locale/__init__.py +++ b/betty/locale/__init__.py @@ -950,7 +950,7 @@ def locales(self) -> Iterator[str]: if self._locales is None: self._locales = set() self._locales.add(DEFAULT_LOCALE) - for assets_directory_path, __ in reversed(self._assets.paths): + for assets_directory_path in reversed(self._assets.assets_directory_paths): for po_file_path in assets_directory_path.glob("locale/*/betty.po"): self._locales.add(po_file_path.parent.name) yield from self._locales @@ -979,7 +979,7 @@ async def get_negotiated(self, *preferred_locales: str) -> Localizer: async def _build_translation(self, locale: str) -> Localizer: translations = gettext.NullTranslations() - for assets_directory_path, __ in reversed(self._assets.paths): + for assets_directory_path in reversed(self._assets.assets_directory_paths): opened_translations = await self._open_translations( locale, assets_directory_path ) @@ -1041,7 +1041,7 @@ async def coverage(self, locale: Localey) -> tuple[int, int]: return len(translations), len(translatables) async def _get_translatables(self) -> AsyncIterator[str]: - for assets_directory_path, __ in self._assets.paths: + for assets_directory_path in self._assets.assets_directory_paths: with suppress(FileNotFoundError): async with aiofiles.open( assets_directory_path / "betty.pot" @@ -1051,7 +1051,7 @@ async def _get_translatables(self) -> AsyncIterator[str]: yield entry.msgid_with_context async def _get_translations(self, locale: str) -> AsyncIterator[str]: - for assets_directory_path, __ in reversed(self._assets.paths): + for assets_directory_path in reversed(self._assets.assets_directory_paths): with suppress(FileNotFoundError): async with aiofiles.open( assets_directory_path / "locale" / locale / "betty.po", diff --git a/betty/project/__init__.py b/betty/project/__init__.py index 76df8a201..57ed190e7 100644 --- a/betty/project/__init__.py +++ b/betty/project/__init__.py @@ -1129,14 +1129,14 @@ def assets(self) -> AssetRepository: """ if self._assets is None: self._assert_bootstrapped() - self._assets = AssetRepository() - # Mimic :py:attr:`betty.app.App.assets`. - self._assets.prepend(fs.ASSETS_DIRECTORY_PATH, "utf-8") + asset_paths = [self.configuration.assets_directory_path] for extension in self.extensions.flatten(): extension_assets_directory_path = extension.assets_directory_path() if extension_assets_directory_path is not None: - self._assets.prepend(extension_assets_directory_path, "utf-8") - self._assets.prepend(self.configuration.assets_directory_path) + asset_paths.append(extension_assets_directory_path) + # Mimic :py:attr:`betty.app.App.assets`. + asset_paths.append(fs.ASSETS_DIRECTORY_PATH) + self._assets = AssetRepository(*asset_paths) return self._assets @property diff --git a/betty/tests/locale/test___init__.py b/betty/tests/locale/test___init__.py index 40b1b73c9..8d2ac4402 100644 --- a/betty/tests/locale/test___init__.py +++ b/betty/tests/locale/test___init__.py @@ -623,7 +623,7 @@ async def test_get(self) -> None: locale = "nl-NL" async with TemporaryDirectory() as assets_directory_path_str: assets_directory_path = Path(assets_directory_path_str) - fs = AssetRepository((assets_directory_path, None)) + assets = AssetRepository(assets_directory_path) lc_messages_directory_path = assets_directory_path / "locale" / locale lc_messages_directory_path.mkdir(parents=True) po = """ @@ -653,6 +653,6 @@ async def test_get(self) -> None: """ async with aiofiles.open(lc_messages_directory_path / "betty.po", "w") as f: await f.write(po) - sut = LocalizerRepository(fs) + sut = LocalizerRepository(assets) actual = (await sut.get(locale))._("Subject") assert actual == "Onderwerp" diff --git a/betty/tests/project/test___init__.py b/betty/tests/project/test___init__.py index 02f291606..c53947858 100644 --- a/betty/tests/project/test___init__.py +++ b/betty/tests/project/test___init__.py @@ -1365,7 +1365,7 @@ async def test_app(self, new_temporary_app: App) -> None: async def test_assets(self, new_temporary_app: App) -> None: sut = Project(new_temporary_app) async with sut: - assert len(sut.assets.paths) > 0 + assert len(sut.assets.assets_directory_paths) > 0 async def test_discover_extension_types(self, new_temporary_app: App) -> None: sut = Project(new_temporary_app) diff --git a/betty/tests/test_assets.py b/betty/tests/test_assets.py index 3f6fe3f09..7d1fe3610 100644 --- a/betty/tests/test_assets.py +++ b/betty/tests/test_assets.py @@ -1,159 +1,97 @@ from pathlib import Path -import aiofiles -import pytest from aiofiles.tempfile import TemporaryDirectory from betty.assets import AssetRepository class TestAssetRepository: - async def test_open(self) -> None: + async def test_assets_directory_paths(self) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - async with aiofiles.open(source_path_1 / "apples", "w") as f: - await f.write("apples") - async with aiofiles.open(source_path_2 / "apples", "w") as f: - await f.write("notapples") - async with aiofiles.open(source_path_1 / "oranges", "w") as f: - await f.write("oranges") - async with aiofiles.open(source_path_2 / "bananas", "w") as f: - await f.write("bananas") + sut = AssetRepository(source_path_1, source_path_2) + assert sut.assets_directory_paths == (source_path_1, source_path_2) - sut = AssetRepository((source_path_1, None), (source_path_2, None)) - - async with sut.open(Path("apples")) as f: - assert await f.read() == "apples" - async with sut.open(Path("oranges")) as f: - assert await f.read() == "oranges" - async with sut.open(Path("bananas")) as f: - assert await f.read() == "bananas" - - with pytest.raises(FileNotFoundError): - async with sut.open(Path("mangos")): - pass - - async def test_open_with_first_file_path_alternative_first_source_path( + async def test___getitem___with_override( self, ) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - async with aiofiles.open(source_path_1 / "pinkladies", "w") as f: - await f.write("pinkladies") - async with aiofiles.open(source_path_2 / "pinkladies", "w") as f: - await f.write("notpinkladies") - async with aiofiles.open(source_path_1 / "apples", "w") as f: - await f.write("notpinkladies") - async with aiofiles.open(source_path_2 / "apples", "w") as f: - await f.write("notpinkladies") + (source_path_1 / "apples").touch() + (source_path_2 / "apples").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert sut[Path("apples")] == source_path_1 / "apples" - sut = AssetRepository((source_path_1, None), (source_path_2, None)) - - async with sut.open(Path("pinkladies"), Path("apples")) as f: - assert await f.read() == "pinkladies" - - async def test_open_with_first_file_path_alternative_second_source_path( + async def test___getitem___without_override( self, ) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - async with aiofiles.open(source_path_2 / "pinkladies", "w") as f: - await f.write("pinkladies") - async with aiofiles.open(source_path_1 / "apples", "w") as f: - await f.write("notpinkladies") - async with aiofiles.open(source_path_2 / "apples", "w") as f: - await f.write("notpinkladies") - - sut = AssetRepository((source_path_1, None), (source_path_2, None)) + (source_path_1 / "apples").touch() + (source_path_2 / "oranges").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert sut[Path("oranges")] == source_path_2 / "oranges" - async with sut.open(Path("pinkladies"), Path("apples")) as f: - assert await f.read() == "pinkladies" - - async def test_open_with_second_file_path_alternative_first_source_path( - self, - ) -> None: + async def test_walk_with_override(self) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - async with aiofiles.open(source_path_1 / "apples", "w") as f: - await f.write("apples") - async with aiofiles.open(source_path_2 / "apples", "w") as f: - await f.write("notapples") + (source_path_1 / "apples").touch() + (source_path_2 / "apples").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert set(sut.walk()) == {Path("apples")} - sut = AssetRepository((source_path_1, None), (source_path_2, None)) - - async with sut.open(Path("pinkladies"), Path("apples")) as f: - assert await f.read() == "apples" - - async def test_copy2(self) -> None: + async def test_walk_without_override(self) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - async with aiofiles.open(source_path_1 / "apples", "w") as f: - await f.write("apples") - async with aiofiles.open(source_path_2 / "apples", "w") as f: - await f.write("notapples") - async with aiofiles.open(source_path_1 / "oranges", "w") as f: - await f.write("oranges") - async with aiofiles.open(source_path_2 / "bananas", "w") as f: - await f.write("bananas") - - async with TemporaryDirectory() as destination_path_str: - destination_path = Path(destination_path_str) - sut = AssetRepository((source_path_1, None), (source_path_2, None)) - - await sut.copy2(Path("apples"), destination_path) - await sut.copy2(Path("oranges"), destination_path) - await sut.copy2(Path("bananas"), destination_path) - - async with sut.open(destination_path / "apples") as f: - assert await f.read() == "apples" - async with sut.open(destination_path / "oranges") as f: - assert await f.read() == "oranges" - async with sut.open(destination_path / "bananas") as f: - assert await f.read() == "bananas" - - with pytest.raises(FileNotFoundError): - await sut.copy2(Path("mangos"), destination_path) - - async def test_copytree(self) -> None: + (source_path_1 / "apples").touch() + (source_path_2 / "oranges").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert set(sut.walk()) == { + Path("apples"), + Path("oranges"), + } + + async def test_walk_with_override_with_filter(self) -> None: async with TemporaryDirectory() as source_path_str_1: source_path_1 = Path(source_path_str_1) - (source_path_1 / "basket").mkdir() + (source_path_1 / "fruits").mkdir() + (source_path_1 / "vegetables").mkdir() async with TemporaryDirectory() as source_path_str_2: source_path_2 = Path(source_path_str_2) - (source_path_2 / "basket").mkdir() - async with aiofiles.open(source_path_1 / "basket" / "apples", "w") as f: - await f.write("apples") - async with aiofiles.open(source_path_2 / "basket" / "apples", "w") as f: - await f.write("notapples") - async with aiofiles.open( - source_path_1 / "basket" / "oranges", "w" - ) as f: - await f.write("oranges") - async with aiofiles.open( - source_path_2 / "basket" / "bananas", "w" - ) as f: - await f.write("bananas") - - async with TemporaryDirectory() as destination_path_str: - destination_path = Path(destination_path_str) - sut = AssetRepository((source_path_1, None), (source_path_2, None)) - - async for _ in sut.copytree(Path(), destination_path): - pass - - async with sut.open(destination_path / "basket" / "apples") as f: - assert await f.read() == "apples" - async with sut.open(destination_path / "basket" / "oranges") as f: - assert await f.read() == "oranges" - async with sut.open(destination_path / "basket" / "bananas") as f: - assert await f.read() == "bananas" + (source_path_2 / "fruits").mkdir() + (source_path_2 / "vegetables").mkdir() + (source_path_1 / "fruits" / "apples").touch() + (source_path_2 / "fruits" / "apples").touch() + (source_path_1 / "vegetables" / "peppers").touch() + (source_path_2 / "vegetables" / "peppers").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert set(sut.walk(Path("fruits"))) == {Path("fruits") / "apples"} + + async def test_walk_without_override_with_filter(self) -> None: + async with TemporaryDirectory() as source_path_str_1: + source_path_1 = Path(source_path_str_1) + (source_path_1 / "fruits").mkdir() + (source_path_1 / "vegetables").mkdir() + async with TemporaryDirectory() as source_path_str_2: + source_path_2 = Path(source_path_str_2) + (source_path_2 / "fruits").mkdir() + (source_path_2 / "vegetables").mkdir() + (source_path_1 / "fruits" / "apples").touch() + (source_path_2 / "fruits" / "oranges").touch() + (source_path_1 / "vegetables" / "peppers").touch() + (source_path_2 / "vegetables" / "oranges").touch() + sut = AssetRepository(source_path_1, source_path_2) + assert set(sut.walk(Path("fruits"))) == { + Path("fruits") / "apples", + Path("fruits") / "oranges", + } diff --git a/documentation/conf.py b/documentation/conf.py index 0cffd9aaa..d86f2ccb1 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -14,8 +14,7 @@ betty_replacements: dict[str, str] = {} -assets = AssetRepository() -assets.prepend(fs.ASSETS_DIRECTORY_PATH, "utf-8") +assets = AssetRepository(fs.ASSETS_DIRECTORY_PATH) localizers = LocalizerRepository(assets) for locale in localizers.locales: coverage = wait_to_thread(localizers.coverage(locale))