Make site generation use multiprocessing

betty/cache/_base.py

@@ -48,9 +48,9 @@ def __init__(
         self._scopes = scopes or ()
         self._scoped_caches: dict[str, Self] = {}
         self._locks: MutableMapping[str, _Lock] = defaultdict(
-            AsynchronizedLock.threading
+            AsynchronizedLock.multiprocessing
         )
-        self._locks_lock = AsynchronizedLock.threading()
+        self._locks_lock = AsynchronizedLock.multiprocessing()
 
     async def _lock(self, cache_item_id: str) -> _Lock:
         async with self._locks_lock:

betty/concurrent.py

@@ -3,6 +3,7 @@
 """
 
 import asyncio
+import multiprocessing
 import threading
 import time
 from abc import ABC, abstractmethod
@@ -81,6 +82,13 @@ async def acquire(self, *, wait: bool = True) -> bool:
     def release(self) -> None:
         self._lock.release()
 
+    @classmethod
+    def multiprocessing(cls) -> Self:
+        """
+        Create a new multiprocessing-safe, asynchronous lock.
+        """
+        return cls(multiprocessing.Manager().Lock())
+
     @classmethod
     def threading(cls) -> Self:
         """

betty/generate/__init__.py

@@ -0,0 +1,92 @@
+"""
+Provide the Generation API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import shutil
+from contextlib import suppress
+from pathlib import Path
+
+from aiofiles.os import makedirs
+
+from betty.job import Context
+from betty.project import Project
+from betty.project import ProjectEvent
+
+
+class GenerateSiteEvent(ProjectEvent):
+    """
+    Dispatched to generate a project's site.
+    """
+
+    def __init__(self, job_context: GenerationContext):
+        super().__init__(job_context.project)
+        self._job_context = job_context
+
+    @property
+    def job_context(self) -> GenerationContext:
+        """
+        The site generation job context.
+        """
+        return self._job_context
+
+
+class GenerationContext(Context):
+    """
+    A site generation job context.
+    """
+
+    def __init__(self, project: Project):
+        super().__init__()
+        self._project = project
+
+    @property
+    def project(self) -> Project:
+        """
+        The Betty project this job context is run within.
+        """
+        return self._project
+
+
+async def generate(project: Project) -> None:
+    """
+    Generate a new site.
+    """
+    from betty.generate.pool import _GenerationProcessPool
+    from betty.generate.task import _generate_delegate, _generate_static_public
+
+    logger = logging.getLogger(__name__)
+    job_context = GenerationContext(project)
+    app = project.app
+
+    logger.info(
+        app.localizer._("Generating your site to {output_directory}.").format(
+            output_directory=project.configuration.output_directory_path
+        )
+    )
+    with suppress(FileNotFoundError):
+        await asyncio.to_thread(
+            shutil.rmtree, project.configuration.output_directory_path
+        )
+    await makedirs(project.configuration.output_directory_path, exist_ok=True)
+
+    # The static public assets may be overridden depending on the number of locales rendered, so ensure they are
+    # generated before anything else.
+    await _generate_static_public(job_context)
+
+    async with _GenerationProcessPool(project) as process_pool:
+        await _generate_delegate(project, process_pool)
+
+    project.configuration.output_directory_path.chmod(0o755)
+    for directory_path_str, subdirectory_names, file_names in os.walk(
+        project.configuration.output_directory_path
+    ):
+        directory_path = Path(directory_path_str)
+        for subdirectory_name in subdirectory_names:
+            (directory_path / subdirectory_name).chmod(0o755)
+        for file_name in file_names:
+            (directory_path / file_name).chmod(0o644)

betty/generate/file.py

@@ -0,0 +1,39 @@
+"""
+File utilities for site generation.
+"""
+
+from __future__ import annotations
+
+from typing import AsyncContextManager, cast, TYPE_CHECKING
+
+import aiofiles
+from aiofiles.os import makedirs
+from aiofiles.threadpool.text import AsyncTextIOWrapper
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+async def create_file(path: Path) -> AsyncContextManager[AsyncTextIOWrapper]:
+    """
+    Create the file for a resource.
+    """
+    await makedirs(path.parent, exist_ok=True)
+    return cast(
+        AsyncContextManager[AsyncTextIOWrapper],
+        aiofiles.open(path, "w", encoding="utf-8"),
+    )
+
+
+async def create_html_resource(path: Path) -> AsyncContextManager[AsyncTextIOWrapper]:
+    """
+    Create the file for an HTML resource.
+    """
+    return await create_file(path / "index.html")
+
+
+async def create_json_resource(path: Path) -> AsyncContextManager[AsyncTextIOWrapper]:
+    """
+    Create the file for a JSON resource.
+    """
+    return await create_file(path / "index.json")