Merge pull request #75 from smkent/manage-cookie

Add manage-cookie utility for keeping templated projects up to date

Author: smkent

.github/workflows/cd.yml

@@ -24,6 +24,9 @@ jobs:
       - name: 💾 Check out repository
         uses: actions/checkout@v3
 
+      - name: 📜 Fetch full repository history for unit tests
+        run: git fetch --unshallow
+
       - name: 🐍 Set up Python project with Poetry
         uses: ./.github/workflows/actions/python-poetry
         with:

.github/workflows/ci.yml

@@ -32,6 +32,9 @@ jobs:
       - name: 💾 Check out repository
         uses: actions/checkout@v3
 
+      - name: 📜 Fetch full repository history for unit tests
+        run: git fetch --unshallow
+
       - name: 🐍 Set up Python project with Poetry
         uses: ./.github/workflows/actions/python-poetry
         with:

cookie_python/manage/__init__.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import argparse
+from enum import Enum
+from typing import Callable, Optional
+
+from .update import update_action
+
+
+class Action(str, Enum):
+    UPDATE = (
+        "update",
+        update_action,
+        "Update repository cruft and dependencies",
+    )
+
+    def __new__(
+        cls,
+        value: str,
+        func: Optional[Callable[[argparse.Namespace], None]] = None,
+        description: str = "",
+    ) -> Action:
+        obj = str.__new__(cls, value)
+        obj._value_ = value
+        obj.func = func  # type: ignore
+        obj.description = description  # type: ignore
+        return obj
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "action",
+        type=lambda value: Action(str(value)),
+        choices=list(Action),
+    )
+    ap.add_argument("repo", nargs="+", help="Repository URL")
+    ap.add_argument(
+        "-p",
+        "--pretend",
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Dry run",
+    )
+    args = ap.parse_args()
+    args.action.func(args)

cookie_python/manage/main.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from functools import cached_property, partial
+from pathlib import Path
+from types import TracebackType
+from typing import Any, Optional
+
+
+class RepoSandbox:
+    def __init__(self, repo: str, dry_run: bool = False) -> None:
+        self._stack = contextlib.ExitStack()
+        self.repo = repo
+        self.branch = "update-cookie"
+        self.dry_run = dry_run
+
+    def __enter__(self) -> RepoSandbox:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]] = None,
+        exc_val: Optional[BaseException] = None,
+        exc_tb: Optional[TracebackType] = None,
+    ) -> None:
+        self._stack.close()
+
+    @cached_property
+    def tempdir(self) -> Path:
+        return Path(
+            self._stack.enter_context(
+                tempfile.TemporaryDirectory(suffix=".manage_cookie")
+            )
+        )
+
+    @cached_property
+    def clone_path(self) -> Path:
+        subprocess.run(
+            ["git", "clone", self.repo, "repo"], cwd=self.tempdir, check=True
+        )
+        clone_path = self.tempdir / "repo"
+        run = partial(subprocess.run, cwd=clone_path, check=True)
+        if (
+            run(
+                ["git", "ls-remote", "origin", self.branch],
+                capture_output=True,
+            )
+            .stdout.decode()  # type: ignore
+            .strip()
+        ):
+            raise Exception(f'Branch "{self.branch}" already exists on remote')
+        run(["git", "checkout", "-b", self.branch])
+        run(["git", "reset", "--hard", "origin/main"])
+        return clone_path
+
+    def cruft_attr(self, attr: str) -> str:
+        with open(self.clone_path / ".cruft.json") as f:
+            cruft = json.load(f)
+        value = cruft[attr]
+        assert isinstance(value, str)
+        return value
+
+    def run(
+        self, *popenargs: Any, check: bool = True, **kwargs: Any
+    ) -> subprocess.CompletedProcess:
+        kwargs.setdefault("cwd", self.clone_path)
+        return subprocess.run(*popenargs, check=check, **kwargs)
+
+    def shell(self) -> None:
+        if sys.__stdin__.isatty():
+            print('Starting shell. Run "exit 1" to abort.')
+            self.run([os.environ.get("SHELL", "/bin/bash")])
+
+    def commit_changes(self, message: str) -> None:
+        self.run(["git", "add", "--", "."])
+        self.run(
+            ["git", "commit", "--no-verify", "-F", "-"],
+            input=message.replace("```\n", "").encode(),
+        )
+        if self.dry_run:
+            self.run(
+                [
+                    "git",
+                    "--no-pager",
+                    "show",
+                    "--",
+                    ".",
+                    ":!poetry.lock",
+                    ":!.cruft.json",
+                ]
+            )
+        self.lint_test()
+
+    def lint_test(self) -> None:
+        self.run(["poetry", "run", "poe", "lint"], check=False)
+        with contextlib.suppress(subprocess.CalledProcessError):
+            self.run(["git", "add", "--", "."])
+            self.run(["git", "commit", "-m", "Apply automatic linting fixes"])
+        try:
+            self.run(["poetry", "run", "poe", "test"])
+        except subprocess.CalledProcessError as e:
+            print(e)
+            print("Resolve errors and exit shell to continue")
+            self.shell()
+
+    def open_pr(self, message: str) -> None:
+        if self.dry_run:
+            return
+        self.run(["git", "push", "origin", self.branch])
+        commit_title, _, *commit_body = message.splitlines()
+        self.run(
+            [
+                "gh",
+                "pr",
+                "create",
+                "--title",
+                commit_title.strip(),
+                "--body-file",
+                "-",
+                "--base",
+                "main",
+                "--head",
+                self.branch,
+            ],
+            input=os.linesep.join(commit_body).encode("utf-8"),
+        )

cookie_python/manage/repo.py

@@ -0,0 +1,119 @@
+import os
+from argparse import Namespace
+from pathlib import Path
+from typing import Optional
+
+from .repo import RepoSandbox
+
+
+def update_cruft(repo: RepoSandbox) -> Optional[str]:
+    before_ref = repo.cruft_attr("commit")
+    repo.run(["poetry", "env", "remove", "--all"], check=False)
+    repo.run(["poetry", "env", "use", "/usr/bin/python3"])
+    repo.run(["poetry", "install"])
+    repo.run(["poetry", "run", "cruft", "update", "-y"])
+    after_ref = repo.cruft_attr("commit")
+    if before_ref == after_ref:
+        return None
+    for try_count in range(1):
+        rej_files = [
+            fn.strip()
+            for fn in repo.run(
+                ["find", ".", "-iname", "*.rej"],
+                capture_output=True,
+                check=True,
+            )
+            .stdout.decode()
+            .splitlines()
+        ]
+        conflicts = any(
+            line.startswith("U")
+            for line in repo.run(
+                ["git", "status", "--porcelain"],
+                capture_output=True,
+                check=True,
+            )
+            .stdout.decode()
+            .splitlines()
+        )
+        if rej_files or conflicts:
+            if try_count == 0:
+                print(f">>> Conflicts found: {rej_files}")
+                print("Resolve conflicts and exit shell to continue")
+                repo.shell()
+                continue
+            raise Exception(f"Unresolved conflicts: {rej_files}")
+
+    cruft_repo = repo.cruft_attr("template")
+    range_prefix = None
+    if cruft_repo.startswith("https://github.com"):
+        range_prefix = f"{cruft_repo}/compare/"
+    if Path(cruft_repo).is_dir():
+        template_repo_path = Path(cruft_repo)
+    else:
+        template_repo_path = repo.tempdir / "cookie_repo"
+        repo.run(
+            ["git", "clone", cruft_repo, template_repo_path], cwd=repo.tempdir
+        )
+    compare_cmd = [
+        "git",
+        "log",
+        "--oneline",
+        "--graph",
+        f"{before_ref}...{after_ref}",
+    ]
+    graph_output = repo.run(
+        compare_cmd, cwd=template_repo_path, capture_output=True, check=True
+    ).stdout.decode()
+    return (
+        "Applied updates from upstream project template commits:\n\n"
+        f"{range_prefix or ''}{before_ref}...{after_ref}\n\n"
+        f"```\n{graph_output.strip()}\n```\n\n"
+    )
+
+
+def update_dependencies(repo: RepoSandbox) -> Optional[str]:
+    repo.run(["poetry", "run", "pre-commit", "autoupdate"])
+    updates = repo.run(
+        ["poetry", "update", "--no-cache"], capture_output=True
+    ).stdout.decode("utf-8")
+    print(updates)
+    try:
+        while (
+            not updates.splitlines()[0]
+            .lower()
+            .startswith("package operations")
+        ):
+            updates = os.linesep.join(updates.splitlines()[1:])
+    except IndexError:
+        print("No updates in output detected")
+        return None
+    update_lines = [u.strip().replace("•", "-") for u in updates.splitlines()]
+    if len(update_lines) < 3:
+        print("No updates in output detected")
+        return None
+    return (
+        "Updated project dependencies via `poetry update`:"
+        + os.linesep * 2
+        + os.linesep.join(update_lines)
+    )
+
+
+def update_action(args: Namespace) -> None:
+    for repo_url in args.repo:
+        with RepoSandbox(repo_url, args.dry_run) as repo:
+            actions = []
+            msg_body = ""
+            cruft_msg = update_cruft(repo)
+            if cruft_msg:
+                msg_body += cruft_msg
+                actions.append("project template cruft")
+            deps_msg = update_dependencies(repo)
+            if deps_msg:
+                msg_body += deps_msg
+                actions.append("dependencies")
+            if not msg_body:
+                return None
+            message = f"Update {', '.join(actions)}\n\n{msg_body}"
+            repo.commit_changes(message)
+            repo.open_pr(message)

cookie_python/manage/update.py

@@ -48,7 +48,8 @@ pytest-sugar = "*"
 types-PyYAML = "*"
 
 [tool.poetry.scripts]
-new-cookie = "cookie_python.main:main"
+new-cookie = "cookie_python.new:main"
+manage-cookie = "cookie_python.manage.main:main"
 
 [tool.poetry-dynamic-versioning]
 enable = false

cookie_python/main.py renamed to cookie_python/new.py

@@ -1,3 +1,7 @@
+import os
+from tempfile import TemporaryDirectory
+from typing import Iterator
+
 import pytest
 
 
@@ -16,3 +20,9 @@ def opt_update_expected_outputs(request: pytest.FixtureRequest) -> bool:
     value = request.config.getoption("--update-expected-outputs")
     assert isinstance(value, bool)
     return value
+
+
+@pytest.fixture
+def temp_dir() -> Iterator[str]:
+    with TemporaryDirectory(prefix="cookie-python.unittest.") as td:
+        yield os.path.join(td)