Create dependabot changelogs at release time (#15481)

* Ditch dependabot changelog workflow

* Summarise dependabot commits in release script

* Changelog

* Update scripts-dev/release.py

Author: David Robertson

.github/workflows/dependabot_changelog.yml

@@ -0,0 +1 @@
+Create dependabot changelogs at release time.

changelog.d/15481.misc

@@ -260,15 +260,17 @@ doesn't require poetry. (It's what we use in CI too). However, you could try
 
 ## ...handle a Dependabot pull request?
 
-Synapse uses Dependabot to keep the `poetry.lock` file up-to-date. When it
-creates a pull request a GitHub Action will run to automatically create a changelog
-file. Ensure that:
+Synapse uses Dependabot to keep the `poetry.lock` and `Cargo.lock` file 
+up-to-date with the latest releases of our dependencies. The changelog check is
+omitted for Dependabot PRs; the release script will include them in the 
+changelog.
+
+When reviewing a dependabot PR, ensure that:
 
 * the lockfile changes look reasonable;
 * the upstream changelog file (linked in the description) doesn't include any
   breaking changes;
-* continuous integration passes (due to permissions, the GitHub Actions run on
-  the changelog commit will fail, look at the initial commit of the pull request);
+* continuous integration passes.
 
 In particular, any updates to the type hints (usually packages which start with `types-`)
 should be safe to merge if linting passes.

docs/development/dependencies.md

@@ -27,7 +27,7 @@
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import Any, List, Optional
+from typing import Any, List, Match, Optional, Union
 
 import attr
 import click
@@ -233,7 +233,7 @@ def _prepare() -> None:
     subprocess.check_output(["poetry", "version", new_version])
 
     # Generate changelogs.
-    generate_and_write_changelog(current_version, new_version)
+    generate_and_write_changelog(synapse_repo, current_version, new_version)
 
     # Generate debian changelogs
     if parsed_new_version.pre is not None:
@@ -814,7 +814,7 @@ class VersionSection:
 
 
 def generate_and_write_changelog(
-    current_version: version.Version, new_version: str
+    repo: Repo, current_version: version.Version, new_version: str
 ) -> None:
     # We do this by getting a draft so that we can edit it before writing to the
     # changelog.
@@ -827,6 +827,10 @@ def generate_and_write_changelog(
     new_changes = new_changes.replace(
         "No significant changes.", f"No significant changes since {current_version}."
     )
+    new_changes += build_dependabot_changelog(
+        repo,
+        current_version,
+    )
 
     # Prepend changes to changelog
     with open("CHANGES.md", "r+") as f:
@@ -841,5 +845,47 @@ def generate_and_write_changelog(
         os.remove(filename)
 
 
+def build_dependabot_changelog(repo: Repo, current_version: version.Version) -> str:
+    """Summarise dependabot commits between `current_version` and `release_branch`.
+
+    Returns an empty string if there have been no such commits; otherwise outputs a
+    third-level markdown header followed by an unordered list."""
+    last_release_commit = repo.tag("v" + str(current_version)).commit
+    rev_spec = f"{last_release_commit.hexsha}.."
+    commits = list(git.objects.Commit.iter_items(repo, rev_spec))
+    messages = []
+    for commit in reversed(commits):
+        if commit.author.name == "dependabot[bot]":
+            message: Union[str, bytes] = commit.message
+            if isinstance(message, bytes):
+                message = message.decode("utf-8")
+            messages.append(message.split("\n", maxsplit=1)[0])
+
+    if not messages:
+        print(f"No dependabot commits in range {rev_spec}", file=sys.stderr)
+        return ""
+
+    messages.sort()
+
+    def replacer(match: Match[str]) -> str:
+        desc = match.group(1)
+        number = match.group(2)
+        return f"* {desc}. ([\\#{number}](https://github.com/matrix-org/synapse/issues/{number}))"
+
+    for i, message in enumerate(messages):
+        messages[i] = re.sub(r"(.*) \(#(\d+)\)$", replacer, message)
+    messages.insert(0, "### Updates to locked dependencies\n")
+    return "\n".join(messages)
+
+
+@cli.command()
+@click.argument("since")
+def test_dependabot_changelog(since: str) -> None:
+    """Test building the dependabot changelog.
+
+    Summarises all dependabot commits between the SINCE tag and the current git HEAD."""
+    print(build_dependabot_changelog(git.Repo("."), version.Version(since)))
+
+
 if __name__ == "__main__":
     cli()