diff --git a/dev/releases/make_archives.py b/dev/releases/make_archives.py index 7a8252e319..5b06f8e0f3 100755 --- a/dev/releases/make_archives.py +++ b/dev/releases/make_archives.py @@ -24,15 +24,13 @@ import subprocess import sys import tarfile +from typing import List, Optional from utils import ( download_with_sha256, error, - get_makefile_var, notice, patchfile, - run_with_log, - safe_git_fetch_tags, verify_command_available, verify_git_clean, verify_git_repo, @@ -43,6 +41,27 @@ if sys.version_info < (3, 6): error("Python 3.6 or newer is required") + +# helper for extracting values of variables set in the GAP Makefiles.rules +def get_makefile_var(var: str) -> str: + res = subprocess.run(["make", f"print-{var}"], check=True, capture_output=True) + kv = res.stdout.decode("ascii").strip().split("=") + assert len(kv) == 2 + assert kv[0] == var + return kv[1] + + +# Run what ever command and create appropriate log file +def run_with_log(args: List[str], name: str, msg: Optional[str] = None) -> None: + if not msg: + msg = name + with open("../" + name + ".log", "w", encoding="utf-8") as fp: + try: + subprocess.run(args, check=True, stdout=fp, stderr=fp) + except subprocess.CalledProcessError: + error(msg + " failed. See " + name + ".log.") + + notice("Checking prerequisites") verify_command_available("curl") verify_command_available("git") @@ -52,7 +71,11 @@ verify_git_clean() # fetch tags, so we can properly detect -safe_git_fetch_tags() +try: + subprocess.run(["git", "fetch", "--tags"], check=True) +except subprocess.CalledProcessError: + error("failed to fetch tags, you may have to do \n" + "git fetch --tags -f") + # Creating tmp directory tmpdir = os.getcwd() + "/tmp" diff --git a/dev/releases/make_github_release.py b/dev/releases/make_github_release.py index 8de5f4b8a1..35cc42cf99 100755 --- a/dev/releases/make_github_release.py +++ b/dev/releases/make_github_release.py @@ -8,13 +8,10 @@ ## ## SPDX-License-Identifier: GPL-2.0-or-later ## -## This script makes a github release and uploads all tar balls as assets. -## The name of the target repository CURRENT_REPO_NAME is defined in -## utils.py. -## -## If we do import * from utils, then initialize_github can't overwrite the -## global CURRENT_REPO variables. +## This script makes a github release and uploads all tarballs as assets. ## +import re +import subprocess import sys import utils @@ -24,26 +21,65 @@ if len(sys.argv) != 3: error("usage: " + sys.argv[0] + " ") + +def is_possible_gap_release_tag(tag: str) -> bool: + return re.fullmatch(r"v[1-9]+\.[0-9]+\.[0-9]+(-.+)?", tag) is not None + + +def verify_is_possible_gap_release_tag(tag: str) -> None: + if not is_possible_gap_release_tag(tag): + error(f"{tag} does not look like the tag of a GAP release version") + + +# lightweight vs annotated +# https://stackoverflow.com/questions/40479712/how-can-i-tell-if-a-given-git-tag-is-annotated-or-lightweight#40499437 +def is_annotated_git_tag(tag: str) -> bool: + res = subprocess.run( + ["git", "for-each-ref", "refs/tags/" + tag], + capture_output=True, + text=True, + check=False, + ) + return res.returncode == 0 and res.stdout.split()[1] == "tag" + + +def check_git_tag_for_release(tag: str) -> None: + if not is_annotated_git_tag(tag): + error(f"There is no annotated tag {tag}") + # check that tag points to HEAD + tag_commit = subprocess.run( + ["git", "rev-parse", tag + "^{}"], check=True, capture_output=True, text=True + ).stdout.strip() + head = subprocess.run( + ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True + ).stdout.strip() + if tag_commit != head: + error( + f"The tag {tag} does not point to the current commit {head} but" + + f" instead points to {tag_commit}" + ) + + TAG_NAME = sys.argv[1] PATH_TO_RELEASE = sys.argv[2] VERSION = TAG_NAME[1:] # strip 'v' prefix utils.verify_git_clean() -utils.verify_is_possible_gap_release_tag(TAG_NAME) -utils_github.initialize_github() +verify_is_possible_gap_release_tag(TAG_NAME) +repo = utils_github.initialize_github() -# Error if the tag TAG_NAME hasn't been pushed to CURRENT_REPO yet. -if not any(tag.name == TAG_NAME for tag in utils_github.CURRENT_REPO.get_tags()): - error(f"Repository {utils_github.CURRENT_REPO_NAME} has no tag '{TAG_NAME}'") +# Error if the tag TAG_NAME hasn't been pushed out yet. +if not any(tag.name == TAG_NAME for tag in repo.get_tags()): + error(f"Repository {repo.full_name} has no tag '{TAG_NAME}'") # make sure that TAG_NAME # - exists # - is an annotated tag # - points to current HEAD -utils.check_git_tag_for_release(TAG_NAME) +check_git_tag_for_release(TAG_NAME) # Error if this release has been already created on GitHub -if any(r.tag_name == TAG_NAME for r in utils_github.CURRENT_REPO.get_releases()): +if any(r.tag_name == TAG_NAME for r in repo.get_releases()): error(f"Github release with tag '{TAG_NAME}' already exists!") # Create release @@ -52,9 +88,7 @@ + f"[CHANGES.md](https://github.com/gap-system/gap/blob/{TAG_NAME}/CHANGES.md) file." ) notice(f"Creating release {TAG_NAME}") -RELEASE = utils_github.CURRENT_REPO.create_git_release( - TAG_NAME, TAG_NAME, RELEASE_NOTE, prerelease=True -) +RELEASE = repo.create_git_release(TAG_NAME, TAG_NAME, RELEASE_NOTE, prerelease=True) with utils.working_directory(PATH_TO_RELEASE): manifest_filename = "MANIFEST" diff --git a/dev/releases/release_notes.py b/dev/releases/release_notes.py index 309a66c6b1..2bc47af243 100755 --- a/dev/releases/release_notes.py +++ b/dev/releases/release_notes.py @@ -29,7 +29,7 @@ from typing import Any, Dict, List, TextIO import requests -from utils import download_with_sha256, error, is_existing_tag, notice, warning +from utils import download_with_sha256, error, notice, warning def usage(name: str) -> None: @@ -37,6 +37,13 @@ def usage(name: str) -> None: sys.exit(1) +def is_existing_tag(tag: str) -> bool: + res = subprocess.run( + ["git", "show-ref", "--quiet", "--verify", "refs/tags/" + tag], check=False + ) + return res.returncode == 0 + + def find_previous_version(version: str) -> str: major, minor, patchlevel = map(int, version.split(".")) if major != 4: diff --git a/dev/releases/update_website.py b/dev/releases/update_website.py index 558b7bab13..0e526fa0c3 100755 --- a/dev/releases/update_website.py +++ b/dev/releases/update_website.py @@ -9,10 +9,8 @@ ## SPDX-License-Identifier: GPL-2.0-or-later ## ## -## This script is intended to implement step 7 of -## , i.e.: -## -## 7. Update the website +## This script updates the website when there is a new GAP release. +## It is to be run from inside a clone of the gap-system/GapWWW repository. import argparse import datetime @@ -25,9 +23,9 @@ import tarfile import tempfile +import github import requests import utils -import utils_github from utils import error, notice if sys.version_info < (3, 6): @@ -42,13 +40,8 @@ (most likely the master branch of github.com/gap-system/GapWWW). \ The script modifies the working directory according to the information \ on GitHub.""", - epilog="""Notes: -* To learn how to create a GitHub access token, please consult \ - https://help.github.com/articles/creating-an-access-token-for-command-line-use -""", ) group = parser.add_argument_group("Repository details and access") -group.add_argument("--token", type=str, help="GitHub access token") group.add_argument( "--gap-fork", type=str, @@ -68,6 +61,7 @@ # Downloads the asset with name from the current GitHub release # (global variable , with assets ) to . def download_asset_by_name(asset_name: str, writedir: str) -> None: + global assets try: url = [x for x in assets if x.name == asset_name][0].browser_download_url except: @@ -114,130 +108,76 @@ def download_and_extract_json_gz_asset(asset_name: str, dest: str) -> None: ################################################################################ -# Get all releases from 4.11.0 onwards, that are not a draft or prerelease -utils_github.CURRENT_REPO_NAME = f"{args.gap_fork}/gap" -utils_github.initialize_github(args.token) +# Get latest GAP release notice(f"Will use temporary directory: {tmpdir}") -releases = [ - x - for x in utils_github.CURRENT_REPO.get_releases() - if not x.draft - and not x.prerelease - and utils.is_possible_gap_release_tag(x.tag_name) - and ( - int(x.tag_name[1:].split(".")[0]) > 4 - or ( - int(x.tag_name[1:].split(".")[0]) == 4 - and int(x.tag_name[1:].split(".")[1]) >= 11 - ) - ) -] -if releases: - notice(f"Found {len(releases)} published GAP releases >= v4.11.0") -else: - notice("Found no published GAP releases >= v4.11.0") - sys.exit(0) +g = github.Github() # no token required as we just read a little data +repo = g.get_repo(f"{args.gap_fork}/gap") +release = repo.get_latest_release() -# Sort by version number, biggest to smallest -releases.sort(key=lambda s: list(map(int, s.tag_name[1:].split(".")))) -releases.reverse() +notice(f"Latest GAP release is {release.title} at tag {release.tag_name}") +if release.tag_name[0] != "v": + error("Tag name has unexpected format") +version = release.tag_name[1:] -################################################################################ -# For each release, extract the appropriate information -for release in releases: - version = release.tag_name[1:] - version_safe = version.replace(".", "-") # Safe for the Jekyll website - notice(f"\nProcessing GAP {version}...") - - # Work out the relevance of this release - known_release = os.path.isfile(f"_Releases/{version}.html") - newest_release = releases.index(release) == 0 - if known_release: - notice("I have seen this release before") - elif newest_release: - notice("This is a new release to me, and it has the biggest version number") - else: - notice( - "This is a new release to me, but I know about releases with bigger version numbers" - ) +# Determine which version is currently in GapWWW +with open("_data/release.json", "r", encoding="utf-8") as f: + release_json = json.load(f) +www_release = release_json["version"] - # For all releases, record the assets (in case they were deleted/updated/added) - notice(f"Collecting GitHub release asset data in _data/assets/{version_safe}.json") - assets = release.get_assets() - asset_data = [] - for asset in assets: - if asset.name.endswith(".sha256") or asset.name.endswith(".json.gz"): - continue - request = requests.get(f"{asset.browser_download_url}.sha256") - try: - request.raise_for_status() - sha256 = request.text.strip() - except: - error(f"Failed to download {asset.browser_download_url}.sha256") - filtered_asset = { - "bytes": asset.size, - "name": asset.name, - "sha256": sha256, - "url": asset.browser_download_url, - } - asset_data.append(filtered_asset) - asset_data.sort(key=lambda s: list(map(str, s["name"]))) - with open(f"{pwd}/_data/assets/{version_safe}.json", "wb") as outfile: - outfile.write(json.dumps(asset_data, indent=2).encode("utf-8")) - - # For new-to-me releases create a file in _Releases/ and _data/package-infos/ - if not known_release: - # When we find a previously unknown release, we extract the release date - # from the configure.ac file contained in gap-{version}-core.tar.gz. - # This date is set by the make_archives.py script. - # First download gap-X.Y.Z-core.tar.gz, extract, and fetch the date. - tarball = f"gap-{version}-core.tar.gz" - download_asset_by_name(tarball, tmpdir) - with utils.working_directory(tmpdir): - extract_tarball(tarball) - date = get_date_from_configure_ac(f"{tmpdir}/gap-{version}") - notice(f"Using release date {date} for GAP {version}") - - notice(f"Writing the file _Releases/{version}.html") - with open(f"{pwd}/_Releases/{version}.html", "wb") as outfile: - outfile.write( - f"---\nversion: {version}\ndate: '{date}'\n---\n".encode("utf-8") - ) - - notice(f"Writing the file _data/package-infos/{version_safe}.json") - download_and_extract_json_gz_asset( - "package-infos.json.gz", f"{pwd}/_data/package-infos/{version_safe}.json" - ) +notice(f"GAP release in GapWWW is {www_release}") - # For a new-to-me release with biggest version number, also set this is the - # 'default'/'main' version on the website (i.e. the most prominent release). - # Therefore update _data/release.json, _data/help.json, and _Packages/. - if not known_release and newest_release: - notice("Rewriting the _data/release.json file") - release_data = { - "version": version, - "version-safe": version_safe, - "date": date, - } - with open(f"{pwd}/_data/release.json", "wb") as outfile: - outfile.write(json.dumps(release_data, indent=2).encode("utf-8")) - - notice("Overwriting _data/help.json with the contents of help-links.json.gz") - download_and_extract_json_gz_asset( - "help-links.json.gz", f"{pwd}/_data/help.json" - ) +# TODO: abort if identical or even older - notice( - "Repopulating _Packages/ with one HTML file for each package in packages-info.json" - ) - shutil.rmtree("_Packages") - os.mkdir("_Packages") - with open(f"{pwd}/_data/package-infos/{version_safe}.json", "rb") as infile: - data = json.loads(infile.read()) - for pkg in data: - with open( - f"{pwd}/_Packages/{pkg}.html", "w+", encoding="utf-8" - ) as pkg_file: - pkg_file.write(f"---\ntitle: {data[pkg]['PackageName']}\n---\n") + +# For all releases, record the assets (in case they were deleted/updated/added) +notice(f"Collecting GitHub release asset data in _data/assets.json") +assets = release.get_assets() +asset_data = [] +for asset in assets: + if asset.name.endswith(".sha256") or asset.name.endswith(".json.gz"): + continue + request = requests.get(f"{asset.browser_download_url}.sha256") + try: + request.raise_for_status() + sha256 = request.text.strip() + except: + error(f"Failed to download {asset.browser_download_url}.sha256") + filtered_asset = { + "bytes": asset.size, + "name": asset.name, + "sha256": sha256, + "url": asset.browser_download_url, + } + asset_data.append(filtered_asset) +asset_data.sort(key=lambda s: list(map(str, s["name"]))) +with open(f"{pwd}/_data/assets.json", "wb") as outfile: + outfile.write(json.dumps(asset_data, indent=2).encode("utf-8")) + +# Extract the release date from the configure.ac file contained in +# gap-{version}-core.tar.gz. +# This date is set by the make_archives.py script. +# First download gap-X.Y.Z-core.tar.gz, extract, and fetch the date. +tarball = f"gap-{version}-core.tar.gz" +download_asset_by_name(tarball, tmpdir) +with utils.working_directory(tmpdir): + extract_tarball(tarball) +date = get_date_from_configure_ac(f"{tmpdir}/gap-{version}") +notice(f"Using release date {date} for GAP {version}") + +notice(f"Writing the file assets/package-infos.json") +download_and_extract_json_gz_asset( + "package-infos.json.gz", f"{pwd}/assets/package-infos.json" +) + +notice("Rewriting the _data/release.json file") +release_data = { + "version": version, + "date": date, +} +with open(f"{pwd}/_data/release.json", "wb") as outfile: + outfile.write(json.dumps(release_data, indent=2).encode("utf-8")) + +notice("Overwriting _data/help.json with the contents of help-links.json.gz") +download_and_extract_json_gz_asset("help-links.json.gz", f"{pwd}/_data/help.json") diff --git a/dev/releases/utils.py b/dev/releases/utils.py index 2ddbef83ce..e38c0ba72f 100644 --- a/dev/releases/utils.py +++ b/dev/releases/utils.py @@ -14,7 +14,7 @@ import shutil import subprocess import sys -from typing import Iterator, List, NoReturn, Optional +from typing import Iterator, NoReturn import requests @@ -38,9 +38,6 @@ def error(msg: str) -> NoReturn: def verify_command_available(cmd: str) -> None: if shutil.which(cmd) is None: error(f"the '{cmd}' command was not found, please install it") - # TODO: do the analog of this in ReleaseTools bash script: - # command -v curl >/dev/null 2>&1 || - # error "the 'curl' command was not found, please install it" def verify_git_repo() -> None: @@ -77,15 +74,6 @@ def working_directory(path: str) -> Iterator[None]: os.chdir(prev_cwd) -# helper for extracting values of variables set in the GAP Makefiles.rules -def get_makefile_var(var: str) -> str: - res = subprocess.run(["make", f"print-{var}"], check=True, capture_output=True) - kv = res.stdout.decode("ascii").strip().split("=") - assert len(kv) == 2 - assert kv[0] == var - return kv[1] - - # compute the sha256 checksum of a file def sha256file(path: str) -> str: h = hashlib.sha256() @@ -138,67 +126,3 @@ def download_with_sha256(url: str, dst: str) -> None: error( f"checksum for '{dst}' expected to be {expected_checksum} but got {actual_checksum}" ) - - -# Run what ever command and create appropriate log file -def run_with_log(args: List[str], name: str, msg: Optional[str] = None) -> None: - if not msg: - msg = name - with open("../" + name + ".log", "w", encoding="utf-8") as fp: - try: - subprocess.run(args, check=True, stdout=fp, stderr=fp) - except subprocess.CalledProcessError: - error(msg + " failed. See " + name + ".log.") - - -def is_possible_gap_release_tag(tag: str) -> bool: - return re.fullmatch(r"v[1-9]+\.[0-9]+\.[0-9]+(-.+)?", tag) is not None - - -def verify_is_possible_gap_release_tag(tag: str) -> None: - if not is_possible_gap_release_tag(tag): - error(f"{tag} does not look like the tag of a GAP release version") - - -def is_existing_tag(tag: str) -> bool: - res = subprocess.run( - ["git", "show-ref", "--quiet", "--verify", "refs/tags/" + tag], check=False - ) - return res.returncode == 0 - - -# Error checked git fetch of tags -def safe_git_fetch_tags() -> None: - try: - subprocess.run(["git", "fetch", "--tags"], check=True) - except subprocess.CalledProcessError: - error("failed to fetch tags, you may have to do \n" + "git fetch --tags -f") - - -# lightweight vs annotated -# https://stackoverflow.com/questions/40479712/how-can-i-tell-if-a-given-git-tag-is-annotated-or-lightweight#40499437 -def is_annotated_git_tag(tag: str) -> bool: - res = subprocess.run( - ["git", "for-each-ref", "refs/tags/" + tag], - capture_output=True, - text=True, - check=False, - ) - return res.returncode == 0 and res.stdout.split()[1] == "tag" - - -def check_git_tag_for_release(tag: str) -> None: - if not is_annotated_git_tag(tag): - error(f"There is no annotated tag {tag}") - # check that tag points to HEAD - tag_commit = subprocess.run( - ["git", "rev-parse", tag + "^{}"], check=True, capture_output=True, text=True - ).stdout.strip() - head = subprocess.run( - ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True - ).stdout.strip() - if tag_commit != head: - error( - f"The tag {tag} does not point to the current commit {head} but" - + f" instead points to {tag_commit}" - ) diff --git a/dev/releases/utils_github.py b/dev/releases/utils_github.py index fb19aea970..02f3a8c8ea 100644 --- a/dev/releases/utils_github.py +++ b/dev/releases/utils_github.py @@ -9,27 +9,15 @@ ## import os import subprocess +from typing import Optional import github from utils import error, notice, sha256file -CURRENT_REPO_NAME = os.environ.get("GITHUB_REPOSITORY", "gap-system/gap") -# Initialized by initialize_github -GITHUB_INSTANCE = None -CURRENT_REPO = None - - -# sets the global variables GITHUB_INSTANCE and CURRENT_REPO # If no token is provided, this uses the value of the environment variable # GITHUB_TOKEN. -def initialize_github(token=None) -> None: - global GITHUB_INSTANCE, CURRENT_REPO - if GITHUB_INSTANCE is not None or CURRENT_REPO is not None: - error( - "Global variables GITHUB_INSTANCE and CURRENT_REPO" - + " are already initialized." - ) +def initialize_github(token: Optional[str] = None) -> github.Repository.Repository: if token is None and "GITHUB_TOKEN" in os.environ: token = os.environ["GITHUB_TOKEN"] if token is None: @@ -50,11 +38,11 @@ def initialize_github(token=None) -> None: token = token_file.read().strip() if token is None: error("Error: no access token found or provided") - g = github.Github(token) - GITHUB_INSTANCE = g - notice(f"Accessing repository {CURRENT_REPO_NAME}") + g = github.Github(auth=github.Auth.Token(token)) + repo_name = os.environ.get("GITHUB_REPOSITORY", "gap-system/gap") + notice(f"Accessing repository {repo_name}") try: - CURRENT_REPO = GITHUB_INSTANCE.get_repo(CURRENT_REPO_NAME) + return g.get_repo(repo_name) except github.GithubException: error("Error: the access token may be incorrect") @@ -74,7 +62,9 @@ def verify_via_checksumfile(filename: str) -> None: # just to be safe, in the latter case). Then upload the files and # .sha256 as assets to the GitHub . # Files already ending in ".sha256" are ignored. -def upload_asset_with_checksum(release, filename: str) -> None: +def upload_asset_with_checksum( + release: github.GitRelease.GitRelease, filename: str +) -> None: if not os.path.isfile(filename): error(f"{filename} not found")