release-gap-package

#!/bin/sh
#
# ReleaseTools - a set of shells script for making GAP package releases
#
# Please always use the latest version of this tool, available from
# <https://github.com/gap-system/ReleaseTools>.
#
# Copyright (c) 2013-2020 Max Horn <max@quendi.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#

set -e

######################################################################
#
# Usage information
#
help() {
cat <<EOF
Usage: $0 [OPTIONS]

A tool for making releases of GAP packages on GitHub.

Run this from within a git clone of your package repository, checked out
at the revision you want to release. This tool extracts relevant data
from the PackageInfo.g file, and performs the releases process.

Actions
  -h,  --help                      display this help text and exit
  -p,  --push                      perform the final push, completing the release [Default]
  -P,  --no-push                   do not perform the final push
  -f,  --force                     if a release with the same name already exists: overwrite it
  --skip-existing-release          if a release with the same name already exists: exit without error

Paths
  --srcdir <path>                  path of directory containing PackageInfo.g [Default: current directory]
  --tmpdir <path>                  path of temporary directory [Default: tmp subdirectory of srcdir]
  --webdir <path>                  path of web directory [Default: gh-pages subdirectory of srcdir]
  --update-script <file>           path of the update script [Default: update.g in webdir]

Custom settings
  --token <oauth>                  GitHub access token

Notes:
* The package name and version, the list of archive formats, and the GitHub repository
  are extracted from PackageInfo.g.
* To learn how to create a GitHub access token, please consult
  https://help.github.com/articles/creating-an-access-token-for-command-line-use/
* Without the --push option, all steps are performed, except for the final push
  of the gh-pages changes. These changes are what make the release visible
  to the GAP package distribution system.
* Please consult the README for more information.
EOF
    exit 0
}

######################################################################
#
# Various little helper functions


# print notices in green
notice() {
    printf '\033[32m%s\033[0m\n' "$*"
}

# print warnings in yellow
warning() {
    printf '\033[33mWARNING: %s\033[0m\n' "$*"
}

# print error in red and exit
error() {
    printf '\033[31mERROR: %s\033[0m\n' "$*"
    exit 1
}

# check for uncommitted changes
verify_git_clean() {
    git update-index --refresh || error "uncommitted changes detected"
    git diff-index --quiet HEAD -- || error "uncommitted changes detected"
}

# execute GAP commands from a heredoc, and export its output in the shell
# variable `gap_output`; abort if GAP reported any errors
run_gap() {
  # Invoke GAP. The final call to 'tr' removes newlines and is a workaround
  # for people whose gap script is broken (by not honoring -A, thus loading
  # Browse, which inserts that extra character). I am looking at you,
  # Sebastian!
  gap_output=$( \
    (echo 'OnBreak:=function() Print("FATAL ERROR"); FORCE_QUIT_GAP(1); end;;' ; cat - ; echo ; echo "FORCE_QUIT_GAP(0);") \
    | $GAP -A -q -b 2>&1 \
    | tr -d '\r' )
  mkdir -p "$TMP_DIR"
  echo "$gap_output" > "$TMP_DIR/gap-error.log"
  if echo "$gap_output" | grep -q '\(Error\|FATAL ERROR\|Syntax \)' ; then
    error "there was an error running GAP, see $TMP_DIR/gap-error.log"
  fi
}

# helper function for parsing GitHub's JSON output. Right now,
# we only extract the value of a single key at a time. This means
# we may end up parsing the same JSON data two times, but that
# doesn't really matter as it is tiny.
json_get_key() {
    echo "$response" | python -c 'import json,sys;obj=json.load(sys.stdin);print(obj.get("'"$1"'",""))'
}

# On Mac OS X, tar stores extended attributes in ._FOO files inside archives.
# Setting COPYFILE_DISABLE prevents that. See <http://superuser.com/a/260264>
export COPYFILE_DISABLE=1


######################################################################
#
# Command line processing
#
SRC_DIR="$PWD"
TMP_DIR="$PWD/tmp"
PackageInfoURL=
README_URL=

PUSH=yes
FORCE=no
ONLY_TARBALL=no
SKIP_EXISTING_RELEASE=no
while [ x"$1" != x ]; do
  option="$1" ; shift
  case "$option" in
    -h | --help ) help ;;

    --srcdir ) SRC_DIR="$1"; shift ;;
    --webdir ) WEB_DIR="$1"; shift ;;
    --tmpdir ) TMP_DIR="$1"; shift ;;
    --update-script ) UPDATE_SCRIPT="$1"; shift ;;

    --srcdir=*) SRC_DIR=${option#--srcdir=}; shift ;;
    --webdir=*) WEB_DIR=${option#--webdir=}; shift ;;
    --tmpdir=*) TMP_DIR=${option#--tmpdir=}; shift ;;
    --update-script=*) UPDATE_SCRIPT=${option#--update-script=}; shift ;;

    --token ) TOKEN="$1"; shift ;;

    -p | --push ) PUSH=yes ;;
    -P | --no-push ) PUSH=no ;;

    -f | --force ) FORCE=yes ;;
    --no-force ) FORCE=no ;;

    --only-tarball ) ONLY_TARBALL=yes ;;

    --skip-existing-release ) SKIP_EXISTING_RELEASE=yes ;;

    -- ) break ;;
    * ) error "unknown option '$option'" ;;
  esac
done


######################################################################
#
# Some initial sanity checks
#

cd "$SRC_DIR"

if [ ! -f PackageInfo.g ] ; then
    error "unable to read PackageInfo.g file, use --help for instructions"
fi

# Check for webdir presence
if [ "x$WEB_DIR" = x ] ; then
    WEB_DIR="$SRC_DIR/gh-pages"
fi
if [ ! -d "$WEB_DIR" ] ; then
    error "could not find 'webdir' with clone of your gh-pages branch"
fi

# Check for presence of the update script
if [ "x$UPDATE_SCRIPT" = x ] ; then
    UPDATE_SCRIPT="$WEB_DIR/update.g"
fi
if [ ! -f "$UPDATE_SCRIPT" ] ; then
    error "could not find update script \"${UPDATE_SCRIPT}\""
fi

# Check whether GAP is usable
GAP=${GAP:-gap}
command -v "$GAP" >/dev/null 2>&1 ||
    error "could not find GAP (perhaps set the GAP environment variable?)"

command -v curl >/dev/null 2>&1 ||
    error "the 'curl' command was not found, please install it"

command -v git >/dev/null 2>&1 ||
    error "the 'git' command was not found, please install it"

command -v python >/dev/null 2>&1 ||
    error "the 'python' command was not found, please install it"

verify_git_clean


######################################################################
#
# Determine package name and version, and other meta data
#
notice "Extracting information from PackageInfo.g..."
run_gap <<EOF
Read("PackageInfo.g");
Print("PKG=\"",GAPInfo.PackageInfoCurrent.PackageName,"\"\n");
Print("VERSION=\"",GAPInfo.PackageInfoCurrent.Version,"\"\n");
tmp:=GAPInfo.PackageInfoCurrent.Date;; # either YYYY-MM-DD or DD/MM/YYYY format
Print("DATE=\"",tmp,"\"\n");
Print("PackageInfoURL=\"",GAPInfo.PackageInfoCurrent.PackageInfoURL,"\"\n");
Print("README_URL=\"",GAPInfo.PackageInfoCurrent.README_URL,"\"\n");
# extract archive formats; these can be comma or space separated, we want
# to output them space separated
tmp := GAPInfo.PackageInfoCurrent.ArchiveFormats;;
tmp := JoinStringsWithSeparator(SplitString(tmp, ", "), " ");;
Print("ARCHIVE_FORMATS=\"",tmp,"\"\n");
tmp := SplitString(GAPInfo.PackageInfoCurrent.ArchiveURL, "/");;
if Length(tmp) = 9 and tmp{[1,2,3,6,7]} = ["https:","","github.com","releases","download"] then
    Print("REPO=\"",Concatenation(tmp[4],"/",tmp[5]),"\"\n");
    Print("TAG=\"",tmp[8],"\"\n");
    len := Length(tmp[9]) - Length(GAPInfo.PackageInfoCurrent.Version);
    if len > 0 then
        Print("BASENAME=\"",tmp[9]{[1..len]},"\"\n");
    fi;
else
    tmp := GAPInfo.PackageInfoCurrent.ArchiveURL;
    Print("GAP_ERROR=\"The ArchiveURL has unexpected value '",tmp,"'\"\n");
fi;
Print("PDFFile=\"",GAPInfo.PackageInfoCurrent.PackageDoc.PDFFile,"\"\n");
EOF

# evaluate the output of GAP, which should be valid shell script code
eval "$gap_output"
if [ x"$GAP_ERROR" != "x" ] ; then
    error "$GAP_ERROR"
fi

notice "Package $PKG $VERSION"
case "$VERSION" in
  *dev) error "Must not use 'dev' versions for package releases" ;;
esac

######################################################################
#
# Run makedoc.g (if present), as that also can uncover mistakes (e.g.
# for projects which generate doc/title.xml from PackageInfo.g, it is
# easy to forget that when making the release commit).
#
if [ -f makedoc.g ] ; then
    notice "Building GAP package documentation (using makedoc.g)"
    run_gap <<GAPInput
if not IsPackageMarkedForLoading("$PKG", "") then
  SetPackagePath("$PKG", ".");
fi;
Read("makedoc.g");
GAPInput
elif [ -f doc/make_doc ] ; then
    notice "Building GAP package documentation (using doc/make_doc)"
    cd doc && ./make_doc && cd ..
else
    warning "Could not build manual (no makedoc.g and no doc/make_doc found)"
fi

# check if building the manual modified any files
verify_git_clean


######################################################################
#
# Determine the basename for the package archives
#
#
if [ x"$BASENAME" = "x" ] ; then
    BASENAME="$PKG-"
fi
BASENAME="$BASENAME$VERSION"
notice "Using archive basename $BASENAME"


######################################################################
#
# Fetch GitHub oauth token, used to authenticate the following commands.
# See https://help.github.com/articles/git-automation-with-oauth-tokens/
#
if [ "x$TOKEN" = x ] ; then
    TOKEN=$(git config --get github.token || echo)
fi
if [ "x$TOKEN" = x ] && [ -r ~/.github_shell_token ] ; then
    TOKEN=$(cat ~/.github_shell_token)
fi
if [ "x$TOKEN" = x ] ; then
    error "could not determine GitHub access token, please consult the README"
fi


######################################################################
#
# Determine GitHub repository and username, and the current branch
#
notice "Using GitHub repository $REPO"

GITHUB_USER=$(dirname "$REPO")
notice "Using GitHub username $GITHUB_USER"

BRANCH=$(git symbolic-ref -q --short HEAD)
notice "Using branch $BRANCH"


######################################################################
#
# Derive API urls
#
API_URL=https://api.github.com/repos/$REPO/releases
UPLOAD_URL=https://uploads.github.com/repos/$REPO/releases


######################################################################
#
# Determine the tag
#
verify_git_clean

if git show-ref -q "$TAG" ; then
    notice "Using git tag $TAG (derived from ArchiveURL in in PackageInfo.g)"
else
    notice "Creating git tag $TAG"
    git tag "$TAG"
fi;

######################################################################
#
# Check if a GitHub release for this tag already exists
#
response=$(curl -s -S -X GET "$API_URL/tags/$TAG" -H "Authorization: token $TOKEN")
MESSAGE=$(json_get_key message)
RELEASE_ID=$(json_get_key id)

if [ "$MESSAGE" = "Not Found" ] ; then
    MESSAGE=  # release does not yet exist -> that's how we like it
elif [ x"$RELEASE_ID" != x ] ; then
    # release already exists -> skip, error out or delete it
    if [ "x$SKIP_EXISTING_RELEASE" = xyes ] ; then
        notice "release $TAG already exists on GitHub, skipping release"
        exit 0
    elif [ "x$FORCE" = xyes ] ; then
        notice "Deleting existing release $TAG from GitHub"
        response=$(curl --fail -s -S -X DELETE "$API_URL/$RELEASE_ID" -H "Authorization: token $TOKEN")
        MESSAGE=
    else
        error "release $TAG already exists on GitHub, aborting (use --force to override this)"
    fi
fi

if [ x"$MESSAGE" != x ] ; then
    error "accessing GitHub failed: $MESSAGE"
fi

######################################################################
#
# Validate the tag
#
HEAD_REF=$(git rev-parse --verify HEAD)
TAG_REF=$(git rev-parse --verify "$TAG^{}")

if [ "x$TAG_REF" != "x$HEAD_REF" ] ; then
    error "tag $TAG is not the HEAD commit -- did you tag the right commit?"
fi


echo ""


######################################################################
#
# Get fresh (unmodified) copies of the files, and generate some stuff
#

# Clean any remains of previous export attempts
mkdir -p "$TMP_DIR"
rm -rf "${TMP_DIR:?}/$BASENAME"*

# Set umask to ensure the file permissions in the release
# archives are sane.
umask 0022

notice "Exporting repository content for tag '$TAG'"
git archive --prefix="$BASENAME/" "$TAG" . | tar xf - -C "$TMP_DIR"


# Build the package documentation, run autoconf, etc.
cd "$TMP_DIR/$BASENAME"

# adjust date
# Note that we cannot use sed's `-i` option for in-place editing, as
# that is a non-portable extension of POSIX, which works differently in
# BSD and GNU make.
sed "s;Date := .*;Date := \"$(date +%d/%m/%Y)\",;" PackageInfo.g > PackageInfo.g.bak
mv PackageInfo.g.bak PackageInfo.g

notice "Removing unnecessary files"
# Remove recursively in case there is a .github directory
rm -rf .git* .hg* .cvs* .circleci
rm -f .appveyor.yml .codecov.yml .travis.yml

# execute .release script, if present
if [ -f .release ] ; then
    # the .release script can perform additional preparation, e.g.:
    # * add files for distribution which are not part of the repository;
    # * remove further files not intended for distribution (e.g. scripts/ directory);
    # * build the package manual in a custom way;
    # * perform additional sanity checks;
    # * ...
    . ./.release
    rm -f .release
fi

if [ -x autogen.sh ] ; then
    notice "Generating build system files"
    sh autogen.sh
    rm -rf autom4te.cache
fi

if [ -f makedoc.g ] ; then
    notice "Building GAP package documentation for archives (using makedoc.g)"
    run_gap <<GAPInput
if not IsPackageMarkedForLoading("$PKG", "") then
  SetPackagePath("$PKG", ".");
fi;
PushOptions(rec(relativePath:="../../.."));
Read("makedoc.g");
GAPInput
    rm -f doc/*.tex
    rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst
elif [ -f doc/make_doc ] ; then
    notice "Copying GAP package documentation for archives (using doc/make_doc)"
    cp -r "$SRC_DIR/doc" .
    cp -r "$SRC_DIR/htm" .
    rm -f doc/*.aux doc/*.bbl doc/*.blg doc/*.brf doc/*.idx doc/*.ilg doc/*.ind doc/*.log doc/*.out doc/*.pnr doc/*.tst
fi

# make sure every file is readable
chmod -R a+r .

# replace links to packages which are possibly referenced in the documentation, keep this in sync with `Tests.yml.j2`
for package in CAP_project/CAP CAP_project/CompilerForCAP CAP_project/FreydCategoriesForCAP CAP_project/MonoidalCategories HigherHomologicalAlgebra/ToolsForHigherHomologicalAlgebra homalg_project/homalg homalg_project/Modules Toposes; do
    
    # adjust links to other manuals
    # Note that we cannot use sed's `-i` option for in-place editing, as
    # that is a non-portable extension of POSIX, which works differently in
    # BSD and GNU make.
    for f in ./*/*.htm* ; do
        sed "s;href=\"/home/gap/.gap/pkg/$package/doc/;href=\"https://homalg-project.github.io/$package/doc/;g" "$f" > "$f.bak"
        mv "$f.bak" "$f"
    done
    
done

# basic sanity check
fgrep -q -r '<a href="/' */*.htm* &&
    error "HTML manual contains absolute paths"


######################################################################
#
# Validate PackageInfo.g
#
echo ""
notice "Validating PackageInfo.g..."
run_gap <<GAPInput || error "validation failed"
if not ValidatePackageInfo("PackageInfo.g") then
    FORCE_QUIT_GAP(1);
fi;
GAPInput


######################################################################
#
# Verify README_URL refers to an existing file
#
README_NAME=$(basename "$README_URL")
[ -f "$README_NAME" ] ||
    error "No file named '$README_NAME' exists; is your README_URL valid?"


######################################################################
#
# Create all archive files requested by ARCHIVE_FORMATS
#
cd "$TMP_DIR"
echo ""
for EXT in $ARCHIVE_FORMATS ; do
    ARCHIVENAME=$BASENAME$EXT
    FULLNAME="$TMP_DIR/$ARCHIVENAME"
    notice "Creating $ARCHIVENAME ..."
    case $EXT in
    .tar.gz)  tar cf - "$BASENAME" | gzip -9c > "$ARCHIVENAME" ;;
    .tar.bz2) tar cf - "$BASENAME" | bzip2 -9c > "$ARCHIVENAME" ;;
    .zip)     zip -r9 --quiet "$ARCHIVENAME" "$BASENAME" ;;
    *)
        warning "unsupported archive format $EXT"
        continue
        ;;
    esac
    if [ ! -f "$FULLNAME" ] ; then
        error "failed creating $FULLNAME"
    fi
done


######################################################################
#
# Abort early if user requested only creation of tarballs
#
if [ "x$ONLY_TARBALL" = xyes ] ; then
    exit 0
fi

######################################################################
#
# Push commits to GitHub
#

cd "$SRC_DIR"

# construct GitHub URL for pushing
REMOTE="https://$GITHUB_USER:$TOKEN@github.com/$REPO"

# Make sure the branch is on the server
notice "Pushing your branch to GitHub"
git push "$REMOTE" "$BRANCH"

# Make sure the tag is on the server
notice "Pushing your tag to GitHub"
if [ "x$FORCE" = xyes ] ; then
    git push --force "$REMOTE" "$TAG"
else
    git push "$REMOTE" "$TAG"
fi


######################################################################
#
# Create the GitHub release
#

# Create the release by sending suitable JSON
DATA=$(cat <<EOF
{
  "tag_name": "$TAG",
  "name": "$PKG $VERSION",
  "body": "Release for $PKG",
  "draft": false,
  "prerelease": false
}
EOF
)

notice "Creating new release $TAG on GitHub"
response=$(curl -s -S -H "Content-Type: application/json" \
 -X POST --data "$DATA" "$API_URL" -H "Authorization: token $TOKEN")

MESSAGE=$(json_get_key message)
if [ x"$MESSAGE" != x ] ; then
    error "creating release on GitHub failed: $MESSAGE"
fi
RELEASE_ID=$(json_get_key id)
if [ x"$RELEASE_ID" = x ] ; then
    error "creating release on GitHub failed: no release id"
fi


######################################################################
#
# Upload all archive files requested by ARCHIVE_FORMATS
#
cd "$TMP_DIR"
echo ""
for EXT in $ARCHIVE_FORMATS ; do
    ARCHIVENAME=$BASENAME$EXT
    FULLNAME="$TMP_DIR/$ARCHIVENAME"
    case $EXT in
    .tar.gz)  MIMETYPE="application/x-gzip" ;;
    .tar.bz2) MIMETYPE="application/x-bzip2" ;;
    .zip)     MIMETYPE="application/zip" ;;
    *)        continue ;;
    esac
    if [ ! -f "$FULLNAME" ] ; then
        error "failed creating $FULLNAME"
    fi
    notice "Uploading $ARCHIVENAME with mime type $MIMETYPE"
    response=$(curl --fail --progress-bar -o "$TMP_DIR/upload.log" \
        -X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$ARCHIVENAME" \
        -H "Accept: application/vnd.github.v3+json" \
        -H "Authorization: token $TOKEN" \
        -H "Content-Type: $MIMETYPE" \
        --data-binary @"$FULLNAME")
done


######################################################################
#
# Upload PDF
#
cd "$TMP_DIR"
echo ""

FULLNAME="$TMP_DIR/$BASENAME/$PDFFile"
if [ ! -f "$FULLNAME" ] ; then
	error "could not find PDF"
fi
notice "Uploading PDF"
response=$(curl --fail --progress-bar -o "$TMP_DIR/upload_pdf.log" \
	-X POST "$UPLOAD_URL/$RELEASE_ID/assets?name=$BASENAME.pdf" \
	-H "Accept: application/vnd.github.v3+json" \
	-H "Authorization: token $TOKEN" \
	-H "Content-Type: application/pdf" \
	--data-binary @"$FULLNAME")


######################################################################
#
# Update the website
#
notice "Updating website"
cd "$WEB_DIR"
git pull --ff-only

cp "$TMP_DIR/$BASENAME/$README_NAME" .
cp "$TMP_DIR/$BASENAME/PackageInfo.g" .
rm -rf doc/ htm/
if [ -f "$TMP_DIR/$BASENAME/doc/chap0.html" ] ; then
    mkdir -p doc/
    for ext in css html js txt ; do
        cp "$TMP_DIR/$BASENAME/doc"/*.$ext doc/
    done
fi
if [ -d "$TMP_DIR/$BASENAME/htm" ] ; then
    cp -r "$TMP_DIR/$BASENAME/htm" .
fi

# adjust links to the GAP manuals
# Note that we cannot use sed's `-i` option for in-place editing, as
# that is a non-portable extension of POSIX, which works differently in
# BSD and GNU make.
for f in ./*/*.htm* ; do
  sed 's;href="../../../doc/;href="https://www.gap-system.org/Manuals/doc/;g' "$f" > "$f.bak"
  mv "$f.bak" "$f"
done

run_gap <<GAPInput
Read("$UPDATE_SCRIPT");
GAPInput

git add -A
git commit -m "Update website for $PKG $VERSION"

if [ "x$PUSH" = xyes ] ; then
    notice "Pushing website changes"
    git push "$REMOTE"
    notice "Done"
    notice "Your PackageInfo.g is now at $PackageInfoURL"
else
    warning "To complete your release, run 'git push' in your gh-pages directory"
    notice "Afterwards, your PackageInfo.g will be at $PackageInfoURL"
fi

exit 0