Add gitlab builder for creating releases, update build to include pyt…

…hon wheels

Initial changes by Lawrence Nahum <lawrence@greenaddress.it> and
Jeremy Kauffman <jeremy@blockstream.io>.

.gitignore

@@ -66,3 +66,4 @@ wallycore.egg-info/
 wally*.gz
 wally*.sha256
 wally*.asc
+wally*.whl

.gitlab-ci.yml

@@ -0,0 +1,28 @@
+build_tag_release:
+  image: greenaddress/wallycore:fc18366c4da94ea7fe328650709e39a1133595f4b998b9c992f700c4f392c1e7
+  artifacts:
+    name: wallycore-android-jni
+    when: on_success
+    paths:
+    - wallycore-android-jni.tar.gz
+    - wallycore-android-jni.tar.gz.sha256
+    - wallycore-0.2.0-py27-none-linux_x86_64.whl
+    - wallycore-0.2.0-py27-none-linux_x86_64.whl.sha256
+    - wallycore-0.2.0-py35-none-linux_x86_64.whl
+    - wallycore-0.2.0-py35-none-linux_x86_64.whl.sha256
+    - wallycore-linux_x86_64-python2.tar.gz
+    - wallycore-linux_x86_64-python2.tar.gz.sha256
+    - wallycore-linux_x86_64-python3.tar.gz
+    - wallycore-linux_x86_64-python3.tar.gz.sha256
+  tags:
+    - ga
+  script:
+    - cd $CI_PROJECT_DIR
+    - ./tools/build_python_eggs.sh
+    - ./tools/build_android_libraries.sh
+    - mv release wallycore-android-jni
+    - tar cvf wallycore-android-jni.tar wallycore-android-jni
+    - gzip -9 wallycore-android-jni.tar
+    - sha256sum wallycore-android-jni.tar.gz > wallycore-android-jni.tar.gz.sha256
+  only:
+    - tag

README.md

@@ -80,19 +80,27 @@ is under heavy development.
 
 If you wish to explicitly choose the python version to use, set the
 `PYTHON_VERSION` environment variable (to e.g. `2`, `2.7`, `3` etc) before
-running `./configure`.
+running `setup.py` or (when compiling manually) `./configure`.
 
-You can also install the binary wally releases using the released egg files
-without having to compile the library. e.g. Assuming you have untarred the
-x86_64 python3 release into the wallycore-linux-x86_64-64-python3 directory,
-you can install it using:
+You can also install the binary wally releases using the released egg or
+wheel files without having to compile the library.
+
+For wheel releases you can install directly using the released wheel files
+and pip install, e.g.:
+
+```
++pip install wallycore-0.3.0-py27-none-linux_x86_64.whl
+```
+
+For egg releases, untar the tarball which will create a directory of the
+form `wallcore-<platform>-<python version>`. You can install using:
 
 ```
-python3 wallycore-linux-x86_64-64-python3/setup.py easy_install wallycore-linux-x86_64-64-python3/*.egg
+python3 <dir>/setup.py easy_install <dir>/*.egg
 ```
 
-The script `tools/build_python_eggs.sh` contains python code to compute the
-name of the release file if you wish to fetch it dynamically.
+The script `tools/build_python_eggs.sh` builds the release files and can be
+used as an example for your own deployments.
 
 ### Android
 

contrib/Dockerfile

@@ -0,0 +1,6 @@
+FROM debian:stretch
+COPY stretch_deps.sh /deps.sh
+RUN /deps.sh && rm /deps.sh
+VOLUME /wallycore
+ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+ENV ANDROID_NDK=/opt/android-ndk-r14b

contrib/stretch_deps.sh

@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+set -e
+
+export NDK_FILENAME=android-ndk-r14b-linux-x86_64.zip
+
+dpkg --add-architecture i386
+sed -i 's/deb.debian.org/httpredir.debian.org/g' /etc/apt/sources.list
+
+apt-get update -qq
+apt-get upgrade -yqq
+apt-get install python{,3}-distutils-extra python{,3}-dev build-essential libffi-dev swig autoconf libtool pkg-config lib32z1 openjdk-8-jdk ca-certificates-java unzip curl libc6:i386 libc6-dev:i386 libncurses5:i386 libstdc++6:i386 lib32z1 virtualenv python{,3}-setuptools -yqq
+update-java-alternatives -s java-1.8.0-openjdk-amd64
+
+cd /opt && curl -sSO https://dl.google.com/android/repository/${NDK_FILENAME} && unzip -qq ${NDK_FILENAME} && rm ${NDK_FILENAME}
+
+apt-get remove --purge curl unzip -yqq
+apt-get -yqq autoremove
+apt-get -yqq clean
+rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/* /usr/share/locale/* /usr/share/man /usr/share/doc /lib/xtables/libip6*

tools/build_android_libraries.sh

@@ -0,0 +1,50 @@
+#! /usr/bin/env bash
+#
+# Build native android libraries with JNI bindings, for use with C or Java.
+# Requires JAVA_HOME and ANDROID_NDK to be set.
+#
+set -e
+
+if [ -z "$ANDROID_NDK" ]; then
+    export ANDROID_NDK=$(dirname `which ndk-build 2>/dev/null`)
+fi
+echo ${ANDROID_NDK:?}
+if [ -z "$JAVA_HOME" ]; then
+    export JAVA_HOME=$JAVA7_HOME
+fi
+echo ${JAVA_HOME:?}
+
+source $PWD/tools/android_helpers.sh
+
+$PWD/tools/cleanup.sh && $PWD/tools/autogen.sh
+
+# Build everything unless the user passed a single target name
+ARCH_LIST=$(android_get_arch_list)
+if [ -n "$1" ]; then
+    ARCH_LIST="$1"
+fi
+
+for arch in $ARCH_LIST; do
+    # Use API level 14 for non-64 bit targets for better device coverage
+    api="14"
+    if [[ $arch == *"64"* ]]; then
+        api="21"
+    fi
+
+    # Location of the NDK tools to build with
+    toolsdir="$PWD/toolchain-$arch"
+
+    # What we want built
+    useropts="--enable-swig-java"
+
+    # Configure and build with the above options
+    android_build_wally $arch $toolsdir $api $useropts
+
+    # Copy the build result
+    mkdir -p $PWD/release/lib/$arch
+    $toolsdir/bin/*-strip -o $PWD/release/lib/$arch/libwallycore.so $PWD/src/.libs/libwallycore.so
+done
+
+mkdir -p $PWD/release/include $PWD/release/src/swig_java/src/com/blockstream/libwally
+cp $PWD/include/*.h $PWD/release/include
+cp $PWD/src/swig_java/src/com/blockstream/libwally/Wally.java $PWD/release/src/swig_java/src/com/blockstream/libwally

tools/build_python_eggs.sh

@@ -1,25 +1,42 @@
-#!/bin/bash
+#! /usr/bin/env bash
 
-# Creates tarballs with a binary egg file and setup.py for python2/3.
+# Creates tarballs with a binary egg file and setup.py for python2/3,
+# as well as binary wheel files.
+#
 # After unpacking, the resulting egg file can be installed with
 # python setup.py easy_install wallycore*.egg
+#
+# The wheel file can be installed with
+# pip install wallycore*.whl
+#
 # Like all tools/ scripts, this should be run from the project root.
 
-PLATFORM=$(python -c 'import platform; print(platform.system().lower())')
-BITS=$(python -c 'import sys; print("64" if sys.maxsize > 2**32 else "32")')
-MACHINE=$(python -c 'import platform; print(platform.machine().lower())')
-NAME="wallycore-$PLATFORM-$MACHINE-$BITS"
+set -e
+
+PLATFORM=$(python -c "from distutils import util; print(util.get_platform().replace('-','_').replace('.','_'))")
+NAME="wallycore-$PLATFORM"
 
 function build {
     ./tools/cleanup.sh
     virtualenv -p $1 .venv
     source .venv/bin/activate
+
+    # Create an egg from a new wally build
     PYTHONDONTWRITEBYTECODE= $1 setup.py install
     cp setup.py dist
     mv dist $NAME-$1
     tar czf $NAME-$1.tar.gz $NAME-$1
     sha256sum $NAME-$1.tar.gz >$NAME-$1.tar.gz.sha256
+
+    # Create a wheel from the .egg
+    cd $NAME-$1
+    wheel convert *.egg
+    WHEEL_NAME=$(ls *whl | sed "s/-any/-$PLATFORM/")
+    mv *.whl ../$WHEEL_NAME
+    cd ..
+    sha256sum $WHEEL_NAME > $WHEEL_NAME.sha256
     #gpg --armor --output $NAME-$1.tar.gz.asc --detach-sign $NAME-$1.tar.gz
+    #gpg --armor --output $WHEEL_NAME.asc --detach-sign $WHEEL_NAME
     rm -r $NAME-$1
     deactivate
 }
@@ -28,4 +45,3 @@ build python2
 build python3
 
 ./tools/cleanup.sh
-

tools/cleanup.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#! /usr/bin/env bash
 
 # Clean up all generated files
 make -o configure distclean >/dev/null 2>&1

tools/coverage.sh

@@ -1,4 +1,4 @@
-#/bin/bash
+#! /usr/bin/env bash
 
 # Helper to generate coverage reports.
 # ./tools/coverage.sh clean : Sets coverage stats to 0.

tools/swigjavapost.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#! /usr/bin/env bash
 
 sed_exe=$1
 

tools/travis_build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#! /usr/bin/env bash
 
 ENABLE_SWIG_PYTHON="--enable-swig-python"
 ENABLE_SWIG_JAVA="--enable-swig-java"

tools/travis_install.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#! /usr/bin/env bash
 
 if [ "$TRAVIS_OS_NAME" = "osx" ]; then
     brew update