Reproducible buildsystem

Makefile

@@ -2,7 +2,7 @@
 
 PACKAGES_NOSIMULATION=$(shell go list ./... | grep -v '/simulation')
 PACKAGES_SIMTEST=$(shell go list ./... | grep '/simulation')
-VERSION := $(shell echo $(shell git describe) | sed 's/^v//')
+VERSION := $(shell echo $(shell git describe --always) | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 LEDGER_ENABLED ?= true
 BINDIR ?= $(GOPATH)/bin
@@ -86,12 +86,35 @@ include contrib/devtools/Makefile
 build: go.sum
 	go install -mod=readonly ./...
 
-build-simd: go.sum
+simd:
 	mkdir -p $(BUILDDIR)
 	go build -mod=readonly $(BUILD_FLAGS) -o $(BUILDDIR) ./simapp/simd
 
+build-simd-all: go.sum
+	$(if $(shell docker inspect -f '{{ .Id }}' cosmossdk/rbuilder 2>/dev/null),$(info found image cosmossdk/rbuilder),docker pull cosmossdk/rbuilder:latest)
+	docker rm latest-build || true
+	docker run --volume=$(CURDIR):/sources:ro \
+        --env TARGET_OS='darwin linux windows' \
+        --env APP=simd \
+        --env VERSION=$(VERSION) \
+        --env COMMIT=$(COMMIT) \
+        --env LEDGER_ENABLED=$(LEDGER_ENABLED) \
+        --name latest-build cosmossdk/rbuilder:latest
+	docker cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/
+
 build-simd-linux: go.sum
-	LEDGER_ENABLED=false GOOS=linux GOARCH=amd64 $(MAKE) build-simd
+	$(if $(shell docker inspect -f '{{ .Id }}' cosmossdk/rbuilder 2>/dev/null),$(info found image cosmossdk/rbuilder),docker pull cosmossdk/rbuilder:latest)
+	docker rm latest-build || true
+	docker run --volume=$(CURDIR):/sources:ro \
+        --env TARGET_OS='linux' \
+        --env APP=simd \
+        --env VERSION=$(VERSION) \
+        --env COMMIT=$(COMMIT) \
+        --env LEDGER_ENABLED=false \
+        --name latest-build cosmossdk/rbuilder:latest
+	docker cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/
+	mkdir -p $(BUILDDIR)
+	cp artifacts/simd-*-linux-amd64 $(BUILDDIR)/simd
 
 cosmovisor:
 	$(MAKE) -C cosmovisor cosmovisor
@@ -119,7 +142,9 @@ distclean: clean
     .gitian-builder-cache/
 
 clean:
-	rm -rf $(BUILDDIR)/
+	rm -rf \
+    $(BUILDDIR)/ \
+    artifacts/
 
 .PHONY: distclean clean
 

build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -ue
+
+# Expect the following envvars to be set:
+# - APP
+# - VERSION
+# - COMMIT
+# - TARGET_OS
+# - LEDGER_ENABLED
+# - DEBUG
+
+# Source builder's functions library
+. /usr/local/share/cosmos-sdk/buildlib.sh
+
+# These variables are now available
+# - BASEDIR
+# - OUTDIR
+
+# Build for each os-architecture pair
+for os in ${TARGET_OS} ; do
+    archs="`f_build_archs ${os}`"
+    exe_file_extension="`f_binary_file_ext ${os}`"
+    for arch in ${archs} ; do
+        make clean
+        GOOS="${os}" GOARCH="${arch}" GOROOT_FINAL="$(go env GOROOT)" \
+        make ${APP} \
+            LDFLAGS=-buildid=${VERSION} \
+            VERSION=${VERSION} \
+            COMMIT=${COMMIT} \
+            LEDGER_ENABLED=${LEDGER_ENABLED}
+        mv ./build/${APP}${exe_file_extension} ${OUTDIR}/${APP}-${VERSION}-${os}-${arch}${exe_file_extension}
+    done
+    unset exe_file_extension
+done
+
+# Generate and display build report
+f_generate_build_report ${OUTDIR}
+cat ${OUTDIR}/build_report

contrib/images/Makefile

@@ -3,4 +3,7 @@ all: simd-env
 simd-env:
 	docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) --tag cosmossdk/simd-env simd-env
 
-.PHONY: all simd-env
+rbuilder:
+	docker build --tag cosmossdk/rbuilder rbuilder
+
+.PHONY: all simd-env rbuilder

contrib/images/rbuilder/Dockerfile

@@ -0,0 +1,19 @@
+FROM golang:1.15.0-buster
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get --no-install-recommends -y install \
+    pciutils build-essential git wget \
+    lsb-release dpkg-dev curl bsdmainutils fakeroot
+RUN mkdir -p /usr/local/share/cosmos-sdk/
+COPY buildlib.sh /usr/local/share/cosmos-sdk/
+RUN useradd -ms /bin/bash -U builder
+ARG APP
+ARG DEBUG
+ENV APP ${APP:-cosmos-sdk}
+ENV DEBUG ${DEBUG}
+ENV VERSION unknown
+ENV COMMIT unknown
+ENV LEDGER_ENABLE true
+USER builder:builder
+WORKDIR /sources
+VOLUME [ "/sources" ]
+ENTRYPOINT [ "/sources/build.sh" ]

contrib/images/rbuilder/buildlib.sh

@@ -0,0 +1,80 @@
+#/bin/bash
+
+f_make_release_tarball() {
+    SOURCEDIST=${BASEDIR}/${APP}-${VERSION}.tar.gz
+
+    git archive --format tar.gz --prefix "${APP}-${VERSION}/" -o "${SOURCEDIST}" HEAD
+
+    l_tempdir="$(mktemp -d)"
+    pushd "${l_tempdir}" >/dev/null
+    tar xf "${SOURCEDIST}"
+    rm "${SOURCEDIST}"
+    find ${APP}-* | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > "${SOURCEDIST}"
+    popd >/dev/null
+    rm -rf "${l_tempdir}"
+}
+
+f_setup_pristine_src_dir() {
+    cd ${pristinesrcdir}
+    tar --strip-components=1 -xf "${SOURCEDIST}"
+    go mod download
+}
+
+f_build_archs() {
+    local l_os
+
+    l_os=$1
+
+    case "${l_os}" in
+    darwin | windows)
+        echo 'amd64'
+        ;;
+    linux)
+        echo 'amd64 arm64'
+        ;;
+    *)
+        echo "unknown OS -- ${l_os}" >&2
+        return 1
+    esac
+}
+
+f_binary_file_ext() {
+   [ $1 = windows ] && printf '%s' '.exe' || printf ''
+}
+
+f_generate_build_report() {
+    local l_tempfile
+
+    l_tempfile="$(mktemp)"
+
+    pushd "${OUTDIR}" >/dev/null
+    cat >>"${l_tempfile}" <<EOF
+App: ${APP}
+Version: ${VERSION}
+Commit: ${COMMIT}
+EOF
+    echo 'Files:' >> "${l_tempfile}"
+    md5sum * | sed 's/^/ /' >> "${l_tempfile}"
+    echo 'Checksums-Sha256:' >> "${l_tempfile}"
+    sha256sum * | sed 's/^/ /' >> "${l_tempfile}"
+    mv "${l_tempfile}" build_report
+    popd >/dev/null
+}
+
+[ "x${DEBUG}" = "x" ] || set -x
+
+BASEDIR="$(mktemp -d)"
+OUTDIR=$HOME/artifacts
+rm -rfv ${OUTDIR}/
+mkdir -p ${OUTDIR}/
+pristinesrcdir=${BASEDIR}/buildsources
+mkdir -p ${pristinesrcdir}
+
+# Make release tarball
+f_make_release_tarball
+
+# Extract release tarball and cache dependencies
+f_setup_pristine_src_dir
+
+# Move the release tarball to the out directory
+mv ${SOURCEDIST} ${OUTDIR}/