Skip to content

Commit

Permalink
Multiplatform docker based builds (#7792)
Browse files Browse the repository at this point in the history
* Add dockerized multi-architecture build files

* Add android arm64 build
  • Loading branch information
larroy authored and piiswrong committed Oct 13, 2017
1 parent 43234d0 commit 3b36217
Show file tree
Hide file tree
Showing 19 changed files with 1,036 additions and 0 deletions.
2 changes: 2 additions & 0 deletions docker_multiarch/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
mxnet/
build/
77 changes: 77 additions & 0 deletions docker_multiarch/Dockerfile.build.android.arm64
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# -*- mode: dockerfile -*-
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllarroy@amazon.com"

# The cross-compiling emulator
RUN apt-get update && apt-get install -y \
qemu-user \
qemu-user-static \
unzip

ENV CROSS_TRIPLE=aarch64-linux-android
ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld

ENV ANDROID_NDK_REVISION 15c
RUN mkdir -p /build && \
cd /build && \
curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
cd android-ndk-r${ANDROID_NDK_REVISION} && \
./build/tools/make_standalone_toolchain.py \
--stl=libc++ \
--arch arm64 \
--api 21 \
--install-dir=${CROSS_ROOT} && \
cd / && \
rm -rf /build && \
find ${CROSS_ROOT} -exec chmod a+r '{}' \; && \
find ${CROSS_ROOT} -executable -exec chmod a+x '{}' \;


ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm

# COPY Toolchain.cmake ${CROSS_ROOT}/
# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake

# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
ARG VCS_REF
ARG VCS_URL
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name=$IMAGE \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url=$VCS_URL \
org.label-schema.schema-version="1.0"

ENV ARCH aarch64

# Build OpenBLAS
# https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android
RUN git clone https://github.com/xianyi/OpenBLAS.git && \
cd OpenBLAS && \
make -j$(nproc) TARGET=ARMV8 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs

ENV OPENBLAS_ROOT /work/OpenBLAS
ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH
ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH
WORKDIR /work

ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0"

# Build MXNet
ADD mxnet mxnet
ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
RUN cd mxnet && \
make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
78 changes: 78 additions & 0 deletions docker_multiarch/Dockerfile.build.android.armv7
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# -*- mode: dockerfile -*-
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllarroy@amazon.com"

# The cross-compiling emulator
RUN apt-get update && apt-get install -y \
qemu-user \
qemu-user-static \
unzip

ENV CROSS_TRIPLE=arm-linux-androideabi
ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld

ENV ANDROID_NDK_REVISION 15c
RUN mkdir -p /build && \
cd /build && \
curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
cd android-ndk-r${ANDROID_NDK_REVISION} && \
./build/tools/make_standalone_toolchain.py \
--stl=libc++ \
--arch arm \
--api 16 \
--install-dir=${CROSS_ROOT} && \
cd / && \
rm -rf /build && \
find ${CROSS_ROOT} -exec chmod a+r '{}' \; && \
find ${CROSS_ROOT} -executable -exec chmod a+x '{}' \;


ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm

# COPY Toolchain.cmake ${CROSS_ROOT}/
# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake

# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
ARG VCS_REF
ARG VCS_URL
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name=$IMAGE \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url=$VCS_URL \
org.label-schema.schema-version="1.0"

ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-gcc
ENV CXX /usr/arm-linux-androideabi/bin/arm-linux-androideabi-g++

# Build OpenBLAS
# https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android
RUN git clone https://github.com/xianyi/OpenBLAS.git && \
cd OpenBLAS && \
make -j$(nproc) TARGET=ARMV7 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs

ENV OPENBLAS_ROOT /work/OpenBLAS
ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH
ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH
WORKDIR /work

ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang
ENV CXX /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang++
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0"

# Build MXNet
ADD mxnet mxnet
ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
RUN cd mxnet && \
make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
37 changes: 37 additions & 0 deletions docker_multiarch/Dockerfile.build.arm64
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# -*- mode: dockerfile -*-
# dockerfile to build libmxnet.so for armv7
FROM dockcross/linux-arm64

ENV ARCH aarch64
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
ENV CC /usr/bin/aarch64-linux-gnu-gcc
ENV CXX /usr/bin/aarch64-linux-gnu-g++
ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
ENV HOSTCC gcc

WORKDIR /work

# Build OpenBLAS
ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master /tmp/openblas_version.json
RUN git clone https://github.com/xianyi/OpenBLAS.git && \
cd OpenBLAS && \
make -j$(nproc) TARGET=ARMV8 && \
make install && \
ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/libopenblas.so && \
ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/libopenblas.a && \
ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a

ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include

# Build MXNet
#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

WORKDIR /work/mxnet
ADD arm.crosscompile.mk make/config.mk
RUN make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
38 changes: 38 additions & 0 deletions docker_multiarch/Dockerfile.build.armv6
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# -*- mode: dockerfile -*-
# dockerfile to build libmxnet.so for armv7
FROM dockcross/linux-armv6

ENV ARCH armv6l
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
ENV CC /usr/bin/arm-linux-gnueabihf-gcc
ENV CXX /usr/bin/arm-linux-gnueabihf-g++
ENV FC /usr/bin/arm-linux-gnueabihf-gfortran
ENV HOSTCC gcc

WORKDIR /work

# Build OpenBLAS
ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master openblas_version.json
RUN git clone https://github.com/xianyi/OpenBLAS.git && \
cd OpenBLAS && \
make -j$(nproc) TARGET=ARMV6 && \
make install && \
ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.so && \
ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.a && \
ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/liblapack.a && \
ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a

ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include

# Build MXNet
#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

WORKDIR /work/mxnet
ADD arm.crosscompile.mk make/config.mk
RUN make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
26 changes: 26 additions & 0 deletions docker_multiarch/Dockerfile.build.armv7
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# -*- mode: dockerfile -*-
# dockerfile to build libmxnet.so for armv7
FROM dockcross/linux-armv7

RUN apt-get update && \
apt-get install -y libopenblas-dev:armhf && \
rm -rf /var/lib/apt/lists/*

ENV ARCH armv71
ENV CC /usr/bin/arm-linux-gnueabihf-gcc
ENV CXX /usr/bin/arm-linux-gnueabihf-g++
ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0"

# Build MXNet

WORKDIR /work
#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

WORKDIR /work/mxnet
ADD arm.crosscompile.mk make/config.mk
RUN make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
37 changes: 37 additions & 0 deletions docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
FROM ubuntu:17.04


RUN apt-get update &&\
apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
build-essential git pkgconf\
libopenblas-dev liblapack-dev\
maven default-jdk\
ninja-build\
libgtest-dev\
&&\
rm -rf /var/lib/apt/lists/*



###########################
# Build gtest
WORKDIR /work/googletest
RUN cmake /usr/src/googletest/googletest/ -GNinja
RUN ninja
RUN cp libgtest.a /usr/lib
###########################



WORKDIR /work
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

WORKDIR mxnet/build
RUN cmake -DUSE_CUDA=OFF -DUSE_OPENCV=OFF -GNinja ..
RUN ninja


# Copy artifacts
RUN mkdir -p /work/build
RUN cp *.a *.so /work/build
32 changes: 32 additions & 0 deletions docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
FROM nvidia/cuda:8.0-cudnn5-devel

RUN apt-get update &&\
apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
build-essential git pkgconf\
libopenblas-dev liblapack-dev\
maven default-jdk\
&&\
rm -rf /var/lib/apt/lists/*

WORKDIR /work
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

# Compile MxNet
ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1"
WORKDIR /work/mxnet
RUN make -j$(nproc) $BUILD_OPTS

WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .

# Scala packag
#WORKDIR /work
#RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
#RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb

#WORKDIR /work/mxnet
#RUN make scalapkg $BUILD_OPTS

#WORKDIR /work/build/scala_gpu
#RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-gpu/target/*.jar .
30 changes: 30 additions & 0 deletions docker_multiarch/Dockerfile.build.ubuntu-17.04
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#
# Base image to build MXNet from source in ubuntu
#
# Other images depend on it, so build it like:
#
# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
#
FROM ubuntu:17.04


RUN apt-get update &&\
apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
build-essential git pkgconf\
libopenblas-dev liblapack-dev\
maven default-jdk

RUN rm -rf /var/lib/apt/lists/*

WORKDIR /work
#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
ADD mxnet mxnet

# Compile MxNet
ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas"
WORKDIR /work/mxnet
RUN make -j$(nproc) $BUILD_OPTS

# Copy artifacts
WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
17 changes: 17 additions & 0 deletions docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Before building this image you would need to build MXNet by executing:
# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
# if you haven't done it before.

FROM mxnet.build.ubuntu-17.04

# Scala package
WORKDIR /work
RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb

WORKDIR /work/mxnet
RUN make scalapkg $BUILD_OPTS

WORKDIR /work/build/scala
RUN cp /work/mxnet/scala-package/core/target/*.jar .
RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-cpu/target/*.jar .
Loading

0 comments on commit 3b36217

Please sign in to comment.