From 424fafe58785520cfb8e3ad0e6ce26f9e9db5e24 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Wed, 25 Jul 2018 00:09:51 -0700 Subject: [PATCH] [MXNET-710] Change POM files to be able to regularly publish to Apache Release & Maven Central Repo (#11862) * pom file changes for maven builds --- Makefile | 23 +++- .../assembly/linux-x86_64-cpu/pom.xml | 36 +++++- .../assembly/linux-x86_64-gpu/pom.xml | 36 +++++- scala-package/assembly/osx-x86_64-cpu/pom.xml | 41 +++++++ scala-package/assembly/pom.xml | 9 +- scala-package/core/pom.xml | 7 ++ scala-package/dev/compile-mxnet-backend.sh | 108 ++++++++++++++++++ scala-package/infer/pom.xml | 7 ++ .../init-native/linux-x86_64/pom.xml | 9 +- scala-package/init-native/osx-x86_64/pom.xml | 11 +- scala-package/init-native/pom.xml | 13 +++ scala-package/init/pom.xml | 33 ++++++ scala-package/macros/pom.xml | 7 ++ scala-package/native/linux-x86_64-cpu/pom.xml | 9 +- scala-package/native/linux-x86_64-gpu/pom.xml | 9 +- scala-package/native/osx-x86_64-cpu/pom.xml | 9 +- scala-package/native/pom.xml | 13 +++ scala-package/pom.xml | 75 ++++++------ 18 files changed, 402 insertions(+), 53 deletions(-) create mode 100755 scala-package/dev/compile-mxnet-backend.sh diff --git a/Makefile b/Makefile index 5816637f989a..88f7dd9278cb 100644 --- a/Makefile +++ b/Makefile @@ -589,6 +589,7 @@ scalaclean: scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dcurrent_libdir="$(ROOTDIR)/lib" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -608,12 +609,32 @@ scalaintegrationtest: scalainstall: (cd $(ROOTDIR)/scala-package; \ mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") +scalarelease-dryrun: + (cd $(ROOTDIR)/scala-package; \ + mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ + -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + +scalarelease-prepare: + (cd $(ROOTDIR)/scala-package; \ + mvn release:clean release:prepare -DautoVersionSubmodules=true \ + -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + +scalarelease-perform: + (cd $(ROOTDIR)/scala-package; \ + mvn release:perform -DautoVersionSubmodules=true \ + -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + scaladeploy: (cd $(ROOTDIR)/scala-package; \ - mvn deploy -Prelease,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \ + mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index 5ae050c68b44..b410de10f56e 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -26,15 +26,49 @@ 1.3.0-SNAPSHOT so + + org.apache.mxnet + mxnet-infer_${scala.binary.version} + 1.3.0-SNAPSHOT + + + org.codehaus.mojo + flatten-maven-plugin + + + + remove + + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-deploy-plugin + false - true + false diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index c4d7aa98b128..0a1e7953848a 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -26,15 +26,49 @@ 1.3.0-SNAPSHOT so + + org.apache.mxnet + mxnet-infer_${scala.binary.version} + 1.3.0-SNAPSHOT + + + org.codehaus.mojo + flatten-maven-plugin + + + + remove + + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-deploy-plugin + false - true + false diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index 69e87b7b16e6..8a12d80878a6 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -26,10 +26,51 @@ 1.3.0-SNAPSHOT jnilib + + org.apache.mxnet + mxnet-infer_${scala.binary.version} + 1.3.0-SNAPSHOT + + + org.codehaus.mojo + flatten-maven-plugin + + + + remove + + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + + + + org.apache.maven.plugins + maven-deploy-plugin + false + + false + + org.apache.maven.plugins maven-assembly-plugin diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 83d6c732a4b4..aef50ce57cff 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -37,6 +37,13 @@ release + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-source-plugin @@ -47,7 +54,7 @@ jar-no-fork - true> + true diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 79da63b7c980..134e0a59da1f 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -48,6 +48,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-jar-plugin diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh new file mode 100755 index 000000000000..b065e01afc8e --- /dev/null +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (Yizhi) This is mainly inspired by the script in apache/spark. +# I did some modificaiton to get it with our project. +# + +set -e +echo "Compiling MXNet Backend, Hang tight!....." + +if [[ ($# -ne 2) || ( $1 == "--help") || $1 == "-h" ]]; then + echo "Usage: [-h|--help] " 1>&2 + exit 1 +fi +PLATFORM=$1 +MXNETDIR=$2 + + +# below routine shamelessly copied from +# https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh +# This routine executes a command, +# prints error message on the console on non-zero exit codes and +# returns the exit code to the caller. +chkret() { + cmd=$* + echo "$cmd" + $cmd + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: $cmd" + echo " " + exit 1 + fi +} # chkret() + +UNAME=`uname -s` +chkret pushd $MXNETDIR +chkret git submodule update --init --recursive + +# don't want to overwrite an existing config file +cp make/config.mk ./config.mk + +if [[ $PLATFORM == "osx-x86_64-cpu" ]]; +then + echo "Building MXNet Backend on MAC OS" + echo "ADD_CFLAGS += -I/usr/local/opt/opencv/include" >> ./config.mk + echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/opt/opencv/lib" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk + echo "USE_OPENMP = 0" >> ./config.mk + echo "USE_LAPACK_PATH = /usr/local/opt/lapack/lib" >> ./config.mk + make -j$(sysctl -n hw.ncpu) +elif [[ $PLATFORM == "linux-x86_64-cpu" ]]; +then + echo "Building MXNet Backend on Linux CPU" + echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk + echo "USE_OPENCV=1" >> ./config.mk + echo "USE_OPENMP=1" >> ./config.mk + echo "USE_BLAS=openblas" >> ./config.mk + echo "USE_LAPACK=1" >> ./config.mk + echo "USE_DIST_KVSTORE=1" >> ./config.mk + echo "USE_S3=1" >> ./config.mk + make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1) +elif [[ $PLATFORM == "linux-x86_64-gpu" ]] +then + echo "Building MXNet Backend on Linux GPU" + echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk + echo "USE_OPENCV=1" >> ./config.mk + echo "USE_OPENMP=1" >> ./config.mk + echo "USE_BLAS=openblas" >> ./config.mk + echo "USE_LAPACK=1" >> ./config.mk + echo "USE_DIST_KVSTORE=1" >> ./config.mk + echo "USE_S3=1" >> ./config.mk + echo "USE_CUDA=1" >> ./config.mk + echo "USE_CUDNN=1" >> ./config.mk + echo "ADD_CFLAGS += -I/usr/local/cuda/include" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/cuda/lib64/ " >> ./config.mk + #update th nccl version approriately + echo "ADD_LDFLAGS += -L/lib/nccl/cuda-9.0/lib " >> ./config.mk + eval "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/lib/nccl/cuda-9.0/lib" + eval "export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" + make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1) + echo "Building MXNet Backend on Linux GPU" +else + echo "MY ALIEN OVERLOADS HAVE NOT TOLD WHAT TO DO FOR INVALID INPUT !!!" + echo "Currently supported platforms: osx-x86_64-cpu or linux-x86_64-cpu or linux-x86_64-gpu" +fi +chkret popd +echo "done building MXNet Backend" +exit 0 diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 573684d2dd1d..3425579acabc 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -48,6 +48,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-jar-plugin diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index 2ddeaba7acbf..7a6d90866442 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -28,6 +28,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin @@ -55,7 +62,7 @@ -std=c++0x - -I../../../include + -I${project.basedir}/../../../include ${all_includes} ${cflags} diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index 120854986afb..0eaf4e343655 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -28,6 +28,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin @@ -55,7 +62,7 @@ -std=c++0x - -I../../../include + -I${project.basedir}/../../../include ${cflags} @@ -66,7 +73,7 @@ -Wl,-exported_symbol,_Java_* -Wl,-x ${lddeps} - -force_load ../../../lib/libmxnet.a + -force_load ${project.basedir}/../../../lib/libmxnet.a ${ldflags} diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 8ac369d1d71b..1cd79a84093e 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -34,4 +34,17 @@ + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index ef1d67b5dda7..2c00ca5ffbca 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -32,5 +32,38 @@ linux-x86_64-gpu + + apache-release + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + compile-mxnet-backend + compile + + exec + + + bash + ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 15f13f3940e8..795900987fb2 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -73,6 +73,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-jar-plugin diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 2504b1f31553..c45635eb6f83 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -28,6 +28,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin @@ -55,7 +62,7 @@ -std=c++0x - -I../../../include + -I${project.basedir}/../../../include ${all_includes} ${cflags} diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index aca290f6d1fc..a1f5ec34d4ba 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -28,6 +28,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin @@ -55,7 +62,7 @@ -std=c++0x - -I../../../include + -I${project.basedir}/../../../include ${all_includes} ${cflags} diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 15033535e4a1..3f66fe68e041 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -28,6 +28,13 @@ + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-compiler-plugin @@ -66,8 +73,6 @@ -Wl,-exported_symbol,_Java_* -Wl,-x ${lddeps} - -force_load ../../../lib/libmxnet.a - -force_load ../../../3rdparty/tvm/nnvm/lib/libnnvm.a ${ldflags} diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 54ba2b57afa4..485b69f098ee 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -34,4 +34,17 @@ + + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index d931a82a664e..c4f162008fac 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -3,31 +3,42 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + org.apache + apache + 19 + org.apache.mxnet mxnet-parent_2.11 1.3.0-SNAPSHOT MXNet Scala Package - Parent https://github.com/apache/incubator-mxnet/tree/master/scala-package - MXNet Scala Package + + Scala Package for Apache MXNet (Incubating) - flexible and efficient library for deep learning. + - Distributed (Deep) Machine Learning Community - http://dmlc.ml + The Apache Software Foundation + https://www.apache.org/ - The Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + - scm:git:git@github.com:dmlc/mxnet.git - scm:git:git@github.com:dmlc/mxnet.git + scm:git:git@github.com:apache/incubator-mxnet.git + scm:git:git@github.com:apache/incubator-mxnet.git https://github.com/apache/incubator-mxnet + HEAD 2.11.8 2.11 + pom @@ -65,29 +76,6 @@ - - org.apache.maven.plugins - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - ossrh - https://oss.sonatype.org/ - true - - @@ -161,15 +149,23 @@ - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - + + org.apache.maven.plugins + maven-release-plugin + + true + false + + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + org.apache.maven.plugins maven-source-plugin @@ -213,11 +209,6 @@ maven-assembly-plugin 2.5.5 - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - org.apache.maven.plugins maven-surefire-plugin