[MXNET-16] Move mshadow/ps-lite/dlpack to 3rdparty apache#10132 (apac…

…he#10138)

* move

* update cmake/make

* remove commment

* update license

* ps-lite move

* update license

* move mshadow for make

* mshadow cmake

* update license

* update readme

* Update Jenkinsfile

* update license file with paths

.gitmodules

@@ -1,17 +1,17 @@
 [submodule "mshadow"]
-	path = mshadow
+	path = 3rdparty/mshadow
 	url = https://github.com/dmlc/mshadow.git
 [submodule "dmlc-core"]
 	path = dmlc-core
 	url = https://github.com/dmlc/dmlc-core.git
 [submodule "ps-lite"]
-	path = ps-lite
+	path = 3rdparty/ps-lite
 	url = https://github.com/dmlc/ps-lite
 [submodule "nnvm"]
 	path = nnvm
 	url = https://github.com/dmlc/nnvm
 [submodule "dlpack"]
-	path = dlpack
+	path = 3rdparty/dlpack
 	url = https://github.com/dmlc/dlpack
 [submodule "3rdparty/openmp"]
 	path = 3rdparty/openmp

CMakeLists.txt

@@ -221,14 +221,14 @@ endif()
 
 if(USE_CUDA AND FIRST_CUDA)
   include(cmake/ChooseBlas.cmake)
-  include(mshadow/cmake/Utils.cmake)
+  include(3rdparty/mshadow/cmake/Utils.cmake)
   include(cmake/FirstClassLangCuda.cmake)
   include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
 else()
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
-    include(mshadow/cmake/mshadow.cmake)
-    include(mshadow/cmake/Utils.cmake)
-    include(mshadow/cmake/Cuda.cmake)
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+    include(3rdparty/mshadow/cmake/mshadow.cmake)
+    include(3rdparty/mshadow/cmake/Utils.cmake)
+    include(3rdparty/mshadow/cmake/Cuda.cmake)
   else()
     include(mshadowUtils)
     include(Cuda)
@@ -243,16 +243,16 @@ foreach(var ${C_CXX_INCLUDE_DIRECTORIES})
 endforeach()
 
 include_directories("include")
-include_directories("mshadow")
+include_directories("3rdparty/mshadow")
 include_directories("3rdparty/cub")
 include_directories("nnvm/include")
 include_directories("nnvm/tvm/include")
 include_directories("dmlc-core/include")
-include_directories("dlpack/include")
+include_directories("3rdparty/dlpack/include")
 
 # commented out until PR goes through
-#if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dlpack)
-#  add_subdirectory(dlpack)
+#if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/dlpack)
+#  add_subdirectory(3rdparty/dlpack)
 #endif()
 
 # Prevent stripping out symbols (operator registrations, for example)
@@ -394,8 +394,8 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dmlc-core/cmake)
   add_subdirectory("dmlc-core")
 endif()
 
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mshadow/cmake)
-  add_subdirectory("mshadow")
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/mshadow/cmake)
+  add_subdirectory("3rdparty/mshadow")
 endif()
 FILE(GLOB_RECURSE SOURCE "src/*.cc" "src/*.h" "include/*.h")
 FILE(GLOB_RECURSE CUDA "src/*.cu" "src/*.cuh")
@@ -412,15 +412,15 @@ FILE(GLOB_RECURSE NNVMSOURCE
 list(APPEND SOURCE ${NNVMSOURCE})
 
 # add mshadow file
-FILE(GLOB_RECURSE MSHADOWSOURCE "mshadow/mshadow/*.h")
-FILE(GLOB_RECURSE MSHADOW_CUDASOURCE "mshadow/mshadow/*.cuh")
+FILE(GLOB_RECURSE MSHADOWSOURCE "3rdparty/mshadow/mshadow/*.h")
+FILE(GLOB_RECURSE MSHADOW_CUDASOURCE "3rdparty/mshadow/mshadow/*.cuh")
 list(APPEND SOURCE ${MSHADOWSOURCE})
 list(APPEND CUDA ${MSHADOW_CUDASOURCE})
 
 # add source group
 FILE(GLOB_RECURSE GROUP_SOURCE "src/*.cc" "nnvm/*.cc" "plugin/*.cc")
-FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "mshadow/mshadow/*.h" "plugin/*.h")
-FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "mshadow/mshadow/*.cuh" "plugin/*.cu"
+FILE(GLOB_RECURSE GROUP_Include "src/*.h" "nnvm/*.h" "3rdparty/mshadow/mshadow/*.h" "plugin/*.h")
+FILE(GLOB_RECURSE GROUP_CUDA "src/*.cu" "src/*.cuh" "3rdparty/mshadow/mshadow/*.cuh" "plugin/*.cu"
   "plugin/*.cuh" "3rdparty/cub/cub/*.cuh")
 assign_source_group("Source" ${GROUP_SOURCE})
 assign_source_group("Include" ${GROUP_Include})
@@ -593,8 +593,8 @@ if(USE_CUDA)
   endif()
 endif()
 if(USE_DIST_KVSTORE)
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt)
-    add_subdirectory("ps-lite")
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/ps-lite/CMakeLists.txt)
+    add_subdirectory("3rdparty/ps-lite")
     list(APPEND pslite_LINKER_LIBS pslite protobuf)
     target_link_libraries(mxnet PUBLIC debug ${pslite_LINKER_LIBS_DEBUG})
     target_link_libraries(mxnet PUBLIC optimized ${pslite_LINKER_LIBS_RELEASE})

Jenkinsfile

@@ -279,7 +279,7 @@ try {
                 xcopy python pkg_vc14_cpu\\python /E /I /Y
                 xcopy include pkg_vc14_cpu\\include /E /I /Y
                 xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
-                xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
+                xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
                 xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y
                 del /Q *.7z
                 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\
@@ -312,7 +312,7 @@ try {
               xcopy python pkg_vc14_gpu\\python /E /I /Y
               xcopy include pkg_vc14_gpu\\include /E /I /Y
               xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
-              xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
+              xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
               xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
               del /Q *.7z
               7z.exe a vc14_gpu.7z pkg_vc14_gpu\\

LICENSE

@@ -219,13 +219,13 @@
     2. MXNet rcnn - For details, see, example/rcnn/LICENSE
     3. scala-package - For details, see, scala-package/LICENSE
     4. Warp-CTC - For details, see, src/operator/contrib/ctc_include/LICENSE
-    5. dlpack - For details, see, dlpack/LICENSE
+    5. 3rdparty/dlpack - For details, see, 3rdparty/dlpack/LICENSE
     6. dmlc-core - For details, see, dmlc-core/LICENSE
-    7. mshadow - For details, see, mshadow/LICENSE
+    7. 3rdparty/mshadow - For details, see, 3rdparty/mshadow/LICENSE
     8. nnvm/dmlc-core - For details, see, nnvm/dmlc-core/LICENSE
     9. nnvm - For details, see, nnvm/LICENSE
     10. nnvm-fusion - For details, see, nnvm/plugin/nnvm-fusion/LICENSE
-    11. ps-lite - For details, see, ps-lite/LICENSE
+    11. 3rdparty/ps-lite - For details, see, 3rdparty/ps-lite/LICENSE
     12. nnvm/tvm - For details, see, nnvm/tvm/LICENSE
     13. googlemock scripts/generator - For details, see, 3rdparty/googletest/googlemock/scripts/generator/LICENSE
 

Makefile

@@ -16,6 +16,7 @@
 # under the License.
 
 ROOTDIR = $(CURDIR)
+TPARTYDIR = $(ROOTDIR)/3rdparty
 
 SCALA_VERSION_PROFILE := scala-2.11
 
@@ -45,7 +46,7 @@ ifndef NNVM_PATH
 endif
 
 ifndef DLPACK_PATH
-	DLPACK_PATH = $(ROOTDIR)/dlpack
+	DLPACK_PATH = $(ROOTDIR)/3rdparty/dlpack
 endif
 
 ifndef AMALGAMATION_PATH
@@ -73,7 +74,7 @@ ifeq ($(USE_MKLDNN), 1)
 	export USE_MKLML = 1
 endif
 
-include mshadow/make/mshadow.mk
+include $(TPARTYDIR)/mshadow/make/mshadow.mk
 include $(DMLC_CORE)/make/dmlc.mk
 
 # all tge possible warning tread
@@ -91,7 +92,7 @@ ifeq ($(DEBUG), 1)
 else
 	CFLAGS += -O3 -DNDEBUG=1
 endif
-CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
+CFLAGS += -I$(TPARTYDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -I$(NNVM_PATH)/tvm/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
 ifeq ($(DEBUG), 1)
 	NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
@@ -293,7 +294,7 @@ $(info Running CUDA_ARCH: $(CUDA_ARCH))
 endif
 
 # ps-lite
-PS_PATH=$(ROOTDIR)/ps-lite
+PS_PATH=$(ROOTDIR)/3rdparty/ps-lite
 DEPS_PATH=$(shell pwd)/deps
 include $(PS_PATH)/make/ps.mk
 ifeq ($(USE_DIST_KVSTORE), 1)

amalgamation/Makefile

@@ -16,6 +16,7 @@
 # under the License.
 
 export MXNET_ROOT=`pwd`/..
+export TPARTYDIR=`pwd`/../3rdparty
 
 # Change this to path or specify in make command
 ifndef OPENBLAS_ROOT
@@ -78,9 +79,9 @@ dmlc.d: dmlc-minimum0.cc
 
 mxnet_predict0.d: mxnet_predict0.cc nnvm.d dmlc.d
 	${CXX} ${CFLAGS} -M -MT mxnet_predict0.o \
-	-I ${MXNET_ROOT}/ -I ${MXNET_ROOT}/mshadow/ -I ${MXNET_ROOT}/dmlc-core/include -I ${MXNET_ROOT}/dmlc-core/src \
+	-I ${MXNET_ROOT}/ -I ${TPARTYDIR}/mshadow/ -I ${MXNET_ROOT}/dmlc-core/include -I ${MXNET_ROOT}/dmlc-core/src \
 	-I ${MXNET_ROOT}/nnvm/include \
-	-I ${MXNET_ROOT}/dlpack/include \
+	-I ${MXNET_ROOT}/3rdparty/dlpack/include \
 	-I ${MXNET_ROOT}/include \
 	-D__MIN__=$(MIN) mxnet_predict0.cc > mxnet_predict0.d
 	cat dmlc.d >> mxnet_predict0.d

docs/install/index.md

@@ -1213,7 +1213,7 @@ Edit the Makefile to install the MXNet with CUDA bindings to leverage the GPU on
     echo "USE_CUDNN=1" >> config.mk
 ```
 
-Edit the Mshadow Makefile to ensure MXNet builds with Pascal's hardware level low precision acceleration by editing mshadow/make/mshadow.mk and adding the following after line 122:
+Edit the Mshadow Makefile to ensure MXNet builds with Pascal's hardware level low precision acceleration by editing 3rdparty/mshadow/make/mshadow.mk and adding the following after line 122:
 ```bash
 MSHADOW_CFLAGS += -DMSHADOW_USE_PASCAL=1
 ```

tools/license_header.py

@@ -63,12 +63,9 @@
 _WHITE_LIST = ['R-package/',
                'cub/',
                'docker/Dockerfiles',
-               'dlpack/',
                'dmlc-core/',
-               'mshadow/',
                'nnvm',
                '3rdparty',
-               'ps-lite',
                'src/operator/mkl/',
                'src/operator/special_functions-inl.h',
                'src/operator/nn/pool.h',