Merge from remotes/internal/release_1.1.1

CMakeLists.txt

@@ -44,12 +44,22 @@ caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only i
 caffe_option(USE_SYSTEMTAP "Build for SystemTap" OFF)
 caffe_option(PERFORMANCE_MONITORING "Build Caffe with PERFORMANCE_MONITORING " OFF)
 #caffe_option(USE_GITHUB_MKLDNN "Download and use MKL-DNN available on github" OFF)
-
+caffe_option(CODE_COVERAGE "Build with code coverage instrumentation" OFF)
+caffe_option(CO_SIM "Build readonly cpu blob data/diff pycaffe interface" OFF)
+caffe_option(USE_MLSL "Build Caffe with Intel(r) Machine Learning Scaling Library for multinode training" ON)
+
 # ---[ Dependencies
 include(cmake/MKLDNN.cmake)
 include(cmake/DLCP.cmake)
 include(cmake/Dependencies.cmake)
 
+if(CODE_COVERAGE)
+  # code coverage necessary
+  include(CodeCoverage)
+  APPEND_CCOV_COMPILER_FLAGS()
+endif()
+#end code coverage
+
 # ---[ Flags
 if(UNIX OR APPLE)
 
@@ -81,14 +91,16 @@ if(UNIX OR APPLE)
     # Same as above, with exception that ICC compilation crashes with -fPIE option, even 
     # though it uses -pie linker option that require -fPIE during compilation. Checksec
     # shows that it generates correct PIE anyway if only -pie is provided.
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector -wd2196")
     # Auto enable SGD Fusion if use intel compiler
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SGD_FUSION")
   endif()
 
   # Generic flags.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -fno-operator-names -Wformat -Wformat-security -Wall")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+  if(NOT CODE_COVERAGE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Werror")
+  endif()
   # Dot not forward c++11 flag to GPU beucause it is not supported
   set( CUDA_PROPAGATE_HOST_FLAGS OFF )
   set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -D_FORTIFY_SOURCE=2")

Makefile

@@ -387,14 +387,14 @@ endif
 
 # Compiler flags
 ifneq (,$(findstring icpc,$(CXX)))
-	CXX_HARDENING_FLAGS += -fstack-protector
+	CXX_HARDENING_FLAGS += -fstack-protector -wd2196
 	#Enable SGD FUSION if use intel compiler
 	COMMON_FLAGS += -DENABLE_SGD_FUSION
 
 else ifneq (,$(findstring clang++,$(CXX)))
 	CXX_HARDENING_FLAGS += -fPIE -fstack-protector
 else ifneq (,$(findstring g++,$(CXX)))
-	ifeq ($(shell echo | awk '{ print $(GCCVERSION) >= 4.9 }'), 1)
+	ifeq ($(shell echo | awk '{print $(GCCVERSION) < 4.9;}'), 0)
 		CXX_HARDENING_FLAGS += -fPIE -fstack-protector-strong
 		#Enable SGD FUSION if gcc version >= 4.9
 		COMMON_FLAGS += -DENABLE_SGD_FUSION
@@ -415,7 +415,7 @@ else
 endif
 
 # Generic flags
-CXX_HARDENING_FLAGS += -fPIC -fno-operator-names -Wformat -Wformat-security -Wall
+CXX_HARDENING_FLAGS += -fPIC -fno-operator-names -Wformat -Wformat-security -Wall -Werror
 LINKER_EXEC_HARDENING_FLAGS += -pie
 
 # Release-only flag
@@ -493,11 +493,26 @@ ifeq ($(DISABLE_BN_FOLDING), 1)
 	COMMON_FLAGS += -DDISABLE_BN_FOLDING
 endif
 
+# Disable Conv + ReLU fusion
+ifeq ($(DISABLE_CONV_RELU_FUSION), 1)
+    COMMON_FLAGS += -DDISABLE_CONV_RELU_FUSION
+endif
+
+# Disable Bn + ReLU fusion
+ifeq ($(DISABLE_BN_RELU_FUSION), 1)
+    COMMON_FLAGS += -DDISABLE_BN_RELU_FUSION
+endif
+
 # Disable the conv/eltwise/relu layer fusion
 ifeq ($(DISABLE_CONV_SUM_FUSION), 1)
 	COMMON_FLAGS += -DDISABLE_CONV_SUM_FUSION
 endif
 
+# Disable sparse
+ifeq ($(DISABLE_SPARSE), 1)
+	COMMON_FLAGS += -DDISABLE_SPARSE
+endif
+
 # Performance monitoring
 ifeq ($(PERFORMANCE_MONITORING), 1)
 	CXXFLAGS += -DPERFORMANCE_MONITORING
@@ -515,7 +530,9 @@ include Makefile.dlcp
 BOOST_LDFLAGS += $(foreach boost_lib,$(BOOST_LIBRARIES),-l$(boost_lib))
 ifneq ($(origin BOOST_ROOT), undefined)
 	INCLUDE_DIRS += $(BOOST_ROOT)
+	INCLUDE_DIRS += $(BOOST_ROOT)/include
 	BOOST_LDFLAGS+=-L$(BOOST_ROOT)/stage/lib -Wl,-rpath,$(BOOST_ROOT)/stage/lib
+	BOOST_LDFLAGS+=-L$(BOOST_ROOT)/lib -Wl,-rpath,$(BOOST_ROOT)/lib
 endif
 
 # BLAS configuration (default = MKL)

Makefile.config.example

@@ -79,11 +79,25 @@ USE_MKLDNN_AS_DEFAULT_ENGINE := 1
 # Use remove batch norm optimization to boost inference
 DISABLE_BN_FOLDING := 0
 
-#Use conv/eltwise/relu layer fusion to boost inference.
+# Use Conv + Relu fusion to boost inference
+DISABLE_CONV_RELU_FUSION:= 0
+
+# Use Bn + ReLU fusion to boost inference
+DISABLE_BN_RELU_FUSION := 0
+
+# Use Conv + Eltwise + Relu layer fusion to boost inference.
 DISABLE_CONV_SUM_FUSION := 0
+
+# Use sparse to boost inference.
+DISABLE_SPARSE := 0
+
+# Intel(R) Math Kernel Library for Deep Neural Networks (Intel(R) MKL-DNN) 
+# Uncomment to disable MKLDNN download by customized setting
+# DISABLE_MKLDNN_DOWNLOAD := 1
+
 # Intel(r) Machine Learning Scaling Library (uncomment to build
 # with MLSL for multi-node training)
-# USE_MLSL :=1
+USE_MLSL := 1
 
 # CUDA directory contains bin/ and lib/ directories that we need.
 CUDA_DIR := /usr/local/cuda

Makefile.mkldnn

@@ -24,6 +24,17 @@ endif
 MKLDNN_GITHUB := https://github.com/01org/mkl-dnn.git
 MKLDNN_CMAKE_FLAGS += $(MKLDNN_SRCDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNN_INSTALLDIR) -DMKLROOT=${MKL_ROOTDIR} -B$(MKLDNN_BUILDDIR) -DCMAKE_CXX_COMPILER="$(MKLDNN_CXX)" -DCMAKE_C_COMPILER="$(MKLDNN_CC)"
 
+# Disable MKLDNN download by customized setting
+ifeq ($(DISABLE_MKLDNN_DOWNLOAD), 1)
+disable_mkldnn_download_info:
+	@echo "MKLDNN download is disabled by customized setting! If MKLDNNROOT is not set in the ENV, may cause compile error in the following steps!"
+mkldnn_download:
+mkldnn_build:
+
+else
+disable_mkldnn_download_info:
+	@echo "MKLDNN download is enabled by customized setting!"
+
 ifeq ("$(wildcard $(MKLDNN_INSTALLDIR)/include/mkldnn.hpp)", "")
 mkldnn_download:
 	git clone --no-checkout $(MKLDNN_GITHUB) $(MKLDNN_TMPDIR)
@@ -38,6 +49,7 @@ else
 mkldnn_download:
 mkldnn_build:
 endif
+endif
 
 mkldnn_clean:
 	@rm -rf $(MKLDNN_SRCDIR) $(MKLDNN_BUILDDIR) $(MKLDNN_INSTALLDIR) $(MKLDNN_TMPDIR)
@@ -55,10 +67,14 @@ mkldnnroot_notset: mkldnn_build
 	$(eval MKLDNN_LDFLAGS += -lmkldnn) \
 	$(eval MKLDNN_LDFLAGS += -L$(MKLDNN_INSTALLDIR)/lib -Wl,-rpath,${MKLDNN_INSTALLDIR}/lib)
 
+
+
 ifneq ($(origin MKLDNNROOT), undefined)
 ifdef MKLDNNROOT
 mkldnn: mkldnnroot_set
+mkldnn: disable_mkldnn_download_info
 endif
 else
 mkldnn: mkldnnroot_notset
+mkldnn: disable_mkldnn_download_info
 endif

cmake/Dependencies.cmake

@@ -25,6 +25,11 @@ if(PERFORMANCE_MONITORING)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPERFORMANCE_MONITORING")
 endif()
 
+# ---[ CPU DATA READONLY CO SIM
+if(CO_SIM)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCO_SIM")
+endif()
+
 # ---[ Google-glog
 include("cmake/External/glog.cmake")
 include_directories(SYSTEM ${GLOG_INCLUDE_DIRS})
@@ -191,8 +196,10 @@ if(NOT APPLE)
 	set(MKL_INCLUDE_DIR "${MKL_ROOT_DIR}/include/")
 	if( ${MKL_EXTERNAL} EQUAL 1 )
 	  set(MKL_LIBRARIES "${MKL_ROOT_DIR}/lib/lib${MKL_LIBRARIES}.so")
+          set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${MKL_ROOT_DIR}/lib/)
 	else()
 	  set(MKL_LIBRARIES "${MKL_ROOT_DIR}/lib/intel64/lib${MKL_LIBRARIES}.so")
+          set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${MKL_ROOT_DIR}/lib/intel64/)
 	endif()
 	message(STATUS "Found MKL: ${MKL_INCLUDE_DIR}")
 	message(STATUS "Found MKL (include: ${MKL_INCLUDE_DIR}, lib: ${MKL_LIBRARIES}")	
@@ -237,6 +244,14 @@ endif()
 
 # ---[ MKLDNN
 if(DEFINED ENV{MKLDNNROOT})
+  if(DISABLE_MKLDNN_DOWNLOAD)
+    message(STATUS "MKLDNN download is disabled by customized setting!")
+    message(STATUS "MKLDNNROOT is already set in the ENV, will use ENV MKLDNNROOT in the following compile steps!")
+    add_definitions("-DDISABLE_MKLDNN_DOWNLOAD=1")
+  else()
+    message(STATUS "MKLDNN download is enabled by customized setting!")
+    add_definitions("-DDISABLE_MKLDNN_DOWNLOAD=0")
+  endif()
   set(MKLDNNROOT_DIR $ENV{MKLDNNROOT})
   if(NOT ${MKLDNNROOT_DIR} STREQUAL "")
     set(MKLDNNROOT_INCLUDE_DIR "${MKLDNNROOT_DIR}/include/")
@@ -249,6 +264,7 @@ if(DEFINED ENV{MKLDNNROOT})
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_MKLDNN_AS_DEFAULT_ENGINE")
       endif()
       list(APPEND Caffe_LINKER_LIBS "${MKLDNNROOT_DIR}/lib/libmkldnn.so")
+      set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${MKLDNNROOT_DIR}/lib/)
 	    include_directories(SYSTEM ${MKLDNNROOT_INCLUDE_DIR})
     else()
       message(STATUS "MKLDNN not found. MKLDNN_INCLUDE_DIR = ${MKLDNNROOT_INCLUDE_DIR}")
@@ -259,18 +275,27 @@ if(DEFINED ENV{MKLDNNROOT})
     endif()
   endif()
 else()
-  Download_MKLDNN()
+  if(DISABLE_MKLDNN_DOWNLOAD)
+    message(STATUS "MKLDNN download is disabled by customized setting!")
+    message(STATUS "MKLDNNROOT is not set in the ENV, will cause compile error in the following steps!")
+    add_definitions("-DDISABLE_MKLDNN_DOWNLOAD=1")
+  else()
+    message(STATUS "MKLDNN download is enabled by customized setting!")
+    add_definitions("-DDISABLE_MKLDNN_DOWNLOAD=0")
+    Download_MKLDNN()
 
-  message(STATUS "MKLDNN will be downloaded from github and installed in ${MKLDNN_INSTALL_DIR}")
-  message(STATUS "MKLDNN include directory: ${MKLDNN_INCLUDE_DIR}")
-  set(MKLDNN_SUPPORTED ON)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMKLDNN_SUPPORTED -std=c++11")
-  if(USE_MKLDNN_AS_DEFAULT_ENGINE)
-    message(STATUS "MKLDNN engine will be used as a default engine")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_MKLDNN_AS_DEFAULT_ENGINE")
+    message(STATUS "MKLDNN will be downloaded from github and installed in ${MKLDNN_INSTALL_DIR}")
+    message(STATUS "MKLDNN include directory: ${MKLDNN_INCLUDE_DIR}")
+    set(MKLDNN_SUPPORTED ON)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMKLDNN_SUPPORTED -std=c++11")
+    if(USE_MKLDNN_AS_DEFAULT_ENGINE)
+      message(STATUS "MKLDNN engine will be used as a default engine")
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_MKLDNN_AS_DEFAULT_ENGINE")
+    endif()
+    list(APPEND Caffe_LINKER_LIBS "${MKLDNN_INSTALL_DIR}/lib/libmkldnn.so")
+    set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH}:${MKLDNN_INSTALL_DIR}/lib/)
+      include_directories(${MKLDNN_INCLUDE_DIR})
   endif()
-  list(APPEND Caffe_LINKER_LIBS "${MKLDNN_INSTALL_DIR}/lib/libmkldnn.so")
-	include_directories(${MKLDNN_INCLUDE_DIR})
 endif()
 
 # ---[ Python

cmake/Misc.cmake

@@ -11,11 +11,25 @@ if(DISABLE_BN_FOLDING)
 	add_definitions("-DDISABLE_BN_FOLDING")
 endif()
 
+if(DISABLE_CONV_RELU_FUSION)
+	message(STATUS "CONV + ReLU fusion is disabled!")
+	add_definitions("-DDISABLE_CONV_RELU_FUSION")
+endif()
+
+if(DISABLE_BN_RELU_FUSION)
+    message(STATUS "Bn + ReLU fusion is disabled!")
+    add_definitions("-DDISABLE_BN_RELU_FUSION")
+endif()
+
 if(DISABLE_CONV_SUM_FUSION)
 	message(STATUS "conv/eltwise/relu fusion is disabled!")
 	add_definitions("-DDISABLE_CONV_SUM_FUSION")
 endif()
 
+if(DISABLE_SPARSE)
+	message(STATUS "sparse is disabled!")
+	add_definitions("-DDISABLE_SPARSE")
+endif()
 # --[ If user doesn't specify build type then assume release
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Release)

cmake/Modules/CodeCoverage.cmake

@@ -0,0 +1,19 @@
+include(CMakeParseArguments)
+
+find_program( GCOV gcov )
+if(NOT GCOV)
+    message(FATAL_ERROR "command gcov not found!")
+endif()
+
+set(CCOV_COMPILER_FLAGS "-g -O0 --coverage -fprofile-arcs -ftest-coverage"
+    CACHE INTERNAL "")
+
+if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message(WARNING "Non debug build may lead to wrong code coverage result.")
+endif()
+
+function(APPEND_CCOV_COMPILER_FLAGS)
+    set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${CCOV_COMPILER_FLAGS}" PARENT_SCOPE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CCOV_COMPILER_FLAGS}" PARENT_SCOPE)
+    message(STATUS "Setting code coverage compiler flags: ${CCOV_COMPILER_FLAGS}")
+endfunction()

cmake/Summary.cmake

@@ -121,6 +121,7 @@ function(caffe_print_configuration_summary)
   caffe_status("  ALLOW_LMDB_NOLOCK       :   ${ALLOW_LMDB_NOLOCK}")
   caffe_status("  USE_SYSTEMTAP           :   ${USE_SYSTEMTAP}")
   caffe_status("  PERFORMANCE_MONITORING  :   ${PERFORMANCE_MONITORING}")
+  caffe_status("  CO_SIM  :   ${CO_SIM}")
   caffe_status("")
   caffe_status("Dependencies:")
   caffe_status("  BLAS              : " APPLE THEN "Yes (vecLib)" ELSE "Yes (${BLAS})")

docker/standalone/cpu-ubuntu/Dockerfile

@@ -5,11 +5,22 @@ MAINTAINER caffe-maint@googlegroups.com
 #ENV https_proxy proxy:port
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+	cpio \
         build-essential \
         cmake \
         git \
         wget \
         ssh \
+        openssh-server \
+        numactl \
+        vim \
+        net-tools \
+        iputils-ping \
+        screen \
+        libmlx4-1 libmlx5-1 ibutils  rdmacm-utils libibverbs1 ibverbs-utils perftest infiniband-diags \
+        openmpi-bin libopenmpi-dev \
+        ufw \
+        iptables \
         libboost-all-dev \
         libgflags-dev \
         libgoogle-glog-dev \
@@ -38,12 +49,27 @@ RUN pip install --upgrade pip
 RUN git clone -b ${CLONE_TAG} --depth 1 https://github.com/intel/caffe.git . && \
     for req in $(cat python/requirements.txt) pydot; do pip install $req; done && \
     mkdir build && cd build && \
-    cmake -DCPU_ONLY=1 -DCMAKE_BUILD_TYPE=Release .. && \
+    cmake -DCPU_ONLY=1 -DUSE_MLSL=1 -DCMAKE_BUILD_TYPE=Release .. && \
     make all -j"$(nproc)"
 
 ENV PYCAFFE_ROOT $CAFFE_ROOT/python
 ENV PYTHONPATH $PYCAFFE_ROOT:$PYTHONPATH
 ENV PATH $CAFFE_ROOT/build/tools:$PYCAFFE_ROOT:$PATH
 RUN echo "$CAFFE_ROOT/build/lib" >> /etc/ld.so.conf.d/caffe.conf && ldconfig
 
-WORKDIR /workspace
+WORKDIR /opt/caffe
+
+RUN mkdir /var/run/sshd
+RUN echo 'root:123456' | chpasswd
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -i 's/Port 22/Port 10010/' /etc/ssh/sshd_config
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+ENV NOTVISIBLE "in users profile"
+RUN echo "export VISIBLE=now" >> /etc/profile
+
+EXPOSE 10010
+
+CMD ["/usr/sbin/sshd", "-D"]