[aten] Move submodules to third_party (pytorch#6866)

* [aten] Move submodules to third_party

* [aten] Update aten_mirror.sh script for third_party

* [aten] Move ATen submodules def to root and rename

* [aten] Update cpuinfo cmake build

* [aten] Fix cpuinfo cmake build

* Update third_party/cpuinfo to d03d5d296063063c66877fb559cf34469734e3e1

* [aten] Fix JIT test reference to catch

.gitmodules

@@ -1,12 +1,9 @@
-[submodule "aten/src/ATen/cpu/cpuinfo"]
-	path = aten/src/ATen/cpu/cpuinfo
-	url = https://github.com/Maratyszcza/cpuinfo
-[submodule "aten/src/ATen/cpu/tbb/tbb_remote"]
-	path = aten/src/ATen/cpu/tbb/tbb_remote
+[submodule "third_party/tbb"]
+	path = third_party/tbb
 	url = https://github.com/01org/tbb
 	branch = tbb_2018
-[submodule "aten/src/ATen/utils/catch"]
-	path = aten/src/ATen/utils/catch
+[submodule "third_party/catch"]
+	path = third_party/catch
 	url = https://github.com/catchorg/Catch2.git
 [submodule "third_party/nanopb"]
 	path = third_party/nanopb

.gitmodules.aten

@@ -0,0 +1,10 @@
+[submodule "third_party/tbb"]
+	path = third_party/tbb
+	url = https://github.com/01org/tbb
+	branch = tbb_2018
+[submodule "third_party/catch"]
+	path = third_party/catch
+	url = https://github.com/catchorg/Catch2.git
+[submodule "third-party/cpuinfo"]
+	path = third_party/cpuinfo
+	url = https://github.com/Maratyszcza/cpuinfo.git

aten/CMakeLists.txt

@@ -506,7 +506,7 @@ ${CMAKE_CURRENT_SOURCE_DIR}/src/THCUNN)
 add_subdirectory(src/ATen)
 include_directories(
 ${CMAKE_CURRENT_SOURCE_DIR}/src
-${CMAKE_CURRENT_SOURCE_DIR}/src/ATen/utils/catch/single_include
+${CMAKE_CURRENT_SOURCE_DIR}/../third_party/catch/single_include
 ${CMAKE_CURRENT_BINARY_DIR}/src/ATen)
 if(NOT NO_CUDA)
   include_directories(${CUDA_INCLUDE_DIRS})

aten/src/ATen/CMakeLists.txt

@@ -237,7 +237,7 @@ ELSE()
 ENDIF()
 ADD_DEPENDENCIES(ATen aten_files_are_generated)
 
-set(TBB_ROOT_DIR "${PROJECT_SOURCE_DIR}/src/ATen/cpu/tbb/tbb_remote")
+set(TBB_ROOT_DIR "${PROJECT_SOURCE_DIR}/../third_party/tbb")
 set(TBB_BUILD_STATIC ON CACHE BOOL " " FORCE)
 set(TBB_BUILD_SHARED OFF CACHE BOOL " " FORCE)
 set(TBB_BUILD_TBBMALLOC OFF CACHE BOOL " " FORCE)
@@ -323,14 +323,33 @@ if(MKLDNN_FOUND)
   target_link_libraries(ATen ${MKLDNN_LIBRARIES})
 endif(MKLDNN_FOUND)
 
+# Directory where cpuinfo will download and build all dependencies
+set(CONFU_DEPENDENCIES_BINARY_DIR ${PROJECT_BINARY_DIR}/confu-deps
+  CACHE PATH "Confu-style dependencies binary directory")
+
 # ---[ Configure cpuinfo
-IF(NOT TARGET cpuinfo)
-  SET(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_UNIT_TESTS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_MOCK_TESTS OFF CACHE BOOL "")
-  SET(CPUINFO_BUILD_BENCHMARKS OFF CACHE BOOL "")
-  ADD_SUBDIRECTORY("cpu/cpuinfo")
-ENDIF()
+if (NOT TARGET cpuinfo)
+  if (NOT DEFINED CPUINFO_SOURCE_DIR)
+    set(CPUINFO_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../third_party/cpuinfo" CACHE STRING "cpuinfo source directory")
+  endif()
+
+  set(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "")
+  set(CPUINFO_BUILD_UNIT_TESTS OFF CACHE BOOL "")
+  set(CPUINFO_BUILD_MOCK_TESTS OFF CACHE BOOL "")
+  set(CPUINFO_BUILD_BENCHMARKS OFF CACHE BOOL "")
+  set(CPUINFO_LIBRARY_TYPE "static" CACHE STRING "")
+  if(MSVC)
+    # ATen is built with default MSVC runtime linking, which is /MD (DLL)
+    # so no need to check *_USE_MSVC_STATIC_RUNTIME or some equivalent.
+    set(CPUINFO_RUNTIME_TYPE "shared" CACHE STRING "")
+  endif()
+  add_subdirectory(
+    "${CPUINFO_SOURCE_DIR}"
+    "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo")
+  # We build static version of cpuinfo but link
+  # them into a shared library for Caffe2, so they need PIC.
+  set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON)
+endif()
 TARGET_LINK_LIBRARIES(ATen cpuinfo)
 
 IF(CUDA_FOUND)

aten/src/ATen/native/cpu/CapabilityDispatch.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "ATen/cpu/cpuinfo/include/cpuinfo.h"
+#include <cpuinfo.h>
 #include <type_traits>
 #include <iostream>
 

setup.py

@@ -272,9 +272,9 @@ def check_file(f):
         check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt"))
         check_file(os.path.join(third_party_path, "nanopb", "CMakeLists.txt"))
         check_file(os.path.join(third_party_path, "pybind11", "CMakeLists.txt"))
-        check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'cpuinfo', 'CMakeLists.txt'))
-        check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'tbb', 'tbb_remote', 'Makefile'))
-        check_file(os.path.join('aten', 'src', 'ATen', 'utils', 'catch', 'CMakeLists.txt'))
+        check_file(os.path.join(third_party_path, 'cpuinfo', 'CMakeLists.txt'))
+        check_file(os.path.join(third_party_path, 'tbb', 'Makefile'))
+        check_file(os.path.join(third_party_path, 'catch', 'CMakeLists.txt'))
 
         check_pydep('yaml', 'pyyaml')
         check_pydep('typing', 'typing')

tools/aten_mirror.sh

@@ -27,7 +27,7 @@ git fetch fullrepo
 git checkout -b temporary-split-branch fullrepo/master
 # Cribbed from https://stackoverflow.com/questions/2982055/detach-many-subdirectories-into-a-new-separate-git-repository
 # and https://stackoverflow.com/questions/42355621/git-filter-branch-moving-a-folder-with-index-filter-does-not-work
-git filter-branch -f --index-filter 'git rm --cached -qr --ignore-unmatch -- . && git reset -q $GIT_COMMIT -- aten cmake third_party/cpuinfo && (git ls-files -s | sed "s-.travis.aten.yml-.travis.yml-" | git update-index --index-info)'
+git filter-branch -f --index-filter 'git rm --cached -qr --ignore-unmatch -- . && git reset -q $GIT_COMMIT -- aten cmake third_party/tbb third_party/catch third_party/cpuinfo && (git ls-files -s | sed "s-.travis.aten.yml-.travis.yml-" | sed "s-.gitmodules.aten-.gitmodules-" | git update-index --index-info)'
 git checkout master
 git merge temporary-split-branch
 git push

tools/cpp_build/libtorch/CMakeLists.txt

@@ -195,7 +195,7 @@ target_link_libraries(test_jit torch)
 
 target_include_directories(test_jit PUBLIC
   "${TORCH_SRC_DIR}/../"
-  "${TORCH_SRC_DIR}/../aten/src/ATen/utils/catch/single_include"
+  "${TORCH_SRC_DIR}/../third_party/catch/single_include"
   "${COMMON_INCLUDES}")
 
 install(TARGETS test_jit