Fix out-of-tree build of torch-mlir-dialects (llvm#726)

Follows up on llvm#623 for out-of-tree builds of torch-mlir, which
added building `torch-mir-dialects` as a subdirectory.

Our goal is to support both in-tree and out-of-tree builds of
`torch-mlir` with minimum hassle, for instance by using the same
variable names in both setups.

Specific changes to `externals/llvm-external-projects/torch-mlir-dialects/CMakeLists.txt`:
- We use `MLIR_FOUND` to detect that it is being build as a subdirectory
and the llvm+mlir cmake infrastructure is already set up (via
find_package in the parent build) as opposed to an in-tree build.
- For in-tree, the setting of variables and loading of llvm+mlir cmake
infrastructure is now conditionally performed.
- For in-tree, the names of cmake variables being defined for are
adjusted to match those `llvm-project` makes available through
`find_package(MLIR REQUIRED CONFIG)`, under the assumption that those
are the more "standardized" names.

Co-authored-by: Clément Fournier <clement.fournier@amd.com>

Co-authored-by: Liam Fitzpatrick <liam.fitzpatrick@xilinx.com>

CMakeLists.txt

@@ -108,8 +108,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
 
 function(torch_mlir_target_includes target)
   set(_dirs
-    $<BUILD_INTERFACE:${MLIR_INCLUDE_DIR}>
-    $<BUILD_INTERFACE:${MLIR_GENERATED_INCLUDE_DIR}>
+    $<BUILD_INTERFACE:${MLIR_INCLUDE_DIRS}>
     $<BUILD_INTERFACE:${TORCH_MLIR_SOURCE_DIR}/include>
     $<BUILD_INTERFACE:${TORCH_MLIR_BINARY_DIR}/include>
   )

externals/llvm-external-projects/torch-mlir-dialects/CMakeLists.txt

@@ -1,8 +1,11 @@
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
-message(FATAL_ERROR
-  "This project is intended to be built as part of LLVM via "
-  "-DLLVM_EXTERNAL_PROJECTS=torch-mlir-dialects "
-  "-DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
+  # The cmake configuration to build torch-mlir-dialects as
+  # an out-of-tree project has not been implemented. It can
+  # be built as part of LLVM or as a subdirectory of torch-mlir.
+  message(FATAL_ERROR
+    "This project is intended to be built as part of LLVM via "
+    "-DLLVM_EXTERNAL_PROJECTS=torch-mlir-dialects "
+    "-DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
 endif()
 
 option(MLIR_ENABLE_BINDINGS_PYTHON "Enables MLIR Python Bindings" OFF)
@@ -11,20 +14,32 @@ set(TORCH_MLIR_DIALECTS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(TORCH_MLIR_DIALECTS_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 message(STATUS "Building torch-mlir-dialects project at ${TORCH_MLIR_DIALECTS_SOURCE_DIR} (into ${TORCH_MLIR_DIALECTS_BINARY_DIR})")
 
-# TODO: Fix this upstream so that global include directories are not needed.
-set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir)
-set(MLIR_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/../mlir/include)
-set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include)
+if(MLIR_FOUND)
+  message(STATUS "LLVM and MLIR packages have already been configured.")
+else()
+  message(STATUS "torch-mlir-dialect is being built in-tree")
+  # An in-tree build, LLVM hasn't been installed yet.
+  # We compute these properties manually, they're otherwise
+  # contributed by find_package(MLIR...)
+  set(MLIR_MAIN_SRC_DIR ${LLVM_MAIN_SRC_DIR}/../mlir)
+  set(MLIR_INCLUDE_DIR ${LLVM_MAIN_SRC_DIR}/../mlir/include)
+  set(MLIR_GENERATED_INCLUDE_DIR ${LLVM_BINARY_DIR}/tools/mlir/include)
+  set(MLIR_INCLUDE_DIRS ${MLIR_INCLUDE_DIR} ${MLIR_GENERATED_INCLUDE_DIR})
+  set(LLVM_INCLUDE_DIRS ${LLVM_MAIN_INCLUDE_DIR})
 
-# TODO: Needed for tablegen. Remove.
-include_directories(SYSTEM ${MLIR_INCLUDE_DIR})
-include_directories(SYSTEM ${MLIR_GENERATED_INCLUDE_DIR})
-include_directories(SYSTEM ${TORCH_MLIR_DIALECTS_SOURCE_DIR}/include)
+  # Configure CMake and tablegen.
+  list(APPEND CMAKE_MODULE_PATH ${MLIR_MAIN_SRC_DIR}/cmake/modules)
+  list(APPEND CMAKE_MODULE_PATH ${LLVM_MAIN_SRC_DIR}/cmake)
+  set(MLIR_TABLEGEN_EXE mlir-tblgen)
+
+  include(TableGen)
+  include(AddLLVM)
+  include(AddMLIR)
+endif()
 
 function(torch_mlir_dialects_target_includes target)
   set(_dirs
-    $<BUILD_INTERFACE:${MLIR_INCLUDE_DIR}>
-    $<BUILD_INTERFACE:${MLIR_GENERATED_INCLUDE_DIR}>
+    $<BUILD_INTERFACE:${MLIR_INCLUDE_DIRS}>
     $<BUILD_INTERFACE:${TORCH_MLIR_DIALECTS_SOURCE_DIR}/include>
     $<BUILD_INTERFACE:${TORCH_MLIR_DIALECTS_BINARY_DIR}/include>
   )
@@ -39,14 +54,10 @@ function(torch_mlir_dialects_target_includes target)
   endif()
 endfunction()
 
-# Configure CMake and tablegen.
-list(APPEND CMAKE_MODULE_PATH ${MLIR_MAIN_SRC_DIR}/cmake/modules)
-list(APPEND CMAKE_MODULE_PATH ${LLVM_MAIN_SRC_DIR}/cmake)
-set(MLIR_TABLEGEN_EXE mlir-tblgen)
-
-include(TableGen)
-include(AddLLVM)
-include(AddMLIR)
+# TODO: Needed for tablegen. Remove.
+include_directories(SYSTEM ${MLIR_INCLUDE_DIRS})
+include_directories(SYSTEM ${LLVM_INCLUDE_DIRS})
+include_directories(SYSTEM ${TORCH_MLIR_DIALECTS_SOURCE_DIR}/include)
 
 add_subdirectory(include)
 add_subdirectory(lib)

externals/llvm-external-projects/torch-mlir-dialects/test/CMakeLists.txt

@@ -1,3 +1,5 @@
+# lit needs to find the tools, and the location differs for in-tree and out-of-tree builds.
+get_target_property(TORCH_MLIR_DIALECTS_TOOLS_DIR torch-mlir-dialects-opt RUNTIME_OUTPUT_DIRECTORY)
 configure_lit_site_cfg(
         ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
         ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py

externals/llvm-external-projects/torch-mlir-dialects/test/lit.cfg.py

@@ -60,9 +60,9 @@
 
 # Tweak the PATH to include the tools dir.
 llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
-
-tool_dirs = [config.llvm_tools_dir]
+tool_dirs = [config.torch_mlir_dialects_tools_dir, config.llvm_tools_dir]
 tools = [
+    "torch-mlir-dialects-opt",
     ToolSubst('%PYTHON', config.python_executable, unresolved='ignore'),
 ]
 

externals/llvm-external-projects/torch-mlir-dialects/test/lit.site.cfg.py.in

@@ -9,6 +9,7 @@
 import sys
 
 config.torch_mlir_dialects_obj_root = "@TORCH_MLIR_DIALECTS_BINARY_DIR@"
+config.torch_mlir_dialects_tools_dir = "@TORCH_MLIR_DIALECTS_TOOLS_DIR@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"