diff --git a/CMakeLists.txt b/CMakeLists.txt
index ac62423db582..deeb99c20216 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -118,10 +118,6 @@ else()
 endif()
 
 if (TORCH_MLIR_ENABLE_STABLEHLO)
-  set(STABLEHLO_BUILD_EMBEDDED ON)
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/externals/stablehlo
-    ${CMAKE_CURRENT_BINARY_DIR}/stablehlo
-    EXCLUDE_FROM_ALL)
   include_directories(${CMAKE_CURRENT_SOURCE_DIR}/externals/stablehlo)
 endif()
 
@@ -226,3 +222,18 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
             COMPONENT torch-mlir-headers)
   endif()
 endif()
+
+# Important: If loading StableHLO in this fashion, it must come last,
+# after all of our libraries and test targets have been defined.
+# It seems that they both abuse upstream CMake macros that accumulate
+# properties.
+# Getting this wrong results in building large parts of the stablehlo
+# project that we don't actually depend on. Further some of those parts
+# do not even compile on all platforms.
+if (TORCH_MLIR_ENABLE_STABLEHLO)
+  set(STABLEHLO_BUILD_EMBEDDED ON)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/externals/stablehlo
+    ${CMAKE_CURRENT_BINARY_DIR}/stablehlo
+    EXCLUDE_FROM_ALL)
+  include_directories(${CMAKE_CURRENT_SOURCE_DIR}/externals/stablehlo)
+endif()
diff --git a/tools/torch-mlir-opt/CMakeLists.txt b/tools/torch-mlir-opt/CMakeLists.txt
index 5cfaa82b06c3..94c547d0eb2d 100644
--- a/tools/torch-mlir-opt/CMakeLists.txt
+++ b/tools/torch-mlir-opt/CMakeLists.txt
@@ -9,6 +9,11 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 get_property(extension_libs GLOBAL PROPERTY MLIR_EXTENSION_LIBS)
 
+set(dependency_libraries)
+if(TORCH_MLIR_ENABLE_STABLEHLO)
+  list(APPEND dependency_libraries StablehloRegister)
+endif()
+
 target_link_libraries(torch-mlir-opt PRIVATE
   MLIROptLib
   TorchMLIRInitAll
@@ -17,4 +22,5 @@ target_link_libraries(torch-mlir-opt PRIVATE
   ${dialect_libs}
   ${conversion_libs}
   ${extension_libs}
+  ${dependency_libraries}
 )