create and validate build_variables.bzl

First step of #8268 -- we are moving from buck-extracted filelist to using one shared filelist, and the first step is to create the shared filelist and validate it against the buck generation.

ghstack-source-id: c4195ce
ghstack-comment-id: 2644414497
Pull Request resolved: #8326

Author: swolchok

CMakeLists.txt

@@ -44,6 +44,7 @@
 
 cmake_minimum_required(VERSION 3.19)
 project(executorch)
+include(build/Codegen.cmake)
 include(build/Utils.cmake)
 include(CMakeDependentOption)
 
@@ -401,6 +402,7 @@ if(NOT EXECUTORCH_SRCS_FILE)
   message(STATUS "executorch: Generating source lists")
   set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/executorch_srcs.cmake")
   extract_sources(${EXECUTORCH_SRCS_FILE})
+  validate_build_variables()
 endif()
 
 # This file defines the `_<target>__srcs` variables used below.

build/Codegen.cmake

@@ -9,7 +9,7 @@
 
 # Selective build. See codegen/tools/gen_oplist.py for how to use these
 # arguments.
-include(${EXECUTORCH_ROOT}/build/Utils.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/Utils.cmake)
 
 function(gen_selected_ops)
   set(arg_names LIB_NAME OPS_SCHEMA_YAML ROOT_OPS INCLUDE_ALL_OPS)
@@ -97,7 +97,7 @@ function(generate_bindings_for_kernels)
       --tags-path=${site-packages-out}/torchgen/packaged/ATen/native/tags.yaml
       --aten-yaml-path=${site-packages-out}/torchgen/packaged/ATen/native/native_functions.yaml
       --op-selection-yaml-path=${_oplist_yaml}
-    )
+  )
   if(GEN_ADD_EXCEPTION_BOUNDARY)
     set(_gen_command "${_gen_command}" --add-exception-boundary)
   endif()
@@ -217,3 +217,127 @@ function(merge_yaml)
     WORKING_DIRECTORY ${EXECUTORCH_ROOT}
   )
 endfunction()
+
+# Append the file list in the variable named `name` in build/build_variables.bzl
+# to the variable named `outputvar` in the caller's scope.
+function(append_filelist name outputvar)
+  # configure_file adds its input to the list of CMAKE_RERUN dependencies
+  configure_file(
+    ${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl
+    ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY
+  )
+  execute_process(
+    COMMAND
+      "${PYTHON_EXECUTABLE}" -c
+      "exec(open('${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl').read());print(';'.join(${name}))"
+    WORKING_DIRECTORY "${_rootdir}"
+    RESULT_VARIABLE _retval
+    OUTPUT_VARIABLE _tempvar
+    ERROR_VARIABLE _stderr
+  )
+  if(NOT _retval EQUAL 0)
+    message(
+      FATAL_ERROR
+        "Failed to fetch filelist ${name} from build_variables.bzl with output ${_tempvar} and stderr ${_stderr}"
+    )
+  endif()
+  string(REPLACE "\n" "" _tempvar "${_tempvar}")
+  list(APPEND ${outputvar} ${_tempvar})
+  set(${outputvar}
+      "${${outputvar}}"
+      PARENT_SCOPE
+  )
+endfunction()
+
+# Fail the build if the src lists in build_variables.bzl do not match the src
+# lists extracted from Buck and placed into EXECUTORCH_SRCS_FILE. This is
+# intended to be a safety mechanism while we are in the process of removing Buck
+# from the CMake build and replacing it with build_variables.bzl; if you are
+# seeing failures after you have intentionally changed Buck srcs, then simply
+# update build_variables.bzl. If you are seeing failures after changing
+# something about the build system, make sure your changes will work both before
+# and after we finish replacing Buck with build_variables.bzl, which should
+# involve getting these lists to match!
+function(validate_build_variables)
+  include(${EXECUTORCH_SRCS_FILE})
+  set(BUILD_VARIABLES_FILELISTS
+      EXECUTORCH_SRCS
+      EXECUTORCH_CORE_SRCS
+      PORTABLE_KERNELS_SRCS
+      OPTIMIZED_KERNELS_SRCS
+      QUANTIZED_KERNELS_SRCS
+      PROGRAM_SCHEMA_SRCS
+      OPTIMIZED_CPUBLAS_SRCS
+      OPTIMIZED_NATIVE_CPU_OPS_SRCS
+      EXTENSION_DATA_LOADER_SRCS
+      EXTENSION_MODULE_SRCS
+      EXTENSION_RUNNER_UTIL_SRCS
+      EXTENSION_LLM_RUNNER_SRCS
+      EXTENSION_TENSOR_SRCS
+      EXTENSION_THREADPOOL_SRCS
+      EXTENSION_TRAINING_SRCS
+      TRAIN_XOR_SRCS
+      EXECUTOR_RUNNER_SRCS
+      SIZE_TEST_SRCS
+      MPS_EXECUTOR_RUNNER_SRCS
+      MPS_BACKEND_SRCS
+      MPS_SCHEMA_SRCS
+      XNN_EXECUTOR_RUNNER_SRCS
+      XNNPACK_BACKEND_SRCS
+      XNNPACK_SCHEMA_SRCS
+      VULKAN_SCHEMA_SRCS
+      CUSTOM_OPS_SRCS
+      LLAMA_RUNNER_SRCS
+  )
+  set(BUILD_VARIABLES_VARNAMES
+      _executorch__srcs
+      _executorch_core__srcs
+      _portable_kernels__srcs
+      _optimized_kernels__srcs
+      _quantized_kernels__srcs
+      _program_schema__srcs
+      _optimized_cpublas__srcs
+      _optimized_native_cpu_ops__srcs
+      _extension_data_loader__srcs
+      _extension_module__srcs
+      _extension_runner_util__srcs
+      _extension_llm_runner__srcs
+      _extension_tensor__srcs
+      _extension_threadpool__srcs
+      _extension_training__srcs
+      _train_xor__srcs
+      _executor_runner__srcs
+      _size_test__srcs
+      _mps_executor_runner__srcs
+      _mps_backend__srcs
+      _mps_schema__srcs
+      _xnn_executor_runner__srcs
+      _xnnpack_backend__srcs
+      _xnnpack_schema__srcs
+      _vulkan_schema__srcs
+      _custom_ops__srcs
+      _llama_runner__srcs
+  )
+  foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS
+                                  BUILD_VARIABLES_VARNAMES
+  )
+    if("${filelist_and_varname_1}" STREQUAL "_custom_ops__srcs")
+      continue()
+    endif()
+    append_filelist(
+      ${filelist_and_varname_0}
+      "${filelist_and_varname_1}_from_build_variables"
+    )
+    if(NOT ${filelist_and_varname_1} STREQUAL
+       ${filelist_and_varname_1}_from_build_variables
+    )
+      message(
+        FATAL_ERROR
+          "Buck-generated ${filelist_and_varname_1} does not match hardcoded "
+          "${filelist_and_varname_0} in build_variables.bzl. Left: "
+          "${${filelist_and_varname_1}}\n "
+          "Right: ${${filelist_and_varname_1}_from_build_variables}"
+      )
+    endif()
+  endforeach()
+endfunction()