[OpenMP][libomp] Remove Perl in favor of Python (llvm#95307)

* Removes all Perl scripts and modules * Adds Python3 scripts which mimic the behavior of the Perl scripts * Removes Perl from CMake; Adds Python3 requirement to CMake * The check-instruction-set.pl script is Knights Corner specific. The script is removed and not replicated with a corresponding Python3 script. Relevant Discourse: https://discourse.llvm.org/t/error-compiling-clang-with-offloading-support/79223/4 Fixes: llvm#62289
Artem-B · Jun 20, 2024 · 88dae3d · 88dae3d
1 parent 67da89c
commit 88dae3d
Show file tree

Hide file tree

Showing 20 changed files with 987 additions and 5,618 deletions.
diff --git a/openmp/runtime/cmake/LibompExports.cmake b/openmp/runtime/cmake/LibompExports.cmake
@@ -28,15 +28,22 @@ endif()
 string(REPLACE ";" "" libomp_suffix "${libomp_suffix}")
 
 # Set exports locations
+if(WIN32)
+  set(LIBOMP_SHORT_OS win)
+elseif(APPLE)
+  set(LIBOMP_SHORT_OS mac)
+else()
+  set(LIBOMP_SHORT_OS lin)
+endif()
 if(${MIC})
-  set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_${LIBOMP_MIC_ARCH}") # e.g., lin_knf, lin_knc
+  set(libomp_platform "${LIBOMP_SHORT_OS}_${LIBOMP_MIC_ARCH}") # e.g., lin_knf, lin_knc
 else()
   if(${IA32})
-    set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_32")
+    set(libomp_platform "${LIBOMP_SHORT_OS}_32")
   elseif(${INTEL64})
-    set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_32e")
+    set(libomp_platform "${LIBOMP_SHORT_OS}_32e")
   else()
-    set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_${LIBOMP_ARCH}") # e.g., lin_arm, lin_ppc64
+    set(libomp_platform "${LIBOMP_SHORT_OS}_${LIBOMP_ARCH}") # e.g., lin_arm, lin_ppc64
   endif()
 endif()
 set(LIBOMP_EXPORTS_DIR "${LIBOMP_BASE_DIR}/exports")

diff --git a/openmp/runtime/cmake/LibompHandleFlags.cmake b/openmp/runtime/cmake/LibompHandleFlags.cmake
@@ -170,7 +170,7 @@ function(libomp_get_fflags fflags)
   set(${fflags} ${fflags_local} PARENT_SCOPE)
 endfunction()
 
-# Perl generate-defs.pl flags (For Windows only)
+# Python generate-defs.py flags (For Windows only)
 function(libomp_get_gdflags gdflags)
   set(gdflags_local)
   if(${IA32})

diff --git a/openmp/runtime/cmake/LibompMicroTests.cmake b/openmp/runtime/cmake/LibompMicroTests.cmake
@@ -25,12 +25,7 @@
 #  - Fails if stack is executable. Should only be readable and writable. Not executable.
 #  - Program dependencies: perl, readelf
 #  - Available for Unix dynamic library builds. Not available otherwise.
-# (4) test-instr (Intel(R) MIC Architecture only)
-#  - Tests Intel(R) MIC Architecture libraries for valid instruction set
-#  - Fails if finds invalid instruction for Intel(R) MIC Architecture (wasn't compiled with correct flags)
-#  - Program dependencies: perl, objdump
-#  - Available for Intel(R) MIC Architecture and i386 builds. Not available otherwise.
-# (5) test-deps
+# (4) test-deps
 #  - Tests newly created libomp for library dependencies
 #  - Fails if sees a dependence not listed in td_exp variable below
 #  - Program dependencies: perl, (unix)readelf, (mac)otool[64], (windows)link.exe
@@ -93,7 +88,6 @@ endif()
 macro(libomp_test_touch_recipe test_touch_dir)
   set(libomp_test_touch_dependencies ${LIBOMP_SRC_DIR}/test-touch.c omp)
   set(libomp_test_touch_exe ${test_touch_dir}/test-touch${CMAKE_EXECUTABLE_SUFFIX})
-  set(libomp_test_touch_obj ${test_touch_dir}/test-touch${CMAKE_C_OUTPUT_EXTENSION})
   if(WIN32)
     if(${RELEASE_BUILD} OR ${RELWITHDEBINFO_BUILD})
       if(${test_touch_dir} MATCHES "test-touch-mt")
@@ -108,13 +102,13 @@ macro(libomp_test_touch_recipe test_touch_dir)
         libomp_append(libomp_test_touch_cflags /MDd)
       endif()
     endif()
-    set(libomp_test_touch_out_flags -Fe${libomp_test_touch_exe} -Fo${libomp_test_touch_obj})
+    set(libomp_test_touch_out_flags -Fe${libomp_test_touch_exe})
     list(APPEND libomp_test_touch_dependencies ompimp)
   else()
     set(libomp_test_touch_out_flags -o ${libomp_test_touch_exe})
   endif()
   add_custom_command(
-    OUTPUT  ${test_touch_dir}/.success ${libomp_test_touch_exe} ${libomp_test_touch_obj}
+    OUTPUT  ${test_touch_dir}/.success ${libomp_test_touch_exe}
     COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${test_touch_dir}
     COMMAND ${CMAKE_COMMAND} -E remove -f ${test_touch_dir}/*
     COMMAND ${libomp_test_touch_compiler} ${libomp_test_touch_out_flags} ${libomp_test_touch_cflags}
@@ -152,22 +146,10 @@ set_target_properties(libomp-test-execstack PROPERTIES FOLDER "OpenMP/Tests")
 add_custom_command(
   OUTPUT  test-execstack/.success
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-execstack
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.pl
-    --arch=${LIBOMP_PERL_SCRIPT_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
+  COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.py
+          ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
   COMMAND ${CMAKE_COMMAND} -E touch test-execstack/.success
-  DEPENDS omp
-)
-
-# test-instr
-add_custom_target(libomp-test-instr DEPENDS test-instr/.success)
-set_target_properties(libomp-test-instr PROPERTIES FOLDER "OpenMP/Tests")
-add_custom_command(
-  OUTPUT  test-instr/.success
-  COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-instr
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_PERL_SCRIPT_ARCH} --show --mic-arch=${LIBOMP_MIC_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
-  COMMAND ${CMAKE_COMMAND} -E touch test-instr/.success
-  DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl
+  DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-execstack.py
 )
 
 # test-deps
@@ -187,7 +169,15 @@ elseif(APPLE)
   set(libomp_expected_library_deps /usr/lib/libSystem.B.dylib)
 elseif(WIN32)
   set(libomp_expected_library_deps kernel32.dll)
-  libomp_append(libomp_expected_library_deps psapi.dll LIBOMP_OMPT_SUPPORT)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-convert-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-environment-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-heap-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-runtime-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-stdio-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-string-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps api-ms-win-crt-utility-l1-1-0.dll)
+  libomp_append(libomp_expected_library_deps vcruntime140.dll)
+  libomp_append(libomp_expected_library_deps psapi.dll)
 else()
   if(${MIC})
     set(libomp_expected_library_deps libc.so.6 libpthread.so.0 libdl.so.2)
@@ -202,9 +192,11 @@ else()
     if(${IA32})
       libomp_append(libomp_expected_library_deps libc.so.6)
       libomp_append(libomp_expected_library_deps ld-linux.so.2)
+      libomp_append(libomp_expected_library_deps librt.so.1)
     elseif(${INTEL64})
       libomp_append(libomp_expected_library_deps libc.so.6)
       libomp_append(libomp_expected_library_deps ld-linux-x86-64.so.2)
+      libomp_append(libomp_expected_library_deps librt.so.1)
     elseif(${ARM})
       libomp_append(libomp_expected_library_deps libc.so.6)
       libomp_append(libomp_expected_library_deps libffi.so.6)
@@ -232,13 +224,14 @@ else()
   libomp_append(libomp_expected_library_deps libstdc++.so.6 LIBOMP_USE_STDCPPLIB)
   libomp_append(libomp_expected_library_deps libm.so.6 LIBOMP_STATS)
 endif()
-# Perl script expects comma separated list
+# Check depends script expects comma separated list
 string(REPLACE ";" "," libomp_expected_library_deps "${libomp_expected_library_deps}")
 add_custom_command(
   OUTPUT  test-deps/.success
   COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-deps
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-depends.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --arch=${LIBOMP_PERL_SCRIPT_ARCH} --expected="${libomp_expected_library_deps}" ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
+  COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-depends.py
+          --expected="${libomp_expected_library_deps}"
+          ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE}
   COMMAND ${CMAKE_COMMAND} -E touch test-deps/.success
-  DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-depends.pl
+  DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-depends.py
 )
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
@@ -219,30 +219,10 @@ if (IA32 OR INTEL64)
   set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
 endif()
 
-# Find perl executable
-# Perl is used to create omp.h (and other headers) along with kmp_i18n_id.inc and kmp_i18n_default.inc
-find_package(Perl REQUIRED)
-# The perl scripts take the --os=/--arch= flags which expect a certain format for operating systems and arch's.
-# Until the perl scripts are removed, the most portable way to handle this is to have all operating systems that
-# are neither Windows nor Mac (Most Unix flavors) be considered lin to the perl scripts.  This is rooted
-# in that all the Perl scripts check the operating system and will fail if it isn't "valid".  This
-# temporary solution lets us avoid trying to enumerate all the possible OS values inside the Perl modules.
-if(WIN32)
-  set(LIBOMP_PERL_SCRIPT_OS win)
-elseif(APPLE)
-  set(LIBOMP_PERL_SCRIPT_OS mac)
-else()
-  set(LIBOMP_PERL_SCRIPT_OS lin)
-endif()
-if(IA32)
-  set(LIBOMP_PERL_SCRIPT_ARCH 32)
-elseif(MIC)
-  set(LIBOMP_PERL_SCRIPT_ARCH mic)
-elseif(INTEL64)
-  set(LIBOMP_PERL_SCRIPT_ARCH 32e)
-else()
-  set(LIBOMP_PERL_SCRIPT_ARCH ${LIBOMP_ARCH})
-endif()
+# Find python3 executable
+# Python3 is used to create kmp_i18n_id.inc and
+# kmp_i18n_default.inc and for Windows the *.def files.
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
 
 # Checking features
 # Check if version symbol assembler directives are supported

diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
@@ -28,15 +28,15 @@ endif()
 # Generate message catalog files: kmp_i18n_id.inc and kmp_i18n_default.inc
 add_custom_command(
   OUTPUT  kmp_i18n_id.inc
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --prefix=kmp_i18n --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
-  DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl
+  COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.py
+          --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
+  DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.py
 )
 add_custom_command(
   OUTPUT  kmp_i18n_default.inc
-  COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS}
-    --prefix=kmp_i18n --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
-  DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl
+  COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.py
+          --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt
+  DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.py
 )
 
 # Set the -D definitions for all sources
@@ -301,11 +301,12 @@ if(WIN32)
   # Create the main def file with ordinals to use for building the runtime dll to maintain backwards compatible exports order
   libomp_get_gdflags(LIBOMP_GDFLAGS)
   libomp_string_to_list("${LIBOMP_GDFLAGS}" LIBOMP_GDFLAGS)
+
   add_custom_command(
     OUTPUT  ${LIBOMP_GENERATED_DEF_FILE}
-    COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.pl ${LIBOMP_GDFLAGS} -D NAME=${LIBOMP_LIB_FILE}
+    COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.py ${LIBOMP_GDFLAGS} --name ${LIBOMP_LIB_FILE}
             -o ${LIBOMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports
-    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.pl
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.py
   )
 
   if(MSVC)
@@ -317,9 +318,10 @@ if(WIN32)
     set_target_properties(libompimp-needed-def-file PROPERTIES FOLDER "OpenMP/Resources")
     add_custom_command(
       OUTPUT  ${LIBOMPIMP_GENERATED_DEF_FILE}
-      COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.pl ${LIBOMP_GDFLAGS} -D NAME=${LIBOMP_LIB_FILE} -D NOORDINALS
-          -o ${LIBOMPIMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports
-      DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.pl
+      COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.py ${LIBOMP_GDFLAGS}
+              --name ${LIBOMP_LIB_FILE} --no-ordinals
+              -o ${LIBOMPIMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports
+      DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.py
     )
     # while this merely generates an import library off a def file, CMAKE still requires it to have a "source" so feed it a dummy one,
     # making it a .txt which CMAKE will filter out from the librarian (a .cpp will make lib.exe punt trying to resolve the .def symbols)
@@ -396,9 +398,6 @@ endif()
 if(NOT WIN32 AND NOT APPLE)
   add_dependencies(libomp-micro-tests libomp-test-execstack)
 endif()
-if(${MIC})
-  add_dependencies(libomp-micro-tests libomp-test-instr)
-endif()
 add_dependencies(libomp-micro-tests libomp-test-deps)
 
 # `omp` needs to be exported if in-tree build.