From 88dae3d5d0230747f3cbabdde9ac5ae9e5dc3f8d Mon Sep 17 00:00:00 2001 From: Jonathan Peyton Date: Thu, 20 Jun 2024 12:54:49 -0500 Subject: [PATCH] [OpenMP][libomp] Remove Perl in favor of Python (#95307) * Removes all Perl scripts and modules * Adds Python3 scripts which mimic the behavior of the Perl scripts * Removes Perl from CMake; Adds Python3 requirement to CMake * The check-instruction-set.pl script is Knights Corner specific. The script is removed and not replicated with a corresponding Python3 script. Relevant Discourse: https://discourse.llvm.org/t/error-compiling-clang-with-offloading-support/79223/4 Fixes: https://github.com/llvm/llvm-project/issues/62289 --- openmp/runtime/cmake/LibompExports.cmake | 15 +- openmp/runtime/cmake/LibompHandleFlags.cmake | 2 +- openmp/runtime/cmake/LibompMicroTests.cmake | 51 +- openmp/runtime/cmake/config-ix.cmake | 28 +- openmp/runtime/src/CMakeLists.txt | 27 +- openmp/runtime/tools/check-depends.pl | 505 ----- openmp/runtime/tools/check-depends.py | 168 ++ openmp/runtime/tools/check-execstack.pl | 145 -- openmp/runtime/tools/check-execstack.py | 65 + openmp/runtime/tools/check-instruction-set.pl | 320 --- openmp/runtime/tools/generate-def.pl | 331 --- openmp/runtime/tools/generate-def.py | 244 ++ openmp/runtime/tools/lib/Build.pm | 263 --- openmp/runtime/tools/lib/LibOMP.pm | 84 - openmp/runtime/tools/lib/Platform.pm | 502 ----- openmp/runtime/tools/lib/Uname.pm | 646 ------ openmp/runtime/tools/lib/tools.pm | 1976 ----------------- openmp/runtime/tools/libomputils.py | 65 + openmp/runtime/tools/message-converter.pl | 774 ------- openmp/runtime/tools/message-converter.py | 394 ++++ 20 files changed, 987 insertions(+), 5618 deletions(-) delete mode 100755 openmp/runtime/tools/check-depends.pl create mode 100644 openmp/runtime/tools/check-depends.py delete mode 100755 openmp/runtime/tools/check-execstack.pl create mode 100644 openmp/runtime/tools/check-execstack.py delete mode 100755 openmp/runtime/tools/check-instruction-set.pl delete mode 100755 openmp/runtime/tools/generate-def.pl create mode 100644 openmp/runtime/tools/generate-def.py delete mode 100644 openmp/runtime/tools/lib/Build.pm delete mode 100644 openmp/runtime/tools/lib/LibOMP.pm delete mode 100644 openmp/runtime/tools/lib/Platform.pm delete mode 100644 openmp/runtime/tools/lib/Uname.pm delete mode 100644 openmp/runtime/tools/lib/tools.pm create mode 100644 openmp/runtime/tools/libomputils.py delete mode 100755 openmp/runtime/tools/message-converter.pl create mode 100644 openmp/runtime/tools/message-converter.py diff --git a/openmp/runtime/cmake/LibompExports.cmake b/openmp/runtime/cmake/LibompExports.cmake index dbeb18f358f1a..461e47d449157 100644 --- a/openmp/runtime/cmake/LibompExports.cmake +++ b/openmp/runtime/cmake/LibompExports.cmake @@ -28,15 +28,22 @@ endif() string(REPLACE ";" "" libomp_suffix "${libomp_suffix}") # Set exports locations +if(WIN32) + set(LIBOMP_SHORT_OS win) +elseif(APPLE) + set(LIBOMP_SHORT_OS mac) +else() + set(LIBOMP_SHORT_OS lin) +endif() if(${MIC}) - set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_${LIBOMP_MIC_ARCH}") # e.g., lin_knf, lin_knc + set(libomp_platform "${LIBOMP_SHORT_OS}_${LIBOMP_MIC_ARCH}") # e.g., lin_knf, lin_knc else() if(${IA32}) - set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_32") + set(libomp_platform "${LIBOMP_SHORT_OS}_32") elseif(${INTEL64}) - set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_32e") + set(libomp_platform "${LIBOMP_SHORT_OS}_32e") else() - set(libomp_platform "${LIBOMP_PERL_SCRIPT_OS}_${LIBOMP_ARCH}") # e.g., lin_arm, lin_ppc64 + set(libomp_platform "${LIBOMP_SHORT_OS}_${LIBOMP_ARCH}") # e.g., lin_arm, lin_ppc64 endif() endif() set(LIBOMP_EXPORTS_DIR "${LIBOMP_BASE_DIR}/exports") diff --git a/openmp/runtime/cmake/LibompHandleFlags.cmake b/openmp/runtime/cmake/LibompHandleFlags.cmake index 1aba1fbea8290..cb7e488652230 100644 --- a/openmp/runtime/cmake/LibompHandleFlags.cmake +++ b/openmp/runtime/cmake/LibompHandleFlags.cmake @@ -170,7 +170,7 @@ function(libomp_get_fflags fflags) set(${fflags} ${fflags_local} PARENT_SCOPE) endfunction() -# Perl generate-defs.pl flags (For Windows only) +# Python generate-defs.py flags (For Windows only) function(libomp_get_gdflags gdflags) set(gdflags_local) if(${IA32}) diff --git a/openmp/runtime/cmake/LibompMicroTests.cmake b/openmp/runtime/cmake/LibompMicroTests.cmake index 6fcde37259931..0d48246cb6ec5 100644 --- a/openmp/runtime/cmake/LibompMicroTests.cmake +++ b/openmp/runtime/cmake/LibompMicroTests.cmake @@ -25,12 +25,7 @@ # - Fails if stack is executable. Should only be readable and writable. Not executable. # - Program dependencies: perl, readelf # - Available for Unix dynamic library builds. Not available otherwise. -# (4) test-instr (Intel(R) MIC Architecture only) -# - Tests Intel(R) MIC Architecture libraries for valid instruction set -# - Fails if finds invalid instruction for Intel(R) MIC Architecture (wasn't compiled with correct flags) -# - Program dependencies: perl, objdump -# - Available for Intel(R) MIC Architecture and i386 builds. Not available otherwise. -# (5) test-deps +# (4) test-deps # - Tests newly created libomp for library dependencies # - Fails if sees a dependence not listed in td_exp variable below # - Program dependencies: perl, (unix)readelf, (mac)otool[64], (windows)link.exe @@ -93,7 +88,6 @@ endif() macro(libomp_test_touch_recipe test_touch_dir) set(libomp_test_touch_dependencies ${LIBOMP_SRC_DIR}/test-touch.c omp) set(libomp_test_touch_exe ${test_touch_dir}/test-touch${CMAKE_EXECUTABLE_SUFFIX}) - set(libomp_test_touch_obj ${test_touch_dir}/test-touch${CMAKE_C_OUTPUT_EXTENSION}) if(WIN32) if(${RELEASE_BUILD} OR ${RELWITHDEBINFO_BUILD}) if(${test_touch_dir} MATCHES "test-touch-mt") @@ -108,13 +102,13 @@ macro(libomp_test_touch_recipe test_touch_dir) libomp_append(libomp_test_touch_cflags /MDd) endif() endif() - set(libomp_test_touch_out_flags -Fe${libomp_test_touch_exe} -Fo${libomp_test_touch_obj}) + set(libomp_test_touch_out_flags -Fe${libomp_test_touch_exe}) list(APPEND libomp_test_touch_dependencies ompimp) else() set(libomp_test_touch_out_flags -o ${libomp_test_touch_exe}) endif() add_custom_command( - OUTPUT ${test_touch_dir}/.success ${libomp_test_touch_exe} ${libomp_test_touch_obj} + OUTPUT ${test_touch_dir}/.success ${libomp_test_touch_exe} COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${test_touch_dir} COMMAND ${CMAKE_COMMAND} -E remove -f ${test_touch_dir}/* COMMAND ${libomp_test_touch_compiler} ${libomp_test_touch_out_flags} ${libomp_test_touch_cflags} @@ -152,22 +146,10 @@ set_target_properties(libomp-test-execstack PROPERTIES FOLDER "OpenMP/Tests") add_custom_command( OUTPUT test-execstack/.success COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-execstack - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.pl - --arch=${LIBOMP_PERL_SCRIPT_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-execstack.py + ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} COMMAND ${CMAKE_COMMAND} -E touch test-execstack/.success - DEPENDS omp -) - -# test-instr -add_custom_target(libomp-test-instr DEPENDS test-instr/.success) -set_target_properties(libomp-test-instr PROPERTIES FOLDER "OpenMP/Tests") -add_custom_command( - OUTPUT test-instr/.success - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-instr - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl --os=${LIBOMP_PERL_SCRIPT_OS} - --arch=${LIBOMP_PERL_SCRIPT_ARCH} --show --mic-arch=${LIBOMP_MIC_ARCH} ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} - COMMAND ${CMAKE_COMMAND} -E touch test-instr/.success - DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-instruction-set.pl + DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-execstack.py ) # test-deps @@ -187,7 +169,15 @@ elseif(APPLE) set(libomp_expected_library_deps /usr/lib/libSystem.B.dylib) elseif(WIN32) set(libomp_expected_library_deps kernel32.dll) - libomp_append(libomp_expected_library_deps psapi.dll LIBOMP_OMPT_SUPPORT) + libomp_append(libomp_expected_library_deps api-ms-win-crt-convert-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-environment-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-heap-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-runtime-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-stdio-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-string-l1-1-0.dll) + libomp_append(libomp_expected_library_deps api-ms-win-crt-utility-l1-1-0.dll) + libomp_append(libomp_expected_library_deps vcruntime140.dll) + libomp_append(libomp_expected_library_deps psapi.dll) else() if(${MIC}) set(libomp_expected_library_deps libc.so.6 libpthread.so.0 libdl.so.2) @@ -202,9 +192,11 @@ else() if(${IA32}) libomp_append(libomp_expected_library_deps libc.so.6) libomp_append(libomp_expected_library_deps ld-linux.so.2) + libomp_append(libomp_expected_library_deps librt.so.1) elseif(${INTEL64}) libomp_append(libomp_expected_library_deps libc.so.6) libomp_append(libomp_expected_library_deps ld-linux-x86-64.so.2) + libomp_append(libomp_expected_library_deps librt.so.1) elseif(${ARM}) libomp_append(libomp_expected_library_deps libc.so.6) libomp_append(libomp_expected_library_deps libffi.so.6) @@ -232,13 +224,14 @@ else() libomp_append(libomp_expected_library_deps libstdc++.so.6 LIBOMP_USE_STDCPPLIB) libomp_append(libomp_expected_library_deps libm.so.6 LIBOMP_STATS) endif() -# Perl script expects comma separated list +# Check depends script expects comma separated list string(REPLACE ";" "," libomp_expected_library_deps "${libomp_expected_library_deps}") add_custom_command( OUTPUT test-deps/.success COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/test-deps - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-depends.pl --os=${LIBOMP_PERL_SCRIPT_OS} - --arch=${LIBOMP_PERL_SCRIPT_ARCH} --expected="${libomp_expected_library_deps}" ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/check-depends.py + --expected="${libomp_expected_library_deps}" + ${LIBOMP_OUTPUT_DIRECTORY}/${LIBOMP_LIB_FILE} COMMAND ${CMAKE_COMMAND} -E touch test-deps/.success - DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-depends.pl + DEPENDS omp ${LIBOMP_TOOLS_DIR}/check-depends.py ) diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake index 337fe2e599648..0568474f1d74e 100644 --- a/openmp/runtime/cmake/config-ix.cmake +++ b/openmp/runtime/cmake/config-ix.cmake @@ -219,30 +219,10 @@ if (IA32 OR INTEL64) set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) endif() -# Find perl executable -# Perl is used to create omp.h (and other headers) along with kmp_i18n_id.inc and kmp_i18n_default.inc -find_package(Perl REQUIRED) -# The perl scripts take the --os=/--arch= flags which expect a certain format for operating systems and arch's. -# Until the perl scripts are removed, the most portable way to handle this is to have all operating systems that -# are neither Windows nor Mac (Most Unix flavors) be considered lin to the perl scripts. This is rooted -# in that all the Perl scripts check the operating system and will fail if it isn't "valid". This -# temporary solution lets us avoid trying to enumerate all the possible OS values inside the Perl modules. -if(WIN32) - set(LIBOMP_PERL_SCRIPT_OS win) -elseif(APPLE) - set(LIBOMP_PERL_SCRIPT_OS mac) -else() - set(LIBOMP_PERL_SCRIPT_OS lin) -endif() -if(IA32) - set(LIBOMP_PERL_SCRIPT_ARCH 32) -elseif(MIC) - set(LIBOMP_PERL_SCRIPT_ARCH mic) -elseif(INTEL64) - set(LIBOMP_PERL_SCRIPT_ARCH 32e) -else() - set(LIBOMP_PERL_SCRIPT_ARCH ${LIBOMP_ARCH}) -endif() +# Find python3 executable +# Python3 is used to create kmp_i18n_id.inc and +# kmp_i18n_default.inc and for Windows the *.def files. +find_package(Python3 REQUIRED COMPONENTS Interpreter) # Checking features # Check if version symbol assembler directives are supported diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt index 62c35c19e6b45..60641e6f0fe5d 100644 --- a/openmp/runtime/src/CMakeLists.txt +++ b/openmp/runtime/src/CMakeLists.txt @@ -28,15 +28,15 @@ endif() # Generate message catalog files: kmp_i18n_id.inc and kmp_i18n_default.inc add_custom_command( OUTPUT kmp_i18n_id.inc - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS} - --prefix=kmp_i18n --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt - DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.py + --enum=kmp_i18n_id.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt + DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.py ) add_custom_command( OUTPUT kmp_i18n_default.inc - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.pl --os=${LIBOMP_PERL_SCRIPT_OS} - --prefix=kmp_i18n --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt - DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.pl + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/message-converter.py + --default=kmp_i18n_default.inc ${LIBOMP_SRC_DIR}/i18n/en_US.txt + DEPENDS ${LIBOMP_SRC_DIR}/i18n/en_US.txt ${LIBOMP_TOOLS_DIR}/message-converter.py ) # Set the -D definitions for all sources @@ -301,11 +301,12 @@ if(WIN32) # Create the main def file with ordinals to use for building the runtime dll to maintain backwards compatible exports order libomp_get_gdflags(LIBOMP_GDFLAGS) libomp_string_to_list("${LIBOMP_GDFLAGS}" LIBOMP_GDFLAGS) + add_custom_command( OUTPUT ${LIBOMP_GENERATED_DEF_FILE} - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.pl ${LIBOMP_GDFLAGS} -D NAME=${LIBOMP_LIB_FILE} + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.py ${LIBOMP_GDFLAGS} --name ${LIBOMP_LIB_FILE} -o ${LIBOMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.pl + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.py ) if(MSVC) @@ -317,9 +318,10 @@ if(WIN32) set_target_properties(libompimp-needed-def-file PROPERTIES FOLDER "OpenMP/Resources") add_custom_command( OUTPUT ${LIBOMPIMP_GENERATED_DEF_FILE} - COMMAND ${PERL_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.pl ${LIBOMP_GDFLAGS} -D NAME=${LIBOMP_LIB_FILE} -D NOORDINALS - -o ${LIBOMPIMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.pl + COMMAND ${Python3_EXECUTABLE} ${LIBOMP_TOOLS_DIR}/generate-def.py ${LIBOMP_GDFLAGS} + --name ${LIBOMP_LIB_FILE} --no-ordinals + -o ${LIBOMPIMP_GENERATED_DEF_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/dllexports + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/dllexports ${LIBOMP_TOOLS_DIR}/generate-def.py ) # while this merely generates an import library off a def file, CMAKE still requires it to have a "source" so feed it a dummy one, # making it a .txt which CMAKE will filter out from the librarian (a .cpp will make lib.exe punt trying to resolve the .def symbols) @@ -396,9 +398,6 @@ endif() if(NOT WIN32 AND NOT APPLE) add_dependencies(libomp-micro-tests libomp-test-execstack) endif() -if(${MIC}) - add_dependencies(libomp-micro-tests libomp-test-instr) -endif() add_dependencies(libomp-micro-tests libomp-test-deps) # `omp` needs to be exported if in-tree build. diff --git a/openmp/runtime/tools/check-depends.pl b/openmp/runtime/tools/check-depends.pl deleted file mode 100755 index aca888b331e2b..0000000000000 --- a/openmp/runtime/tools/check-depends.pl +++ /dev/null @@ -1,505 +0,0 @@ -#!/usr/bin/env perl - -# -#//===----------------------------------------------------------------------===// -#// -#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -#// See https://llvm.org/LICENSE.txt for license information. -#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#// -#//===----------------------------------------------------------------------===// -# - -use strict; -use warnings; - -use FindBin; -use lib "$FindBin::Bin/lib"; - -use tools; - -our $VERSION = "0.005"; -my $target_os; -my $target_arch; - -# -------------------------------------------------------------------------------------------------- -# Output parse error. -# $tool -- Name of tool. -# @bulk -- Output of the tool. -# $n -- Number of line caused parse error. -sub parse_error($\@$) { - my ( $tool, $bulk, $n ) = @_; - my @bulk; - for ( my $i = 0; $i < @$bulk; ++ $i ) { - push( @bulk, ( $i == $n ? ">>> " : " " ) . $bulk->[ $i ] ); - }; # for $i - runtime_error( "Fail to parse $tool output:", @bulk, "(eof)" ); -}; # sub parse_error - - -# -------------------------------------------------------------------------------------------------- -# Linux* OS version of get_deps() parses output of ldd: -# -# $ ldd libname.so -# libc.so.6 => /lib64/libc.so.6 (0x00002b60fedd8000) -# libdl.so.2 => /lib64/libdl.so.2 (0x00002b60ff12b000) -# libpthread.so.0 => /lib64/libpthread.so.0 (0x00002b60ff32f000) -# /lib64/ld-linux-x86-64.so.2 (0x0000003879400000) -# -# Note: ldd printd all the dependencies, direct and indirect. (For example, if specified library -# requires libdl.so, and libdl.so requires /lib/ld-linux.so, ldd prints both libdl.so and -# /lib/ld-linux.so). If you do not want indirect dependencies, look at readelf tool. -# -sub get_deps_ldd($) { - - my $lib = shift ( @_ ); - my $tool = "ldd"; - my @bulk; - my @deps; - - execute( [ $tool, $lib ], -stdout => \@bulk ); - debug( @bulk, "(eof)" ); - - foreach my $i ( 0 .. @bulk - 1 ) { - my $line = $bulk[ $i ]; - if ( $line !~ m{^\s*(?:([_a-z0-9.+-/]*)\s+=>\s+)?([_a-z0-9.+-/]*)\s+\(0x[0-9a-z]*\)$}i ) { - parse_error( $tool, @bulk, $i ); - }; # if - my $dep = ( defined( $1 ) ? $1 : $2 ); - push( @deps, $dep ); - }; # foreach $i - - return @deps; - -}; # sub get_deps_ldd - - -# -------------------------------------------------------------------------------------------------- -# Another Linux* OS version of get_deps() parses output of readelf: -# -# $ readelf -d exports/lin_32e/lib/libomp.so -# -# Dynamic segment at offset 0x87008 contains 24 entries: -# Tag Type Name/Value -# 0x0000000000000001 (NEEDED) Shared library: [libc.so.6] -# 0x0000000000000001 (NEEDED) Shared library: [libdl.so.2] -# 0x0000000000000001 (NEEDED) Shared library: [libpthread.so.0] -# 0x000000000000000e (SONAME) Library soname: [libomp.so] -# 0x000000000000000d (FINI) 0x51caa -# 0x0000000000000004 (HASH) 0x158 -# 0x0000000000000005 (STRTAB) 0x9350 -# ... -# -# Note: In contrast to ldd, readelf shows only direct dependencies. -# -sub get_deps_readelf($) { - - my $file = shift ( @_ ); - my $tool; - my @bulk; - my @deps; - - if($target_arch eq "mic") { - $tool = "x86_64-k1om-linux-readelf"; - } else { - $tool = "readelf"; - } - - # Force the readelf call to be in English. For example, when readelf - # is called on a french localization, it will find "Librairie partagees" - # instead of shared library - $ENV{ LANG } = "C"; - - execute( [ $tool, "-d", $file ], -stdout => \@bulk ); - debug( @bulk, "(eof)" ); - - my $i = 0; - # Parse header. - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - if ( $i == @bulk - 1 and $bulk[ $i ] =~ m{^There is no dynamic section in this file\.\s*$} ) { - # This is not dynamic executable => no dependencies. - return @deps; - }; # if - ( $i < @bulk and $bulk[ $i ] =~ m{^Dynamic (?:segment|section) at offset 0x[0-9a-f]+ contains \d+ entries:\s*$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*Tag\s+Type\s+Name/Value\s*$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - # Parse body. - while ( $i < @bulk ) { - my $line = $bulk[ $i ]; - if ( $line !~ m{^\s*0x[0-9a-f]+\s+\(?([_A-Z0-9]+)\)?\s+(.*)\s*$}i ) { - parse_error( $tool, @bulk, $i ); - }; # if - my ( $type, $value ) = ( $1, $2 ); - if ( $type eq "NEEDED" ) { - if ( $value !~ m{\AShared library: \[(.*)\]\z} ) { - parse_error( $tool, @bulk, $i ); - }; # if - my $dep = $1; - push( @deps, $dep ); - }; # if - ++ $i; - }; # foreach $i - - return @deps; - -}; # sub get_deps_readelf - - -# -------------------------------------------------------------------------------------------------- -# OS X* version of get_deps() parses output of otool: -# -# $ otool -L libname.dylib -# exports/mac_32/lib.thin/libomp.dylib: -# libomp.dylib (compatibility version 5.0.0, current version 5.0.0) -# /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 88.1.3) -# -sub get_deps_otool($) { - - my $file = shift ( @_ ); - my $name = get_file( $file ); - my $tool = "otool"; - my @bulk; - my @deps; - - if ( $target_arch eq "32e" ) { - # On older (Tiger) systems otool does not recognize 64-bit binaries, so try to locate - # otool64. - my $path = which( "otool64" ); - if ( defined ( $path ) ) { - $tool = "otool64"; - }; # if - }; # if - - execute( [ $tool, "-L", $file ], -stdout => \@bulk ); - debug( @bulk, "(eof)" ); - - my $i = 0; - # Parse the first one or two lines separately. - ( $i < @bulk and $bulk[ $i ] =~ m{^\Q$file\E:$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - if ( $name =~ m{\.dylib\z} ) { - # Added "@rpath/" enables dynamic load of the library designated at link time. - $name = '@rpath/' . $name; - # In case of dynamic library otool print the library itself as a dependent library. - ( $i < @bulk and $bulk[ $i ] =~ m{^\s+\Q$name\E\s+\(compatibility version.*\)$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - }; # if - - # Then parse the rest. - while ( $i < @bulk ) { - my $line = $bulk[ $i ]; - if ( $line !~ m/^\s*(.*)\s+\(compatibility version\s.*\)$/ ) { - parse_error( $tool, @bulk, $i ); - }; # if - my ( $dep ) = ( $1 ); - push( @deps, $dep ); - ++ $i; - }; # while - - return @deps; - -}; # sub get_deps_otool - - -# -------------------------------------------------------------------------------------------------- -# Windows* OS version of get_deps() parses output of link: -# -# > link -dump -dependents libname.dll -# Microsoft (R) COFF/PE Dumper Version 8.00.40310.39 -# Copyright (C) Microsoft Corporation. All rights reserved. -# Dump of file S:\Projects.OMP\users\omalyshe\omp\libomp\exports\win_64\lib\libompmd.dll -# File Type: DLL -# Image has the following dependencies: -# KERNEL32.dll -# Summary -# C000 .data -# 6000 .pdata -# 18000 .rdata -# ... -# -# > link -dump -directives libname.lib -# Microsoft (R) COFF/PE Dumper Version 8.00.40310.39 -# Copyright (C) Microsoft Corporation. All rights reserved. -# Dump of file S:\Projects.OMP\users\omalyshe\omp\libomp\exports\win_32e\lib\libimp5mt.lib -# File Type: LIBRARY -# Linker Directives -# ----------------- -# -defaultlib:"uuid.lib" -# -defaultlib:"uuid.lib" -# ..... -# Summary -# 3250 .bss -# 3FBC .data -# 34 .data1 -# .... -sub get_deps_link($) { - - my ( $lib ) = @_; - my $tool = "link"; - my @bulk; - my @deps; - - my $ext = lc( get_ext( $lib ) ); - if ( $ext !~ m{\A\.(?:lib|dll|exe)\z}i ) { - runtime_error( "Incorrect file is specified: `$lib'; only `lib', `dll' or `exe' file expected" ); - }; # if - - execute( - [ $tool, "/dump", ( $ext eq ".lib" ? "/directives" : "/dependents" ), $lib ], - -stdout => \@bulk - ); - - debug( @bulk, "(eof)" ); - - my $i = 0; - ( $i < @bulk and $bulk[ $i ] =~ m{^Microsoft \(R\) COFF\/PE Dumper Version.*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^Copyright \(C\) Microsoft Corporation\..*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^Dump of file\s\Q$lib\E$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^File Type:\s(.*)$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - ( $i < @bulk and $bulk[ $i ] =~ m{^\s*$} ) or parse_error( $tool, @bulk, $i ); ++ $i; - - if ( $ext eq ".lib" ) { - - my %deps; - while ( $i < @bulk ) { - my $line = $bulk[ $i ]; - if ( 0 ) { - } elsif ( $line =~ m{^\s*[-/]defaultlib\:(.*)\s*$}i ) { - my $dep = $1; - # Normalize library name: - $dep = lc( $1 ); # Convert to lower case. - $dep =~ s{\A"(.*)"\z}{$1}; # Drop surrounding quotes (if any). - $dep =~ s{\.lib\z}{}; # Drop .lib suffix (if any). - $deps{ $dep } = 1; - } elsif ( $line =~ m{^\s*Linker Directives\s*$} ) { - } elsif ( $line =~ m{^\s*-+\s*$} ) { - } elsif ( $line =~ m{^\s*/alternatename\:.*$} ) { - } elsif ( $line =~ m{^\s*$} ) { - } elsif ( $line =~ m{^\s*/FAILIFMISMATCH\:.*$} ) { - # This directive is produced only by _MSC_VER=1600 - } elsif ( $line =~ m{^\s*Summary\s*$} ) { - last; - } else { - parse_error( $tool, @bulk, $i ); - }; # if - ++ $i; - } # while - @deps = keys( %deps ); - - } else { - - ( $i < @bulk and $bulk[ $i ] =~ m{\s*Image has the following dependencies\:$} ) - or parse_error( $tool, @bulk, $i ); - ++ $i; - while ( $i < @bulk ) { - my $line = $bulk[ $i ]; - if ( 0 ) { - } elsif ( $line =~ m{^\s*$} ) { - # Ignore empty lines. - } elsif ( $line =~ m{^\s*(.*\.dll)$}i ) { - my $dep = lc( $1 ); - push( @deps, $dep ); - } elsif ( $line =~ m{^\s*Summary$} ) { - last; - } else { - parse_error( $tool, @bulk, $i ); - }; # if - ++ $i; - }; # while - - }; # if - - return @deps; - -}; # sub get_deps_link - - -# -------------------------------------------------------------------------------------------------- -# Main. -# -------------------------------------------------------------------------------------------------- - -# Parse command line. -my $expected; -my $bare; -Getopt::Long::Configure( "permute" ); -get_options( - "os=s" => \$target_os, - "arch=s" => \$target_arch, - "bare" => \$bare, - "expected=s" => \$expected, -); -my @expected; -if ( defined( $expected ) ) { - if ( $expected ne "none" ) { - @expected = sort( split( ",", $expected ) ); - if ( $target_os eq "win" ) { - @expected = map( lc( $_ ), @expected ); - }; # if - }; # if -}; # if -if ( @ARGV < 1 ) { - cmdline_error( "Specify a library name to check for dependencies" ); -}; # if -if ( @ARGV > 1 ) { - cmdline_error( "Too many arguments" ); -}; # if -my $lib = shift( @ARGV ); -if ( not -e $lib ){ - runtime_error( "Specified file does not exist: \"$lib\"" ); -}; # if - -# Select appropriate get_deps implementation. -if ( 0 ) { -} elsif ( $target_os eq "lin" ) { - *get_deps = \*get_deps_readelf; -} elsif ( $target_os eq "mac" ) { - *get_deps = \*get_deps_otool; -} elsif ( $target_os eq "win" ) { - *get_deps = \*get_deps_link; -} else { - runtime_error( "OS \"$target_os\" not supported" ); -}; # if - -# Do the work. -my @deps = sort( get_deps( $lib ) ); -if ( $bare ) { - print( map( "$_\n", @deps ) ); -} else { - info( "Dependencies:", @deps ? map( " $_", @deps ) : "(none)" ); -}; # if -if ( defined( $expected ) ) { - my %deps = map( ( $_ => 1 ), @deps ); - foreach my $dep ( @expected ) { - delete( $deps{ $dep } ); - }; # foreach - my @unexpected = sort( keys( %deps ) ); - if ( @unexpected ) { - runtime_error( "Unexpected dependencies:", map( " $_", @unexpected ) ); - }; # if -}; # if - -exit( 0 ); - -__END__ - -=pod - -=head1 NAME - -B -- Check dependencies for a specified library. - -=head1 SYNOPSIS - -B I... I - -=head1 DESCRIPTION - -C finds direct dependencies for a specified library. List of actual dependencies -is sorted alphabetically and printed. If list of expected dependencies is specified, the scripts -checks the library has only allowed dependencies. In case of not expected dependencies. the script -issues error message and exits with non-zero code. - -Linux* OS and OS X*: The script finds dependencies only for dynamic libraries. Windows* OS: The script -finds dependencies for either static or dynamic libraries. - -The script uses external tools. On Linux* OS, it runs F, on OS X* -- F (or F), -on Windows* OS -- F. - -On Windows* OS dependencies are printed in lower case, case of expected dependencies ignored. - -=head1 OPTIONS - -=over - -=item B<--bare> - -Do not use fancy formatting; produce plain, bare output: just a list of libraries, -a library per line. - -=item B<--expected=>I - -I is comma-separated list of expected dependencies (or C). -If C<--expected> option specified, C checks the specified library -has only expected dependencies. - -=item B<--os=>I - -Specify target OS (tool to use) manually. -Useful for cross-build, when host OS is not the same as target OS. -I should be either C, C, or C. - -=back - -=head2 Standard Options - -=over - -=item B<--help> - -Print short help message and exit. - -=item B<--doc> - -=item B<--manual> - -Print full documentation and exit. - -=item B<--quiet> - -Do not output informational messages. - -=item B<--version> - -Print version and exit. - -=back - -=head1 ARGUMENTS - -=over - -=item I - -A name of library to find or check dependencies. - -=back - -=head1 EXAMPLES - -Just print library dependencies (Windows* OS): - - > check-depends.pl exports/win_32/lib/libompmd.dll - check-depends.pl: (i) Dependencies: - check-depends.pl: (i) kernel32.dll - -Print library dependencies, use bare output (Linux* OS): - - $ check-depends.pl --bare exports/lin_32e/lib/libomp_db.so - libc.so.6 - libdl.so.2 - libpthread.so.0 - -Check the library does not have any dependencies (OS X*): - - $ check-depends.pl --expected=none exports/mac_32/lib/libomp.dylib - check-depends.pl: (i) Dependencies: - check-depends.pl: (i) /usr/lib/libSystem.B.dylib - check-depends.pl: (x) Unexpected dependencies: - check-depends.pl: (x) /usr/lib/libSystem.B.dylib - $ echo $? - 2 - -=cut - -# end of file # - diff --git a/openmp/runtime/tools/check-depends.py b/openmp/runtime/tools/check-depends.py new file mode 100644 index 0000000000000..f185900c36d47 --- /dev/null +++ b/openmp/runtime/tools/check-depends.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +# +# //===----------------------------------------------------------------------===// +# // +# // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# // See https://llvm.org/LICENSE.txt for license information. +# // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# // +# //===----------------------------------------------------------------------===// +# + +import argparse +import os +import platform +import re +import sys +from libomputils import ( + ScriptError, + error, + execute_command, + print_info_line, + print_error_line, +) + + +def get_deps_readelf(filename): + """Get list of dependencies from readelf""" + deps = [] + # Force readelf call to be in English + os.environ["LANG"] = "C" + r = execute_command(["readelf", "-d", filename]) + if r.returncode != 0: + error("readelf -d {} failed".format(filename)) + neededRegex = re.compile(r"\(NEEDED\)\s+Shared library: \[([a-zA-Z0-9_.-]+)\]") + for line in r.stdout.split(os.linesep): + match = neededRegex.search(line) + if match: + deps.append(match.group(1)) + return deps + + +def get_deps_otool(filename): + """Get list of dependencies from otool""" + deps = [] + r = execute_command(["otool", "-L", filename]) + if r.returncode != 0: + error("otool -L {} failed".format(filename)) + libRegex = re.compile(r"([^ \t]+)\s+\(compatibility version ") + thisLibRegex = re.compile(r"@rpath/{}".format(os.path.basename(filename))) + for line in r.stdout.split(os.linesep): + match = thisLibRegex.search(line) + if match: + # Don't include the library itself as a needed dependency + continue + match = libRegex.search(line) + if match: + deps.append(match.group(1)) + continue + return deps + + +def get_deps_link(filename): + """Get list of dependecies from link (Windows OS)""" + depsSet = set([]) + f = filename.lower() + args = ["link", "/DUMP"] + if f.endswith(".lib"): + args.append("/DIRECTIVES") + elif f.endswith(".dll") or f.endswith(".exe"): + args.append("/DEPENDENTS") + else: + error("unrecognized file extension: {}".format(filename)) + args.append(filename) + r = execute_command(args) + if r.returncode != 0: + error("{} failed".format(args.command)) + if f.endswith(".lib"): + regex = re.compile(r"\s*[-/]defaultlib:(.*)\s*$") + for line in r.stdout.split(os.linesep): + line = line.lower() + match = regex.search(line) + if match: + depsSet.add(match.group(1)) + else: + started = False + markerStart = re.compile(r"Image has the following depend") + markerEnd = re.compile(r"Summary") + markerEnd2 = re.compile(r"Image has the following delay load depend") + for line in r.stdout.split(os.linesep): + if not started: + if markerStart.search(line): + started = True + continue + else: # Started parsing the libs + line = line.strip() + if not line: + continue + if markerEnd.search(line) or markerEnd2.search(line): + break + depsSet.add(line.lower()) + return list(depsSet) + + +def main(): + parser = argparse.ArgumentParser(description="Check library dependencies") + parser.add_argument( + "--bare", + action="store_true", + help="Produce plain, bare output: just a list" + " of libraries, a library per line", + ) + parser.add_argument( + "--expected", + metavar="CSV_LIST", + help="CSV_LIST is a comma-separated list of expected" + ' dependencies (or "none"). checks the specified' + " library has only expected dependencies.", + ) + + parser.add_argument("library", help="The library file to check") + commandArgs = parser.parse_args() + # Get dependencies + deps = [] + + system = platform.system() + if system == "Windows": + deps = get_deps_link(commandArgs.library) + elif system == "Darwin": + deps = get_deps_otool(commandArgs.library) + else: + deps = get_deps_readelf(commandArgs.library) + deps = sorted(deps) + + # If bare output specified, then just print the dependencies one per line + if commandArgs.bare: + print(os.linesep.join(deps)) + return + + # Calculate unexpected dependencies if expected list specified + unexpected = [] + if commandArgs.expected: + # none => any dependency is unexpected + if commandArgs.expected == "none": + unexpected = list(deps) + else: + expected = [d.strip() for d in commandArgs.expected.split(",")] + unexpected = [d for d in deps if d not in expected] + + # Regular output + print_info_line("Dependencies:") + for dep in deps: + print_info_line(" {}".format(dep)) + if unexpected: + print_error_line("Unexpected Dependencies:") + for dep in unexpected: + print_error_line(" {}".format(dep)) + error("found unexpected dependencies") + + +if __name__ == "__main__": + try: + main() + except ScriptError as e: + print_error_line(str(e)) + sys.exit(1) + +# end of file diff --git a/openmp/runtime/tools/check-execstack.pl b/openmp/runtime/tools/check-execstack.pl deleted file mode 100755 index 7a710072f9724..0000000000000 --- a/openmp/runtime/tools/check-execstack.pl +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env perl - -# -#//===----------------------------------------------------------------------===// -#// -#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -#// See https://llvm.org/LICENSE.txt for license information. -#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#// -#//===----------------------------------------------------------------------===// -# - -use strict; -use warnings; - -use FindBin; -use lib "$FindBin::Bin/lib"; - -use tools; - -our $VERSION = "0.002"; -my $target_arch; - -sub execstack($) { - my ( $file ) = @_; - my @output; - my @stack; - my $tool; - if($target_arch eq "mic") { - $tool = "x86_64-k1om-linux-readelf"; - } else { - $tool = "readelf"; - } - execute( [ $tool, "-l", "-W", $file ], -stdout => \@output ); - @stack = grep( $_ =~ m{\A\s*(?:GNU_)?STACK\s+}, @output ); - if ( not @stack ) { - # Interpret missed "STACK" line as error. - runtime_error( "$file: No stack segment found; looks like stack would be executable." ); - }; # if - if ( @stack > 1 ) { - runtime_error( "$file: More than one stack segment found.", "readelf output:", @output, "(eof)" ); - }; # if - # Typical stack lines are: - # Linux* OS IA-32 architecture: - # GNU_STACK 0x000000 0x00000000 0x00000000 0x00000 0x00000 RWE 0x4 - # Linux* OS Intel(R) 64: - # GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RWE 0x8 - if ( $stack[ 0 ] !~ m{\A\s*(?:GNU_)?STACK(?:\s+0x[0-9a-f]+){5}\s+([R ][W ][E ])\s+0x[0-9a-f]+\s*\z} ) { - runtime_error( "$file: Cannot parse stack segment line:", ">>> $stack[ 0 ]" ); - }; # if - my $attrs = $1; - if ( $attrs =~ m{E} ) { - runtime_error( "$file: Stack is executable" ); - }; # if -}; # sub execstack - -get_options( - "arch=s" => \$target_arch, -); - -foreach my $file ( @ARGV ) { - execstack( $file ); -}; # foreach $file - -exit( 0 ); - -__END__ - -=pod - -=head1 NAME - -B -- Check whether stack is executable, issue an error if so. - -=head1 SYNOPSIS - -B I