From 6489fe94c6edc435f5f0b291032c0e8883212afd Mon Sep 17 00:00:00 2001 From: pcc Date: Wed, 28 Jun 2017 14:30:44 -0700 Subject: [PATCH] Stop building the gold plugin and linking lld against tcmalloc. As far as I am aware, there are no remaining users of the gold plugin (v8, the last user I am aware of, stopped downloading it as part of its build in https://chromium-review.googlesource.com/547058 and removed build support in https://chromium-review.googlesource.com/549300), so we no longer need to build it. Also, now that we have started using ThinLTO in official builds, not only is the perf gain of linking lld against tcmalloc not as important, but I have also measured it to be smaller in relative terms. For base_unittests: With tcmalloc (median of 6): 42.29s With glibc malloc: 44.61s So about 5%, which is about half of what I measured for regular LTO. And of course the absolute delta is smaller as well. I think that justifies removing it. We can re-evaluate for the toolchain as a whole at a later time. BUG=607968 R=thakis@chromium.org,hans@chromium.org Review-Url: https://codereview.chromium.org/2963693002 Cr-Commit-Position: refs/heads/master@{#483149} --- docs/updating_clang.md | 2 - tools/clang/scripts/package.py | 31 ++--------- tools/clang/scripts/update.py | 95 ++++++++++++---------------------- 3 files changed, 37 insertions(+), 91 deletions(-) diff --git a/docs/updating_clang.md b/docs/updating_clang.md index d33708963df220..a08566dd3b21f3 100644 --- a/docs/updating_clang.md +++ b/docs/updating_clang.md @@ -19,8 +19,6 @@ $ for x in Linux_x64 Mac Win ; do \ gsutil cp -n -a public-read gs://chromium-browser-clang-staging/$x/translation_unit-$rev.tgz \ gs://chromium-browser-clang/$x/translation_unit-$rev.tgz ; \ done -$ gsutil cp -n -a public-read gs://chromium-browser-clang-staging/Linux_x64/llvmgold-$rev.tgz \ - gs://chromium-browser-clang/Linux_x64/llvmgold-$rev.tgz ``` 1. Run the goma package update script to push these packages to goma. If you do diff --git a/tools/clang/scripts/package.py b/tools/clang/scripts/package.py index 88e9779116ed29..2f95c5f9121f6f 100755 --- a/tools/clang/scripts/package.py +++ b/tools/clang/scripts/package.py @@ -25,9 +25,7 @@ 'llvm-bootstrap-install') LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') -LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') -BINUTILS_LIB_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64', - 'Release', 'lib') +LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld') STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') @@ -176,7 +174,6 @@ def main(): expected_stamp = GetExpectedStamp() pdir = 'clang-' + expected_stamp - golddir = 'llvmgold-' + expected_stamp print pdir if sys.platform == 'darwin': @@ -187,15 +184,10 @@ def main(): platform = 'Linux_x64' # Check if Google Cloud Storage already has the artifacts we want to build. - if (args.upload and GsutilArchiveExists(pdir, platform) and - (not sys.platform.startswith('linux') or - GsutilArchiveExists(golddir, platform))): + if args.upload and GsutilArchiveExists(pdir, platform): print ('Desired toolchain revision %s is already available ' 'in Google Cloud Storage:') % expected_stamp print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir) - if sys.platform.startswith('linux'): - print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, - golddir) return 0 with open('buildlog.txt', 'w') as log: @@ -230,7 +222,7 @@ def main(): opt_flags = [] if sys.platform.startswith('linux'): - opt_flags += ['--lto-gold-plugin'] + opt_flags += ['--lto-lld'] build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), '--bootstrap', '--force-local-build', '--run-tests'] + opt_flags @@ -325,12 +317,6 @@ def main(): shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), os.path.join(pdir, 'include', 'c++')) - # Copy tcmalloc from the binutils package. - # FIXME: We should eventually be building our own copy. - if sys.platform.startswith('linux'): - shutil.copy(os.path.join(BINUTILS_LIB_DIR, 'libtcmalloc_minimal.so.4'), - os.path.join(pdir, 'lib')) - # Copy buildlog over. shutil.copy('buildlog.txt', pdir) @@ -344,17 +330,6 @@ def main(): MaybeUpload(args, pdir, platform) - # Zip up gold plugin on Linux. - if sys.platform.startswith('linux'): - shutil.rmtree(golddir, ignore_errors=True) - os.makedirs(os.path.join(golddir, 'lib')) - shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'), - os.path.join(golddir, 'lib')) - with tarfile.open(golddir + '.tgz', 'w:gz') as tar: - tar.add(os.path.join(golddir, 'lib'), arcname='lib', - filter=PrintTarProgress) - MaybeUpload(args, golddir, platform) - # Zip up llvm-objdump for sanitizer coverage. objdumpdir = 'llvmobjdump-' + stamp shutil.rmtree(objdumpdir, ignore_errors=True) diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py index a1273615188246..e96e4b558d0bf0 100755 --- a/tools/clang/scripts/update.py +++ b/tools/clang/scripts/update.py @@ -34,7 +34,7 @@ CLANG_REVISION = 'HEAD' # This is incremented when pushing a new build of Clang at the same revision. -CLANG_SUB_REVISION=1 +CLANG_SUB_REVISION=2 PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION) @@ -46,7 +46,7 @@ LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap-install') -LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') +LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld') CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', 'Release+Asserts') @@ -66,10 +66,6 @@ os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) STAMP_FILE = os.path.normpath( os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')) -BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64', 'Release') -BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, 'bin') -BINUTILS_LIB_DIR = os.path.join(BINUTILS_DIR, 'lib') -BFD_PLUGINS_DIR = os.path.join(BINUTILS_LIB_DIR, 'bfd-plugins') VERSION = '5.0.0' ANDROID_NDK_DIR = os.path.join( CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk') @@ -489,16 +485,11 @@ def UpdateClang(args): '-DLLVM_USE_CRT_RELEASE=MT', ] - binutils_incdir = '' - if sys.platform.startswith('linux'): - binutils_incdir = os.path.join(BINUTILS_DIR, 'include') - if args.bootstrap: print 'Building bootstrap compiler' EnsureDirExists(LLVM_BOOTSTRAP_DIR) os.chdir(LLVM_BOOTSTRAP_DIR) bootstrap_args = base_cmake_args + [ - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, '-DLLVM_TARGETS_TO_BUILD=host', '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, '-DCMAKE_C_FLAGS=' + ' '.join(cflags), @@ -537,52 +528,35 @@ def UpdateClang(args): cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] print 'Building final compiler' - # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%. + # Build lld with LTO. That speeds up the linker by ~10%. # We only use LTO for Linux now. - if args.bootstrap and args.lto_gold_plugin: - print 'Building LTO LLVM Gold plugin' - if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR): - RmTree(LLVM_LTO_GOLD_PLUGIN_DIR) - EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR) - os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR) - - # Create a symlink to LLVMgold.so build in the previous step so that ar - # and ranlib could find it while linking LLVMgold.so with LTO. - EnsureDirExists(BFD_PLUGINS_DIR) - RunCommand(['ln', '-sf', - os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'), - os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')]) - - # Link against binutils's copy of tcmalloc to speed up the linker by ~10%. - # In package.py we copy the .so into our package. - tcmalloc_ldflags = ['-L' + BINUTILS_LIB_DIR, '-ltcmalloc_minimal'] - # Make sure that tblgen and the test suite can find tcmalloc. - os.environ['LD_LIBRARY_PATH'] = \ - BINUTILS_LIB_DIR + os.pathsep + os.environ.get('LD_LIBRARY_PATH', '') - - lto_cflags = ['-flto=thin'] - lto_ldflags = ['-fuse-ld=lld'] - if args.gcc_toolchain: - # Tell the bootstrap compiler to use a specific gcc prefix to search - # for standard library headers and shared object files. - lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain] + if args.bootstrap and args.lto_lld: + print 'Building LTO lld' + if os.path.exists(LLVM_LTO_LLD_DIR): + RmTree(LLVM_LTO_LLD_DIR) + EnsureDirExists(LLVM_LTO_LLD_DIR) + os.chdir(LLVM_LTO_LLD_DIR) + + # The linker expects all archive members to have symbol tables, so the + # archiver needs to be able to create symbol tables for bitcode files. + # GNU ar and ranlib don't understand bitcode files, but llvm-ar and + # llvm-ranlib do, so use them. + ar = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ar') + ranlib = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ranlib') + lto_cmake_args = base_cmake_args + [ - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, '-DCMAKE_C_COMPILER=' + cc, '-DCMAKE_CXX_COMPILER=' + cxx, - '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags), - '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags + tcmalloc_ldflags), - '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags), - '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)] - - # We need to use the proper binutils which support LLVM Gold plugin. - lto_env = os.environ.copy() - lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '') + '-DCMAKE_AR=' + ar, + '-DCMAKE_RANLIB=' + ranlib, + '-DLLVM_ENABLE_LTO=thin', + '-DLLVM_USE_LINKER=lld', + '-DCMAKE_C_FLAGS=' + ' '.join(cflags), + '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] RmCmakeCache('.') - RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env) - RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env) + RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR]) + RunCommand(['ninja', 'lld']) # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is @@ -636,7 +610,6 @@ def UpdateClang(args): chrome_tools = list(set(default_tools + args.extra_tools)) cmake_args += base_cmake_args + [ '-DLLVM_ENABLE_THREADS=OFF', - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, '-DCMAKE_C_FLAGS=' + ' '.join(cflags), '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), @@ -663,8 +636,8 @@ def UpdateClang(args): RunCommand(['ninja'], msvc_arch='x64') # Copy LTO-optimized lld, if any. - if args.bootstrap and args.lto_gold_plugin: - CopyFile(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'bin', 'lld'), + if args.bootstrap and args.lto_lld: + CopyFile(os.path.join(LLVM_LTO_LLD_DIR, 'bin', 'lld'), os.path.join(LLVM_BUILD_DIR, 'bin')) if chrome_tools: @@ -841,8 +814,8 @@ def main(): parser.add_argument('--gcc-toolchain', help='set the version for which gcc ' 'version be used for building; --gcc-toolchain=/opt/foo ' 'picks /opt/foo/bin/gcc') - parser.add_argument('--lto-gold-plugin', action='store_true', - help='build LLVM Gold plugin with LTO') + parser.add_argument('--lto-lld', action='store_true', + help='build lld with LTO') parser.add_argument('--llvm-force-head-revision', action='store_true', help=('use the revision in the repo when printing ' 'the revision')) @@ -860,12 +833,12 @@ def main(): default=sys.platform.startswith('linux')) args = parser.parse_args() - if args.lto_gold_plugin and not args.bootstrap: - print '--lto-gold-plugin requires --bootstrap' + if args.lto_lld and not args.bootstrap: + print '--lto-lld requires --bootstrap' return 1 - if args.lto_gold_plugin and not sys.platform.startswith('linux'): - print '--lto-gold-plugin is only effective on Linux. Ignoring the option.' - args.lto_gold_plugin = False + if args.lto_lld and not sys.platform.startswith('linux'): + print '--lto-lld is only effective on Linux. Ignoring the option.' + args.lto_lld = False if args.if_needed: # TODO(thakis): Can probably remove this and --if-needed altogether.