Skip to content

Commit

Permalink
Stop building the gold plugin and linking lld against tcmalloc.
Browse files Browse the repository at this point in the history 
As far as I am aware, there are no remaining users of the gold plugin
(v8, the last user I am aware of, stopped downloading it as part of its
build in https://chromium-review.googlesource.com/547058 and removed
build support in https://chromium-review.googlesource.com/549300),
so we no longer need to build it.

Also, now that we have started using ThinLTO in official builds,
not only is the perf gain of linking lld against tcmalloc not as
important, but I have also measured it to be smaller in relative
terms. For base_unittests:

With tcmalloc (median of 6): 42.29s
With glibc malloc: 44.61s

So about 5%, which is about half of what I measured for regular LTO.
And of course the absolute delta is smaller as well. I think that
justifies removing it. We can re-evaluate for the toolchain as a
whole at a later time.

BUG=607968
R=thakis@chromium.org,hans@chromium.org

Review-Url: https://codereview.chromium.org/2963693002
Cr-Commit-Position: refs/heads/master@{#483149}
  • Loading branch information
pcc authored and Commit Bot committed Jun 28, 2017
1 parent 06129ae commit 6489fe9
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 91 deletions.
2 changes: 0 additions & 2 deletions docs/updating_clang.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ $ for x in Linux_x64 Mac Win ; do \
gsutil cp -n -a public-read gs://chromium-browser-clang-staging/$x/translation_unit-$rev.tgz \
gs://chromium-browser-clang/$x/translation_unit-$rev.tgz ; \
done
$ gsutil cp -n -a public-read gs://chromium-browser-clang-staging/Linux_x64/llvmgold-$rev.tgz \
gs://chromium-browser-clang/Linux_x64/llvmgold-$rev.tgz
```

1. Run the goma package update script to push these packages to goma. If you do
Expand Down
31 changes: 3 additions & 28 deletions tools/clang/scripts/package.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@
'llvm-bootstrap-install')
LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build')
LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts')
LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
BINUTILS_LIB_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64',
'Release', 'lib')
LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld')
STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')


Expand Down Expand Up @@ -176,7 +174,6 @@ def main():

expected_stamp = GetExpectedStamp()
pdir = 'clang-' + expected_stamp
golddir = 'llvmgold-' + expected_stamp
print pdir

if sys.platform == 'darwin':
Expand All @@ -187,15 +184,10 @@ def main():
platform = 'Linux_x64'

# Check if Google Cloud Storage already has the artifacts we want to build.
if (args.upload and GsutilArchiveExists(pdir, platform) and
(not sys.platform.startswith('linux') or
GsutilArchiveExists(golddir, platform))):
if args.upload and GsutilArchiveExists(pdir, platform):
print ('Desired toolchain revision %s is already available '
'in Google Cloud Storage:') % expected_stamp
print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform, pdir)
if sys.platform.startswith('linux'):
print 'gs://chromium-browser-clang-staging/%s/%s.tgz' % (platform,
golddir)
return 0

with open('buildlog.txt', 'w') as log:
Expand Down Expand Up @@ -230,7 +222,7 @@ def main():

opt_flags = []
if sys.platform.startswith('linux'):
opt_flags += ['--lto-gold-plugin']
opt_flags += ['--lto-lld']
build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'),
'--bootstrap', '--force-local-build',
'--run-tests'] + opt_flags
Expand Down Expand Up @@ -325,12 +317,6 @@ def main():
shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'),
os.path.join(pdir, 'include', 'c++'))

# Copy tcmalloc from the binutils package.
# FIXME: We should eventually be building our own copy.
if sys.platform.startswith('linux'):
shutil.copy(os.path.join(BINUTILS_LIB_DIR, 'libtcmalloc_minimal.so.4'),
os.path.join(pdir, 'lib'))

# Copy buildlog over.
shutil.copy('buildlog.txt', pdir)

Expand All @@ -344,17 +330,6 @@ def main():

MaybeUpload(args, pdir, platform)

# Zip up gold plugin on Linux.
if sys.platform.startswith('linux'):
shutil.rmtree(golddir, ignore_errors=True)
os.makedirs(os.path.join(golddir, 'lib'))
shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'),
os.path.join(golddir, 'lib'))
with tarfile.open(golddir + '.tgz', 'w:gz') as tar:
tar.add(os.path.join(golddir, 'lib'), arcname='lib',
filter=PrintTarProgress)
MaybeUpload(args, golddir, platform)

# Zip up llvm-objdump for sanitizer coverage.
objdumpdir = 'llvmobjdump-' + stamp
shutil.rmtree(objdumpdir, ignore_errors=True)
Expand Down
95 changes: 34 additions & 61 deletions tools/clang/scripts/update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
CLANG_REVISION = 'HEAD'

# This is incremented when pushing a new build of Clang at the same revision.
CLANG_SUB_REVISION=1
CLANG_SUB_REVISION=2

PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)

Expand All @@ -46,7 +46,7 @@
LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
'llvm-bootstrap-install')
LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
LLVM_LTO_LLD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-lld')
CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
'Release+Asserts')
Expand All @@ -66,10 +66,6 @@
os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
STAMP_FILE = os.path.normpath(
os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils', 'Linux_x64', 'Release')
BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, 'bin')
BINUTILS_LIB_DIR = os.path.join(BINUTILS_DIR, 'lib')
BFD_PLUGINS_DIR = os.path.join(BINUTILS_LIB_DIR, 'bfd-plugins')
VERSION = '5.0.0'
ANDROID_NDK_DIR = os.path.join(
CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
Expand Down Expand Up @@ -489,16 +485,11 @@ def UpdateClang(args):
'-DLLVM_USE_CRT_RELEASE=MT',
]

binutils_incdir = ''
if sys.platform.startswith('linux'):
binutils_incdir = os.path.join(BINUTILS_DIR, 'include')

if args.bootstrap:
print 'Building bootstrap compiler'
EnsureDirExists(LLVM_BOOTSTRAP_DIR)
os.chdir(LLVM_BOOTSTRAP_DIR)
bootstrap_args = base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DLLVM_TARGETS_TO_BUILD=host',
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
Expand Down Expand Up @@ -537,52 +528,35 @@ def UpdateClang(args):
cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
print 'Building final compiler'

# Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
# Build lld with LTO. That speeds up the linker by ~10%.
# We only use LTO for Linux now.
if args.bootstrap and args.lto_gold_plugin:
print 'Building LTO LLVM Gold plugin'
if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)

# Create a symlink to LLVMgold.so build in the previous step so that ar
# and ranlib could find it while linking LLVMgold.so with LTO.
EnsureDirExists(BFD_PLUGINS_DIR)
RunCommand(['ln', '-sf',
os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])

# Link against binutils's copy of tcmalloc to speed up the linker by ~10%.
# In package.py we copy the .so into our package.
tcmalloc_ldflags = ['-L' + BINUTILS_LIB_DIR, '-ltcmalloc_minimal']
# Make sure that tblgen and the test suite can find tcmalloc.
os.environ['LD_LIBRARY_PATH'] = \
BINUTILS_LIB_DIR + os.pathsep + os.environ.get('LD_LIBRARY_PATH', '')

lto_cflags = ['-flto=thin']
lto_ldflags = ['-fuse-ld=lld']
if args.gcc_toolchain:
# Tell the bootstrap compiler to use a specific gcc prefix to search
# for standard library headers and shared object files.
lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
if args.bootstrap and args.lto_lld:
print 'Building LTO lld'
if os.path.exists(LLVM_LTO_LLD_DIR):
RmTree(LLVM_LTO_LLD_DIR)
EnsureDirExists(LLVM_LTO_LLD_DIR)
os.chdir(LLVM_LTO_LLD_DIR)

# The linker expects all archive members to have symbol tables, so the
# archiver needs to be able to create symbol tables for bitcode files.
# GNU ar and ranlib don't understand bitcode files, but llvm-ar and
# llvm-ranlib do, so use them.
ar = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ar')
ranlib = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'llvm-ranlib')

lto_cmake_args = base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DCMAKE_C_COMPILER=' + cc,
'-DCMAKE_CXX_COMPILER=' + cxx,
'-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags + tcmalloc_ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]

# We need to use the proper binutils which support LLVM Gold plugin.
lto_env = os.environ.copy()
lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
'-DCMAKE_AR=' + ar,
'-DCMAKE_RANLIB=' + ranlib,
'-DLLVM_ENABLE_LTO=thin',
'-DLLVM_USE_LINKER=lld',
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]

RmCmakeCache('.')
RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
RunCommand(['ninja', 'LLVMgold', 'lld'], env=lto_env)
RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR])
RunCommand(['ninja', 'lld'])


# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
Expand Down Expand Up @@ -636,7 +610,6 @@ def UpdateClang(args):
chrome_tools = list(set(default_tools + args.extra_tools))
cmake_args += base_cmake_args + [
'-DLLVM_ENABLE_THREADS=OFF',
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
Expand All @@ -663,8 +636,8 @@ def UpdateClang(args):
RunCommand(['ninja'], msvc_arch='x64')

# Copy LTO-optimized lld, if any.
if args.bootstrap and args.lto_gold_plugin:
CopyFile(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'bin', 'lld'),
if args.bootstrap and args.lto_lld:
CopyFile(os.path.join(LLVM_LTO_LLD_DIR, 'bin', 'lld'),
os.path.join(LLVM_BUILD_DIR, 'bin'))

if chrome_tools:
Expand Down Expand Up @@ -841,8 +814,8 @@ def main():
parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
'version be used for building; --gcc-toolchain=/opt/foo '
'picks /opt/foo/bin/gcc')
parser.add_argument('--lto-gold-plugin', action='store_true',
help='build LLVM Gold plugin with LTO')
parser.add_argument('--lto-lld', action='store_true',
help='build lld with LTO')
parser.add_argument('--llvm-force-head-revision', action='store_true',
help=('use the revision in the repo when printing '
'the revision'))
Expand All @@ -860,12 +833,12 @@ def main():
default=sys.platform.startswith('linux'))
args = parser.parse_args()

if args.lto_gold_plugin and not args.bootstrap:
print '--lto-gold-plugin requires --bootstrap'
if args.lto_lld and not args.bootstrap:
print '--lto-lld requires --bootstrap'
return 1
if args.lto_gold_plugin and not sys.platform.startswith('linux'):
print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
args.lto_gold_plugin = False
if args.lto_lld and not sys.platform.startswith('linux'):
print '--lto-lld is only effective on Linux. Ignoring the option.'
args.lto_lld = False

if args.if_needed:
# TODO(thakis): Can probably remove this and --if-needed altogether.
Expand Down

0 comments on commit 6489fe9

Please sign in to comment.