Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use the improved submodule handling #42081

Merged
merged 5 commits into from
May 26, 2017
Merged

Use the improved submodule handling #42081

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d552e34
Use the improved submodule handling
ishitatsuyuki May 18, 2017
1fea55b
Format bootstrap.py using autopep8
ishitatsuyuki May 19, 2017
5bcf06a
bootstrap.py: Filter instead of iteration
ishitatsuyuki May 19, 2017
46ebd83
bootstrap.py: decode to str
ishitatsuyuki May 22, 2017
15c26c9
bootstrap.py: support verbose for submodules
ishitatsuyuki May 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 78 additions & 86 deletions src/bootstrap/bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ def get(url, path, verbose=False):
return
else:
if verbose:
print("ignoring already-download file " + path + " due to failed verification")
print("ignoring already-download file " +
path + " due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
Expand Down Expand Up @@ -100,8 +101,8 @@ def verify(path, sha_path, verbose):
verified = found == expected
if not verified:
print("invalid checksum:\n"
" found: {}\n"
" expected: {}".format(found, expected))
" found: {}\n"
" expected: {}".format(found, expected))
return verified


Expand All @@ -127,20 +128,21 @@ def unpack(tarball, dst, verbose=False, match=None):
shutil.move(tp, fp)
shutil.rmtree(os.path.join(dst, fname))

def run(args, verbose=False, exception=False, cwd=None, env=None):
def run(args, verbose=False, exception=False, **kwargs):
if verbose:
print("running: " + ' '.join(args))
sys.stdout.flush()
# Use Popen here instead of call() as it apparently allows powershell on
# Windows to not lock up waiting for input presumably.
ret = subprocess.Popen(args, cwd=cwd, env=env)
ret = subprocess.Popen(args, **kwargs)
code = ret.wait()
if code != 0:
err = "failed to run: " + ' '.join(args)
if verbose or exception:
raise RuntimeError(err)
sys.exit(err)


def stage0_data(rust_root):
nightlies = os.path.join(rust_root, "src/stage0.txt")
data = {}
Expand All @@ -153,11 +155,13 @@ def stage0_data(rust_root):
data[a] = b
return data


def format_build_time(duration):
return str(datetime.timedelta(seconds=int(duration)))


class RustBuild(object):

def download_stage0(self):
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, self.stage0_date())
Expand All @@ -172,11 +176,13 @@ def download_stage0(self):
self.print_what_it_means_to_bootstrap()
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
filename = "rust-std-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-std-" + self.build,
verbose=self.verbose)
Expand All @@ -185,20 +191,25 @@ def download_stage0(self):
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rustc", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/rustc")
self.fix_executable(self.bin_root() + "/bin/rustdoc")
with open(self.rustc_stamp(), 'w') as f:
f.write(self.stage0_date())

if "pc-windows-gnu" in self.build:
filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
filename = "rust-mingw-{}-{}.tar.gz".format(
rustc_channel, self.build)
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="rust-mingw", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="rust-mingw", verbose=self.verbose)

if self.cargo().startswith(self.bin_root()) and \
(not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
Expand All @@ -207,8 +218,10 @@ def download_stage0(self):
url = self._download_url + "/dist/" + self.stage0_date()
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
get("{}/{}".format(url, filename),
tarball, verbose=self.verbose)
unpack(tarball, self.bin_root(),
match="cargo", verbose=self.verbose)
self.fix_executable(self.bin_root() + "/bin/cargo")
with open(self.cargo_stamp(), 'w') as f:
f.write(self.stage0_date())
Expand All @@ -218,7 +231,8 @@ def fix_executable(self, fname):

default_encoding = sys.getdefaultencoding()
try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, WindowsError):
return

Expand All @@ -234,7 +248,8 @@ def fix_executable(self, fname):
print("info: you seem to be running NixOS. Attempting to patch " + fname)

try:
interpreter = subprocess.check_output(["patchelf", "--print-interpreter", fname])
interpreter = subprocess.check_output(
["patchelf", "--print-interpreter", fname])
interpreter = interpreter.strip().decode(default_encoding)
except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e)
Expand All @@ -243,7 +258,8 @@ def fix_executable(self, fname):
loader = interpreter.split("/")[-1]

try:
ldd_output = subprocess.check_output(['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = subprocess.check_output(
['ldd', '/run/current-system/sw/bin/sh'])
ldd_output = ldd_output.strip().decode(default_encoding)
except subprocess.CalledProcessError as e:
print("warning: unable to call ldd: %s" % e)
Expand All @@ -261,7 +277,8 @@ def fix_executable(self, fname):
correct_interpreter = loader_path + loader

try:
subprocess.check_output(["patchelf", "--set-interpreter", correct_interpreter, fname])
subprocess.check_output(
["patchelf", "--set-interpreter", correct_interpreter, fname])
except subprocess.CalledProcessError as e:
print("warning: failed to call patchelf: %s" % e)
return
Expand Down Expand Up @@ -371,16 +388,16 @@ def build_bootstrap(self):
env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
os.pathsep + env["PATH"]
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `%s`" % self.cargo())
args = [self.cargo(), "build", "--manifest-path",
Expand All @@ -395,16 +412,6 @@ def build_bootstrap(self):
args.append("--frozen")
run(args, env=env, verbose=self.verbose)

def output(self, args, env=None, cwd=None):
default_encoding = sys.getdefaultencoding()
proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
(out, err) = proc.communicate()
ret = proc.wait()
if ret != 0:
print(out)
sys.exit(ret)
return out.decode(default_encoding)

def build_triple(self):
default_encoding = sys.getdefaultencoding()
config = self.get_toml('build')
Expand All @@ -414,8 +421,10 @@ def build_triple(self):
if config:
return config
try:
ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding)
cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding)
ostype = subprocess.check_output(
['uname', '-s']).strip().decode(default_encoding)
cputype = subprocess.check_output(
['uname', '-m']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError):
if sys.platform == 'win32':
return 'x86_64-pc-windows-msvc'
Expand All @@ -427,7 +436,8 @@ def build_triple(self):
# The goal here is to come up with the same triple as LLVM would,
# at least for the subset of platforms we're willing to target.
if ostype == 'Linux':
os_from_sp = subprocess.check_output(['uname', '-o']).strip().decode(default_encoding)
os_from_sp = subprocess.check_output(
['uname', '-o']).strip().decode(default_encoding)
if os_from_sp == 'Android':
ostype = 'linux-android'
else:
Expand All @@ -451,7 +461,7 @@ def build_triple(self):
# must be used instead.
try:
cputype = subprocess.check_output(['isainfo',
'-k']).strip().decode(default_encoding)
'-k']).strip().decode(default_encoding)
except (subprocess.CalledProcessError, OSError):
err = "isainfo not found"
if self.verbose:
Expand Down Expand Up @@ -544,51 +554,29 @@ def build_triple(self):

def update_submodules(self):
if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
self.get_toml('submodules') == "false" or \
self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
self.get_toml('submodules') == "false" or \
self.get_mk('CFG_DISABLE_MANAGE_SUBMODULES') == "1":
return

print('Updating submodules')
output = self.output(["git", "submodule", "status"], cwd=self.rust_root)
submodules = []
for line in output.splitlines():
# NOTE `git submodule status` output looks like this:
#
# -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc
# +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..)
# e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6)
#
# The first character can be '-', '+' or ' ' and denotes the
# `State` of the submodule Right next to this character is the
# SHA-1 of the submodule HEAD And after that comes the path to the
# submodule
path = line[1:].split(' ')[1]
submodules.append([path, line[0]])

run(["git", "submodule", "sync"], cwd=self.rust_root)

for submod in submodules:
path, status = submod
if path.endswith('llvm') and \
(self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
continue
if path.endswith('jemalloc') and \
(self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
continue
submod_path = os.path.join(self.rust_root, path)

if status == ' ':
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '+':
run(["git", "submodule", "update", path], cwd=self.rust_root)
run(["git", "reset", "--hard"], cwd=submod_path)
run(["git", "clean", "-fdx"], cwd=submod_path)
elif status == '-':
run(["git", "submodule", "init", path], cwd=self.rust_root)
run(["git", "submodule", "update", path], cwd=self.rust_root)
else:
raise ValueError('unknown submodule status: ' + status)
default_encoding = sys.getdefaultencoding()
run(["git", "submodule", "-q", "sync"], cwd=self.rust_root)
submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
["git", "config", "--file", os.path.join(self.rust_root, ".gitmodules"),
"--get-regexp", "path"]
).decode(default_encoding).splitlines()]
submodules = [module for module in submodules
if not ((module.endswith("llvm") and
(self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT'))) or
(module.endswith("jemalloc") and
(self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT'))))
]
run(["git", "submodule", "update",
"--init"] + submodules, cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "-q", "foreach", "git",
"reset", "-q", "--hard"], cwd=self.rust_root, verbose=self.verbose)
run(["git", "submodule", "-q", "foreach", "git",
"clean", "-qdfx"], cwd=self.rust_root, verbose=self.verbose)


def bootstrap():
parser = argparse.ArgumentParser(description='Build rust')
Expand Down Expand Up @@ -641,7 +629,7 @@ def bootstrap():
if rb.use_vendored_sources:
if not os.path.exists('.cargo'):
os.makedirs('.cargo')
with open('.cargo/config','w') as f:
with open('.cargo/config', 'w') as f:
f.write("""
[source.crates-io]
replace-with = 'vendored-sources'
Expand Down Expand Up @@ -681,21 +669,25 @@ def bootstrap():
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
run(args, env=env, verbose=rb.verbose)


def main():
start_time = time()
help_triggered = ('-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
help_triggered = (
'-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
try:
bootstrap()
if not help_triggered:
print("Build completed successfully in %s" % format_build_time(time() - start_time))
print("Build completed successfully in %s" %
format_build_time(time() - start_time))
except (SystemExit, KeyboardInterrupt) as e:
if hasattr(e, 'code') and isinstance(e.code, int):
exit_code = e.code
else:
exit_code = 1
print(e)
if not help_triggered:
print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
print("Build completed unsuccessfully in %s" %
format_build_time(time() - start_time))
sys.exit(exit_code)

if __name__ == '__main__':
Expand Down