Update how Cargo nightlies are downloaded

Author: alexcrichton

master/build-rust-manifest.py

@@ -42,6 +42,8 @@
 # Temporary work directory
 temp_dir = sys.argv[6]
 
+cargo_addy = "https://s3.amazonaws.com/rust-lang-ci/cargo-builds"
+
 print "channel: " + channel
 print "today: " + today
 print "s3_addy: " + s3_addy
@@ -130,9 +132,6 @@
     "i686-pc-windows-gnu",
 ])
 
-# This file defines which cargos go with which rustc on beta/stable
-cargo_revs = "https://raw.githubusercontent.com/rust-lang/rust-packaging/master/cargo-revs.txt"
-
 if os.path.isdir(temp_dir):
     shutil.rmtree(temp_dir)
 os.mkdir(temp_dir)
@@ -154,28 +153,29 @@ def main():
     # on s3.
     rustc_short_version = parse_short_version(rustc_version)
 
-    cargo_date = None
     if channel == "nightly":
-        # Nightly rust is paired with nightly cargo
-        cargo_date = most_recent_build_date("nightly", "cargo")
+        cargo_branch = 'master'
     else:
-        # Beta / stable rust is paired with a cargo specified
-        # by rust-packaging
-        cargo_date = cargo_date_from_packaging(rustc_short_version)
+        cargo_branch = 'rust-' + rustc_short_version
+    if channel == "nightly":
+        cargo_branch = 'master'
+    cargo_rev_url = "https://api.github.com/repos/rust-lang/cargo/commits/" + cargo_branch
+    request = urllib2.Request(cargo_rev_url, headers={"Accept" : "Accept: application/vnd.github.3.sha"})
+    cargo_rev = urllib2.urlopen(request).read()
 
-    cargo_version = version_from_channel("nightly", "cargo", cargo_date)
+    cargo_version = version_from_channel(channel, "cargo", cargo_rev)
     cargo_short_version = parse_short_version(cargo_version)
 
     print "rustc date: " + rustc_date
     print "rustc version: " + rustc_version
     print "rustc short version: " + rustc_short_version
-    print "cargo date: " + cargo_date
+    print "cargo rev: " + cargo_rev
     print "cargo version: " + cargo_version
     print "cargo short version: " + cargo_short_version
 
     # Validate the component artifacts and generate the manifest
     generate_manifest(rustc_date, rustc_version, rustc_short_version,
-                      cargo_date, cargo_version, cargo_short_version)
+                      cargo_rev, cargo_version, cargo_short_version)
 
 # Use the channel-$component-$channel-date.txt file to get the archive
 # date for cargo or rustc. These files are generated by finish_dist
@@ -195,25 +195,31 @@ def most_recent_build_date(channel, component):
 def version_from_channel(channel, component, date):
     dist = dist_folder(component)
 
-    # Load the manifest
-    manifest_url = s3_addy + "/" + dist + "/" + date + "/channel-" + component + "-" + channel
-    print "downloading " + manifest_url
-    response = urllib2.urlopen(manifest_url)
-    if response.getcode() != 200:
-        raise Exception("couldn't download " + manifest_url)
-    manifest = response.read().strip();
+    if component == "cargo":
+        target = "x86_64-unknown-linux-gnu"
+        installer_name = "cargo-nightly-" + target + ".tar.gz"
+        installer_url = cargo_addy + "/" + date + "/" + installer_name
+    else:
+        # Load the manifest
+        manifest_url = s3_addy + "/" + dist + "/" + date + "/channel-" + component + "-" + channel
+        print "downloading " + manifest_url
+        response = urllib2.urlopen(manifest_url)
+        if response.getcode() != 200:
+            raise Exception("couldn't download " + manifest_url)
+        manifest = response.read().strip();
+
+        # Find the installer name
+        installer_name = None
+        for line in manifest.split("\n"):
+            if line.startswith(component) and line.endswith(".tar.gz"):
+                installer_name = line
 
-    # Find the installer name
-    installer_name = None
-    for line in manifest.split("\n"):
-        if line.startswith(component) and line.endswith(".tar.gz"):
-            installer_name = line
+        if installer_name == None:
+            raise Exception("couldn't find installer in manifest for " + component)
 
-    if installer_name == None:
-        raise Exception("couldn't find installer in manifest for " + component)
+        installer_url = s3_addy + "/" + dist + "/" + date + "/" + installer_name
 
     # Download the installer
-    installer_url = s3_addy + "/" + dist + "/" + date + "/" + installer_name
     print "downloading " + installer_url
     response = urllib2.urlopen(installer_url)
     if response.getcode() != 200:
@@ -249,7 +255,7 @@ def dist_folder(component):
         return "cargo-dist"
     return "dist"
 
-def cargo_date_from_packaging(rustc_version):
+def cargo_rev_from_packaging(rustc_version):
     print "downloading " + cargo_revs
     response = urllib2.urlopen(cargo_revs)
     if response.getcode() != 200:
@@ -275,10 +281,10 @@ def parse_short_version(version):
     return v
 
 def generate_manifest(rustc_date, rustc_version, rustc_short_version,
-                      cargo_date, cargo_version, cargo_short_version):
+                      cargo_rev, cargo_version, cargo_short_version):
 
     m = build_manifest(rustc_date, rustc_version, rustc_short_version,
-                       cargo_date, cargo_version, cargo_short_version)
+                       cargo_rev, cargo_version, cargo_short_version)
 
     manifest_name = "channel-rust-" + channel + ".toml"
     manifest_file = rust_package_dir + "/" + manifest_name
@@ -293,7 +299,7 @@ def generate_manifest(rustc_date, rustc_version, rustc_short_version,
     print_summary(m)
 
 def build_manifest(rustc_date, rustc_version, rustc_short_version,
-                   cargo_date, cargo_version, cargo_short_version):
+                   cargo_rev, cargo_version, cargo_short_version):
 
     packages = {}
 
@@ -308,7 +314,7 @@ def build_manifest(rustc_date, rustc_version, rustc_short_version,
     doc_pkg = build_package_def_from_archive("rust-docs", "dist", rustc_date,
                                              rustc_version, rustc_short_version,
                                              host_list)
-    cargo_pkg = build_package_def_from_archive("cargo", "cargo-dist", cargo_date,
+    cargo_pkg = build_package_def_from_archive("cargo", "cargo-dist", cargo_rev,
                                                cargo_version, cargo_short_version,
                                                host_list)
     mingw_pkg = build_package_def_from_archive("rust-mingw", "dist", rustc_date,
@@ -437,17 +443,17 @@ def build_package_def_from_archive(name, dist_dir, archive_date,
 def live_package_url(name, dist_dir, date, version, target):
     # Cargo builds are always named 'nightly'
     maybe_channel = channel
-    if name == "cargo":
-        maybe_channel = "nightly"
 
-    if name == "rust-src":
+    if name == "cargo":
+        url1 = cargo_addy + "/" + date + "/cargo-nightly-" + target + ".tar.gz"
+    elif name == "rust-src":
         # The build system treats source packages as a separate target for `rustc`
         # but for rustup we'd like to treat them as a completely separate package.
         url1 = s3_addy + "/" + dist_dir + "/" + date + "/rust-src-" + version + ".tar.gz"
-        url2 = s3_addy + "/" + dist_dir + "/" + date + "/rust-src-" + maybe_channel + ".tar.gz"
+        url2 = s3_addy + "/" + dist_dir + "/" + date + "/rust-src-" + channel + ".tar.gz"
     else:
         url1 = s3_addy + "/" + dist_dir + "/" + date + "/" + name + "-" + version + "-" + target + ".tar.gz"
-        url2 = s3_addy + "/" + dist_dir + "/" + date + "/" + name + "-" + maybe_channel + "-" + target + ".tar.gz"
+        url2 = s3_addy + "/" + dist_dir + "/" + date + "/" + name + "-" + channel + "-" + target + ".tar.gz"
 
     print "checking " + url1
     request = urllib2.Request(url1)