Ensure the release works before uploading the tarballs

Cargo.toml

@@ -25,3 +25,5 @@ pgp = "0.7.1"
 chrono = "0.4.19"
 git2 = "0.13.11"
 tempfile = "3.1.0"
+hyper = "0.13.8"
+tokio = { version = "0.2.22", features = ["rt-threaded", "macros", "fs", "sync"] }

local/Dockerfile

@@ -26,6 +26,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     ninja-build \
     python-is-python3
 
+# Install rustup while removing the pre-installed stable toolchain.
+RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
+    chmod +x /tmp/rustup-init && \
+    /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
+    /root/.cargo/bin/rustup toolchain remove stable
+ENV PATH=/root/.cargo/bin:$PATH
+
 COPY --from=mc /usr/bin/mc /usr/local/bin/mc
 RUN chmod 0755 /usr/local/bin/mc
 

prod/Dockerfile

@@ -56,6 +56,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     ninja-build \
     python-is-python3
 
+# Install rustup while removing the pre-installed stable toolchain.
+RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
+    chmod +x /tmp/rustup-init && \
+    /tmp/rustup-init -y --no-modify-path --default-toolchain stable && \
+    /root/.cargo/bin/rustup toolchain remove stable
+ENV PATH=/root/.cargo/bin:$PATH
+
 COPY --from=build /tmp/source/target/release/promote-release /usr/local/bin/
 COPY prod/load-gpg-keys.sh /usr/local/bin/load-gpg-keys
 

src/build_manifest.rs

@@ -1,7 +1,7 @@
 use crate::Context;
 use anyhow::{Context as _, Error};
 use std::{
-    collections::HashSet,
+    collections::{HashMap, HashSet},
     fs::File,
     io::BufReader,
     path::{Path, PathBuf},
@@ -35,30 +35,38 @@ impl<'a> BuildManifest<'a> {
         self.tarball_path.is_file()
     }
 
-    pub(crate) fn run(&self) -> Result<Execution, Error> {
+    pub(crate) fn run(&self, upload_base: &str) -> Result<Execution, Error> {
         let config = &self.builder.config;
         let bin = self
             .extract()
             .context("failed to extract build-manifest from the tarball")?;
 
         let metadata_dir = TempDir::new()?;
+        let checksum_cache = metadata_dir.path().join("checksum-cache.json");
         let shipped_files_path = metadata_dir.path().join("shipped-files.txt");
 
+        // Ensure the manifest dir exists but is empty.
+        let manifest_dir = self.builder.manifest_dir();
+        if manifest_dir.is_dir() {
+            std::fs::remove_dir_all(&manifest_dir)?;
+        }
+        std::fs::create_dir_all(&manifest_dir)?;
+
         println!("running build-manifest...");
-        let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir);
         // build-manifest <input-dir> <output-dir> <date> <upload-addr> <channel>
         let status = Command::new(bin.path())
             .arg(self.builder.dl_dir())
-            .arg(self.builder.dl_dir())
+            .arg(self.builder.manifest_dir())
             .arg(&self.builder.date)
-            .arg(upload_addr)
+            .arg(upload_base)
             .arg(config.channel.to_string())
+            .env("BUILD_MANIFEST_CHECKSUM_CACHE", &checksum_cache)
             .env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path)
             .status()
             .context("failed to execute build-manifest")?;
 
         if status.success() {
-            Execution::new(&shipped_files_path)
+            Execution::new(&shipped_files_path, &checksum_cache)
         } else {
             anyhow::bail!("build-manifest failed with status {:?}", status);
         }
@@ -87,10 +95,11 @@ impl<'a> BuildManifest<'a> {
 
 pub(crate) struct Execution {
     pub(crate) shipped_files: Option<HashSet<PathBuf>>,
+    pub(crate) checksum_cache: HashMap<PathBuf, String>,
 }
 
 impl Execution {
-    fn new(shipped_files_path: &Path) -> Result<Self, Error> {
+    fn new(shipped_files_path: &Path, checksum_cache_path: &Path) -> Result<Self, Error> {
         // Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the
         // "shipped files" file is always generated, and we can remove the Option<_>.
         let shipped_files = if shipped_files_path.is_file() {
@@ -105,6 +114,17 @@ impl Execution {
             None
         };
 
-        Ok(Execution { shipped_files })
+        // Once https://github.com/rust-lang/rust/pull/78409 reaches stable we can assume the
+        // checksum cache will always be generated, and we can remove the if branch.
+        let checksum_cache = if checksum_cache_path.is_file() {
+            serde_json::from_slice(&std::fs::read(checksum_cache_path)?)?
+        } else {
+            HashMap::new()
+        };
+
+        Ok(Execution {
+            shipped_files,
+            checksum_cache,
+        })
     }
 }

src/main.rs

@@ -1,6 +1,7 @@
 mod build_manifest;
 mod config;
 mod sign;
+mod smoke_test;
 
 use std::fs::{self, File, OpenOptions};
 use std::io::{self, Read};
@@ -9,13 +10,14 @@ use std::process::Command;
 use std::time::Instant;
 use std::{collections::HashSet, env};
 
+use crate::build_manifest::BuildManifest;
+use crate::sign::Signer;
+use crate::smoke_test::SmokeTester;
 use anyhow::Error;
-use build_manifest::BuildManifest;
 use chrono::Utc;
 use curl::easy::Easy;
 use fs2::FileExt;
 use rayon::prelude::*;
-use sign::Signer;
 use xz2::read::XzDecoder;
 
 use crate::config::{Channel, Config};
@@ -182,10 +184,18 @@ impl Context {
 
         // Ok we've now determined that a release needs to be done.
 
+        let mut signer = Signer::new(&self.config)?;
         let build_manifest = BuildManifest::new(self);
+
         if build_manifest.exists() {
-            // Generate the channel manifest
-            let execution = build_manifest.run()?;
+            let smoke_test = SmokeTester::new(&[self.manifest_dir(), self.dl_dir()])?;
+
+            // First of all, a manifest is generated pointing to the smoke test server. This will
+            // produce the correct checksums and shipped files list, as the only difference from
+            // between the "real" execution and this one is the URLs included in the manifest.
+            let execution =
+                build_manifest.run(&format!("http://{}/dist", smoke_test.server_addr()))?;
+            signer.override_checksum_cache(execution.checksum_cache);
 
             if self.config.wip_prune_unused_files {
                 // Removes files that we are not shipping from the files we're about to upload.
@@ -194,9 +204,29 @@ impl Context {
                 }
             }
 
-            // Generate checksums and sign all the files we're about to ship.
-            let signer = Signer::new(&self.config)?;
+            // Sign both the downloaded artifacts and the generated manifests. The signatures of
+            // the downloaded files are permanent, while the signatures for the generated manifests
+            // will be discarded later (as the manifests point to the smoke test server).
             signer.sign_directory(&self.dl_dir())?;
+            signer.sign_directory(&self.manifest_dir())?;
+
+            // Ensure the release is downloadable from rustup and can execute a basic binary.
+            smoke_test.test(&self.config.channel)?;
+
+            // Generate the real manifests and sign them.
+            build_manifest.run(&format!(
+                "{}/{}",
+                self.config.upload_addr, self.config.upload_dir
+            ))?;
+            signer.sign_directory(&self.manifest_dir())?;
+
+            // Merge the generated manifests with the downloaded artifacts.
+            for entry in std::fs::read_dir(&self.manifest_dir())? {
+                let entry = entry?;
+                if entry.file_type()?.is_file() {
+                    std::fs::rename(entry.path(), self.dl_dir().join(entry.file_name()))?;
+                }
+            }
         } else {
             // For releases using the legacy build-manifest, we need to clone the rustc monorepo
             // and invoke `./x.py dist hash-and-sign` in it. This won't be needed after 1.48.0 is
@@ -677,6 +707,10 @@ upload-addr = \"{}/{}\"
         self.work.join("dl")
     }
 
+    fn manifest_dir(&self) -> PathBuf {
+        self.work.join("manifests")
+    }
+
     fn build_dir(&self) -> PathBuf {
         self.work.join("build")
     }

src/sign.rs

@@ -10,6 +10,7 @@ use pgp::{
 use rayon::prelude::*;
 use sha2::Digest;
 use std::{
+    collections::HashMap,
     fs::File,
     path::{Path, PathBuf},
     time::Instant,
@@ -20,6 +21,7 @@ use crate::config::Config;
 pub(crate) struct Signer {
     gpg_key: SignedSecretKey,
     gpg_password: String,
+    sha256_checksum_cache: HashMap<PathBuf, String>,
 }
 
 impl Signer {
@@ -29,9 +31,14 @@ impl Signer {
         Ok(Signer {
             gpg_key: SignedSecretKey::from_armor_single(&mut key_file)?.0,
             gpg_password,
+            sha256_checksum_cache: HashMap::new(),
         })
     }
 
+    pub(crate) fn override_checksum_cache(&mut self, new: HashMap<PathBuf, String>) {
+        self.sha256_checksum_cache = new;
+    }
+
     pub(crate) fn sign_directory(&self, path: &Path) -> Result<(), Error> {
         let mut paths = Vec::new();
         for entry in std::fs::read_dir(path)? {
@@ -85,9 +92,15 @@ impl Signer {
     }
 
     fn generate_sha256(&self, path: &Path, data: &[u8]) -> Result<(), Error> {
-        let mut digest = sha2::Sha256::default();
-        digest.update(data);
-        let sha256 = hex::encode(digest.finalize());
+        let canonical_path = std::fs::canonicalize(path)?;
+
+        let sha256 = if let Some(cached) = self.sha256_checksum_cache.get(&canonical_path) {
+            cached.clone()
+        } else {
+            let mut digest = sha2::Sha256::default();
+            digest.update(data);
+            hex::encode(digest.finalize())
+        };
 
         std::fs::write(
             add_suffix(path, ".sha256"),

src/smoke_test.rs

@@ -0,0 +1,113 @@
+use anyhow::Error;
+use hyper::{Body, Request, Response, Server, StatusCode};
+use std::{net::SocketAddr, sync::Arc};
+use std::{path::PathBuf, process::Command};
+use tempfile::TempDir;
+use tokio::{runtime::Runtime, sync::oneshot::Sender};
+
+use crate::config::Channel;
+
+pub(crate) struct SmokeTester {
+    runtime: Runtime,
+    server_addr: SocketAddr,
+    shutdown_send: Sender<()>,
+}
+
+impl SmokeTester {
+    pub(crate) fn new(paths: &[PathBuf]) -> Result<Self, Error> {
+        let addr = SocketAddr::from(([127, 0, 0, 1], 0));
+
+        let paths = Arc::new(paths.to_vec());
+        let service = hyper::service::make_service_fn(move |_| {
+            let paths = paths.clone();
+            async move {
+                Ok::<_, Error>(hyper::service::service_fn(move |req| {
+                    server_handler(req, paths.clone())
+                }))
+            }
+        });
+
+        let (shutdown_send, shutdown_recv) = tokio::sync::oneshot::channel::<()>();
+
+        let runtime = Runtime::new()?;
+        let (server, server_addr) = runtime.enter(|| {
+            let server = Server::bind(&addr).serve(service);
+            let server_addr = server.local_addr();
+            let server = server.with_graceful_shutdown(async {
+                shutdown_recv.await.ok();
+            });
+            (server, server_addr)
+        });
+        runtime.spawn(server);
+
+        Ok(Self {
+            runtime,
+            server_addr,
+            shutdown_send,
+        })
+    }
+
+    pub(crate) fn server_addr(&self) -> SocketAddr {
+        self.server_addr
+    }
+
+    pub(crate) fn test(self, channel: &Channel) -> Result<(), Error> {
+        let tempdir = TempDir::new()?;
+        let cargo_dir = tempdir.path().join("sample-crate");
+        std::fs::create_dir_all(&cargo_dir)?;
+
+        let cargo = |args: &[&str]| {
+            crate::run(
+                Command::new("cargo")
+                    .arg(format!("+{}", channel))
+                    .args(args)
+                    .env("USER", "root")
+                    .current_dir(&cargo_dir),
+            )
+        };
+        let rustup = |args: &[&str]| {
+            crate::run(
+                Command::new("rustup")
+                    .env("RUSTUP_DIST_SERVER", format!("http://{}", self.server_addr))
+                    .args(args),
+            )
+        };
+
+        rustup(&["toolchain", "remove", &channel.to_string()])?;
+        rustup(&["toolchain", "install", &channel.to_string()])?;
+        cargo(&["init", "--bin", "."])?;
+        cargo(&["run"])?;
+
+        // Finally shut down the HTTP server and the tokio reactor.
+        self.shutdown_send
+            .send(())
+            .expect("failed to send shutdown message to the server");
+        self.runtime.shutdown_background();
+
+        Ok(())
+    }
+}
+
+async fn server_handler(
+    req: Request<Body>,
+    paths: Arc<Vec<PathBuf>>,
+) -> Result<Response<Body>, Error> {
+    let file_name = match req.uri().path().split('/').last() {
+        Some(file_name) => file_name,
+        None => return not_found(),
+    };
+    for directory in &*paths {
+        let path = directory.join(file_name);
+        if path.is_file() {
+            let content = tokio::fs::read(&path).await?;
+            return Ok(Response::new(content.into()));
+        }
+    }
+    not_found()
+}
+
+fn not_found() -> Result<Response<Body>, Error> {
+    let mut response = Response::new("404: Not Found\n".into());
+    *response.status_mut() = StatusCode::NOT_FOUND;
+    Ok(response)
+}