From 5f864598843f6b69fcbfc9f247051cce8a8734ac Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 28 Oct 2020 19:56:45 +0100 Subject: [PATCH 1/3] sign: reuse checksums calculated during build-manifest This commit starts to use build-manifest's new checksum cache feature, which stores all the checksums it calculated in a JSON file before exiting. This allows to avoid duplicate calculations. --- src/build_manifest.rs | 22 ++++++++++++++++++---- src/main.rs | 3 +++ src/sign.rs | 19 ++++++++++++++++--- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/build_manifest.rs b/src/build_manifest.rs index 421d8c7..c02aff5 100644 --- a/src/build_manifest.rs +++ b/src/build_manifest.rs @@ -1,7 +1,7 @@ use crate::Context; use anyhow::{Context as _, Error}; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fs::File, io::BufReader, path::{Path, PathBuf}, @@ -42,6 +42,7 @@ impl<'a> BuildManifest<'a> { .context("failed to extract build-manifest from the tarball")?; let metadata_dir = TempDir::new()?; + let checksum_cache = metadata_dir.path().join("checksum-cache.json"); let shipped_files_path = metadata_dir.path().join("shipped-files.txt"); println!("running build-manifest..."); @@ -53,12 +54,13 @@ impl<'a> BuildManifest<'a> { .arg(&self.builder.date) .arg(upload_addr) .arg(config.channel.to_string()) + .env("BUILD_MANIFEST_CHECKSUM_CACHE", &checksum_cache) .env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path) .status() .context("failed to execute build-manifest")?; if status.success() { - Execution::new(&shipped_files_path) + Execution::new(&shipped_files_path, &checksum_cache) } else { anyhow::bail!("build-manifest failed with status {:?}", status); } @@ -87,10 +89,11 @@ impl<'a> BuildManifest<'a> { pub(crate) struct Execution { pub(crate) shipped_files: Option>, + pub(crate) checksum_cache: HashMap, } impl Execution { - fn new(shipped_files_path: &Path) -> Result { + fn new(shipped_files_path: &Path, checksum_cache_path: &Path) -> Result { // Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the // "shipped files" file is always generated, and we can remove the Option<_>. let shipped_files = if shipped_files_path.is_file() { @@ -105,6 +108,17 @@ impl Execution { None }; - Ok(Execution { shipped_files }) + // Once https://github.com/rust-lang/rust/pull/78409 reaches stable we can assume the + // checksum cache will always be generated, and we can remove the if branch. + let checksum_cache = if checksum_cache_path.is_file() { + serde_json::from_slice(&std::fs::read(checksum_cache_path)?)? + } else { + HashMap::new() + }; + + Ok(Execution { + shipped_files, + checksum_cache, + }) } } diff --git a/src/main.rs b/src/main.rs index 28a0827..675a0e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -182,10 +182,13 @@ impl Context { // Ok we've now determined that a release needs to be done. + let mut signer = Signer::new(&self.config)?; let build_manifest = BuildManifest::new(self); + if build_manifest.exists() { // Generate the channel manifest let execution = build_manifest.run()?; + signer.override_checksum_cache(execution.checksum_cache); if self.config.wip_prune_unused_files { // Removes files that we are not shipping from the files we're about to upload. diff --git a/src/sign.rs b/src/sign.rs index 2e8176c..2fb578e 100644 --- a/src/sign.rs +++ b/src/sign.rs @@ -10,6 +10,7 @@ use pgp::{ use rayon::prelude::*; use sha2::Digest; use std::{ + collections::HashMap, fs::File, path::{Path, PathBuf}, time::Instant, @@ -20,6 +21,7 @@ use crate::config::Config; pub(crate) struct Signer { gpg_key: SignedSecretKey, gpg_password: String, + sha256_checksum_cache: HashMap, } impl Signer { @@ -29,9 +31,14 @@ impl Signer { Ok(Signer { gpg_key: SignedSecretKey::from_armor_single(&mut key_file)?.0, gpg_password, + sha256_checksum_cache: HashMap::new(), }) } + pub(crate) fn override_checksum_cache(&mut self, new: HashMap) { + self.sha256_checksum_cache = new; + } + pub(crate) fn sign_directory(&self, path: &Path) -> Result<(), Error> { let mut paths = Vec::new(); for entry in std::fs::read_dir(path)? { @@ -85,9 +92,15 @@ impl Signer { } fn generate_sha256(&self, path: &Path, data: &[u8]) -> Result<(), Error> { - let mut digest = sha2::Sha256::default(); - digest.update(data); - let sha256 = hex::encode(digest.finalize()); + let canonical_path = std::fs::canonicalize(path)?; + + let sha256 = if let Some(cached) = self.sha256_checksum_cache.get(&canonical_path) { + cached.clone() + } else { + let mut digest = sha2::Sha256::default(); + digest.update(data); + hex::encode(digest.finalize()) + }; std::fs::write( add_suffix(path, ".sha256"), From 49c3b3a5b571352a2ae49d7fe3a1745ad3d8f41d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 28 Oct 2020 19:38:07 +0100 Subject: [PATCH 2/3] build-manifest: generate files in a separate directory and then merge it This commit changes how build-manifest is invoked, stopping it from generating the manifests directly into the downloads directory. Instead, the files are generated in a separate directory and then merged back. While there is not much point in doing so *right now*, this will be crucial to implement smoke tests for the release, as we need to generate a separate set of manifests with a different download URL to be able to perform the smoke test. Having the generated output in a separate directory will allow to easily discard those temporary manifests and their signatures in a clean way. --- src/build_manifest.rs | 9 ++++++++- src/main.rs | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/build_manifest.rs b/src/build_manifest.rs index c02aff5..85406ac 100644 --- a/src/build_manifest.rs +++ b/src/build_manifest.rs @@ -45,12 +45,19 @@ impl<'a> BuildManifest<'a> { let checksum_cache = metadata_dir.path().join("checksum-cache.json"); let shipped_files_path = metadata_dir.path().join("shipped-files.txt"); + // Ensure the manifest dir exists but is empty. + let manifest_dir = self.builder.manifest_dir(); + if manifest_dir.is_dir() { + std::fs::remove_dir_all(&manifest_dir)?; + } + std::fs::create_dir_all(&manifest_dir)?; + println!("running build-manifest..."); let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir); // build-manifest let status = Command::new(bin.path()) .arg(self.builder.dl_dir()) - .arg(self.builder.dl_dir()) + .arg(self.builder.manifest_dir()) .arg(&self.builder.date) .arg(upload_addr) .arg(config.channel.to_string()) diff --git a/src/main.rs b/src/main.rs index 675a0e5..809678d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,9 +197,20 @@ impl Context { } } - // Generate checksums and sign all the files we're about to ship. - let signer = Signer::new(&self.config)?; + // Generate checksums for all the downloaded files that weren't pruned earlier. This + // does *not* include the manifests, as those are in a different directory. signer.sign_directory(&self.dl_dir())?; + + // Generate checksums for all the files generated by build-manifest. + signer.sign_directory(&self.manifest_dir())?; + + // Merge the manifests directory into the downloads directory. + for entry in std::fs::read_dir(&self.manifest_dir())? { + let entry = entry?; + if entry.file_type()?.is_file() { + std::fs::rename(entry.path(), self.dl_dir().join(entry.file_name()))?; + } + } } else { // For releases using the legacy build-manifest, we need to clone the rustc monorepo // and invoke `./x.py dist hash-and-sign` in it. This won't be needed after 1.48.0 is @@ -680,6 +691,10 @@ upload-addr = \"{}/{}\" self.work.join("dl") } + fn manifest_dir(&self) -> PathBuf { + self.work.join("manifests") + } + fn build_dir(&self) -> PathBuf { self.work.join("build") } From bfac3ba05363ea282b3e326b656a8945e6fb4e49 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 29 Oct 2020 19:35:16 +0100 Subject: [PATCH 3/3] smoke_test: ensure the release works with rustup before uploading This commit changes the release process to execute a "smoke test" before uploading the release to the destination buckets, by generating another set of manifests pointing to a localhost server serving the files we're about to upload. After the localhost web server is up then rustup is invoked to download the toolchain, and cargo is invoked to create a binary project and run the hello world in it. If any of this fails the release is aborted. --- Cargo.lock | 428 ++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 + local/Dockerfile | 7 + prod/Dockerfile | 7 + src/build_manifest.rs | 5 +- src/main.rs | 32 +++- src/smoke_test.rs | 113 +++++++++++ 7 files changed, 566 insertions(+), 28 deletions(-) create mode 100644 src/smoke_test.rs diff --git a/Cargo.lock b/Cargo.lock index a7f2fcb..c6258e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,6 +145,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + [[package]] name = "cast5" version = "0.8.0" @@ -190,7 +196,7 @@ dependencies = [ "num-integer", "num-traits", "time", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -297,7 +303,7 @@ dependencies = [ "openssl-sys", "schannel", "socket2", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -312,7 +318,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -446,7 +452,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -474,7 +480,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -483,6 +489,61 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures-channel" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46" + +[[package]] +name = "futures-sink" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e3ca3f17d6e8804ae5d3df7a7d35b2b3a6fe89dac84b31872720fc3060a0b11" + +[[package]] +name = "futures-task" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c" + +[[package]] +name = "futures-util" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34" +dependencies = [ + "futures-core", + "futures-task", + "pin-project 1.0.1", + "pin-utils", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -519,6 +580,32 @@ dependencies = [ "url", ] +[[package]] +name = "h2" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + [[package]] name = "hermit-abi" version = "0.1.15" @@ -534,6 +621,63 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "httpdate" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" + +[[package]] +name = "hyper" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3afcfae8af5ad0576a31e768415edb627824129e8e5a29b8bfccb2f234e835" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project 0.4.27", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -551,6 +695,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "itoa" version = "0.4.6" @@ -572,6 +735,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -700,6 +873,48 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if", + "libc", + "winapi 0.3.9", +] + [[package]] name = "nom" version = "4.2.3" @@ -888,6 +1103,58 @@ dependencies = [ "zeroize", ] +[[package]] +name = "pin-project" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" +dependencies = [ + "pin-project-internal 0.4.27", +] + +[[package]] +name = "pin-project" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841" +dependencies = [ + "pin-project-internal 1.0.1", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.18" @@ -902,9 +1169,9 @@ checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" [[package]] name = "proc-macro2" -version = "1.0.21" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ "unicode-xid", ] @@ -920,6 +1187,7 @@ dependencies = [ "fs2", "git2", "hex", + "hyper", "pgp", "rand 0.6.5", "rayon", @@ -928,6 +1196,7 @@ dependencies = [ "sha2", "tar", "tempfile", + "tokio", "toml", "xz2", ] @@ -957,7 +1226,7 @@ dependencies = [ "rand_os", "rand_pcg", "rand_xorshift", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1052,7 +1321,7 @@ checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" dependencies = [ "libc", "rand_core 0.4.2", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1066,7 +1335,7 @@ dependencies = [ "libc", "rand_core 0.4.2", "rdrand", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1152,7 +1421,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1207,7 +1476,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" dependencies = [ "lazy_static", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1302,6 +1571,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + [[package]] name = "smallvec" version = "1.4.2" @@ -1317,7 +1592,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1350,9 +1625,9 @@ checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" [[package]] name = "syn" -version = "1.0.40" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ "proc-macro2", "quote", @@ -1394,7 +1669,7 @@ dependencies = [ "rand 0.7.3", "redox_syscall", "remove_dir_all", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1434,7 +1709,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1443,6 +1718,50 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" +[[package]] +name = "tokio" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "slab", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.4.10" @@ -1452,6 +1771,49 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tracing" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project 0.4.27", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "try_from" version = "0.3.2" @@ -1531,6 +1893,16 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -1543,6 +1915,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -1553,6 +1931,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -1565,6 +1949,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "x25519-dalek" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 2e6c758..f1c334e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,3 +25,5 @@ pgp = "0.7.1" chrono = "0.4.19" git2 = "0.13.11" tempfile = "3.1.0" +hyper = "0.13.8" +tokio = { version = "0.2.22", features = ["rt-threaded", "macros", "fs", "sync"] } diff --git a/local/Dockerfile b/local/Dockerfile index c58608d..c050cb3 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -26,6 +26,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ ninja-build \ python-is-python3 +# Install rustup while removing the pre-installed stable toolchain. +RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ + chmod +x /tmp/rustup-init && \ + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \ + /root/.cargo/bin/rustup toolchain remove stable +ENV PATH=/root/.cargo/bin:$PATH + COPY --from=mc /usr/bin/mc /usr/local/bin/mc RUN chmod 0755 /usr/local/bin/mc diff --git a/prod/Dockerfile b/prod/Dockerfile index a8c468a..4636777 100644 --- a/prod/Dockerfile +++ b/prod/Dockerfile @@ -56,6 +56,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ ninja-build \ python-is-python3 +# Install rustup while removing the pre-installed stable toolchain. +RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \ + chmod +x /tmp/rustup-init && \ + /tmp/rustup-init -y --no-modify-path --default-toolchain stable && \ + /root/.cargo/bin/rustup toolchain remove stable +ENV PATH=/root/.cargo/bin:$PATH + COPY --from=build /tmp/source/target/release/promote-release /usr/local/bin/ COPY prod/load-gpg-keys.sh /usr/local/bin/load-gpg-keys diff --git a/src/build_manifest.rs b/src/build_manifest.rs index 85406ac..9aadcde 100644 --- a/src/build_manifest.rs +++ b/src/build_manifest.rs @@ -35,7 +35,7 @@ impl<'a> BuildManifest<'a> { self.tarball_path.is_file() } - pub(crate) fn run(&self) -> Result { + pub(crate) fn run(&self, upload_base: &str) -> Result { let config = &self.builder.config; let bin = self .extract() @@ -53,13 +53,12 @@ impl<'a> BuildManifest<'a> { std::fs::create_dir_all(&manifest_dir)?; println!("running build-manifest..."); - let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir); // build-manifest let status = Command::new(bin.path()) .arg(self.builder.dl_dir()) .arg(self.builder.manifest_dir()) .arg(&self.builder.date) - .arg(upload_addr) + .arg(upload_base) .arg(config.channel.to_string()) .env("BUILD_MANIFEST_CHECKSUM_CACHE", &checksum_cache) .env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path) diff --git a/src/main.rs b/src/main.rs index 809678d..776a49a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod build_manifest; mod config; mod sign; +mod smoke_test; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read}; @@ -9,13 +10,14 @@ use std::process::Command; use std::time::Instant; use std::{collections::HashSet, env}; +use crate::build_manifest::BuildManifest; +use crate::sign::Signer; +use crate::smoke_test::SmokeTester; use anyhow::Error; -use build_manifest::BuildManifest; use chrono::Utc; use curl::easy::Easy; use fs2::FileExt; use rayon::prelude::*; -use sign::Signer; use xz2::read::XzDecoder; use crate::config::{Channel, Config}; @@ -186,8 +188,13 @@ impl Context { let build_manifest = BuildManifest::new(self); if build_manifest.exists() { - // Generate the channel manifest - let execution = build_manifest.run()?; + let smoke_test = SmokeTester::new(&[self.manifest_dir(), self.dl_dir()])?; + + // First of all, a manifest is generated pointing to the smoke test server. This will + // produce the correct checksums and shipped files list, as the only difference from + // between the "real" execution and this one is the URLs included in the manifest. + let execution = + build_manifest.run(&format!("http://{}/dist", smoke_test.server_addr()))?; signer.override_checksum_cache(execution.checksum_cache); if self.config.wip_prune_unused_files { @@ -197,14 +204,23 @@ impl Context { } } - // Generate checksums for all the downloaded files that weren't pruned earlier. This - // does *not* include the manifests, as those are in a different directory. + // Sign both the downloaded artifacts and the generated manifests. The signatures of + // the downloaded files are permanent, while the signatures for the generated manifests + // will be discarded later (as the manifests point to the smoke test server). signer.sign_directory(&self.dl_dir())?; + signer.sign_directory(&self.manifest_dir())?; + + // Ensure the release is downloadable from rustup and can execute a basic binary. + smoke_test.test(&self.config.channel)?; - // Generate checksums for all the files generated by build-manifest. + // Generate the real manifests and sign them. + build_manifest.run(&format!( + "{}/{}", + self.config.upload_addr, self.config.upload_dir + ))?; signer.sign_directory(&self.manifest_dir())?; - // Merge the manifests directory into the downloads directory. + // Merge the generated manifests with the downloaded artifacts. for entry in std::fs::read_dir(&self.manifest_dir())? { let entry = entry?; if entry.file_type()?.is_file() { diff --git a/src/smoke_test.rs b/src/smoke_test.rs new file mode 100644 index 0000000..8cf258c --- /dev/null +++ b/src/smoke_test.rs @@ -0,0 +1,113 @@ +use anyhow::Error; +use hyper::{Body, Request, Response, Server, StatusCode}; +use std::{net::SocketAddr, sync::Arc}; +use std::{path::PathBuf, process::Command}; +use tempfile::TempDir; +use tokio::{runtime::Runtime, sync::oneshot::Sender}; + +use crate::config::Channel; + +pub(crate) struct SmokeTester { + runtime: Runtime, + server_addr: SocketAddr, + shutdown_send: Sender<()>, +} + +impl SmokeTester { + pub(crate) fn new(paths: &[PathBuf]) -> Result { + let addr = SocketAddr::from(([127, 0, 0, 1], 0)); + + let paths = Arc::new(paths.to_vec()); + let service = hyper::service::make_service_fn(move |_| { + let paths = paths.clone(); + async move { + Ok::<_, Error>(hyper::service::service_fn(move |req| { + server_handler(req, paths.clone()) + })) + } + }); + + let (shutdown_send, shutdown_recv) = tokio::sync::oneshot::channel::<()>(); + + let runtime = Runtime::new()?; + let (server, server_addr) = runtime.enter(|| { + let server = Server::bind(&addr).serve(service); + let server_addr = server.local_addr(); + let server = server.with_graceful_shutdown(async { + shutdown_recv.await.ok(); + }); + (server, server_addr) + }); + runtime.spawn(server); + + Ok(Self { + runtime, + server_addr, + shutdown_send, + }) + } + + pub(crate) fn server_addr(&self) -> SocketAddr { + self.server_addr + } + + pub(crate) fn test(self, channel: &Channel) -> Result<(), Error> { + let tempdir = TempDir::new()?; + let cargo_dir = tempdir.path().join("sample-crate"); + std::fs::create_dir_all(&cargo_dir)?; + + let cargo = |args: &[&str]| { + crate::run( + Command::new("cargo") + .arg(format!("+{}", channel)) + .args(args) + .env("USER", "root") + .current_dir(&cargo_dir), + ) + }; + let rustup = |args: &[&str]| { + crate::run( + Command::new("rustup") + .env("RUSTUP_DIST_SERVER", format!("http://{}", self.server_addr)) + .args(args), + ) + }; + + rustup(&["toolchain", "remove", &channel.to_string()])?; + rustup(&["toolchain", "install", &channel.to_string()])?; + cargo(&["init", "--bin", "."])?; + cargo(&["run"])?; + + // Finally shut down the HTTP server and the tokio reactor. + self.shutdown_send + .send(()) + .expect("failed to send shutdown message to the server"); + self.runtime.shutdown_background(); + + Ok(()) + } +} + +async fn server_handler( + req: Request, + paths: Arc>, +) -> Result, Error> { + let file_name = match req.uri().path().split('/').last() { + Some(file_name) => file_name, + None => return not_found(), + }; + for directory in &*paths { + let path = directory.join(file_name); + if path.is_file() { + let content = tokio::fs::read(&path).await?; + return Ok(Response::new(content.into())); + } + } + not_found() +} + +fn not_found() -> Result, Error> { + let mut response = Response::new("404: Not Found\n".into()); + *response.status_mut() = StatusCode::NOT_FOUND; + Ok(response) +}