Skip to content

Commit

Permalink
Merge pull request #25 from rust-lang/smoke-test
Browse files Browse the repository at this point in the history
Ensure the release works before uploading the tarballs
  • Loading branch information
pietroalbini authored Nov 3, 2020
2 parents 2df758c + bfac3ba commit 1370b31
Show file tree
Hide file tree
Showing 8 changed files with 624 additions and 34 deletions.
428 changes: 411 additions & 17 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,5 @@ pgp = "0.7.1"
chrono = "0.4.19"
git2 = "0.13.11"
tempfile = "3.1.0"
hyper = "0.13.8"
tokio = { version = "0.2.22", features = ["rt-threaded", "macros", "fs", "sync"] }
7 changes: 7 additions & 0 deletions local/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
ninja-build \
python-is-python3

# Install rustup while removing the pre-installed stable toolchain.
RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
chmod +x /tmp/rustup-init && \
/tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
/root/.cargo/bin/rustup toolchain remove stable
ENV PATH=/root/.cargo/bin:$PATH

COPY --from=mc /usr/bin/mc /usr/local/bin/mc
RUN chmod 0755 /usr/local/bin/mc

Expand Down
7 changes: 7 additions & 0 deletions prod/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
ninja-build \
python-is-python3

# Install rustup while removing the pre-installed stable toolchain.
RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
chmod +x /tmp/rustup-init && \
/tmp/rustup-init -y --no-modify-path --default-toolchain stable && \
/root/.cargo/bin/rustup toolchain remove stable
ENV PATH=/root/.cargo/bin:$PATH

COPY --from=build /tmp/source/target/release/promote-release /usr/local/bin/
COPY prod/load-gpg-keys.sh /usr/local/bin/load-gpg-keys

Expand Down
36 changes: 28 additions & 8 deletions src/build_manifest.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::Context;
use anyhow::{Context as _, Error};
use std::{
collections::HashSet,
collections::{HashMap, HashSet},
fs::File,
io::BufReader,
path::{Path, PathBuf},
Expand Down Expand Up @@ -35,30 +35,38 @@ impl<'a> BuildManifest<'a> {
self.tarball_path.is_file()
}

pub(crate) fn run(&self) -> Result<Execution, Error> {
pub(crate) fn run(&self, upload_base: &str) -> Result<Execution, Error> {
let config = &self.builder.config;
let bin = self
.extract()
.context("failed to extract build-manifest from the tarball")?;

let metadata_dir = TempDir::new()?;
let checksum_cache = metadata_dir.path().join("checksum-cache.json");
let shipped_files_path = metadata_dir.path().join("shipped-files.txt");

// Ensure the manifest dir exists but is empty.
let manifest_dir = self.builder.manifest_dir();
if manifest_dir.is_dir() {
std::fs::remove_dir_all(&manifest_dir)?;
}
std::fs::create_dir_all(&manifest_dir)?;

println!("running build-manifest...");
let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir);
// build-manifest <input-dir> <output-dir> <date> <upload-addr> <channel>
let status = Command::new(bin.path())
.arg(self.builder.dl_dir())
.arg(self.builder.dl_dir())
.arg(self.builder.manifest_dir())
.arg(&self.builder.date)
.arg(upload_addr)
.arg(upload_base)
.arg(config.channel.to_string())
.env("BUILD_MANIFEST_CHECKSUM_CACHE", &checksum_cache)
.env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path)
.status()
.context("failed to execute build-manifest")?;

if status.success() {
Execution::new(&shipped_files_path)
Execution::new(&shipped_files_path, &checksum_cache)
} else {
anyhow::bail!("build-manifest failed with status {:?}", status);
}
Expand Down Expand Up @@ -87,10 +95,11 @@ impl<'a> BuildManifest<'a> {

pub(crate) struct Execution {
pub(crate) shipped_files: Option<HashSet<PathBuf>>,
pub(crate) checksum_cache: HashMap<PathBuf, String>,
}

impl Execution {
fn new(shipped_files_path: &Path) -> Result<Self, Error> {
fn new(shipped_files_path: &Path, checksum_cache_path: &Path) -> Result<Self, Error> {
// Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the
// "shipped files" file is always generated, and we can remove the Option<_>.
let shipped_files = if shipped_files_path.is_file() {
Expand All @@ -105,6 +114,17 @@ impl Execution {
None
};

Ok(Execution { shipped_files })
// Once https://github.com/rust-lang/rust/pull/78409 reaches stable we can assume the
// checksum cache will always be generated, and we can remove the if branch.
let checksum_cache = if checksum_cache_path.is_file() {
serde_json::from_slice(&std::fs::read(checksum_cache_path)?)?
} else {
HashMap::new()
};

Ok(Execution {
shipped_files,
checksum_cache,
})
}
}
46 changes: 40 additions & 6 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
mod build_manifest;
mod config;
mod sign;
mod smoke_test;

use std::fs::{self, File, OpenOptions};
use std::io::{self, Read};
Expand All @@ -9,13 +10,14 @@ use std::process::Command;
use std::time::Instant;
use std::{collections::HashSet, env};

use crate::build_manifest::BuildManifest;
use crate::sign::Signer;
use crate::smoke_test::SmokeTester;
use anyhow::Error;
use build_manifest::BuildManifest;
use chrono::Utc;
use curl::easy::Easy;
use fs2::FileExt;
use rayon::prelude::*;
use sign::Signer;
use xz2::read::XzDecoder;

use crate::config::{Channel, Config};
Expand Down Expand Up @@ -179,10 +181,18 @@ impl Context {

// Ok we've now determined that a release needs to be done.

let mut signer = Signer::new(&self.config)?;
let build_manifest = BuildManifest::new(self);

if build_manifest.exists() {
// Generate the channel manifest
let execution = build_manifest.run()?;
let smoke_test = SmokeTester::new(&[self.manifest_dir(), self.dl_dir()])?;

// First of all, a manifest is generated pointing to the smoke test server. This will
// produce the correct checksums and shipped files list, as the only difference from
// between the "real" execution and this one is the URLs included in the manifest.
let execution =
build_manifest.run(&format!("http://{}/dist", smoke_test.server_addr()))?;
signer.override_checksum_cache(execution.checksum_cache);

if self.config.wip_prune_unused_files {
// Removes files that we are not shipping from the files we're about to upload.
Expand All @@ -191,9 +201,29 @@ impl Context {
}
}

// Generate checksums and sign all the files we're about to ship.
let signer = Signer::new(&self.config)?;
// Sign both the downloaded artifacts and the generated manifests. The signatures of
// the downloaded files are permanent, while the signatures for the generated manifests
// will be discarded later (as the manifests point to the smoke test server).
signer.sign_directory(&self.dl_dir())?;
signer.sign_directory(&self.manifest_dir())?;

// Ensure the release is downloadable from rustup and can execute a basic binary.
smoke_test.test(&self.config.channel)?;

// Generate the real manifests and sign them.
build_manifest.run(&format!(
"{}/{}",
self.config.upload_addr, self.config.upload_dir
))?;
signer.sign_directory(&self.manifest_dir())?;

// Merge the generated manifests with the downloaded artifacts.
for entry in std::fs::read_dir(&self.manifest_dir())? {
let entry = entry?;
if entry.file_type()?.is_file() {
std::fs::rename(entry.path(), self.dl_dir().join(entry.file_name()))?;
}
}
} else {
// For releases using the legacy build-manifest, we need to clone the rustc monorepo
// and invoke `./x.py dist hash-and-sign` in it. This won't be needed after 1.48.0 is
Expand Down Expand Up @@ -667,6 +697,10 @@ upload-addr = \"{}/{}\"
self.work.join("dl")
}

fn manifest_dir(&self) -> PathBuf {
self.work.join("manifests")
}

fn build_dir(&self) -> PathBuf {
self.work.join("build")
}
Expand Down
19 changes: 16 additions & 3 deletions src/sign.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use pgp::{
use rayon::prelude::*;
use sha2::Digest;
use std::{
collections::HashMap,
fs::File,
path::{Path, PathBuf},
time::Instant,
Expand All @@ -20,6 +21,7 @@ use crate::config::Config;
pub(crate) struct Signer {
gpg_key: SignedSecretKey,
gpg_password: String,
sha256_checksum_cache: HashMap<PathBuf, String>,
}

impl Signer {
Expand All @@ -29,9 +31,14 @@ impl Signer {
Ok(Signer {
gpg_key: SignedSecretKey::from_armor_single(&mut key_file)?.0,
gpg_password,
sha256_checksum_cache: HashMap::new(),
})
}

pub(crate) fn override_checksum_cache(&mut self, new: HashMap<PathBuf, String>) {
self.sha256_checksum_cache = new;
}

pub(crate) fn sign_directory(&self, path: &Path) -> Result<(), Error> {
let mut paths = Vec::new();
for entry in std::fs::read_dir(path)? {
Expand Down Expand Up @@ -85,9 +92,15 @@ impl Signer {
}

fn generate_sha256(&self, path: &Path, data: &[u8]) -> Result<(), Error> {
let mut digest = sha2::Sha256::default();
digest.update(data);
let sha256 = hex::encode(digest.finalize());
let canonical_path = std::fs::canonicalize(path)?;

let sha256 = if let Some(cached) = self.sha256_checksum_cache.get(&canonical_path) {
cached.clone()
} else {
let mut digest = sha2::Sha256::default();
digest.update(data);
hex::encode(digest.finalize())
};

std::fs::write(
add_suffix(path, ".sha256"),
Expand Down
113 changes: 113 additions & 0 deletions src/smoke_test.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
use anyhow::Error;
use hyper::{Body, Request, Response, Server, StatusCode};
use std::{net::SocketAddr, sync::Arc};
use std::{path::PathBuf, process::Command};
use tempfile::TempDir;
use tokio::{runtime::Runtime, sync::oneshot::Sender};

use crate::config::Channel;

pub(crate) struct SmokeTester {
runtime: Runtime,
server_addr: SocketAddr,
shutdown_send: Sender<()>,
}

impl SmokeTester {
pub(crate) fn new(paths: &[PathBuf]) -> Result<Self, Error> {
let addr = SocketAddr::from(([127, 0, 0, 1], 0));

let paths = Arc::new(paths.to_vec());
let service = hyper::service::make_service_fn(move |_| {
let paths = paths.clone();
async move {
Ok::<_, Error>(hyper::service::service_fn(move |req| {
server_handler(req, paths.clone())
}))
}
});

let (shutdown_send, shutdown_recv) = tokio::sync::oneshot::channel::<()>();

let runtime = Runtime::new()?;
let (server, server_addr) = runtime.enter(|| {
let server = Server::bind(&addr).serve(service);
let server_addr = server.local_addr();
let server = server.with_graceful_shutdown(async {
shutdown_recv.await.ok();
});
(server, server_addr)
});
runtime.spawn(server);

Ok(Self {
runtime,
server_addr,
shutdown_send,
})
}

pub(crate) fn server_addr(&self) -> SocketAddr {
self.server_addr
}

pub(crate) fn test(self, channel: &Channel) -> Result<(), Error> {
let tempdir = TempDir::new()?;
let cargo_dir = tempdir.path().join("sample-crate");
std::fs::create_dir_all(&cargo_dir)?;

let cargo = |args: &[&str]| {
crate::run(
Command::new("cargo")
.arg(format!("+{}", channel))
.args(args)
.env("USER", "root")
.current_dir(&cargo_dir),
)
};
let rustup = |args: &[&str]| {
crate::run(
Command::new("rustup")
.env("RUSTUP_DIST_SERVER", format!("http://{}", self.server_addr))
.args(args),
)
};

rustup(&["toolchain", "remove", &channel.to_string()])?;
rustup(&["toolchain", "install", &channel.to_string()])?;
cargo(&["init", "--bin", "."])?;
cargo(&["run"])?;

// Finally shut down the HTTP server and the tokio reactor.
self.shutdown_send
.send(())
.expect("failed to send shutdown message to the server");
self.runtime.shutdown_background();

Ok(())
}
}

async fn server_handler(
req: Request<Body>,
paths: Arc<Vec<PathBuf>>,
) -> Result<Response<Body>, Error> {
let file_name = match req.uri().path().split('/').last() {
Some(file_name) => file_name,
None => return not_found(),
};
for directory in &*paths {
let path = directory.join(file_name);
if path.is_file() {
let content = tokio::fs::read(&path).await?;
return Ok(Response::new(content.into()));
}
}
not_found()
}

fn not_found() -> Result<Response<Body>, Error> {
let mut response = Response::new("404: Not Found\n".into());
*response.status_mut() = StatusCode::NOT_FOUND;
Ok(response)
}

0 comments on commit 1370b31

Please sign in to comment.