Skip to content

Improve? test reliability #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 5, 2024
Merged

Improve? test reliability #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7e75b49
server.c: error handling for bind/listen
ctz May 5, 2024
e623f09
Make `want_for_stdout` more defensive
ctz May 5, 2024
7c6549e
Make server port allocation less manual
ctz May 5, 2024
f91c409
Don't wait forever for subprocess
ctz May 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions rustls-libssl/tests/client.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ int main(int argc, char **argv) {
}

struct addrinfo *result = NULL;
TRACE(getaddrinfo(host, port, NULL, &result));
REQUIRE(0, getaddrinfo(host, port, NULL, &result));

int sock = TRACE(
socket(result->ai_family, result->ai_socktype, result->ai_protocol));
TRACE(connect(sock, result->ai_addr, result->ai_addrlen));
REQUIRE(0, connect(sock, result->ai_addr, result->ai_addrlen));
freeaddrinfo(result);

TRACE(OPENSSL_init_ssl(0, NULL));
Expand Down
10 changes: 10 additions & 0 deletions rustls-libssl/tests/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@ static int trace(int rc, const char *str) {

#define TRACE(fn) trace((fn), #fn)

static int require(int expect_rc, int got_rc, const char *str) {
if (expect_rc != got_rc) {
printf("REQUIRED(%s) failed: wanted=%d, got=%d\n", str, expect_rc, got_rc);
abort();
}
return got_rc;
}

#define REQUIRE(expect, fn) require((expect), (fn), #fn)

static void hexdump(const char *label, const void *buf, int n) {
const uint8_t *ubuf = (const uint8_t *)buf;
printf("%s (%d bytes): ", label, n);
Expand Down
135 changes: 100 additions & 35 deletions rustls-libssl/tests/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use std::io::Read;
use std::ops::Add;
use std::process::{Child, Command, Output, Stdio};
use std::sync::atomic;
use std::{fs, net, thread, time};

/* Note:
Expand Down Expand Up @@ -31,6 +33,8 @@ use std::{fs, net, thread, time};
#[test]
#[ignore]
fn client_unauthenticated() {
let (port, port_str) = choose_port();

let _server = KillOnDrop(Some(
Command::new("openssl")
.args([
Expand All @@ -44,27 +48,27 @@ fn client_unauthenticated() {
"-alpn",
"hello,world",
"-accept",
"localhost:4443",
&format!("localhost:{port}"),
"-rev",
])
.env("LD_LIBRARY_PATH", "")
.spawn()
.expect("failed to start openssl s_server"),
));

wait_for_port(4443);
wait_for_port(port);

// server is unauthenticated
let openssl_insecure_output = Command::new("tests/maybe-valgrind.sh")
.env("LD_LIBRARY_PATH", "")
.args(["target/client", "localhost", "4443", "insecure"])
.args(["target/client", "localhost", &port_str, "insecure"])
.stdout(Stdio::piped())
.output()
.map(print_output)
.unwrap();

let rustls_insecure_output = Command::new("tests/maybe-valgrind.sh")
.args(["target/client", "localhost", "4443", "insecure"])
.args(["target/client", "localhost", &port_str, "insecure"])
.stdout(Stdio::piped())
.output()
.map(print_output)
Expand All @@ -75,14 +79,24 @@ fn client_unauthenticated() {
// server is authenticated, client has no creds
let openssl_secure_output = Command::new("tests/maybe-valgrind.sh")
.env("LD_LIBRARY_PATH", "")
.args(["target/client", "localhost", "4443", "test-ca/rsa/ca.cert"])
.args([
"target/client",
"localhost",
&port_str,
"test-ca/rsa/ca.cert",
])
.stdout(Stdio::piped())
.output()
.map(print_output)
.unwrap();

let rustls_secure_output = Command::new("tests/maybe-valgrind.sh")
.args(["target/client", "localhost", "4443", "test-ca/rsa/ca.cert"])
.args([
"target/client",
"localhost",
&port_str,
"test-ca/rsa/ca.cert",
])
.stdout(Stdio::piped())
.output()
.map(print_output)
Expand All @@ -96,7 +110,7 @@ fn client_unauthenticated() {
.args([
"target/client",
"localhost",
"4443",
&port_str,
"test-ca/rsa/ca.cert",
"test-ca/rsa/client.key",
"test-ca/rsa/client.cert",
Expand All @@ -110,7 +124,7 @@ fn client_unauthenticated() {
.args([
"target/client",
"localhost",
"4443",
&port_str,
"test-ca/rsa/ca.cert",
"test-ca/rsa/client.key",
"test-ca/rsa/client.cert",
Expand All @@ -126,6 +140,8 @@ fn client_unauthenticated() {
#[test]
#[ignore]
fn client_auth() {
let (port, port_str) = choose_port();

let _server = KillOnDrop(Some(
Command::new("openssl")
.args([
Expand All @@ -143,23 +159,23 @@ fn client_auth() {
"-CAfile",
"test-ca/rsa/ca.cert",
"-accept",
"localhost:4444",
&format!("localhost:{port}"),
"-rev",
])
.env("LD_LIBRARY_PATH", "")
.spawn()
.expect("failed to start openssl s_server"),
));

wait_for_port(4444);
wait_for_port(port);

// mutual auth
let openssl_authed_output = Command::new("tests/maybe-valgrind.sh")
.env("LD_LIBRARY_PATH", "")
.args([
"target/client",
"localhost",
"4444",
&port_str,
"test-ca/rsa/ca.cert",
"test-ca/rsa/client.key",
"test-ca/rsa/client.cert",
Expand All @@ -173,7 +189,7 @@ fn client_auth() {
.args([
"target/client",
"localhost",
"4444",
&port_str,
"test-ca/rsa/ca.cert",
"test-ca/rsa/client.key",
"test-ca/rsa/client.cert",
Expand All @@ -188,14 +204,24 @@ fn client_auth() {
// failed auth
let openssl_failed_output = Command::new("tests/maybe-valgrind.sh")
.env("LD_LIBRARY_PATH", "")
.args(["target/client", "localhost", "4444", "test-ca/rsa/ca.cert"])
.args([
"target/client",
"localhost",
&port_str,
"test-ca/rsa/ca.cert",
])
.stdout(Stdio::piped())
.output()
.map(print_output)
.unwrap();

let rustls_failed_output = Command::new("tests/maybe-valgrind.sh")
.args(["target/client", "localhost", "4444", "test-ca/rsa/ca.cert"])
.args([
"target/client",
"localhost",
&port_str,
"test-ca/rsa/ca.cert",
])
.stdout(Stdio::piped())
.output()
.map(print_output)
Expand Down Expand Up @@ -273,14 +299,16 @@ fn ciphers() {
#[test]
#[ignore]
fn server() {
fn curl() {
let (port, port_str) = choose_port();

fn curl(port: u16) {
Command::new("curl")
.env("LD_LIBRARY_PATH", "")
.args([
"-v",
"--cacert",
"test-ca/rsa/ca.cert",
"https://localhost:5555/",
&format!("https://localhost:{port}/"),
])
.stdout(Stdio::piped())
.output()
Expand All @@ -293,7 +321,7 @@ fn server() {
.env("LD_LIBRARY_PATH", "")
.args([
"target/server",
"5555",
&port_str,
"test-ca/rsa/server.key",
"test-ca/rsa/server.cert",
"unauth",
Expand All @@ -304,15 +332,15 @@ fn server() {
.unwrap(),
));
wait_for_stdout(openssl_server.0.as_mut().unwrap(), b"listening\n");
curl();
curl(port);

let openssl_output = print_output(openssl_server.take_inner().wait_with_output().unwrap());
let openssl_output = print_output(openssl_server.wait_with_timeout());

let mut rustls_server = KillOnDrop(Some(
Command::new("tests/maybe-valgrind.sh")
.args([
"target/server",
"5555",
&port_str,
"test-ca/rsa/server.key",
"test-ca/rsa/server.cert",
"unauth",
Expand All @@ -323,20 +351,20 @@ fn server() {
.unwrap(),
));
wait_for_stdout(rustls_server.0.as_mut().unwrap(), b"listening\n");
curl();
curl(port);

let rustls_output = print_output(rustls_server.take_inner().wait_with_output().unwrap());
let rustls_output = print_output(rustls_server.wait_with_timeout());
assert_eq!(openssl_output, rustls_output);
}

fn server_with_key_algorithm(key_type: &str, sig_algs: &str, version_flag: &str) {
fn connect(key_type: &str, sig_algs: &str, version_flag: &str) {
fn connect(port: u16, key_type: &str, sig_algs: &str, version_flag: &str) {
Command::new("openssl")
.env("LD_LIBRARY_PATH", "")
.args([
"s_client",
"-connect",
"localhost:5556",
&format!("localhost:{port}"),
"-sigalgs",
sig_algs,
"-CAfile",
Expand All @@ -351,12 +379,14 @@ fn server_with_key_algorithm(key_type: &str, sig_algs: &str, version_flag: &str)
.unwrap();
}

let (port, port_str) = choose_port();

let mut openssl_server = KillOnDrop(Some(
Command::new("tests/maybe-valgrind.sh")
.env("LD_LIBRARY_PATH", "")
.args([
"target/server",
"5556",
&port_str,
&format!("test-ca/{key_type}/server.key"),
&format!("test-ca/{key_type}/server.cert"),
"unauth",
Expand All @@ -367,15 +397,15 @@ fn server_with_key_algorithm(key_type: &str, sig_algs: &str, version_flag: &str)
.unwrap(),
));
wait_for_stdout(openssl_server.0.as_mut().unwrap(), b"listening\n");
connect(key_type, sig_algs, version_flag);
connect(port, key_type, sig_algs, version_flag);

let openssl_output = print_output(openssl_server.take_inner().wait_with_output().unwrap());
let openssl_output = print_output(openssl_server.wait_with_timeout());

let mut rustls_server = KillOnDrop(Some(
Command::new("tests/maybe-valgrind.sh")
.args([
"target/server",
"5556",
&port_str,
&format!("test-ca/{key_type}/server.key"),
&format!("test-ca/{key_type}/server.cert"),
"unauth",
Expand All @@ -386,9 +416,9 @@ fn server_with_key_algorithm(key_type: &str, sig_algs: &str, version_flag: &str)
.unwrap(),
));
wait_for_stdout(rustls_server.0.as_mut().unwrap(), b"listening\n");
connect(key_type, sig_algs, version_flag);
connect(port, key_type, sig_algs, version_flag);

let rustls_output = print_output(rustls_server.take_inner().wait_with_output().unwrap());
let rustls_output = print_output(rustls_server.wait_with_timeout());
assert_eq!(openssl_output, rustls_output);
}

Expand Down Expand Up @@ -511,6 +541,26 @@ impl KillOnDrop {
fn take_inner(&mut self) -> Child {
self.0.take().unwrap()
}

fn wait_with_timeout(&mut self) -> Output {
let mut child = self.take_inner();

// close stdin in case child is waiting for us
child.stdin.take();

let timeout_secs = 30;
let deadline = time::SystemTime::now().add(time::Duration::from_secs(timeout_secs));

loop {
if time::SystemTime::now() > deadline {
panic!("subprocess did not end within {timeout_secs} seconds");
}
if child.try_wait().expect("subprocess broken").is_some() {
return child.wait_with_output().unwrap();
};
thread::sleep(time::Duration::from_millis(500));
}
}
}

impl Drop for KillOnDrop {
Expand Down Expand Up @@ -557,18 +607,33 @@ fn wait_for_port(port: u16) -> Option<()> {
/// on a given `Child`, this must not read bytes from its `stdout`
/// that appear after `expected`.
fn wait_for_stdout(stream: &mut Child, expected: &[u8]) {
let stdout = stream.stdout.as_mut().unwrap();

let mut buffer = Vec::with_capacity(1024);
let mut buffer = Vec::with_capacity(128);

loop {
let mut input = [0u8];
let new = stdout.read(&mut input).unwrap();
assert_eq!(new, 1);
let new = stream.stdout.as_mut().unwrap().read(&mut input).unwrap();
assert_eq!(new, 1, "stdout EOF -- read so far {buffer:?}");
buffer.push(input[0]);

if buffer.ends_with(expected) {
return;
}

assert!(
buffer.len() < 128,
"{expected:?} did not appear in first part of {stream:?} (instead it was {buffer:?}"
);

match stream.try_wait() {
Ok(Some(status)) => panic!("process exited already with {status}"),
Ok(None) => {}
Err(e) => panic!("subprocess broken {e:?}"),
};
}
}

fn choose_port() -> (u16, String) {
static NEXT_PORT: atomic::AtomicU16 = atomic::AtomicU16::new(5555);
let port = NEXT_PORT.fetch_add(1, atomic::Ordering::SeqCst);
(port, port.to_string())
}
4 changes: 2 additions & 2 deletions rustls-libssl/tests/server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ int main(int argc, char **argv) {
us.sin_family = AF_INET;
us.sin_addr.s_addr = htonl(INADDR_ANY);
us.sin_port = htons(atoi(port));
TRACE(bind(listener, (struct sockaddr *)&us, sizeof(us)));
TRACE(listen(listener, 5));
REQUIRE(0, bind(listener, (struct sockaddr *)&us, sizeof(us)));
REQUIRE(0, listen(listener, 5));
printf("listening\n");
fflush(stdout);
socklen_t them_len = sizeof(them);
Expand Down