From 470c059e691a4e7befa5e2a7ad0110a2b0055b50 Mon Sep 17 00:00:00 2001 From: Jakob Schikowski Date: Thu, 26 Nov 2020 21:15:15 +0100 Subject: [PATCH 1/2] libtest: Print the total time taken to execute a test suite --- library/test/src/console.rs | 12 ++++++++++- library/test/src/formatters/json.rs | 15 +++++++++---- library/test/src/formatters/pretty.rs | 11 ++++++++-- library/test/src/formatters/terse.rs | 11 ++++++++-- library/test/src/tests.rs | 1 + library/test/src/time.rs | 15 +++++++++++-- .../run-make-fulldeps/libtest-json/Makefile | 6 +++--- .../libtest-json/output-default.json | 2 +- .../libtest-json/output-stdout-success.json | 2 +- src/test/rustdoc-ui/cfg-test.rs | 1 + src/test/rustdoc-ui/cfg-test.stdout | 6 +++--- .../rustdoc-ui/doc-test-doctest-feature.rs | 1 + .../doc-test-doctest-feature.stdout | 4 ++-- .../rustdoc-ui/doc-test-rustdoc-feature.rs | 1 + .../doc-test-rustdoc-feature.stdout | 4 ++-- src/test/rustdoc-ui/doctest-output.rs | 1 + src/test/rustdoc-ui/doctest-output.stdout | 8 +++---- .../rustdoc-ui/failed-doctest-compile-fail.rs | 1 + .../failed-doctest-compile-fail.stdout | 8 +++---- .../failed-doctest-missing-codes.rs | 1 + .../failed-doctest-missing-codes.stdout | 10 ++++----- src/test/rustdoc-ui/failed-doctest-output.rs | 1 + .../rustdoc-ui/failed-doctest-output.stdout | 16 +++++++------- .../rustdoc-ui/failed-doctest-should-panic.rs | 1 + .../failed-doctest-should-panic.stdout | 8 +++---- src/test/rustdoc-ui/test-no_std.rs | 1 + src/test/rustdoc-ui/test-no_std.stdout | 4 ++-- src/test/rustdoc-ui/unparseable-doc-test.rs | 1 + .../rustdoc-ui/unparseable-doc-test.stdout | 10 ++++----- src/test/ui/test-panic-abort-nocapture.rs | 1 + .../ui/test-panic-abort-nocapture.run.stderr | 4 ++-- .../ui/test-panic-abort-nocapture.run.stdout | 2 +- src/test/ui/test-panic-abort.rs | 1 + src/test/ui/test-panic-abort.run.stdout | 4 ++-- src/test/ui/test-passed-wasm.rs | 20 ++++++++++++++++++ src/test/ui/test-passed-wasm.run.stdout | 7 +++++++ src/test/ui/test-passed.rs | 21 +++++++++++++++++++ src/test/ui/test-passed.run.stdout | 7 +++++++ src/test/ui/test-thread-capture.rs | 1 + src/test/ui/test-thread-capture.run.stdout | 4 ++-- src/test/ui/test-thread-nocapture.rs | 1 + src/test/ui/test-thread-nocapture.run.stderr | 2 +- src/test/ui/test-thread-nocapture.run.stdout | 2 +- 43 files changed, 176 insertions(+), 64 deletions(-) create mode 100644 src/test/ui/test-passed-wasm.rs create mode 100644 src/test/ui/test-passed-wasm.run.stdout create mode 100644 src/test/ui/test-passed.rs create mode 100644 src/test/ui/test-passed.run.stdout diff --git a/library/test/src/console.rs b/library/test/src/console.rs index ff741e3bd53be..1721c3c14f9a9 100644 --- a/library/test/src/console.rs +++ b/library/test/src/console.rs @@ -3,6 +3,7 @@ use std::fs::File; use std::io; use std::io::prelude::Write; +use std::time::Instant; use super::{ bench::fmt_bench_samples, @@ -14,7 +15,7 @@ use super::{ options::{Options, OutputFormat}, run_tests, test_result::TestResult, - time::TestExecTime, + time::{TestExecTime, TestSuiteExecTime}, types::{NamePadding, TestDesc, TestDescAndFn}, }; @@ -49,6 +50,7 @@ pub struct ConsoleTestState { pub allowed_fail: usize, pub filtered_out: usize, pub measured: usize, + pub exec_time: Option, pub metrics: MetricMap, pub failures: Vec<(TestDesc, Vec)>, pub not_failures: Vec<(TestDesc, Vec)>, @@ -72,6 +74,7 @@ impl ConsoleTestState { allowed_fail: 0, filtered_out: 0, measured: 0, + exec_time: None, metrics: MetricMap::new(), failures: Vec::new(), not_failures: Vec::new(), @@ -277,7 +280,14 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu }; let mut st = ConsoleTestState::new(opts)?; + // Prevent the usage of `Instant` in some cases: + // - It's currently not supported for wasm targets. + // - We disable it for miri because it's not available when isolation is enabled. + let is_instant_supported = !cfg!(target_arch = "wasm32") && !cfg!(miri); + + let start_time = is_instant_supported.then(Instant::now); run_tests(opts, tests, |x| on_test_event(&x, &mut st, &mut *out))?; + st.exec_time = start_time.map(|t| TestSuiteExecTime(t.elapsed())); assert!(st.current_test_count() == st.total); diff --git a/library/test/src/formatters/json.rs b/library/test/src/formatters/json.rs index 4dc4162700c6e..57b6d1a02021f 100644 --- a/library/test/src/formatters/json.rs +++ b/library/test/src/formatters/json.rs @@ -47,7 +47,7 @@ impl JsonFormatter { evt ))?; if let Some(exec_time) = exec_time { - self.write_message(&*format!(r#", "exec_time": "{}""#, exec_time))?; + self.write_message(&*format!(r#", "exec_time": {}"#, exec_time.0.as_secs_f64()))?; } if let Some(stdout) = stdout { self.write_message(&*format!(r#", "stdout": "{}""#, EscapedString(stdout)))?; @@ -162,7 +162,7 @@ impl OutputFormatter for JsonFormatter { } fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { - self.writeln_message(&*format!( + self.write_message(&*format!( "{{ \"type\": \"suite\", \ \"event\": \"{}\", \ \"passed\": {}, \ @@ -170,16 +170,23 @@ impl OutputFormatter for JsonFormatter { \"allowed_fail\": {}, \ \"ignored\": {}, \ \"measured\": {}, \ - \"filtered_out\": {} }}", + \"filtered_out\": {}", if state.failed == 0 { "ok" } else { "failed" }, state.passed, state.failed + state.allowed_fail, state.allowed_fail, state.ignored, state.measured, - state.filtered_out + state.filtered_out, ))?; + if let Some(ref exec_time) = state.exec_time { + let time_str = format!(", \"exec_time\": {}", exec_time.0.as_secs_f64()); + self.write_message(&time_str)?; + } + + self.writeln_message(" }")?; + Ok(state.failed == 0) } } diff --git a/library/test/src/formatters/pretty.rs b/library/test/src/formatters/pretty.rs index 8c90b57b3bac3..6fa369298416b 100644 --- a/library/test/src/formatters/pretty.rs +++ b/library/test/src/formatters/pretty.rs @@ -259,7 +259,7 @@ impl OutputFormatter for PrettyFormatter { let s = if state.allowed_fail > 0 { format!( - ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out", state.passed, state.failed + state.allowed_fail, state.allowed_fail, @@ -269,13 +269,20 @@ impl OutputFormatter for PrettyFormatter { ) } else { format!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out", state.passed, state.failed, state.ignored, state.measured, state.filtered_out ) }; self.write_plain(&s)?; + if let Some(ref exec_time) = state.exec_time { + let time_str = format!("; finished in {}", exec_time); + self.write_plain(&time_str)?; + } + + self.write_plain("\n\n")?; + Ok(success) } } diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs index 1ae7846a99e3a..6f46f7255a47e 100644 --- a/library/test/src/formatters/terse.rs +++ b/library/test/src/formatters/terse.rs @@ -236,7 +236,7 @@ impl OutputFormatter for TerseFormatter { let s = if state.allowed_fail > 0 { format!( - ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out", state.passed, state.failed + state.allowed_fail, state.allowed_fail, @@ -246,13 +246,20 @@ impl OutputFormatter for TerseFormatter { ) } else { format!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out", state.passed, state.failed, state.ignored, state.measured, state.filtered_out ) }; self.write_plain(&s)?; + if let Some(ref exec_time) = state.exec_time { + let time_str = format!("; finished in {}", exec_time); + self.write_plain(&time_str)?; + } + + self.write_plain("\n\n")?; + Ok(success) } } diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs index 85a0705f69c0a..74313cc4438ed 100644 --- a/library/test/src/tests.rs +++ b/library/test/src/tests.rs @@ -669,6 +669,7 @@ fn should_sort_failures_before_printing_them() { allowed_fail: 0, filtered_out: 0, measured: 0, + exec_time: None, metrics: MetricMap::new(), failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], options: Options::new(), diff --git a/library/test/src/time.rs b/library/test/src/time.rs index 96c090f9b0163..130792fa5d7e7 100644 --- a/library/test/src/time.rs +++ b/library/test/src/time.rs @@ -1,8 +1,9 @@ //! Module `time` contains everything related to the time measurement of unit tests //! execution. -//! Two main purposes of this module: +//! The purposes of this module: //! - Check whether test is timed out. //! - Provide helpers for `report-time` and `measure-time` options. +//! - Provide newtypes for executions times. use std::env; use std::fmt; @@ -60,7 +61,7 @@ pub fn get_default_test_timeout() -> Instant { Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S) } -/// The meassured execution time of a unit test. +/// The measured execution time of a unit test. #[derive(Debug, Clone, PartialEq)] pub struct TestExecTime(pub Duration); @@ -70,6 +71,16 @@ impl fmt::Display for TestExecTime { } } +/// The measured execution time of the whole test suite. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TestSuiteExecTime(pub Duration); + +impl fmt::Display for TestSuiteExecTime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:.2}s", self.0.as_secs_f64()) + } +} + /// Structure denoting time limits for test execution. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub struct TimeThreshold { diff --git a/src/test/run-make-fulldeps/libtest-json/Makefile b/src/test/run-make-fulldeps/libtest-json/Makefile index 8339e230bbe92..4f8a24447d1f7 100644 --- a/src/test/run-make-fulldeps/libtest-json/Makefile +++ b/src/test/run-make-fulldeps/libtest-json/Makefile @@ -13,6 +13,6 @@ all: cat $(OUTPUT_FILE_DEFAULT) | "$(PYTHON)" validate_json.py cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_json.py - # Compare to output file - diff output-default.json $(OUTPUT_FILE_DEFAULT) - diff output-stdout-success.json $(OUTPUT_FILE_STDOUT_SUCCESS) + # Normalize the actual output and compare to expected output file + cat $(OUTPUT_FILE_DEFAULT) | sed -r 's/\"exec_time\": [0-9]+(\.[0-9]+)?/\"exec_time\": \$$TIME/' | diff output-default.json - + cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed -r 's/\"exec_time\": [0-9]+(\.[0-9]+)?/\"exec_time\": \$$TIME/' | diff output-stdout-success.json - diff --git a/src/test/run-make-fulldeps/libtest-json/output-default.json b/src/test/run-make-fulldeps/libtest-json/output-default.json index 0cd9ab79e32f3..099b65a23cad6 100644 --- a/src/test/run-make-fulldeps/libtest-json/output-default.json +++ b/src/test/run-make-fulldeps/libtest-json/output-default.json @@ -7,4 +7,4 @@ { "type": "test", "name": "c", "event": "ok" } { "type": "test", "event": "started", "name": "d" } { "type": "test", "name": "d", "event": "ignored" } -{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": 0 } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": $TIME } diff --git a/src/test/run-make-fulldeps/libtest-json/output-stdout-success.json b/src/test/run-make-fulldeps/libtest-json/output-stdout-success.json index dfaf005052e55..fd676799a7664 100644 --- a/src/test/run-make-fulldeps/libtest-json/output-stdout-success.json +++ b/src/test/run-make-fulldeps/libtest-json/output-stdout-success.json @@ -7,4 +7,4 @@ { "type": "test", "name": "c", "event": "ok", "stdout": "thread 'main' panicked at 'assertion failed: false', f.rs:15:5\n" } { "type": "test", "event": "started", "name": "d" } { "type": "test", "name": "d", "event": "ignored" } -{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": 0 } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": $TIME } diff --git a/src/test/rustdoc-ui/cfg-test.rs b/src/test/rustdoc-ui/cfg-test.rs index 597c86a1f19ca..d4ca925858714 100644 --- a/src/test/rustdoc-ui/cfg-test.rs +++ b/src/test/rustdoc-ui/cfg-test.rs @@ -1,6 +1,7 @@ // check-pass // compile-flags:--test --test-args --test-threads=1 // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // Crates like core have doctests gated on `cfg(not(test))` so we need to make // sure `cfg(test)` is not active when running `rustdoc --test`. diff --git a/src/test/rustdoc-ui/cfg-test.stdout b/src/test/rustdoc-ui/cfg-test.stdout index 474f13cfa9843..2960ff8d3b473 100644 --- a/src/test/rustdoc-ui/cfg-test.stdout +++ b/src/test/rustdoc-ui/cfg-test.stdout @@ -1,7 +1,7 @@ running 2 tests -test $DIR/cfg-test.rs - Bar (line 26) ... ok -test $DIR/cfg-test.rs - Foo (line 18) ... ok +test $DIR/cfg-test.rs - Bar (line 27) ... ok +test $DIR/cfg-test.rs - Foo (line 19) ... ok -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/doc-test-doctest-feature.rs b/src/test/rustdoc-ui/doc-test-doctest-feature.rs index 2798804880ad0..0b79aaece8a03 100644 --- a/src/test/rustdoc-ui/doc-test-doctest-feature.rs +++ b/src/test/rustdoc-ui/doc-test-doctest-feature.rs @@ -1,6 +1,7 @@ // check-pass // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // Make sure `cfg(doctest)` is set when finding doctests but not inside // the doctests. diff --git a/src/test/rustdoc-ui/doc-test-doctest-feature.stdout b/src/test/rustdoc-ui/doc-test-doctest-feature.stdout index b1cd74bf8520a..d7de1f105228f 100644 --- a/src/test/rustdoc-ui/doc-test-doctest-feature.stdout +++ b/src/test/rustdoc-ui/doc-test-doctest-feature.stdout @@ -1,6 +1,6 @@ running 1 test -test $DIR/doc-test-doctest-feature.rs - Foo (line 8) ... ok +test $DIR/doc-test-doctest-feature.rs - Foo (line 9) ... ok -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/doc-test-rustdoc-feature.rs b/src/test/rustdoc-ui/doc-test-rustdoc-feature.rs index 2af5782453e6d..bf334c67ecab7 100644 --- a/src/test/rustdoc-ui/doc-test-rustdoc-feature.rs +++ b/src/test/rustdoc-ui/doc-test-rustdoc-feature.rs @@ -1,6 +1,7 @@ // check-pass // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" #![feature(doc_cfg)] diff --git a/src/test/rustdoc-ui/doc-test-rustdoc-feature.stdout b/src/test/rustdoc-ui/doc-test-rustdoc-feature.stdout index f2525c2dbec21..5b07fc4c87af5 100644 --- a/src/test/rustdoc-ui/doc-test-rustdoc-feature.stdout +++ b/src/test/rustdoc-ui/doc-test-rustdoc-feature.stdout @@ -1,6 +1,6 @@ running 1 test -test $DIR/doc-test-rustdoc-feature.rs - Foo (line 9) ... ok +test $DIR/doc-test-rustdoc-feature.rs - Foo (line 10) ... ok -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/doctest-output.rs b/src/test/rustdoc-ui/doctest-output.rs index e0e1e061ac7dc..2670fa572010d 100644 --- a/src/test/rustdoc-ui/doctest-output.rs +++ b/src/test/rustdoc-ui/doctest-output.rs @@ -2,6 +2,7 @@ // aux-build:extern_macros.rs // compile-flags:--test --test-args=--test-threads=1 // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // check-pass //! ``` diff --git a/src/test/rustdoc-ui/doctest-output.stdout b/src/test/rustdoc-ui/doctest-output.stdout index c72bd91d1dd13..35b0e366fb5cc 100644 --- a/src/test/rustdoc-ui/doctest-output.stdout +++ b/src/test/rustdoc-ui/doctest-output.stdout @@ -1,8 +1,8 @@ running 3 tests -test $DIR/doctest-output.rs - (line 7) ... ok -test $DIR/doctest-output.rs - ExpandedStruct (line 23) ... ok -test $DIR/doctest-output.rs - foo::bar (line 17) ... ok +test $DIR/doctest-output.rs - (line 8) ... ok +test $DIR/doctest-output.rs - ExpandedStruct (line 24) ... ok +test $DIR/doctest-output.rs - foo::bar (line 18) ... ok -test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/failed-doctest-compile-fail.rs b/src/test/rustdoc-ui/failed-doctest-compile-fail.rs index 297d6efd45fee..6f2ff5d703a11 100644 --- a/src/test/rustdoc-ui/failed-doctest-compile-fail.rs +++ b/src/test/rustdoc-ui/failed-doctest-compile-fail.rs @@ -3,6 +3,7 @@ // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // failure-status: 101 /// ```compile_fail diff --git a/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout b/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout index 74e33d7beebeb..b8bb5ccb40329 100644 --- a/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout +++ b/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout @@ -1,14 +1,14 @@ running 1 test -test $DIR/failed-doctest-compile-fail.rs - Foo (line 8) ... FAILED +test $DIR/failed-doctest-compile-fail.rs - Foo (line 9) ... FAILED failures: ----- $DIR/failed-doctest-compile-fail.rs - Foo (line 8) stdout ---- +---- $DIR/failed-doctest-compile-fail.rs - Foo (line 9) stdout ---- Test compiled successfully, but it's marked `compile_fail`. failures: - $DIR/failed-doctest-compile-fail.rs - Foo (line 8) + $DIR/failed-doctest-compile-fail.rs - Foo (line 9) -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/failed-doctest-missing-codes.rs b/src/test/rustdoc-ui/failed-doctest-missing-codes.rs index 62102062d4991..57b70b478e624 100644 --- a/src/test/rustdoc-ui/failed-doctest-missing-codes.rs +++ b/src/test/rustdoc-ui/failed-doctest-missing-codes.rs @@ -3,6 +3,7 @@ // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // failure-status: 101 /// ```compile_fail,E0004 diff --git a/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout b/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout index e4ed462232232..7367a7d651919 100644 --- a/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout +++ b/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout @@ -1,12 +1,12 @@ running 1 test -test $DIR/failed-doctest-missing-codes.rs - Foo (line 8) ... FAILED +test $DIR/failed-doctest-missing-codes.rs - Foo (line 9) ... FAILED failures: ----- $DIR/failed-doctest-missing-codes.rs - Foo (line 8) stdout ---- +---- $DIR/failed-doctest-missing-codes.rs - Foo (line 9) stdout ---- error[E0308]: mismatched types - --> $DIR/failed-doctest-missing-codes.rs:9:13 + --> $DIR/failed-doctest-missing-codes.rs:10:13 | LL | let x: () = 5i32; | -- ^^^^ expected `()`, found `i32` @@ -19,7 +19,7 @@ For more information about this error, try `rustc --explain E0308`. Some expected error codes were not found: ["E0004"] failures: - $DIR/failed-doctest-missing-codes.rs - Foo (line 8) + $DIR/failed-doctest-missing-codes.rs - Foo (line 9) -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/failed-doctest-output.rs b/src/test/rustdoc-ui/failed-doctest-output.rs index 90cdb5127bea2..92473b49e1483 100644 --- a/src/test/rustdoc-ui/failed-doctest-output.rs +++ b/src/test/rustdoc-ui/failed-doctest-output.rs @@ -5,6 +5,7 @@ // compile-flags:--test --test-args --test-threads=1 // rustc-env:RUST_BACKTRACE=0 // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // failure-status: 101 // doctest fails at runtime diff --git a/src/test/rustdoc-ui/failed-doctest-output.stdout b/src/test/rustdoc-ui/failed-doctest-output.stdout index ee79ae1a690ec..6dfe648f8549e 100644 --- a/src/test/rustdoc-ui/failed-doctest-output.stdout +++ b/src/test/rustdoc-ui/failed-doctest-output.stdout @@ -1,13 +1,13 @@ running 2 tests -test $DIR/failed-doctest-output.rs - OtherStruct (line 21) ... FAILED -test $DIR/failed-doctest-output.rs - SomeStruct (line 11) ... FAILED +test $DIR/failed-doctest-output.rs - OtherStruct (line 22) ... FAILED +test $DIR/failed-doctest-output.rs - SomeStruct (line 12) ... FAILED failures: ----- $DIR/failed-doctest-output.rs - OtherStruct (line 21) stdout ---- +---- $DIR/failed-doctest-output.rs - OtherStruct (line 22) stdout ---- error[E0425]: cannot find value `no` in this scope - --> $DIR/failed-doctest-output.rs:22:1 + --> $DIR/failed-doctest-output.rs:23:1 | LL | no | ^^ not found in this scope @@ -16,7 +16,7 @@ error: aborting due to previous error For more information about this error, try `rustc --explain E0425`. Couldn't compile the test. ----- $DIR/failed-doctest-output.rs - SomeStruct (line 11) stdout ---- +---- $DIR/failed-doctest-output.rs - SomeStruct (line 12) stdout ---- Test executable failed (exit code 101). stdout: @@ -32,8 +32,8 @@ note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace failures: - $DIR/failed-doctest-output.rs - OtherStruct (line 21) - $DIR/failed-doctest-output.rs - SomeStruct (line 11) + $DIR/failed-doctest-output.rs - OtherStruct (line 22) + $DIR/failed-doctest-output.rs - SomeStruct (line 12) -test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/failed-doctest-should-panic.rs b/src/test/rustdoc-ui/failed-doctest-should-panic.rs index 400fb97804aab..2b8bb31686f2e 100644 --- a/src/test/rustdoc-ui/failed-doctest-should-panic.rs +++ b/src/test/rustdoc-ui/failed-doctest-should-panic.rs @@ -3,6 +3,7 @@ // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // failure-status: 101 /// ```should_panic diff --git a/src/test/rustdoc-ui/failed-doctest-should-panic.stdout b/src/test/rustdoc-ui/failed-doctest-should-panic.stdout index 081b64b50af9b..57a20092a5d6c 100644 --- a/src/test/rustdoc-ui/failed-doctest-should-panic.stdout +++ b/src/test/rustdoc-ui/failed-doctest-should-panic.stdout @@ -1,14 +1,14 @@ running 1 test -test $DIR/failed-doctest-should-panic.rs - Foo (line 8) ... FAILED +test $DIR/failed-doctest-should-panic.rs - Foo (line 9) ... FAILED failures: ----- $DIR/failed-doctest-should-panic.rs - Foo (line 8) stdout ---- +---- $DIR/failed-doctest-should-panic.rs - Foo (line 9) stdout ---- Test executable succeeded, but it's marked `should_panic`. failures: - $DIR/failed-doctest-should-panic.rs - Foo (line 8) + $DIR/failed-doctest-should-panic.rs - Foo (line 9) -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/test-no_std.rs b/src/test/rustdoc-ui/test-no_std.rs index af4843ad32405..ee919985e7a91 100644 --- a/src/test/rustdoc-ui/test-no_std.rs +++ b/src/test/rustdoc-ui/test-no_std.rs @@ -1,5 +1,6 @@ // compile-flags:--test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // check-pass #![no_std] diff --git a/src/test/rustdoc-ui/test-no_std.stdout b/src/test/rustdoc-ui/test-no_std.stdout index 9cdcac2a483ab..8d5a30804c1e2 100644 --- a/src/test/rustdoc-ui/test-no_std.stdout +++ b/src/test/rustdoc-ui/test-no_std.stdout @@ -1,6 +1,6 @@ running 1 test -test $DIR/test-no_std.rs - f (line 9) ... ok +test $DIR/test-no_std.rs - f (line 10) ... ok -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/rustdoc-ui/unparseable-doc-test.rs b/src/test/rustdoc-ui/unparseable-doc-test.rs index 18d6b32bf4037..0cff8cd9a33a8 100644 --- a/src/test/rustdoc-ui/unparseable-doc-test.rs +++ b/src/test/rustdoc-ui/unparseable-doc-test.rs @@ -1,5 +1,6 @@ // compile-flags: --test // normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR" +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // failure-status: 101 // rustc-env: RUST_BACKTRACE=0 diff --git a/src/test/rustdoc-ui/unparseable-doc-test.stdout b/src/test/rustdoc-ui/unparseable-doc-test.stdout index 29cb22e2e4b09..2641c66f25e77 100644 --- a/src/test/rustdoc-ui/unparseable-doc-test.stdout +++ b/src/test/rustdoc-ui/unparseable-doc-test.stdout @@ -1,12 +1,12 @@ running 1 test -test $DIR/unparseable-doc-test.rs - foo (line 6) ... FAILED +test $DIR/unparseable-doc-test.rs - foo (line 7) ... FAILED failures: ----- $DIR/unparseable-doc-test.rs - foo (line 6) stdout ---- +---- $DIR/unparseable-doc-test.rs - foo (line 7) stdout ---- error[E0765]: unterminated double quote string - --> $DIR/unparseable-doc-test.rs:8:1 + --> $DIR/unparseable-doc-test.rs:9:1 | LL | "unterminated | ^^^^^^^^^^^^^ @@ -17,7 +17,7 @@ For more information about this error, try `rustc --explain E0765`. Couldn't compile the test. failures: - $DIR/unparseable-doc-test.rs - foo (line 6) + $DIR/unparseable-doc-test.rs - foo (line 7) -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/ui/test-panic-abort-nocapture.rs b/src/test/ui/test-panic-abort-nocapture.rs index 978732a9ec374..af530cc1a0b8f 100644 --- a/src/test/ui/test-panic-abort-nocapture.rs +++ b/src/test/ui/test-panic-abort-nocapture.rs @@ -4,6 +4,7 @@ // run-fail // check-run-results // exec-env:RUST_BACKTRACE=0 +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // ignore-wasm no panic or subprocess support // ignore-emscripten no panic or subprocess support diff --git a/src/test/ui/test-panic-abort-nocapture.run.stderr b/src/test/ui/test-panic-abort-nocapture.run.stderr index 3388813d5a0bd..727e9691c53a1 100644 --- a/src/test/ui/test-panic-abort-nocapture.run.stderr +++ b/src/test/ui/test-panic-abort-nocapture.run.stderr @@ -1,9 +1,9 @@ thread 'main' panicked at 'assertion failed: `(left == right)` left: `2`, - right: `4`', $DIR/test-panic-abort-nocapture.rs:32:5 + right: `4`', $DIR/test-panic-abort-nocapture.rs:33:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace thread 'main' panicked at 'assertion failed: `(left == right)` left: `2`, - right: `4`', $DIR/test-panic-abort-nocapture.rs:26:5 + right: `4`', $DIR/test-panic-abort-nocapture.rs:27:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace testing321 diff --git a/src/test/ui/test-panic-abort-nocapture.run.stdout b/src/test/ui/test-panic-abort-nocapture.run.stdout index 87a246db5e07b..15b19676a7c2d 100644 --- a/src/test/ui/test-panic-abort-nocapture.run.stdout +++ b/src/test/ui/test-panic-abort-nocapture.run.stdout @@ -19,5 +19,5 @@ failures: failures: it_fails -test result: FAILED. 3 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 3 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/ui/test-panic-abort.rs b/src/test/ui/test-panic-abort.rs index 21e7dc393f512..931b7993c8144 100644 --- a/src/test/ui/test-panic-abort.rs +++ b/src/test/ui/test-panic-abort.rs @@ -4,6 +4,7 @@ // run-fail // check-run-results // exec-env:RUST_BACKTRACE=0 +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // ignore-wasm no panic or subprocess support // ignore-emscripten no panic or subprocess support diff --git a/src/test/ui/test-panic-abort.run.stdout b/src/test/ui/test-panic-abort.run.stdout index 33ddd519030ff..467f834afecbf 100644 --- a/src/test/ui/test-panic-abort.run.stdout +++ b/src/test/ui/test-panic-abort.run.stdout @@ -18,7 +18,7 @@ testing123 testing321 thread 'main' panicked at 'assertion failed: `(left == right)` left: `2`, - right: `5`', $DIR/test-panic-abort.rs:33:5 + right: `5`', $DIR/test-panic-abort.rs:34:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace @@ -26,5 +26,5 @@ failures: it_exits it_fails -test result: FAILED. 3 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 3 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/ui/test-passed-wasm.rs b/src/test/ui/test-passed-wasm.rs new file mode 100644 index 0000000000000..578aa4b1760c4 --- /dev/null +++ b/src/test/ui/test-passed-wasm.rs @@ -0,0 +1,20 @@ +// no-prefer-dynamic +// compile-flags: --test +// run-flags: --test-threads=1 +// run-pass +// check-run-results +// only-wasm32 + +// Tests the output of the test harness with only passed tests. + +#![cfg(test)] + +#[test] +fn it_works() { + assert_eq!(1 + 1, 2); +} + +#[test] +fn it_works_too() { + assert_eq!(1 * 0, 0); +} diff --git a/src/test/ui/test-passed-wasm.run.stdout b/src/test/ui/test-passed-wasm.run.stdout new file mode 100644 index 0000000000000..c3005a77983e3 --- /dev/null +++ b/src/test/ui/test-passed-wasm.run.stdout @@ -0,0 +1,7 @@ + +running 2 tests +test it_works ... ok +test it_works_too ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out + diff --git a/src/test/ui/test-passed.rs b/src/test/ui/test-passed.rs new file mode 100644 index 0000000000000..f65f0003022a1 --- /dev/null +++ b/src/test/ui/test-passed.rs @@ -0,0 +1,21 @@ +// no-prefer-dynamic +// compile-flags: --test +// run-flags: --test-threads=1 +// run-pass +// check-run-results +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" +// ignore-wasm32 no support for `Instant` + +// Tests the output of the test harness with only passed tests. + +#![cfg(test)] + +#[test] +fn it_works() { + assert_eq!(1 + 1, 2); +} + +#[test] +fn it_works_too() { + assert_eq!(1 * 0, 0); +} diff --git a/src/test/ui/test-passed.run.stdout b/src/test/ui/test-passed.run.stdout new file mode 100644 index 0000000000000..17f70d607494e --- /dev/null +++ b/src/test/ui/test-passed.run.stdout @@ -0,0 +1,7 @@ + +running 2 tests +test it_works ... ok +test it_works_too ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME + diff --git a/src/test/ui/test-thread-capture.rs b/src/test/ui/test-thread-capture.rs index 6bec48cd81678..edc972837a3f7 100644 --- a/src/test/ui/test-thread-capture.rs +++ b/src/test/ui/test-thread-capture.rs @@ -3,6 +3,7 @@ // run-flags: --test-threads=1 // check-run-results // exec-env:RUST_BACKTRACE=0 +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // ignore-emscripten no threads support #[test] diff --git a/src/test/ui/test-thread-capture.run.stdout b/src/test/ui/test-thread-capture.run.stdout index 1102aadab02a9..487cfb55eb473 100644 --- a/src/test/ui/test-thread-capture.run.stdout +++ b/src/test/ui/test-thread-capture.run.stdout @@ -10,12 +10,12 @@ fee fie foe fum -thread 'main' panicked at 'explicit panic', $DIR/test-thread-capture.rs:30:5 +thread 'main' panicked at 'explicit panic', $DIR/test-thread-capture.rs:31:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace failures: thready_fail -test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME diff --git a/src/test/ui/test-thread-nocapture.rs b/src/test/ui/test-thread-nocapture.rs index 82df6e77cb1da..8e8e9bbfdf0ed 100644 --- a/src/test/ui/test-thread-nocapture.rs +++ b/src/test/ui/test-thread-nocapture.rs @@ -3,6 +3,7 @@ // run-flags: --test-threads=1 --nocapture // check-run-results // exec-env:RUST_BACKTRACE=0 +// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME" // ignore-emscripten no threads support #[test] diff --git a/src/test/ui/test-thread-nocapture.run.stderr b/src/test/ui/test-thread-nocapture.run.stderr index 98bd96d0abe83..06495681b3eb2 100644 --- a/src/test/ui/test-thread-nocapture.run.stderr +++ b/src/test/ui/test-thread-nocapture.run.stderr @@ -1,2 +1,2 @@ -thread 'main' panicked at 'explicit panic', $DIR/test-thread-nocapture.rs:30:5 +thread 'main' panicked at 'explicit panic', $DIR/test-thread-nocapture.rs:31:5 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace diff --git a/src/test/ui/test-thread-nocapture.run.stdout b/src/test/ui/test-thread-nocapture.run.stdout index 77b42ed88d63f..9d2da50826c25 100644 --- a/src/test/ui/test-thread-nocapture.run.stdout +++ b/src/test/ui/test-thread-nocapture.run.stdout @@ -16,5 +16,5 @@ failures: failures: thready_fail -test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out +test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME From 322f53d8a8da14da5371ca040ade5497112178cf Mon Sep 17 00:00:00 2001 From: Jakob Schikowski Date: Sat, 28 Nov 2020 10:23:36 +0100 Subject: [PATCH 2/2] libtest: Make `sed` arguments compatible with apple --- src/test/run-make-fulldeps/libtest-json/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/run-make-fulldeps/libtest-json/Makefile b/src/test/run-make-fulldeps/libtest-json/Makefile index 4f8a24447d1f7..f61a1aa84100f 100644 --- a/src/test/run-make-fulldeps/libtest-json/Makefile +++ b/src/test/run-make-fulldeps/libtest-json/Makefile @@ -14,5 +14,5 @@ all: cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_json.py # Normalize the actual output and compare to expected output file - cat $(OUTPUT_FILE_DEFAULT) | sed -r 's/\"exec_time\": [0-9]+(\.[0-9]+)?/\"exec_time\": \$$TIME/' | diff output-default.json - - cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed -r 's/\"exec_time\": [0-9]+(\.[0-9]+)?/\"exec_time\": \$$TIME/' | diff output-stdout-success.json - + cat $(OUTPUT_FILE_DEFAULT) | sed 's/"exec_time": [0-9.]*/"exec_time": $$TIME/' | diff output-default.json - + cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed 's/"exec_time": [0-9.]*/"exec_time": $$TIME/' | diff output-stdout-success.json -