Auto merge of #63825 - nathanwhit:check-run-results, r=Mark-Simulacrum

bors · bors · commit 6c18a3d7756c · 2019-09-04T11:37:37.000Z
Allow checking of run-pass execution output in compiletest Closes #63751 Adds a `check-run-results` flag to compiletest headers, which if enabled checks the output of the execution of a run-pass test's binary against expected output.
diff --git a/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.rs
@@ -1,7 +1,5 @@
 // run-pass
-// ignore-cloudabi no processes
-// ignore-emscripten no processes
-// ignore-sgx no processes
+// check-run-results
 
 // Tests ensuring that `dbg!(expr)` has the expected run-time behavior.
 // as well as some compile time properties we expect.
@@ -18,7 +16,7 @@ struct Point<T> {
 #[derive(Debug, PartialEq)]
 struct NoCopy(usize);
 
-fn test() {
+fn main() {
     let a: Unit = dbg!(Unit);
     let _: Unit = dbg!(a);
     // We can move `a` because it's Copy.
@@ -67,81 +65,3 @@ fn test() {
     assert_eq!((1u8, 2u32, "Yeah"), dbg!(1u8, 2u32,
                                          "Yeah",));
 }
-
-fn validate_stderr(stderr: Vec<String>) {
-    assert_eq!(stderr, &[
-        ":22] Unit = Unit",
-
-        ":23] a = Unit",
-
-        ":29] Point{x: 42, y: 24,} = Point {",
-        "    x: 42,",
-        "    y: 24,",
-        "}",
-
-        ":30] b = Point {",
-        "    x: 42,",
-        "    y: 24,",
-        "}",
-
-        ":38]",
-
-        ":42] &a = NoCopy(",
-        "    1337,",
-        ")",
-
-        ":42] dbg!(& a) = NoCopy(",
-        "    1337,",
-        ")",
-        ":47] f(&42) = 42",
-
-        "before",
-        ":52] { foo += 1; eprintln!(\"before\"); 7331 } = 7331",
-
-        ":60] (\"Yeah\",) = (",
-        "    \"Yeah\",",
-        ")",
-
-        ":63] 1 = 1",
-        ":63] 2 = 2",
-
-        ":67] 1u8 = 1",
-        ":67] 2u32 = 2",
-        ":67] \"Yeah\" = \"Yeah\"",
-    ]);
-}
-
-fn main() {
-    // The following is a hack to deal with compiletest's inability
-    // to check the output (to stdout) of run-pass tests.
-    use std::env;
-    use std::process::Command;
-
-    let mut args = env::args();
-    let prog = args.next().unwrap();
-    let child = args.next();
-    if let Some("child") = child.as_ref().map(|s| &**s) {
-        // Only run the test if we've been spawned as 'child'
-        test()
-    } else {
-        // This essentially spawns as 'child' to run the tests
-        // and then it collects output of stderr and checks the output
-        // against what we expect.
-        let out = Command::new(&prog).arg("child").output().unwrap();
-        assert!(out.status.success());
-        assert!(out.stdout.is_empty());
-
-        let stderr = String::from_utf8(out.stderr).unwrap();
-        let stderr = stderr.lines().map(|mut s| {
-            if s.starts_with("[") {
-                // Strip `[` and file path:
-                s = s.trim_start_matches("[");
-                assert!(s.starts_with(file!()));
-                s = s.trim_start_matches(file!());
-            }
-            s.to_owned()
-        }).collect();
-
-        validate_stderr(stderr);
-    }
-}
diff --git a/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr b/src/test/ui/rfc-2361-dbg-macro/dbg-macro-expected-behavior.run.stderr
@@ -0,0 +1,28 @@
+[$DIR/dbg-macro-expected-behavior.rs:20] Unit = Unit
+[$DIR/dbg-macro-expected-behavior.rs:21] a = Unit
+[$DIR/dbg-macro-expected-behavior.rs:27] Point{x: 42, y: 24,} = Point {
+    x: 42,
+    y: 24,
+}
+[$DIR/dbg-macro-expected-behavior.rs:28] b = Point {
+    x: 42,
+    y: 24,
+}
+[$DIR/dbg-macro-expected-behavior.rs:36]
+[$DIR/dbg-macro-expected-behavior.rs:40] &a = NoCopy(
+    1337,
+)
+[$DIR/dbg-macro-expected-behavior.rs:40] dbg!(& a) = NoCopy(
+    1337,
+)
+[$DIR/dbg-macro-expected-behavior.rs:45] f(&42) = 42
+before
+[$DIR/dbg-macro-expected-behavior.rs:50] { foo += 1; eprintln!("before"); 7331 } = 7331
+[$DIR/dbg-macro-expected-behavior.rs:58] ("Yeah",) = (
+    "Yeah",
+)
+[$DIR/dbg-macro-expected-behavior.rs:61] 1 = 1
+[$DIR/dbg-macro-expected-behavior.rs:61] 2 = 2
+[$DIR/dbg-macro-expected-behavior.rs:65] 1u8 = 1
+[$DIR/dbg-macro-expected-behavior.rs:65] 2u32 = 2
+[$DIR/dbg-macro-expected-behavior.rs:65] "Yeah" = "Yeah"
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
@@ -333,10 +333,12 @@ pub fn expected_output_path(
     testpaths.file.with_extension(extension)
 }
 
-pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED];
+pub const UI_EXTENSIONS: &[&str] = &[UI_STDERR, UI_STDOUT, UI_FIXED, UI_RUN_STDERR, UI_RUN_STDOUT];
 pub const UI_STDERR: &str = "stderr";
 pub const UI_STDOUT: &str = "stdout";
 pub const UI_FIXED: &str = "fixed";
+pub const UI_RUN_STDERR: &str = "run.stderr";
+pub const UI_RUN_STDOUT: &str = "run.stdout";
 
 /// Absolute path to the directory where all output for all tests in the given
 /// `relative_dir` group should reside. Example:
diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
@@ -137,6 +137,11 @@ impl EarlyProps {
                    config.parse_needs_sanitizer_support(ln) {
                     props.ignore = Ignore::Ignore;
                 }
+
+                if config.target == "wasm32-unknown-unknown" && config.parse_check_run_results(ln) {
+                    props.ignore = Ignore::Ignore;
+                }
+
             }
 
             if (config.mode == common::DebugInfoGdb || config.mode == common::DebugInfoGdbLldb) &&
@@ -326,6 +331,8 @@ pub struct TestProps {
     pub force_host: bool,
     // Check stdout for error-pattern output as well as stderr
     pub check_stdout: bool,
+    // Check stdout & stderr for output of run-pass test
+    pub check_run_results: bool,
     // For UI tests, allows compiler to generate arbitrary output to stdout
     pub dont_check_compiler_stdout: bool,
     // For UI tests, allows compiler to generate arbitrary output to stderr
@@ -388,6 +395,7 @@ impl TestProps {
             build_aux_docs: false,
             force_host: false,
             check_stdout: false,
+            check_run_results: false,
             dont_check_compiler_stdout: false,
             dont_check_compiler_stderr: false,
             no_prefer_dynamic: false,
@@ -468,6 +476,10 @@ impl TestProps {
                 self.check_stdout = config.parse_check_stdout(ln);
             }
 
+            if !self.check_run_results {
+                self.check_run_results = config.parse_check_run_results(ln);
+            }
+
             if !self.dont_check_compiler_stdout {
                 self.dont_check_compiler_stdout = config.parse_dont_check_compiler_stdout(ln);
             }
@@ -712,6 +724,10 @@ impl Config {
         self.parse_name_directive(line, "check-stdout")
     }
 
+    fn parse_check_run_results(&self, line: &str) -> bool {
+        self.parse_name_directive(line, "check-run-results")
+    }
+
     fn parse_dont_check_compiler_stdout(&self, line: &str) -> bool {
         self.parse_name_directive(line, "dont-check-compiler-stdout")
     }
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
@@ -2,6 +2,7 @@
 
 use crate::common::{CompareMode, PassMode};
 use crate::common::{expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT};
+use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT};
 use crate::common::{output_base_dir, output_base_name, output_testname_unique};
 use crate::common::{Codegen, CodegenUnits, Rustdoc};
 use crate::common::{DebugInfoCdb, DebugInfoGdbLldb, DebugInfoGdb, DebugInfoLldb};
@@ -288,6 +289,11 @@ enum ReadFrom {
     Stdin(String),
 }
 
+enum TestOutput {
+    Compile,
+    Run,
+}
+
 impl<'test> TestCx<'test> {
     /// Code executed for each revision in turn (or, if there are no
     /// revisions, exactly once, with revision == None).
@@ -2930,6 +2936,61 @@ impl<'test> TestCx<'test> {
         }
     }
 
+    fn load_compare_outputs(&self, proc_res: &ProcRes,
+        output_kind: TestOutput, explicit_format: bool) -> usize {
+
+        let (stderr_kind, stdout_kind) = match output_kind {
+            TestOutput::Compile => (UI_STDERR, UI_STDOUT),
+            TestOutput::Run => (UI_RUN_STDERR, UI_RUN_STDOUT)
+        };
+
+        let expected_stderr = self.load_expected_output(stderr_kind);
+        let expected_stdout = self.load_expected_output(stdout_kind);
+
+        let normalized_stdout = match output_kind {
+            TestOutput::Run if self.config.remote_test_client.is_some() => {
+                // When tests are run using the remote-test-client, the string
+                // 'uploaded "$TEST_BUILD_DIR/<test_executable>, waiting for result"'
+                // is printed to stdout by the client and then captured in the ProcRes,
+                // so it needs to be removed when comparing the run-pass test execution output
+                lazy_static! {
+                    static ref REMOTE_TEST_RE: Regex = Regex::new(
+                        "^uploaded \"\\$TEST_BUILD_DIR(/[[:alnum:]_\\-]+)+\", waiting for result\n"
+                    ).unwrap();
+                }
+                REMOTE_TEST_RE.replace(
+                    &self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout),
+                    ""
+                ).to_string()
+            }
+            _ => self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout)
+        };
+
+        let stderr = if explicit_format {
+            proc_res.stderr.clone()
+        } else {
+            json::extract_rendered(&proc_res.stderr)
+        };
+
+        let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr);
+        let mut errors = 0;
+        match output_kind {
+            TestOutput::Compile => {
+                if !self.props.dont_check_compiler_stdout {
+                    errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
+                }
+                if !self.props.dont_check_compiler_stderr {
+                    errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
+                }
+            }
+            TestOutput::Run => {
+                errors += self.compare_output(stdout_kind, &normalized_stdout, &expected_stdout);
+                errors += self.compare_output(stderr_kind, &normalized_stderr, &expected_stderr);
+            }
+        }
+        errors
+    }
+
     fn run_ui_test(&self) {
         // if the user specified a format in the ui test
         // print the output to the stderr file, otherwise extract
@@ -2942,32 +3003,13 @@ impl<'test> TestCx<'test> {
         let proc_res = self.compile_test();
         self.check_if_test_should_compile(&proc_res);
 
-        let expected_stderr = self.load_expected_output(UI_STDERR);
-        let expected_stdout = self.load_expected_output(UI_STDOUT);
         let expected_fixed = self.load_expected_output(UI_FIXED);
 
-        let normalized_stdout =
-            self.normalize_output(&proc_res.stdout, &self.props.normalize_stdout);
-
-        let stderr = if explicit {
-            proc_res.stderr.clone()
-        } else {
-            json::extract_rendered(&proc_res.stderr)
-        };
-
-        let normalized_stderr = self.normalize_output(&stderr, &self.props.normalize_stderr);
-
-        let mut errors = 0;
-        if !self.props.dont_check_compiler_stdout {
-            errors += self.compare_output("stdout", &normalized_stdout, &expected_stdout);
-        }
-        if !self.props.dont_check_compiler_stderr {
-            errors += self.compare_output("stderr", &normalized_stderr, &expected_stderr);
-        }
-
         let modes_to_prune = vec![CompareMode::Nll];
         self.prune_duplicate_outputs(&modes_to_prune);
 
+        let mut errors = self.load_compare_outputs(&proc_res, TestOutput::Compile, explicit);
+
         if self.config.compare_mode.is_some() {
             // don't test rustfix with nll right now
         } else if self.config.rustfix_coverage {
@@ -3045,7 +3087,17 @@ impl<'test> TestCx<'test> {
 
         if self.should_run_successfully() {
             let proc_res = self.exec_compiled_test();
-
+            let run_output_errors = if self.props.check_run_results {
+                self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
+            } else {
+                0
+            };
+            if run_output_errors > 0 {
+                self.fatal_proc_rec(
+                    &format!("{} errors occured comparing run output.", run_output_errors),
+                    &proc_res,
+                );
+            }
             if !proc_res.status.success() {
                 self.fatal_proc_rec("test run failed!", &proc_res);
             }
