Apply suggestions; add test case for non-JSON line

README.md

@@ -220,11 +220,17 @@ gotestsum -- -coverprofile=cover.out ./...
 gotestsum --raw-command -- ./scripts/run_tests.sh
 ```
 
-Note: when using `--raw-command` you must ensure that the stdout produced by
-the script only contains the `test2json` output. Any stderr produced by the script
-will be considered an error (this behaviour is necessary because package build errors
-are only reported by writting to stderr, not the `test2json` stdout). Any stderr
-produced by tests is not considered an error (it will be in the `test2json` stdout).
+Note: when using `--raw-command`, the script must follow a few rules about
+stdout and stderr output:
+
+* The stdout produced by the script must only contain the `test2json` output, or
+  `gotestsum` will fail. If it isn't possible to change the script to avoid
+  non-JSON output, you can use `--ignore-non-json-output-lines` to ignore
+  non-JSON lines and write them to `gotestsum`'s stderr instead.
+* Any stderr produced by the script will be considered an error (this behaviour
+  is necessary because package build errors are only reported by writting to
+  stderr, not the `test2json` stdout). Any stderr produced by tests is not
+  considered an error (it will be in the `test2json` stdout).
 
 **Example: using `TEST_DIRECTORY`**
 ```

cmd/main.go

@@ -61,6 +61,7 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"don't prepend 'go test -json' to the 'go test' command")
 	flags.BoolVar(&opts.ignoreNonJSONOutputLines, "ignore-non-json-output-lines", false,
 		"write non-JSON 'go test' output lines to stderr instead of failing")
+	flags.Lookup("ignore-non-json-output-lines").Hidden = true
 	flags.StringVar(&opts.jsonFile, "jsonfile",
 		lookEnvWithDefault("GOTESTSUM_JSONFILE", ""),
 		"write all TestEvents to file")

cmd/testdata/gotestsum-help-text

@@ -6,7 +6,6 @@ Flags:
       --debug                                       enabled debug logging
   -f, --format string                               print format of test input (default "short")
       --hide-summary summary                        hide sections of the summary: skipped,failed,errors,output (default none)
-      --ignore-non-json-output-lines                write non-JSON 'go test' output lines to stderr instead of failing
       --jsonfile string                             write all TestEvents to file
       --junitfile string                            write a JUnit XML file
       --junitfile-testcase-classname field-format   format the testcase classname field as: full, relative, short (default full)

testjson/execution.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"os"
 	"sort"
 	"strings"
 	"sync"

testjson/execution_test.go

@@ -205,8 +205,34 @@ func TestScanOutput_WithMissingEvents(t *testing.T) {
 	assert.DeepEqual(t, expected, handler.events[start:], cmpTestEventShallow)
 }
 
+func TestScanOutput_WithNonJSONLines(t *testing.T) {
+	source := golden.Get(t, "go-test-json-with-nonjson-stdout.out")
+	nonJSONLine := "|||This line is not valid test2json output.|||"
+
+	// Test that when we ignore non-JSON lines, scanning completes, and test
+	// that when we don't ignore non-JSON lines, scanning fails.
+	for _, ignore := range []bool{true, false} {
+		handler := &captureHandler{}
+		cfg := ScanConfig{
+			Stdout:                   bytes.NewReader(source),
+			Handler:                  handler,
+			IgnoreNonJSONOutputLines: ignore,
+		}
+		_, err := ScanTestOutput(cfg)
+		if ignore {
+			assert.Assert(t, len(handler.errs) == 1)
+			assert.Assert(t, handler.errs[0] == nonJSONLine)
+			assert.NilError(t, err)
+		} else {
+			assert.Assert(t, len(handler.errs) == 0)
+			assert.Error(t, err, "failed to parse test output: "+nonJSONLine+": invalid character '|' looking for beginning of value")
+		}
+	}
+}
+
 type captureHandler struct {
 	events []TestEvent
+	errs   []string
 }
 
 func (s *captureHandler) Event(event TestEvent, _ *Execution) error {
@@ -215,5 +241,6 @@ func (s *captureHandler) Event(event TestEvent, _ *Execution) error {
 }
 
 func (s *captureHandler) Err(text string) error {
+	s.errs = append(s.errs, text)
 	return fmt.Errorf(text)
 }

testjson/testdata/go-test-json-with-nonjson-stdout.out

@@ -0,0 +1,8 @@
+{"Time":"2021-05-12T13:53:07.462687619-05:00","Action":"run","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed"}
+|||This line is not valid test2json output.|||
+{"Time":"2021-05-12T13:53:07.46279664-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Output":"=== RUN   TestPassed\n"}
+{"Time":"2021-05-12T13:53:07.462812837-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Output":"--- PASS: TestPassed (0.00s)\n"}
+{"Time":"2021-05-12T13:53:07.462819251-05:00","Action":"pass","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Elapsed":0}
+{"Time":"2021-05-12T13:53:07.462825108-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Output":"PASS\n"}
+{"Time":"2021-05-12T13:53:07.462848483-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Output":"ok  \tgotest.tools/gotestsum/testjson/internal/good\t0.001s\n"}
+{"Time":"2021-05-12T13:53:07.46309146-05:00","Action":"pass","Package":"gotest.tools/gotestsum/testjson/internal/good","Elapsed":0.001}