support stop-on-assertion-failure behaviour

README.md

@@ -146,6 +146,10 @@ test spec also contains these fields:
   the name of the environment variable to read into the named variable.
 * `assert`: (optional) an object describing the conditions that will be
   asserted about the test action.
+* `assert.require`: (optional) a boolean indicating whether a failed assertion
+  will cause the test scenario's execution to stop. The default behaviour of
+  `gdt` is to continue execution of subsequent test specs in a test scenario when
+  an assertion fails.
 * `assert.exit-code`: (optional) an integer with the expected exit code from the
   executed command. The default successful exit code is 0 and therefore you do
   not need to specify this if you expect a successful exit code.

api/result.go

@@ -16,6 +16,9 @@ package api
 // returned in the Result and the `Scenario.Run` method injects that
 // information into the context that is supplied to the next Spec's `Run`.
 type Result struct {
+	// stopOnFail is an indication to the scenario that if there are any
+	// failures, the scenario should not proceed with test execution.
+	stopOnFail bool
 	// failures is the collection of error messages from assertion failures
 	// that occurred during Eval(). These are *not* `gdterrors.RuntimeError`.
 	failures []error
@@ -39,6 +42,12 @@ func (r *Result) Data() map[string]any {
 	return r.data
 }
 
+// StopOnFail returns true if the test spec indicates that a failure of
+// assertion should stop the execution of the test scenario.
+func (r *Result) StopOnFail() bool {
+	return r.stopOnFail
+}
+
 // Failed returns true if any assertion failed during Eval(), false otherwise.
 func (r *Result) Failed() bool {
 	return len(r.failures) > 0
@@ -95,6 +104,14 @@ func WithData(key string, val any) ResultModifier {
 	}
 }
 
+// WithStopOnFail sets the stopOnFail value for the test spec result.
+// failures
+func WithStopOnFail(val bool) ResultModifier {
+	return func(r *Result) {
+		r.stopOnFail = val
+	}
+}
+
 // WithFailures modifies the Result the supplied collection of assertion
 // failures
 func WithFailures(failures ...error) ResultModifier {

parse/error.go

@@ -179,6 +179,16 @@ func ExpectedScalarOrMapAt(node *yaml.Node) error {
 	}
 }
 
+// ExpectedBoolAt returns a parse error indicating a boolean value was expected
+// and annotated with the line/column of the supplied YAML node.
+func ExpectedBoolAt(node *yaml.Node) error {
+	return &Error{
+		Line:    node.Line,
+		Column:  node.Column,
+		Message: "expected boolean value",
+	}
+}
+
 // ExpectedTimeoutAt returns an ErrExpectedTimeout error annotated
 // with the line/column of the supplied YAML node.
 func ExpectedTimeoutAt(node *yaml.Node) error {

plugin/exec/assertions.go

@@ -18,6 +18,9 @@ import (
 
 // Expect contains the assertions about an Exec Spec's actions
 type Expect struct {
+	// Require indicates that any failed assertion should stop the execution of
+	// the test scenario in which the test spec is contained.
+	Require bool `yaml:"require,omitempty"`
 	// ExitCode is the expected exit code for the executed command. The default
 	// (0) is the universal successful exit code, so you only need to set this
 	// if you expect a non-successful result from executing the command.

plugin/exec/eval.go

@@ -47,5 +47,12 @@ func (s *Spec) Eval(
 			}
 		}
 	}
-	return api.NewResult(api.WithFailures(a.Failures()...)), nil
+	stopOnFail := false
+	if s.Assert != nil {
+		stopOnFail = s.Assert.Require
+	}
+	return api.NewResult(
+		api.WithStopOnFail(stopOnFail),
+		api.WithFailures(a.Failures()...),
+	), nil
 }

plugin/exec/eval_test.go

@@ -463,3 +463,47 @@ func TestVar(t *testing.T) {
 	err = s.Run(ctx, t)
 	require.Nil(err)
 }
+
+func TestFailStopOnFail(t *testing.T) {
+	if !*failFlag {
+		t.Skip("skipping without -fail flag")
+	}
+	require := require.New(t)
+
+	fp := filepath.Join("testdata", "stop-on-fail.yaml")
+	f, err := os.Open(fp)
+	require.Nil(err)
+
+	s, err := scenario.FromReader(
+		f,
+		scenario.WithPath(fp),
+	)
+	require.Nil(err)
+	require.NotNil(s)
+
+	ctx := gdtcontext.New(gdtcontext.WithDebug())
+	err = s.Run(ctx, t)
+	require.Nil(err)
+}
+
+func TestStopOnFail(t *testing.T) {
+	require := require.New(t)
+	target := os.Args[0]
+	failArgs := []string{
+		"-test.v",
+		"-test.run=FailStopOnFail",
+		"-fail",
+	}
+	outerr, err := exec.Command(target, failArgs...).CombinedOutput()
+
+	// The test should have failed...
+	require.NotNil(err)
+
+	debugout := string(outerr)
+	require.Contains(debugout, "assertion failed: not in: expected stdout to contain 1234")
+	// first test spec does not contain the `require: true` field and so the
+	// second test spec should execute (and fail its assertion)
+	require.Contains(debugout, "assertion failed: not in: expected stdout to contain 24")
+	// The third test spec should NOT have been executed...
+	require.NotContains(debugout, "[gdt] [stop-on-fail/2] exec: stdout: 24")
+}

plugin/exec/parse.go

@@ -123,6 +123,16 @@ func (s *Spec) UnmarshalYAML(node *yaml.Node) error {
 				return err
 			}
 			s.Assert = e
+		case "require":
+			if valNode.Kind != yaml.MappingNode {
+				return parse.ExpectedMapAt(valNode)
+			}
+			var e *Expect
+			if err := valNode.Decode(&e); err != nil {
+				return err
+			}
+			e.Require = true
+			s.Assert = e
 		case "on":
 			if valNode.Kind != yaml.MappingNode {
 				return parse.ExpectedMapAt(valNode)
@@ -168,6 +178,16 @@ func (e *Expect) UnmarshalYAML(node *yaml.Node) error {
 		key := keyNode.Value
 		valNode := node.Content[i+1]
 		switch key {
+		case "require", "stop-on-fail", "stop_on_fail", "stop.on.fail",
+			"fail-stop", "fail.stop", "fail_stop":
+			if valNode.Kind != yaml.ScalarNode {
+				return parse.ExpectedScalarAt(valNode)
+			}
+			req, err := strconv.ParseBool(valNode.Value)
+			if err != nil {
+				return parse.ExpectedBoolAt(valNode)
+			}
+			e.Require = req
 		case "exit_code", "exit-code":
 			if valNode.Kind != yaml.ScalarNode {
 				return parse.ExpectedScalarAt(valNode)

plugin/exec/spec.go

@@ -13,6 +13,10 @@ import (
 type Spec struct {
 	api.Spec
 	Action
+	// Require is an object containing the conditions that the Spec will
+	// assert. If any condition fails, the test scenario execution will stop
+	// and be marked as failed.
+	Require *Expect `yaml:"require,omitempty"`
 	// Assert is an object containing the conditions that the Spec will assert.
 	Assert *Expect `yaml:"assert,omitempty"`
 	// On is an object containing actions to take upon certain conditions.

plugin/exec/testdata/stop-on-fail.yaml

@@ -0,0 +1,17 @@
+name: stop-on-fail
+description: |
+    a scenario that tests that a failed assertion for a stop-on-fail
+    test spec prevents subsequent test specs from being executed.
+tests:
+  - exec: echo "meaning of life"
+    assert:
+      out:
+        is: 1234
+  # This test should be executed because we do not use the require: true.
+  - exec: echo 42
+    assert:
+      require: true
+      out:
+        is: 24
+  # This test should not be executed because of the use of require: true above
+  - exec: echo 24

scenario/run.go

@@ -108,6 +108,7 @@ func (s *Scenario) runExternal(ctx context.Context, run *run.Run) error {
 
 	scenCleanups := []func(){}
 	scenOK := true
+outer:
 	for idx, t := range s.Tests {
 		tu := testunit.New(
 			ctx,
@@ -134,11 +135,15 @@ func (s *Scenario) runExternal(ctx context.Context, run *run.Run) error {
 		if res.HasData() {
 			ctx = gdtcontext.SetRun(ctx, res.Data())
 		}
-		if len(res.Failures()) > 0 {
-			tu.FailNow()
-		} else {
-			tu.Finish() // necessary for elapsed timer to stop
+		for _, fail := range res.Failures() {
+			if res.StopOnFail() {
+				tu.Fatal(fail)
+				run.StoreResult(idx, s.Path, tu, res)
+				break outer
+			}
+			tu.Error(fail)
 		}
+		tu.Finish() // necessary for elapsed timer to stop
 		scenOK = scenOK && !tu.Failed()
 
 		run.StoreResult(idx, s.Path, tu, res)
@@ -220,7 +225,10 @@ func (s *Scenario) runGo(ctx context.Context, t *testing.T) error {
 			}
 
 			for _, fail := range res.Failures() {
-				tt.Fatal(fail)
+				if res.StopOnFail() {
+					tt.Fatal(fail)
+				}
+				tt.Error(fail)
 			}
 		}
 	})