runtime: added counter to failpoint.terms field

Each time terms is executed the counter is incremented. Added counter endpoint that gives number of executions on the current terms: GET pkg/failpoint/count
When DELETE /pkg/failpoint is executed, failpoint.terms is nilled, hence counter is reset.

Requested in reference to:
tests/linearizability: added sleep fail point injection #14796

Signed-off-by: Ramil Mirhasanov <ramil600@yahoo.com>

runtime: added test cases for terms counter

runtime/http.go

@@ -72,6 +72,13 @@ func (*httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				lines[i] = fps[i] + "=" + s
 			}
 			w.Write([]byte(strings.Join(lines, "\n") + "\n"))
+		} else if strings.HasSuffix(key, "/count") {
+			fp := key[:len(key)-len("/count")]
+			count, err := StatusCount(fp)
+			if err != nil {
+				http.Error(w, "failed to GET: "+err.Error(), http.StatusNotFound)
+			}
+			w.Write([]byte(count))
 		} else {
 			status, err := Status(key)
 			if err != nil {

runtime/runtime.go

@@ -114,6 +114,25 @@ func Status(failpath string) (string, error) {
 	return t.desc, nil
 }
 
+// StatusCount outputs how many times current term was executed
+func StatusCount(failpath string) (string, error) {
+	failpointsMu.RLock()
+	fp := failpoints[failpath]
+	failpointsMu.RUnlock()
+	if fp == nil {
+		return "", ErrNoExist
+	}
+	fp.mu.RLock()
+	t := fp.t
+	fp.mu.RUnlock()
+	if t == nil {
+		return "", ErrDisabled
+	}
+	count := fp.t.counter
+	return fmt.Sprint(count), nil
+
+}
+
 func List() []string {
 	failpointsMu.RLock()
 	ret := make([]string, 0, len(failpoints))

runtime/terms.go

@@ -41,6 +41,8 @@ type terms struct {
 
 	// mu protects the state of the terms chain
 	mu sync.Mutex
+	// counts executions
+	counter int
 }
 
 // term is an executable unit of the failpoint terms chain
@@ -102,6 +104,7 @@ func (t *terms) eval() (interface{}, error) {
 	defer t.mu.Unlock()
 	for _, term := range t.chain {
 		if term.mods.allow() {
+			t.counter++
 			return term.do(), nil
 		}
 	}

runtime/termscounter_test.go

@@ -0,0 +1,141 @@
+package runtime_test
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"go.etcd.io/gofail/runtime"
+)
+
+var __fp_ExampleString *runtime.Failpoint = runtime.NewFailpoint("runtime_test", "ExampleString") //nolint:stylecheck
+
+func TestTermsCounter(t *testing.T) {
+	testcases := []struct {
+		name      string
+		fp        string
+		desc      string
+		runbefore int
+		runafter  int
+		want      string
+	}{
+		{
+			name:     "Terms limit Failpoint",
+			fp:       "runtime_test/ExampleString",
+			desc:     `10*sleep(10)->1*return("abc")`,
+			runafter: 12,
+			want:     "11",
+		},
+		{
+			name:      "Inbetween Enabling Failpoint",
+			fp:        "runtime_test/ExampleString",
+			desc:      `10*sleep(10)->1*return("abc")`,
+			runbefore: 2,
+			runafter:  3,
+			want:      "3",
+		},
+		{
+			name:      "Before Enabling Failpoint",
+			fp:        "runtime_test/ExampleString",
+			desc:      `10*sleep(10)->1*return("abc")`,
+			runbefore: 2,
+			runafter:  0,
+			want:      "0",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			for i := 0; i < tc.runbefore; i++ {
+				exampleFunc()
+				time.Sleep(10 * time.Millisecond)
+			}
+
+			err := runtime.Enable(tc.fp, tc.desc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer runtime.Disable(tc.fp)
+			for i := 0; i < tc.runafter; i++ {
+				exampleFunc()
+				time.Sleep(10 * time.Millisecond)
+			}
+			count, err := runtime.StatusCount(tc.fp)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if strings.Compare(tc.want, count) != 0 {
+				t.Fatal("counter is not properly incremented")
+			}
+		})
+	}
+}
+
+func TestResetingCounterOnTerm(t *testing.T) {
+	testcases := []struct {
+		name      string
+		fp        string
+		desc      string
+		newdesc   string
+		runbefore int
+		runafter  int
+		want      string
+	}{
+		{
+			name:      "Change and Reset Counter",
+			fp:        "runtime_test/ExampleString",
+			desc:      `10*sleep(10)->1*return("abc")`,
+			newdesc:   "sleep(10)",
+			runbefore: 2,
+			runafter:  3,
+			want:      "3",
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := runtime.Enable(tc.fp, tc.desc)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			for i := 0; i < tc.runbefore; i++ {
+				exampleFunc()
+				time.Sleep(10 * time.Millisecond)
+			}
+			err = runtime.Enable(tc.fp, tc.newdesc)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer runtime.Disable(tc.fp)
+
+			for i := 0; i < tc.runafter; i++ {
+				exampleFunc()
+				time.Sleep(10 * time.Millisecond)
+			}
+			count, err := runtime.StatusCount(tc.fp)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if strings.Compare(tc.want, count) != 0 {
+				t.Fatal("counter is not properly incremented")
+			}
+		})
+	}
+
+}
+
+func exampleFunc() string {
+	if vExampleString, __fpErr := __fp_ExampleString.Acquire(); __fpErr == nil { //nolint:stylecheck
+		defer __fp_ExampleString.Release()                   //nolint:stylecheck
+		ExampleString, __fpTypeOK := vExampleString.(string) //nolint:stylecheck
+		if !__fpTypeOK {                                     //nolint:stylecheck
+			goto __badTypeExampleString //nolint:stylecheck
+		}
+		return ExampleString
+	__badTypeExampleString: //nolint:stylecheck
+		__fp_ExampleString.BadType(vExampleString, "string") //nolint:stylecheck
+	}
+	return "example"
+}