Merge branch 'improbable-eng-prometheus_exporter'

README.md

@@ -248,6 +248,7 @@ package:
 * tap - TAP style
 * junit - JUnit style
 * nagios - Nagios/Sensu compatible output /w exit code 2 for failures.
+* prometheus - Prometheus compatible output.
 * silent - No output. Avoids exposing system information (e.g. when serving tests as a healthcheck endpoint).
 
 ## Community Contributions

docs/manual.md

@@ -290,6 +290,7 @@ The `application/vnd.goss-{output format}` media type can be used in the `Accept
   * `nagios` - Nagios/Sensu compatible output /w exit code 2 for failures
   * `rspecish` **(default)** - Similar to rspec output
   * `tap`
+  * `prometheus` - Prometheus compatible output.
   * `silent` - No output. Avoids exposing system information (e.g. when serving tests as a healthcheck endpoint)
 * `--format-options`, `-o` (output format option)
   * `perfdata` - Outputs Nagios "performance data". Applies to `nagios` output

go.mod

@@ -16,10 +16,11 @@ require (
 	github.com/oleiade/reflections v1.0.1
 	github.com/onsi/gomega v1.19.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/prometheus/client_golang v1.13.0
+	github.com/prometheus/common v0.37.0
 	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli v1.22.9
 	golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
 	golang.org/x/tools v0.1.9 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v2 v2.4.0

integration-tests/run-serve-tests.sh

@@ -64,7 +64,7 @@ args=(
 )
 log_action -e "\nTesting \`${GOSS_BINARY} ${args[*]}\` ...\n"
 "${GOSS_BINARY}" "${args[@]}" &
-url="http://127.0.0.1:${open_port}/healthz"
+base_url="http://127.0.0.1:${open_port}"
 
 assert_response_contains() {
   local url="${1:?"1st arg: url"}"
@@ -92,8 +92,14 @@ failure="false"
 on_test_failure() {
   failure="true"
 }
-assert_response_contains "${url}" "no accept header" "Count: 2, Failed: 0, Skipped: 0" "" || on_test_failure
-assert_response_contains "${url}" "tap accept header" "Count: 2, Failed: 0, Skipped: 0" "application/vnd.goss-documentation" || on_test_failure
-assert_response_contains "${url}" "json accept header" "\"failed-count\":0" "application/json" || on_test_failure
+
+# /healthz endpoint
+assert_response_contains "${base_url}/healthz" "no accept header" "Count: 2, Failed: 0, Skipped: 0" "" || on_test_failure
+assert_response_contains "${base_url}/healthz" "tap accept header" "Count: 2, Failed: 0, Skipped: 0" "application/vnd.goss-documentation" || on_test_failure
+assert_response_contains "${base_url}/healthz" "json accept header" "\"failed-count\":0" "application/json" || on_test_failure
+assert_response_contains "${base_url}/healthz" "prometheus accept header" "goss_tests_outcomes_total" "application/vnd.goss-prometheus" || on_test_failure
+
+# /metrics - specific prometheus metrics endpoint
+assert_response_contains "${base_url}/metrics" "prometheus accept header" "goss_tests_outcomes_total" "" || on_test_failure
 
 [[ "${failure}" == "true" ]] && log_fatal "Test(s) failed, check output above."

outputs/outputs.go

@@ -30,6 +30,7 @@ var (
 		"json":          &Json{},
 		"junit":         &JUnit{},
 		"nagios":        &Nagios{},
+		"prometheus":    &Prometheus{},
 		"rspecish":      &Rspecish{},
 		"structured":    &Structured{},
 		"tap":           &Tap{},

outputs/outputs_test.go

@@ -43,3 +43,17 @@ func TestOutputFormatOptions(t *testing.T) {
 	assert.Contains(t, list, foVerbose)
 	assert.Len(t, list, 3)
 }
+
+func TestOptionsRegistration(t *testing.T) {
+	registeredOutputs := Outputers()
+	assert.Contains(t, registeredOutputs, "documentation")
+	assert.Contains(t, registeredOutputs, "json_oneline")
+	assert.Contains(t, registeredOutputs, "json")
+	assert.Contains(t, registeredOutputs, "junit")
+	assert.Contains(t, registeredOutputs, "nagios")
+	assert.Contains(t, registeredOutputs, "prometheus")
+	assert.Contains(t, registeredOutputs, "rspecish")
+	assert.Contains(t, registeredOutputs, "silent")
+	assert.Contains(t, registeredOutputs, "structured")
+	assert.Contains(t, registeredOutputs, "tap")
+}

outputs/prometheus.go

@@ -0,0 +1,106 @@
+package outputs
+
+import (
+	"io"
+	"strings"
+	"time"
+
+	"github.com/aelsabbahy/goss/resource"
+	"github.com/aelsabbahy/goss/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/common/expfmt"
+)
+
+const (
+	labelType    = "type"
+	labelOutcome = "outcome"
+)
+
+var (
+	testOutcomes = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "outcomes_total",
+		Help:      "The number of test-outcomes from this run.",
+	}, []string{labelType, labelOutcome})
+	testDurations = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "outcomes_duration_milliseconds",
+		Help:      "The duration of tests from this run. Note; tests run concurrently.",
+	}, []string{labelType, labelOutcome})
+	runOutcomes = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "run_outcomes_total",
+		Help:      "The outcomes of this run as a whole.",
+	}, []string{labelOutcome})
+	runDuration = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "run_duration_milliseconds",
+		Help:      "The end-to-end duration of this run.",
+	}, []string{labelOutcome})
+)
+
+// Prometheus renders metrics in prometheus.io text-format https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format
+type Prometheus struct{}
+
+// NewPrometheus creates and initialises a new Prometheus Outputer (to avoid missing metrics)
+func NewPrometheus() *Prometheus {
+	outputer := &Prometheus{}
+	outputer.init()
+	return outputer
+}
+
+func (r *Prometheus) init() {
+	// Avoid missing metrics: https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics
+	for resourceType := range resource.Resources() {
+		for _, outcome := range resource.HumanOutcomes() {
+			testOutcomes.WithLabelValues(resourceType, outcome).Add(0)
+			testDurations.WithLabelValues(resourceType, outcome).Add(0)
+		}
+	}
+	runOutcomes.WithLabelValues(labelOutcome).Add(0)
+	runDuration.WithLabelValues(labelOutcome).Add(0)
+}
+
+// ValidOptions is a list of valid format options for prometheus
+func (r Prometheus) ValidOptions() []*formatOption {
+	return []*formatOption{}
+}
+
+// Output converts the results into the prometheus text-format.
+func (r Prometheus) Output(w io.Writer, results <-chan []resource.TestResult,
+	startTime time.Time, outConfig util.OutputConfig) (exitCode int) {
+	overallOutcome := resource.OutcomeUnknown
+	for resultGroup := range results {
+		for _, tr := range resultGroup {
+			resType := strings.ToLower(tr.ResourceType)
+			outcome := tr.ToOutcome()
+			testOutcomes.WithLabelValues(resType, outcome).Inc()
+			testDurations.WithLabelValues(resType, outcome).Add(float64(tr.Duration.Milliseconds()))
+			if tr.Result != resource.SUCCESS {
+				overallOutcome = tr.ToOutcome()
+			}
+		}
+	}
+
+	runOutcomes.WithLabelValues(overallOutcome).Inc()
+	runDuration.WithLabelValues(overallOutcome).Add(float64(time.Since(startTime).Milliseconds()))
+
+	metricsFamilies, err := prometheus.DefaultGatherer.Gather()
+	if err != nil {
+		return -1
+	}
+	encoder := expfmt.NewEncoder(w, expfmt.FmtText)
+	for _, mf := range metricsFamilies {
+		err := encoder.Encode(mf)
+		if err != nil {
+			return -1
+		}
+	}
+
+	return 0
+}

outputs/prometheus_test.go

@@ -0,0 +1,76 @@
+package outputs
+
+import (
+	"bytes"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aelsabbahy/goss/resource"
+
+	"github.com/aelsabbahy/goss/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPrometheusOutput(t *testing.T) {
+	buf := &bytes.Buffer{}
+	outputer := &Prometheus{}
+	injectedResults := []resource.TestResult{
+		{
+			ResourceType: "Command",
+			Duration:     10 * time.Millisecond,
+			Result:       resource.SUCCESS,
+		},
+		{
+			ResourceType: "Command",
+			Duration:     10 * time.Millisecond,
+			Result:       resource.SUCCESS,
+		},
+		{
+			ResourceType: "Command",
+			Duration:     10 * time.Millisecond,
+			Result:       resource.FAIL,
+		},
+		{
+			ResourceType: "File",
+			Duration:     10 * time.Millisecond,
+			Result:       resource.SKIP,
+		},
+	}
+
+	exitCode := outputer.Output(buf, makeResults(injectedResults...), time.Now().Add(-1*time.Minute), util.OutputConfig{})
+
+	assert.Equal(t, 0, exitCode)
+	output := buf.String()
+	t.Logf(output)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="pass",type="command"} 20`)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="fail",type="command"} 10`)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="skip",type="command"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="pass",type="file"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="fail",type="file"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_duration_milliseconds{outcome="skip",type="file"} 10`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="pass",type="command"} 2`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="fail",type="command"} 1`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="skip",type="command"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="pass",type="file"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="fail",type="file"} 0`)
+	assert.Contains(t, output, `goss_tests_outcomes_total{outcome="skip",type="file"} 1`)
+	assert.Contains(t, output, `goss_tests_run_duration_milliseconds{outcome="skip"} 60000`)
+	assert.Contains(t, output, `goss_tests_run_outcomes_total{outcome="skip"} 1`)
+}
+
+func makeResults(results ...resource.TestResult) <-chan []resource.TestResult {
+	out := make(chan []resource.TestResult)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		out <- append([]resource.TestResult{}, results...)
+	}()
+
+	go func() {
+		wg.Wait()
+		close(out)
+	}()
+	return out
+}

resource/validate.go

@@ -23,8 +23,27 @@ const (
 	SUCCESS = iota
 	FAIL
 	SKIP
+	UNKNOWN
 )
 
+const (
+	OutcomePass    = "pass"
+	OutcomeFail    = "fail"
+	OutcomeSkip    = "skip"
+	OutcomeUnknown = "unknown"
+)
+
+var humanOutcomes map[int]string = map[int]string{
+	UNKNOWN: OutcomeUnknown,
+	SUCCESS: OutcomePass,
+	FAIL:    OutcomeFail,
+	SKIP:    OutcomeSkip,
+}
+
+func HumanOutcomes() map[int]string {
+	return humanOutcomes
+}
+
 const (
 	maxScanTokenSize = 10 * 1024 * 1024
 )
@@ -45,6 +64,20 @@ type TestResult struct {
 	Duration     time.Duration `json:"duration" yaml:"duration"`
 }
 
+// ToOutcome converts the enum to a human-friendly string.
+func (tr TestResult) ToOutcome() string {
+	switch tr.Result {
+	case SUCCESS:
+		return OutcomePass
+	case FAIL:
+		return OutcomeFail
+	case SKIP:
+		return OutcomeSkip
+	default:
+		return OutcomeUnknown
+	}
+}
+
 func skipResult(typeS string, testType int, id string, title string, meta meta, property string, startTime time.Time) TestResult {
 	return TestResult{
 		Successful:   true,

serve.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aelsabbahy/goss/util"
 	"github.com/fatih/color"
 	"github.com/patrickmn/go-cache"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 func Serve(c *util.Config) error {
@@ -24,6 +25,7 @@ func Serve(c *util.Config) error {
 		return err
 	}
 	http.Handle(endpoint, health)
+	http.Handle("/metrics", promhttp.Handler())
 	log.Printf("Starting to listen on: %s", c.ListenAddress)
 	return http.ListenAndServe(c.ListenAddress, nil)
 }