Dogstatsd metrics exporter (#326)

* Add MetricAggregator.Merge() implementations

* Update from feedback

* Type

* Ckpt

* Ckpt

* Add push controller

* Ckpt

* Add aggregator interfaces, stdout encoder

* Modify basic main.go

* Main is working

* Batch stdout output

* Sum udpate

* Rename stdout

* Add stateless/stateful Batcher options

* Undo a for-loop in the example, remove a done TODO

* Update imports

* Add note

* Rename defaultkeys

* Support variable label encoder to speed OpenMetrics/Statsd export

* Lint

* Checkpoint

* Checkpoint

* Doc

* Precommit/lint

* Simplify Aggregator API

* Record->Identifier

* Remove export.Record a.k.a. Identifier

* Checkpoint

* Propagate errors to the SDK, remove a bunch of 'TODO warn'

* Checkpoint

* Introduce export.Labels

* Comments in export/metric.go

* Comment

* More merge

* More doc

* Complete example

* Lint fixes

* Add a testable example

* Lint

* Dogstats

* Let Export return an error

* Checkpoint

* add a basic stdout exporter test

* Add measure test; fix aggregator APIs

* Use JSON numbers, not strings

* Test stdout exporter error

* Add a test for the call to RangeTest

* Add error handler API to improve correctness test; return errors from RecordOne

* Undo the previous -- do not expose errors

* Add simple selector variations, test

* Repair examples

* Test push controller error handling

* Add SDK label encoder tests

* Add a defaultkeys batcher test

* Add an ungrouped batcher test

* Lint new tests

* Respond to krnowak's feedback

* Checkpoint

* Funciontal example using unixgram

* Tidy the example

* Add a packet-split test

* More tests

* Undo comment

* Use concrete receivers for export records and labels, since the constructors return structs not pointers

* Bug fix for stateful batchers; clone an aggregator for long term storage

* Remove TODO addressed in #318

* Add errors to all aggregator interfaces

* Handle ErrNoLastValue case in stdout exporter

* Move aggregator API into sdk/export/metric/aggregator

* Update all aggregator exported-method comments

* Document the aggregator APIs

* More aggregator comments

* Add multiple updates to the ungrouped test

* Fixes for feedback from Gustavo and Liz

* Producer->CheckpointSet; add FinishedCollection

* Process takes an export.Record

* ReadCheckpoint->CheckpointSet

* EncodeLabels->Encode

* Format a better inconsistent type error; add more aggregator API tests

* More RangeTest test coverage

* Make benbjohnson/clock a test-only dependency

* Handle ErrNoLastValue in stress_test

* Update comments; use a pipe vs a unix socket in the example test

* Update test

* Spelling

* Typo fix

* Rename DefaultLabelEncoder to NewDefaultLabelEncoder for clarity

* Rename DefaultLabelEncoder to NewDefaultLabelEncoder for clarity

* Test different adapters; add ForceEncode to statsd label encoder

example/basic/main.go

@@ -65,7 +65,7 @@ func initMeter() *push.Controller {
 	if err != nil {
 		log.Panicf("failed to initialize metric stdout exporter %v", err)
 	}
-	batcher := defaultkeys.New(selector, metricsdk.DefaultLabelEncoder(), true)
+	batcher := defaultkeys.New(selector, metricsdk.NewDefaultLabelEncoder(), true)
 	pusher := push.New(batcher, exporter, time.Second)
 	pusher.Start()
 

exporter/metric/dogstatsd/dogstatsd.go

@@ -0,0 +1,75 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dogstatsd // import "go.opentelemetry.io/otel/exporter/metric/dogstatsd"
+
+import (
+	"bytes"
+
+	"go.opentelemetry.io/otel/exporter/metric/internal/statsd"
+	export "go.opentelemetry.io/otel/sdk/export/metric"
+)
+
+type (
+	Config = statsd.Config
+
+	// Exporter implements a dogstatsd-format statsd exporter,
+	// which encodes label sets as independent fields in the
+	// output.
+	//
+	// TODO: find a link for this syntax.  It's been copied out of
+	// code, not a specification:
+	//
+	// https://github.com/stripe/veneur/blob/master/sinks/datadog/datadog.go
+	Exporter struct {
+		*statsd.Exporter
+		*statsd.LabelEncoder
+
+		ReencodedLabelsCount int
+	}
+)
+
+var (
+	_ export.Exporter     = &Exporter{}
+	_ export.LabelEncoder = &Exporter{}
+)
+
+// New returns a new Dogstatsd-syntax exporter.  This type implements
+// the metric.LabelEncoder interface, allowing the SDK's unique label
+// encoding to be pre-computed for the exporter and stored in the
+// LabelSet.
+func New(config Config) (*Exporter, error) {
+	exp := &Exporter{
+		LabelEncoder: statsd.NewLabelEncoder(),
+	}
+
+	var err error
+	exp.Exporter, err = statsd.NewExporter(config, exp)
+	return exp, err
+}
+
+// AppendName is part of the stats-internal adapter interface.
+func (*Exporter) AppendName(rec export.Record, buf *bytes.Buffer) {
+	_, _ = buf.WriteString(rec.Descriptor().Name())
+}
+
+// AppendTags is part of the stats-internal adapter interface.
+func (e *Exporter) AppendTags(rec export.Record, buf *bytes.Buffer) {
+	encoded, inefficient := e.LabelEncoder.ForceEncode(rec.Labels())
+	_, _ = buf.WriteString(encoded)
+
+	if inefficient {
+		e.ReencodedLabelsCount++
+	}
+}

exporter/metric/dogstatsd/dogstatsd_test.go

@@ -0,0 +1,69 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dogstatsd_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"go.opentelemetry.io/otel/api/core"
+	"go.opentelemetry.io/otel/api/key"
+	"go.opentelemetry.io/otel/exporter/metric/dogstatsd"
+	"go.opentelemetry.io/otel/exporter/metric/internal/statsd"
+	"go.opentelemetry.io/otel/exporter/metric/test"
+	export "go.opentelemetry.io/otel/sdk/export/metric"
+	sdk "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/aggregator/counter"
+)
+
+// TestDogstatsLabels that labels are formatted in the correct style,
+// whether or not the provided labels were encoded by a statsd label
+// encoder.
+func TestDogstatsLabels(t *testing.T) {
+	for inefficientCount, encoder := range []export.LabelEncoder{
+		statsd.NewLabelEncoder(),     // inefficientCount == 0
+		sdk.NewDefaultLabelEncoder(), // inefficientCount == 1
+	} {
+		t.Run(fmt.Sprintf("%T", encoder), func(t *testing.T) {
+			ctx := context.Background()
+			checkpointSet := test.NewCheckpointSet(encoder)
+
+			desc := export.NewDescriptor("test.name", export.CounterKind, nil, "", "", core.Int64NumberKind, false)
+			cagg := counter.New()
+			_ = cagg.Update(ctx, core.NewInt64Number(123), desc)
+			cagg.Checkpoint(ctx, desc)
+
+			checkpointSet.Add(desc, cagg, key.New("A").String("B"))
+
+			var buf bytes.Buffer
+			exp, err := dogstatsd.New(dogstatsd.Config{
+				Writer: &buf,
+			})
+			require.Nil(t, err)
+			require.Equal(t, 0, exp.ReencodedLabelsCount)
+
+			err = exp.Export(ctx, checkpointSet)
+			require.Nil(t, err)
+
+			require.Equal(t, inefficientCount, exp.ReencodedLabelsCount)
+
+			require.Equal(t, "test.name:123|c|#A:B\n", buf.String())
+		})
+	}
+}

exporter/metric/dogstatsd/example_test.go

@@ -0,0 +1,87 @@
+package dogstatsd_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"sync"
+	"time"
+
+	"go.opentelemetry.io/otel/api/key"
+	"go.opentelemetry.io/otel/api/metric"
+	"go.opentelemetry.io/otel/exporter/metric/dogstatsd"
+	"go.opentelemetry.io/otel/sdk/metric/batcher/ungrouped"
+	"go.opentelemetry.io/otel/sdk/metric/controller/push"
+	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
+)
+
+func ExampleNew() {
+	// Create a "server"
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	reader, writer := io.Pipe()
+
+	go func() {
+		defer wg.Done()
+
+		for {
+			var buf [4096]byte
+			n, err := reader.Read(buf[:])
+			if err == io.EOF {
+				return
+			} else if err != nil {
+				log.Fatal("Read err: ", err)
+			} else if n >= len(buf) {
+				log.Fatal("Read small buffer: ", n)
+			} else {
+				fmt.Print(string(buf[0:n]))
+			}
+		}
+	}()
+
+	// Create a meter
+	selector := simple.NewWithExactMeasure()
+	exporter, err := dogstatsd.New(dogstatsd.Config{
+		// The Writer field provides test support.
+		Writer: writer,
+
+		// In real code, use the URL field:
+		//
+		// URL: fmt.Sprint("unix://", path),
+	})
+	if err != nil {
+		log.Fatal("Could not initialize dogstatsd exporter:", err)
+	}
+	// The ungrouped batcher ensures that the export sees the full
+	// set of labels as dogstatsd tags.
+	batcher := ungrouped.New(selector, false)
+
+	// The pusher automatically recognizes that the exporter
+	// implements the LabelEncoder interface, which ensures the
+	// export encoding for labels is encoded in the LabelSet.
+	pusher := push.New(batcher, exporter, time.Hour)
+	pusher.Start()
+
+	ctx := context.Background()
+
+	key := key.New("key")
+
+	// pusher implements the metric.MeterProvider interface:
+	meter := pusher.GetMeter("example")
+
+	// Create and update a single counter:
+	counter := meter.NewInt64Counter("a.counter", metric.WithKeys(key))
+	labels := meter.Labels(key.String("value"))
+
+	counter.Add(ctx, 100, labels)
+
+	// Flush the exporter, close the pipe, and wait for the reader.
+	pusher.Stop()
+	writer.Close()
+	wg.Wait()
+
+	// Output:
+	// a.counter:100|c|#key:value
+}