Introduce sampling package as reference implementation for OTEP 235

.chloggen/add_pkg_sampling.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg_sampling
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Package of code for parsing OpenTelemetry tracestate probability sampling fields.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [29738]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.github/CODEOWNERS

@@ -134,6 +134,7 @@ pkg/ottl/                                                @open-telemetry/collect
 pkg/pdatatest/                                           @open-telemetry/collector-contrib-approvers @djaglowski @fatsheep9146
 pkg/pdatautil/                                           @open-telemetry/collector-contrib-approvers @dmitryax
 pkg/resourcetotelemetry/                                 @open-telemetry/collector-contrib-approvers @mx-psi
+pkg/sampling/                                            @open-telemetry/collector-contrib-approvers @jmacd @kentquirk
 pkg/stanza/                                              @open-telemetry/collector-contrib-approvers @djaglowski
 pkg/translator/azure/                                    @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @atoulme @cparkins
 pkg/translator/jaeger/                                   @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @frzifus

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -131,6 +131,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -125,6 +125,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

.github/ISSUE_TEMPLATE/other.yaml

@@ -125,6 +125,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

pkg/sampling/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

pkg/sampling/README.md

@@ -0,0 +1,179 @@
+# pkg/sampling
+
+## Overview
+
+This package contains utilities for parsing and interpreting the W3C
+[TraceState](https://www.w3.org/TR/trace-context/#tracestate-header)
+and all sampling-relevant fields specified by OpenTelemetry that may
+be found in the OpenTelemetry section of the W3C TraceState.
+
+This package implements the draft specification in [OTEP
+235](https://github.com/open-telemetry/oteps/pull/235), which
+specifies two fields used by the OpenTelemetry consistent probability
+sampling scheme.
+
+These are:
+
+- `th`: the Threshold used to determine whether a TraceID is sampled
+- `rv`: an explicit randomness value, which overrides randomness in the TraceID
+
+[OTEP 235](https://github.com/open-telemetry/oteps/pull/235) contains
+details on how to interpret these fields.  The are not meant to be
+human readable, with a few exceptions.  The tracestate entry `ot=th:0`
+indicates 100% sampling.
+
+## API guide
+
+### TraceState handling types
+
+The TraceState handling routines in this package preserve unrecognized
+key values which may be present in the OpenTelemetry `tracestate` header.
+
+- `W3CTraceState`: Represents the outer `tracestate` field, contains OTel and extra values.
+- `OTelTraceState`: Represents the inner `tracestate` field, contains T-Value, R-Value, and extra values.
+- `Threshold`: Represents an exact sampling probability.
+- `Randomness`: Randomness used for sampling decisions.
+- `Probability`: a float64 in the range `[MinSamplingProbability, 1.0]`.
+
+#### W3C TraceState
+
+A W3C `tracestate` header can be parsed to extract sampling parameters
+and pass through unrecognized fields.
+
+- `NewW3CTraceState`: Parse a W3C `tracestate` header with syntax like
+  `k1=v1,v2=v2`.
+- `W3CTraceState.Serialize`: After modifying a TraceState with a
+  sampling update, this produces a new string encoding to be placed
+  back into the span.
+
+The `W3CTraceState` accessor methods are:
+
+- `W3CTraceState.HasOTelValue`: Returns true when the `tracestate` is
+  contains an `ot` field representing the OpenTelemetry trace state.
+  Callers may opt to take a fast path when the OpenTelemetry
+  tracestate is empty.
+- `W3CTraceState.OTelValue`: Returns a mutable `*OTelTraceState`
+  belonging to this object.
+
+`W3CTraceState` objects also support the common TraceState members.
+
+#### OpenTelemetry TraceState
+
+An OpenTelemetry `tracestate` field is extracted using the `ot` field
+of the W3C TraceState and is parsed to extract sampling parameters and
+pass through unrecognized fields.
+
+- `NewOTelTraceState`: Parse an OpenTelemetry `tracestate` header with
+  syntax like `k1:v1;v2:v2`.
+- `OTelTraceState.Serialize`: After modifying a TraceState with a
+  sampling update, this produces a new `ot` value.
+
+There are `OTelTraceState` accessor methods associated with `Threshold` and
+`Randomness` discussed below.  `OTelTraceState` objects also support
+the common TraceState members.
+
+#### Common TraceState members
+
+Both W3C and OpenTelemety TraceState values allow additional,
+syntactically valid fields to be stored and re-serialized, that
+effectively pass through samplers as a result.
+
+- `HasAnyValue`: Returns true when the `tracestate` is not empty.
+  Callers may opt to take a fast path when a tracestate is empty.
+- `HasExtraValues`: Returns true when there are unrecognized values
+  contained in a TraceState for manual inspection.
+- `ExtraValues`: Returns a slice of `KV` corresponding with additional
+  values.  This may be by callers to inspect arbitrary fields, such as
+  the legacy `p` value representing power-of-two sampling probability.
+- `KV`: Represents unrecognized key-value pairs from in a TraceState
+  object.
+
+#### Sampling Threshold
+
+The `Threshold` type represents a parsed T-value.  Samplers that use a
+fixed probability can store these to make `ShouldSample` an
+inexpensive operation.  Samplers that use dynamic probability can
+compute them on the fly.  The constructors are:
+
+- `TValueToThreshold`: Parse an encoded `Threshold` from an encoded
+  OpenTelemetry TraceState `tv` field.
+- `ProbabilityToThreshold`: Translate a `float64` probability into a
+  `Threshold` with up to 14 hex-digits of precision.
+- `ProbabilityToThresholdWithPrecision`: Translate a `float64`
+  probability into a `Threshold` with a limited number of hex-digits
+  of precision.
+
+The `Threshold` methods are:
+
+- `Threshold.TValue`: Encode the T-value corresponding with `Threshold`.
+- `Threshold.ShouldSample`: Make a sampling decision for a given randomness object.
+- `Threshold.Probability`: Convert a `Threshold` to an exact probability.
+- `ThresholdGreater` and `ThresholdLessThan`: Threshold objects are
+  inversely proportional with probability (i.e., smaller threshold
+  values indicate larger sampling probabilities).  These comparison
+  functions allow a sampler to detect when sampling will be
+  inconsistent or have no change, depending on the application.  If
+  two Thresholds are neither less-than or greater to each other, they
+  are necessarily equal.
+
+The `OTelTraceState` accessor methods are:
+
+- `OTelTraceState.HasTValue`: Returns true when the tracestate contains a T-value,
+  indicating that `TValueThreshold` is valid.
+- `OTelTraceState.SetTValue`: Updates the T-value, unconditioanlly modifying
+  information about probability sampling.
+- `OTelTraceState.ClearTValue`: Allows the sampler to clear an inconsistent T-value.
+- `OTelTraceState.TValueThreshold`: This represents a sampling threshold, primarily
+  supports the `ShouldSample(Randomness)` interface.
+- `OTelTraceState.UpdateTValueWithSampling`: Performs a logically-consistent sampling
+  update, which modifies T-value but does not permit raising the sampling
+  probability.  When successful, this results in an change in `AdjustedCount()`.
+- `OTelTraceState.AdjustedCount`: If T-value is present, this returns the inverse
+  sampling probability, which should be interpreted as an effective
+  count for the item.  If T-value is not present, this returns 0.
+
+#### Sampling Randomness
+
+The `Randomness` type represents a parsed R-value.  Samplers may
+interpret randomness from two sources.  The constructors are:
+
+- `RValueToRandomness`: Parse an encoded Randomness value from the
+  OpenTelemetry TraceState `rv` field.  This should be used when the
+  TraceState contains an `rv` field.
+- `TraceIDToRandomness`: Extract a Randomness value from a TraceID.
+  This should be used to make a sampling decision when there is
+  otherwise no randomness available.
+
+The `Randomness` methods are:
+
+- `Randomness.RValue`: Encode a `rv` field value from `Randomness`.
+
+The `OTelTraceState` accessor methods are:
+
+- `OTelTraceState.RValueRandomness`: Returns the pre-parsed `Randomness` value
+  corresponding with the R-value present in the TraceState.  When this
+  field is present, it will be used instead of TraceID randomness.
+- `OTelTraceState.HasRValue`: Returns true when the tracestate contains an explicit
+  R-value, in which case `RValueRandomness()` returns a valid result.
+- `OTelTraceState.SetRValue`: Modifies the TraceState R-value, which allows a sampler
+  to make randomness explicit.
+- `OTelTraceState.ClearRValue`: Allows the sampler to clear an inconsistent R-value.
+
+#### Error values
+
+- `ErrTraceStateSize`: This is returned when parsing a TraceState with an over-size-limit field.
+- `ErrTValueEmpty`: Returned when T-value has size 0.
+- `ErrTValueSize`: Returned when T-value has size above 14 hex digits.
+- `ErrRValueSize`: Returned when R-value has size not equal to 14 hex digits.
+- `ErrInconsistentSampling`: Indicates an attempt to raise effective sampling probability.
+- `ErrProbabilityRange`: Sampling probability greater than 100% or less than `MinSamplingProbability`.
+- `ErrPrecisionUnderflow`: Sampling precision is too low for the stated probability.
+
+#### Exported values and constants
+
+- `AlwaysSampleThreshold`: Represents 100% sampling, which is T-value 0.
+- `MaxAdjustedCount`: Equals 2 to the power 56.
+- `MinSamplingProbability`: Equals `1/MaxAdjustedCount` (i.e., `0x1p-56`).
+- `NumHexDigits`: Equals 14, the number of hex digits required to
+  encode 56 bits. This is the maximum supported precision, the maximum
+  length of a T-value, and the expected length of an R-value.

pkg/sampling/common.go

@@ -0,0 +1,127 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package sampling // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling"
+
+import (
+	"errors"
+	"io"
+	"strings"
+
+	"go.uber.org/multierr"
+)
+
+// KV represents a key-value parsed from a section of the TraceState.
+type KV struct {
+	Key   string
+	Value string
+}
+
+var (
+	// ErrTraceStateSize is returned when a TraceState is over its
+	// size limit, as specified by W3C.
+	ErrTraceStateSize = errors.New("invalid tracestate size")
+)
+
+// keyValueScanner defines distinct scanner behaviors for lists of
+// key-values.
+type keyValueScanner struct {
+	// maxItems is 32 or -1
+	maxItems int
+	// trim is set if OWS (optional whitespace) should be removed
+	trim bool
+	// separator is , or ;
+	separator byte
+	// equality is = or :
+	equality byte
+}
+
+// commonTraceState is embedded in both W3C and OTel trace states.
+type commonTraceState struct {
+	kvs []KV
+}
+
+func (cts commonTraceState) HasExtraValues() bool {
+	return len(cts.kvs) != 0
+}
+
+func (cts commonTraceState) ExtraValues() []KV {
+	return cts.kvs
+}
+
+// trimOws removes optional whitespace on both ends of a string.
+// this uses the strict definition for optional whitespace tiven
+// in https://www.w3.org/TR/trace-context/#tracestate-header-field-values
+func trimOws(input string) string {
+	return strings.Trim(input, " \t")
+}
+
+// scanKeyValues is common code to scan either W3C or OTel tracestate
+// entries, as parameterized in the keyValueScanner struct.
+func (s keyValueScanner) scanKeyValues(input string, f func(key, value string) error) error {
+	var rval error
+	items := 0
+	for input != "" {
+		items++
+		if s.maxItems > 0 && items >= s.maxItems {
+			// W3C specifies max 32 entries, tested here
+			// instead of via the regexp.
+			return ErrTraceStateSize
+		}
+
+		sep := strings.IndexByte(input, s.separator)
+
+		var member string
+		if sep < 0 {
+			member = input
+			input = ""
+		} else {
+			member = input[:sep]
+			input = input[sep+1:]
+		}
+
+		if s.trim {
+			// Trim only required for W3C; OTel does not
+			// specify whitespace for its value encoding.
+			member = trimOws(member)
+		}
+
+		if member == "" {
+			// W3C allows empty list members.
+			continue
+		}
+
+		eq := strings.IndexByte(member, s.equality)
+		if eq < 0 {
+			// We expect to find the `s.equality`
+			// character in this string because we have
+			// already validated the whole input syntax
+			// before calling this parser.  I.e., this can
+			// never happen, and if it did, the result
+			// would be to skip malformed entries.
+			continue
+		}
+		if err := f(member[:eq], member[eq+1:]); err != nil {
+			rval = multierr.Append(rval, err)
+		}
+	}
+	return rval
+}
+
+// serializer assists with checking and combining errors from
+// (io.StringWriter).WriteString().
+type serializer struct {
+	writer io.StringWriter
+	err    error
+}
+
+// write handles errors from io.StringWriter.
+func (ser *serializer) write(str string) {
+	_, err := ser.writer.WriteString(str)
+	ser.check(err)
+}
+
+// check handles errors (e.g., from another serializer).
+func (ser *serializer) check(err error) {
+	ser.err = multierr.Append(ser.err, err)
+}

pkg/sampling/doc.go

@@ -0,0 +1,9 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Package sampling hanldes probability sampling information in the W3C tracestate.
+package sampling // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling"
+
+// Note that the following does not work for non-component packages:
+//
+//	//go:generate mdatagen metadata.yaml