Introduce sampling package as reference implementation for OTEP 235

.chloggen/add_pkg_sampling.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg_sampling
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Package of code for parsing OpenTelemetry tracestate probability sampling fields.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [29738]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

.github/CODEOWNERS

@@ -136,6 +136,7 @@ pkg/ottl/                                                @open-telemetry/collect
 pkg/pdatatest/                                           @open-telemetry/collector-contrib-approvers @djaglowski @fatsheep9146
 pkg/pdatautil/                                           @open-telemetry/collector-contrib-approvers @dmitryax
 pkg/resourcetotelemetry/                                 @open-telemetry/collector-contrib-approvers @mx-psi
+pkg/sampling/                                            @open-telemetry/collector-contrib-approvers @jmacd @kentquirk
 pkg/stanza/                                              @open-telemetry/collector-contrib-approvers @djaglowski
 pkg/translator/azure/                                    @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @atoulme @cparkins
 pkg/translator/jaeger/                                   @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers @frzifus

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -133,6 +133,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -127,6 +127,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

.github/ISSUE_TEMPLATE/other.yaml

@@ -127,6 +127,7 @@ body:
       - pkg/pdatatest
       - pkg/pdatautil
       - pkg/resourcetotelemetry
+      - pkg/sampling
       - pkg/stanza
       - pkg/translator/azure
       - pkg/translator/jaeger

pkg/sampling/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

pkg/sampling/README.md

@@ -0,0 +1,23 @@
+# pkg/sampling
+
+## Overview
+
+This package contains utilities for parsing and interpreting the W3C
+[TraceState](https://www.w3.org/TR/trace-context/#tracestate-header)
+and all sampling-relevant fields specified by OpenTelemetry that may
+be found in the OpenTelemetry section of the W3C TraceState.
+
+This package implements the draft specification in [OTEP
+235](https://github.com/open-telemetry/oteps/pull/235), which
+specifies two fields used by the OpenTelemetry consistent probability
+sampling scheme.
+
+These are:
+
+- `th`: the Threshold used to determine whether a TraceID is sampled
+- `rv`: an explicit randomness value, which overrides randomness in the TraceID
+
+[OTEP 235](https://github.com/open-telemetry/oteps/pull/235) contains
+details on how to interpret these fields.  The are not meant to be
+human readable, with a few exceptions.  The tracestate entry `ot=th:0`
+indicates 100% sampling.

pkg/sampling/common.go

@@ -0,0 +1,127 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package sampling // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling"
+
+import (
+	"errors"
+	"io"
+	"strings"
+
+	"go.uber.org/multierr"
+)
+
+// KV represents a key-value parsed from a section of the TraceState.
+type KV struct {
+	Key   string
+	Value string
+}
+
+var (
+	// ErrTraceStateSize is returned when a TraceState is over its
+	// size limit, as specified by W3C.
+	ErrTraceStateSize = errors.New("invalid tracestate size")
+)
+
+// keyValueScanner defines distinct scanner behaviors for lists of
+// key-values.
+type keyValueScanner struct {
+	// maxItems is 32 or -1
+	maxItems int
+	// trim is set if OWS (optional whitespace) should be removed
+	trim bool
+	// separator is , or ;
+	separator byte
+	// equality is = or :
+	equality byte
+}
+
+// commonTraceState is embedded in both W3C and OTel trace states.
+type commonTraceState struct {
+	kvs []KV
+}
+
+func (cts commonTraceState) HasExtraValues() bool {
+	return len(cts.kvs) != 0
+}
+
+func (cts commonTraceState) ExtraValues() []KV {
+	return cts.kvs
+}
+
+// trimOws removes optional whitespace on both ends of a string.
+// this uses the strict definition for optional whitespace tiven
+// in https://www.w3.org/TR/trace-context/#tracestate-header-field-values
+func trimOws(input string) string {
+	return strings.Trim(input, " \t")
+}
+
+// scanKeyValues is common code to scan either W3C or OTel tracestate
+// entries, as parameterized in the keyValueScanner struct.
+func (s keyValueScanner) scanKeyValues(input string, f func(key, value string) error) error {
+	var rval error
+	items := 0
+	for input != "" {
+		items++
+		if s.maxItems > 0 && items >= s.maxItems {
+			// W3C specifies max 32 entries, tested here
+			// instead of via the regexp.
+			return ErrTraceStateSize
+		}
+
+		sep := strings.IndexByte(input, s.separator)
+
+		var member string
+		if sep < 0 {
+			member = input
+			input = ""
+		} else {
+			member = input[:sep]
+			input = input[sep+1:]
+		}
+
+		if s.trim {
+			// Trim only required for W3C; OTel does not
+			// specify whitespace for its value encoding.
+			member = trimOws(member)
+		}
+
+		if member == "" {
+			// W3C allows empty list members.
+			continue
+		}
+
+		eq := strings.IndexByte(member, s.equality)
+		if eq < 0 {
+			// We expect to find the `s.equality`
+			// character in this string because we have
+			// already validated the whole input syntax
+			// before calling this parser.  I.e., this can
+			// never happen, and if it did, the result
+			// would be to skip malformed entries.
+			continue
+		}
+		if err := f(member[:eq], member[eq+1:]); err != nil {
+			rval = multierr.Append(rval, err)
+		}
+	}
+	return rval
+}
+
+// serializer assists with checking and combining errors from
+// (io.StringWriter).WriteString().
+type serializer struct {
+	writer io.StringWriter
+	err    error
+}
+
+// write handles errors from io.StringWriter.
+func (ser *serializer) write(str string) {
+	_, err := ser.writer.WriteString(str)
+	ser.check(err)
+}
+
+// check handles errors (e.g., from another serializer).
+func (ser *serializer) check(err error) {
+	ser.err = multierr.Append(ser.err, err)
+}

pkg/sampling/doc.go

@@ -0,0 +1,160 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// ### TraceState handling types
+//
+// The TraceState handling routines in this package preserve unrecognized
+// key values which may be present in the OpenTelemetry `tracestate` header.
+
+// - `Threshold`: Represents an exact sampling probability.
+// - `Randomness`: Randomness used for sampling decisions.
+// - `Probability`: a float64 in the range `[MinSamplingProbability, 1.0]`.
+
+// #### W3C TraceState
+
+// A W3C `tracestate` header can be parsed to extract sampling parameters
+// and pass through unrecognized fields.
+
+// - `NewW3CTraceState`: Parse a W3C `tracestate` header with syntax like
+//   `k1=v1,v2=v2`.
+// - `W3CTraceState.Serialize`: After modifying a TraceState with a
+//   sampling update, this produces a new string encoding to be placed
+//   back into the span.
+
+// The `W3CTraceState` accessor methods are:
+
+// - `W3CTraceState.HasOTelValue`: Returns true when the `tracestate`
+//   contains an `ot` field representing the OpenTelemetry trace state.
+//   Callers may opt to take a fast path when the OpenTelemetry
+//   tracestate is empty.
+// - `W3CTraceState.OTelValue`: Returns a mutable `*OpenTelemetryTraceState`
+//   belonging to this object.
+
+// `W3CTraceState` objects also support the common TraceState members.
+
+// #### OpenTelemetry TraceState
+
+// An OpenTelemetry `tracestate` field is extracted using the `ot` field
+// of the W3C TraceState and is parsed to extract sampling parameters and
+// pass through unrecognized fields.
+
+// - `NewOpenTelemetryTraceState`: Parse an OpenTelemetry `tracestate` header with
+//   syntax like `k1:v1;v2:v2`.
+// - `OpenTelemetryTraceState.Serialize`: After modifying a TraceState with a
+//   sampling update, this produces a new `ot` value.
+
+// There are `OpenTelemetryTraceState` accessor methods associated with `Threshold` and
+// `Randomness` discussed below.  `OpenTelemetryTraceState` objects also support
+// the common TraceState members.
+
+// #### Common TraceState members
+
+// Both W3C and OpenTelemety TraceState values allow additional,
+// syntactically valid fields to be stored and re-serialized, that
+// effectively pass through samplers as a result.
+
+// - `HasAnyValue`: Returns true when the `tracestate` is not empty.
+//   Callers may opt to take a fast path when a tracestate is empty.
+// - `HasExtraValues`: Returns true when there are unrecognized values
+//   contained in a TraceState for manual inspection.
+// - `ExtraValues`: Returns a slice of `KV` corresponding with additional
+//   values.  This may be by callers to inspect arbitrary fields, such as
+//   the legacy `p` value representing power-of-two sampling probability.
+// - `KV`: Represents unrecognized key-value pairs from in a TraceState
+//   object.
+
+// #### Sampling Threshold
+
+// The `Threshold` type represents a parsed T-value.  Samplers that use a
+// fixed probability can store these to make `ShouldSample` an
+// inexpensive operation.  Samplers that use dynamic probability can
+// compute them on the fly.  The constructors are:
+
+// - `TValueToThreshold`: Parse an encoded `Threshold` from an encoded
+//   OpenTelemetry TraceState `tv` field.
+// - `ProbabilityToThreshold`: Translate a `float64` probability into a
+//   `Threshold` with up to 14 hex-digits of precision.
+// - `ProbabilityToThresholdWithPrecision`: Translate a `float64`
+//   probability into a `Threshold` with a limited number of hex-digits
+//   of precision.
+
+// The `Threshold` methods are:
+
+// - `Threshold.TValue`: Encode the T-value corresponding with `Threshold`.
+// - `Threshold.ShouldSample`: Make a sampling decision for a given randomness object.
+// - `Threshold.Probability`: Convert a `Threshold` to an exact probability.
+// - `ThresholdGreater` and `ThresholdLessThan`: Threshold objects are
+//   inversely proportional with probability (i.e., smaller threshold
+//   values indicate larger sampling probabilities).  These comparison
+//   functions allow a sampler to detect when sampling will be
+//   inconsistent or have no change, depending on the application.  If
+//   two Thresholds are neither less-than or greater to each other, they
+//   are necessarily equal.
+
+// The `OpenTelemetryTraceState` accessor methods are:
+
+// - `OpenTelemetryTraceState.HasTValue`: Returns true when the tracestate contains a T-value,
+//   indicating that `TValueThreshold` is valid.
+// - `OpenTelemetryTraceState.SetTValue`: Updates the T-value, unconditioanlly modifying
+//   information about probability sampling.
+// - `OpenTelemetryTraceState.ClearTValue`: Allows the sampler to clear an inconsistent T-value.
+// - `OpenTelemetryTraceState.TValueThreshold`: This represents a sampling threshold, primarily
+//   supports the `ShouldSample(Randomness)` interface.
+// - `OpenTelemetryTraceState.UpdateTValueWithSampling`: Performs a logically-consistent sampling
+//   update, which modifies T-value but does not permit raising the sampling
+//   probability.  When successful, this results in an change in `AdjustedCount()`.
+// - `OpenTelemetryTraceState.AdjustedCount`: If T-value is present, this returns the inverse
+//   sampling probability, which should be interpreted as an effective
+//   count for the item.  If T-value is not present, this returns 0.
+
+// #### Sampling Randomness
+
+// The `Randomness` type represents a parsed R-value.  Samplers may
+// interpret randomness from two sources.  The constructors are:
+
+// - `RValueToRandomness`: Parse an encoded Randomness value from the
+//   OpenTelemetry TraceState `rv` field.  This should be used when the
+//   TraceState contains an `rv` field.
+// - `TraceIDToRandomness`: Extract a Randomness value from a TraceID.
+//   This should be used to make a sampling decision when there is
+//   otherwise no randomness available.
+
+// The `Randomness` methods are:
+
+// - `Randomness.RValue`: Encode a `rv` field value from `Randomness`.
+
+// The `OpenTelemetryTraceState` accessor methods are:
+
+// - `OpenTelemetryTraceState.RValueRandomness`: Returns the pre-parsed `Randomness` value
+//   corresponding with the R-value present in the TraceState.  When this
+//   field is present, it will be used instead of TraceID randomness.
+// - `OpenTelemetryTraceState.HasRValue`: Returns true when the tracestate contains an explicit
+//   R-value, in which case `RValueRandomness()` returns a valid result.
+// - `OpenTelemetryTraceState.SetRValue`: Modifies the TraceState R-value, which allows a sampler
+//   to make randomness explicit.
+// - `OpenTelemetryTraceState.ClearRValue`: Allows the sampler to clear an inconsistent R-value.
+
+// #### Error values
+
+// - `ErrTraceStateSize`: This is returned when parsing a TraceState with an over-size-limit field.
+// - `ErrTValueEmpty`: Returned when T-value has size 0.
+// - `ErrTValueSize`: Returned when T-value has size above 14 hex digits.
+// - `ErrRValueSize`: Returned when R-value has size not equal to 14 hex digits.
+// - `ErrInconsistentSampling`: Indicates an attempt to raise effective sampling probability.
+// - `ErrProbabilityRange`: Sampling probability greater than 100% or less than `MinSamplingProbability`.
+// - `ErrPrecisionUnderflow`: Sampling precision is too low for the stated probability.
+
+// #### Exported values and constants
+
+// - `AlwaysSampleThreshold`: Represents 100% sampling, which is T-value 0.
+// - `MaxAdjustedCount`: Equals 2 to the power 56.
+// - `MinSamplingProbability`: Equals `1/MaxAdjustedCount` (i.e., `0x1p-56`).
+// - `NumHexDigits`: Equals 14, the number of hex digits required to
+//   encode 56 bits. This is the maximum supported precision, the maximum
+//   length of a T-value, and the expected length of an R-value.
+
+package sampling // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling"
+
+// Note that the following does not work for non-component packages:
+//
+//	//go:generate mdatagen metadata.yaml