Allow setting the Sampler via environment variables (#2517)

* Allow setting the Sampler via environment variables (#2305) * Add changelog entry. * Replace t.Setenv with internaltest/SetEnvVariables for Go <= 1.6. * Handle the lack of a sampler argument without logging errors. * Add additional test cases and error checks. * Refactor documentation. Co-authored-by: Joshua MacDonald <jmacd@users.noreply.github.com> * emitBatchOverhead should only be used for splitting spans into batches (#2512) * emitBatchOverhead should only be used for splitting spans into batches (#2503) * limit max packet size parameter * Add additional errors types, simplify abstractions and error handling * Make error comparisons less fragile. * Fix typo in jaeger example (#2524) * Fix some typos in docs for Go libraries (#2520) Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> * Fix getting-started.md Run function (#2527) * Fix getting-started.md Run function, it assigns this new context to a variable shared between connections in to accept loop. Thus creating a growing chain of contexts. so every calculate fibonacci request, all spans in a trace. * add a comment explaining the reason for that new variable * update example fib * Bump github.com/google/go-cmp from 0.5.6 to 0.5.7 across the project (#2545) * update go-cmp to 0.5.7 * fixes go.sums Co-authored-by: Aaron Clawson <MadVikingGod@users.noreply.github.com> * Un-escape url coding when parsing baggage. (#2529) * un-escape url coding when parsing baggage. * Added changelog Co-authored-by: Aaron Clawson <MadVikingGod@users.noreply.github.com> Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> * Bump go.opentelemetry.io/proto/otlp from 0.11.0 to 0.12.0 (#2546) * Update go.opentelemetry.io/proto/otlp to v0.12.0 * Changelog * Update CHANGELOG.md Fix's md linting Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> Co-authored-by: Aaron Clawson <MadVikingGod@users.noreply.github.com> Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> * Remove unused sdk/internal/santize (#2549) * Add links to code examples and docs (#2551) * Bump github.com/prometheus/client_golang from 1.11.0 to 1.12.0 in /exporters/prometheus (#2541) * Bump github.com/prometheus/client_golang in /exporters/prometheus Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.11.0 to 1.12.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](prometheus/client_golang@v1.11.0...v1.12.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * go mod tidy Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tyler Yahn <codingalias@gmail.com> Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> * Optimize evictedQueue implementation and use (#2556) * Optimize evictedQueue impl and use Avoid unnecessary allocations in the recordingSpan by using an evictedQueue type instead of a pointer to one. Lazy allocate the evictedQueue queue to prevent unnecessary operations for spans without any use of the queue. Document the evictedQueue * Fix grammar * Add env support for batch span processor (#2515) * Add env support for batch span processor * Update changelog * lint * Bump golang.org/x/tools from 0.1.8 to 0.1.9 in /internal/tools (#2566) * Bump golang.org/x/tools from 0.1.8 to 0.1.9 in /internal/tools Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.1.8 to 0.1.9. - [Release notes](https://github.com/golang/tools/releases) - [Commits](golang/tools@v0.1.8...v0.1.9) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Bump github.com/golangci/golangci-lint from 1.43.0 to 1.44.0 in /internal/tools (#2567) * Bump github.com/golangci/golangci-lint in /internal/tools Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.43.0 to 1.44.0. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](golangci/golangci-lint@v1.43.0...v1.44.0) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Bump github.com/prometheus/client_golang from 1.12.0 to 1.12.1 in /exporters/prometheus (#2570) * Bump github.com/prometheus/client_golang in /exporters/prometheus Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.12.0 to 1.12.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](prometheus/client_golang@v1.12.0...v1.12.1) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Fix TestBackoffRetry in otlp/internal/retry package (#2562) * Fix TestBackoffRetry in otlp retry pkg The delay of the retry is within two times a randomization factor (the back-off time is delay * random number within [1 - factor, 1 + factor]. This means the waitFunc in TestBackoffRetry needs to check the delay is within an appropriate delta, not equal to configure initial delay. * Fix delta value * Fix delta Co-authored-by: Aaron Clawson <Aaron.Clawson@gmail.com> * Bump google.golang.org/grpc from 1.43.0 to 1.44.0 in /exporters/otlp/otlptrace (#2568) * Bump google.golang.org/grpc in /exporters/otlp/otlptrace Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.43.0 to 1.44.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](grpc/grpc-go@v1.43.0...v1.44.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Bump google.golang.org/grpc from 1.43.0 to 1.44.0 in /example/otel-collector (#2565) * Bump google.golang.org/grpc in /example/otel-collector Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.43.0 to 1.44.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](grpc/grpc-go@v1.43.0...v1.44.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Bump google.golang.org/grpc from 1.43.0 to 1.44.0 in /exporters/otlp/otlpmetric (#2572) * Bump google.golang.org/grpc in /exporters/otlp/otlpmetric Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.43.0 to 1.44.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](grpc/grpc-go@v1.43.0...v1.44.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * Auto-fix go.sum changes in dependent modules Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: MrAlias <MrAlias@users.noreply.github.com> * Change Options to accept type not pointer (#2558) * Change trace options to accept type not pointer Add benchmark to show allocation improvement. * Update CONTRIBUTING.md guidelines * Update all Option iface * Fix grammar in CONTRIBUTING * Do not store TracerProvider or Tracer fields in SDK recordingSpan (#2575) * Do not store TracerProvider fields in span Instead of keeping a reference to the span's Tracer, and therefore also it's TracerProvider, and the associated resource and spanLimits just keep the reference to the Tracer. Refer to the TracerProvider fields when needed instead. * Make span refer to the inst lib via the Tracer Instead of holding a field in the span, refer to the field in the parent Tracer. * [website_docs] fix page meta-links (#2580) Contributes to open-telemetry/opentelemetry.io#1096 /cc @cartermp @austinlparker * Validate members once, in `NewMember` (#2522) * use NewMember, or specify if the member is not validated when creating new ones * expect members to already be validated when creating a new package * add changelog entry * add an isEmpty field to member and property for quick validation * rename isEmpty to hasData So by default, an empty struct really is marked as having no data * Update baggage/baggage_test.go Co-authored-by: Aaron Clawson <Aaron.Clawson@gmail.com> * don't validate the member in parseMember, we alredy ran that validation We also don't want to use NewMember, as that runs the property validation again, making the benchmark quite slower * move changelog entry to the fixed section * provide the member/property data when returning an invalid error Co-authored-by: Aaron Clawson <Aaron.Clawson@gmail.com> * Fix link to Zipkin exporter (#2581) Currently it is linked to the old package that was moved. * Unexport EnvBatchSpanProcessor* constants (#2583) * Move BSP env support to internal * Use pkg name * Update env test * Use internal/env in sdk/trace * Avoid an extra allocation in applyTracerProviderEnvConfigs. * Add additional errors for ratio > 1.0. * Add test cases for ratio > 1.0. * Update CHANGELOG.md Co-authored-by: Joshua MacDonald <jmacd@users.noreply.github.com> Co-authored-by: jaychung <ken8203@gmail.com> Co-authored-by: Ben Wells <b.v.wells@gmail.com> Co-authored-by: Jeremy Kaplan <jeremy@stytch.com> Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com> Co-authored-by: thinkgo <49174849+thinkgos@users.noreply.github.com> Co-authored-by: Aaron Clawson <Aaron.Clawson@gmail.com> Co-authored-by: Aaron Clawson <MadVikingGod@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Tyler Yahn <codingalias@gmail.com> Co-authored-by: Chao Weng <19381524+sincejune@users.noreply.github.com> Co-authored-by: Patrice Chalin <chalin@users.noreply.github.com> Co-authored-by: Damien Mathieu <42@dmathieu.com>
open-telemetry · Mar 21, 2022 · fdbcf71 · fdbcf71
1 parent 2f8698c
commit fdbcf71
Show file tree

Hide file tree

Showing 4 changed files with 301 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,7 @@ Code instrumented with the `go.opentelemetry.io/otel/metric` will need to be mod
 ### Added
 
 - Add go 1.18 to our compatibility tests. (#2679)
+- Allow configuring the Sampler with the `OTEL_TRACES_SAMPLER` and `OTEL_TRACES_SAMPLER_ARG` environment variables. (#2305, #2517)
 
 ### Changed
 

diff --git a/sdk/trace/provider.go b/sdk/trace/provider.go
@@ -99,6 +99,7 @@ func NewTracerProvider(opts ...TracerProviderOption) *TracerProvider {
 	o := tracerProviderConfig{
 		spanLimits: NewSpanLimits(),
 	}
+	o = applyTracerProviderEnvConfigs(o)
 
 	for _, opt := range opts {
 		o = opt.apply(o)
@@ -335,7 +336,10 @@ func WithIDGenerator(g IDGenerator) TracerProviderOption {
 // Tracers the TracerProvider creates to make their sampling decisions for the
 // Spans they create.
 //
-// If this option is not used, the TracerProvider will use a
+// This option overrides the Sampler configured through the OTEL_TRACES_SAMPLER
+// and OTEL_TRACES_SAMPLER_ARG environment variables. If this option is not used
+// and the sampler is not configured through environment variables or the environment
+// contains invalid/unsupported configuration, the TracerProvider will use a
 // ParentBased(AlwaysSample) Sampler by default.
 func WithSampler(s Sampler) TracerProviderOption {
 	return traceProviderOptionFunc(func(cfg tracerProviderConfig) tracerProviderConfig {
@@ -408,6 +412,29 @@ func WithRawSpanLimits(limits SpanLimits) TracerProviderOption {
 	})
 }
 
+func applyTracerProviderEnvConfigs(cfg tracerProviderConfig) tracerProviderConfig {
+	for _, opt := range tracerProviderOptionsFromEnv() {
+		cfg = opt.apply(cfg)
+	}
+
+	return cfg
+}
+
+func tracerProviderOptionsFromEnv() []TracerProviderOption {
+	var opts []TracerProviderOption
+
+	sampler, err := samplerFromEnv()
+	if err != nil {
+		otel.Handle(err)
+	}
+
+	if sampler != nil {
+		opts = append(opts, WithSampler(sampler))
+	}
+
+	return opts
+}
+
 // ensureValidTracerProviderConfig ensures that given TracerProviderConfig is valid.
 func ensureValidTracerProviderConfig(cfg tracerProviderConfig) tracerProviderConfig {
 	if cfg.sampler == nil {

diff --git a/sdk/trace/provider_test.go b/sdk/trace/provider_test.go
@@ -17,10 +17,14 @@ package trace
 import (
 	"context"
 	"errors"
+	"fmt"
+	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
+	ottest "go.opentelemetry.io/otel/internal/internaltest"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -94,3 +98,164 @@ func TestSchemaURL(t *testing.T) {
 	tracerStruct := tracerIface.(*tracer)
 	assert.EqualValues(t, schemaURL, tracerStruct.instrumentationLibrary.SchemaURL)
 }
+
+func TestTracerProviderSamplerConfigFromEnv(t *testing.T) {
+	type testCase struct {
+		sampler             string
+		samplerArg          string
+		argOptional         bool
+		description         string
+		errorType           error
+		invalidArgErrorType interface{}
+	}
+
+	randFloat := rand.Float64()
+
+	tests := []testCase{
+		{
+			sampler:             "invalid-sampler",
+			argOptional:         true,
+			description:         ParentBased(AlwaysSample()).Description(),
+			errorType:           errUnsupportedSampler("invalid-sampler"),
+			invalidArgErrorType: func() *errUnsupportedSampler { e := errUnsupportedSampler("invalid-sampler"); return &e }(),
+		},
+		{
+			sampler:     "always_on",
+			argOptional: true,
+			description: AlwaysSample().Description(),
+		},
+		{
+			sampler:     "always_off",
+			argOptional: true,
+			description: NeverSample().Description(),
+		},
+		{
+			sampler:     "traceidratio",
+			samplerArg:  fmt.Sprintf("%g", randFloat),
+			description: TraceIDRatioBased(randFloat).Description(),
+		},
+		{
+			sampler:     "traceidratio",
+			samplerArg:  fmt.Sprintf("%g", -randFloat),
+			description: TraceIDRatioBased(1.0).Description(),
+			errorType:   errNegativeTraceIDRatio,
+		},
+		{
+			sampler:     "traceidratio",
+			samplerArg:  fmt.Sprintf("%g", 1+randFloat),
+			description: TraceIDRatioBased(1.0).Description(),
+			errorType:   errGreaterThanOneTraceIDRatio,
+		},
+		{
+			sampler:             "traceidratio",
+			argOptional:         true,
+			description:         TraceIDRatioBased(1.0).Description(),
+			invalidArgErrorType: new(samplerArgParseError),
+		},
+		{
+			sampler:     "parentbased_always_on",
+			argOptional: true,
+			description: ParentBased(AlwaysSample()).Description(),
+		},
+		{
+			sampler:     "parentbased_always_off",
+			argOptional: true,
+			description: ParentBased(NeverSample()).Description(),
+		},
+		{
+			sampler:     "parentbased_traceidratio",
+			samplerArg:  fmt.Sprintf("%g", randFloat),
+			description: ParentBased(TraceIDRatioBased(randFloat)).Description(),
+		},
+		{
+			sampler:     "parentbased_traceidratio",
+			samplerArg:  fmt.Sprintf("%g", -randFloat),
+			description: ParentBased(TraceIDRatioBased(1.0)).Description(),
+			errorType:   errNegativeTraceIDRatio,
+		},
+		{
+			sampler:     "parentbased_traceidratio",
+			samplerArg:  fmt.Sprintf("%g", 1+randFloat),
+			description: ParentBased(TraceIDRatioBased(1.0)).Description(),
+			errorType:   errGreaterThanOneTraceIDRatio,
+		},
+		{
+			sampler:             "parentbased_traceidratio",
+			argOptional:         true,
+			description:         ParentBased(TraceIDRatioBased(1.0)).Description(),
+			invalidArgErrorType: new(samplerArgParseError),
+		},
+	}
+
+	handler.Reset()
+
+	for _, test := range tests {
+		t.Run(test.sampler, func(t *testing.T) {
+			envVars := map[string]string{
+				"OTEL_TRACES_SAMPLER": test.sampler,
+			}
+
+			if test.samplerArg != "" {
+				envVars["OTEL_TRACES_SAMPLER_ARG"] = test.samplerArg
+			}
+			envStore, err := ottest.SetEnvVariables(envVars)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				handler.Reset()
+				require.NoError(t, envStore.Restore())
+			})
+
+			stp := NewTracerProvider(WithSyncer(NewTestExporter()))
+			assert.Equal(t, test.description, stp.sampler.Description())
+			if test.errorType != nil {
+				testStoredError(t, test.errorType)
+			} else {
+				assert.Empty(t, handler.errs)
+			}
+
+			if test.argOptional {
+				t.Run("invalid sampler arg", func(t *testing.T) {
+					envStore, err := ottest.SetEnvVariables(map[string]string{
+						"OTEL_TRACES_SAMPLER":     test.sampler,
+						"OTEL_TRACES_SAMPLER_ARG": "invalid-ignored-string",
+					})
+					require.NoError(t, err)
+					t.Cleanup(func() {
+						handler.Reset()
+						require.NoError(t, envStore.Restore())
+					})
+
+					stp := NewTracerProvider(WithSyncer(NewTestExporter()))
+					t.Cleanup(func() {
+						require.NoError(t, stp.Shutdown(context.Background()))
+					})
+					assert.Equal(t, test.description, stp.sampler.Description())
+
+					if test.invalidArgErrorType != nil {
+						testStoredError(t, test.invalidArgErrorType)
+					} else {
+						assert.Empty(t, handler.errs)
+					}
+				})
+			}
+		})
+	}
+}
+
+func testStoredError(t *testing.T, target interface{}) {
+	t.Helper()
+
+	if assert.Len(t, handler.errs, 1) && assert.Error(t, handler.errs[0]) {
+		err := handler.errs[0]
+
+		require.Implements(t, (*error)(nil), target)
+		require.NotNil(t, target.(error))
+
+		defer handler.Reset()
+		if errors.Is(err, target.(error)) {
+			return
+		}
+
+		assert.ErrorAs(t, err, target)
+	}
+}
diff --git a/sdk/trace/sampler_env.go b/sdk/trace/sampler_env.go
@@ -0,0 +1,107 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package trace // import "go.opentelemetry.io/otel/sdk/trace"
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	tracesSamplerKey    = "OTEL_TRACES_SAMPLER"
+	tracesSamplerArgKey = "OTEL_TRACES_SAMPLER_ARG"
+
+	samplerAlwaysOn                = "always_on"
+	samplerAlwaysOff               = "always_off"
+	samplerTraceIDRatio            = "traceidratio"
+	samplerParentBasedAlwaysOn     = "parentbased_always_on"
+	samplerParsedBasedAlwaysOff    = "parentbased_always_off"
+	samplerParentBasedTraceIDRatio = "parentbased_traceidratio"
+)
+
+type errUnsupportedSampler string
+
+func (e errUnsupportedSampler) Error() string {
+	return fmt.Sprintf("unsupported sampler: %s", string(e))
+}
+
+var (
+	errNegativeTraceIDRatio       = errors.New("invalid trace ID ratio: less than 0.0")
+	errGreaterThanOneTraceIDRatio = errors.New("invalid trace ID ratio: greater than 1.0")
+)
+
+type samplerArgParseError struct {
+	parseErr error
+}
+
+func (e samplerArgParseError) Error() string {
+	return fmt.Sprintf("parsing sampler argument: %s", e.parseErr.Error())
+}
+
+func (e samplerArgParseError) Unwrap() error {
+	return e.parseErr
+}
+
+func samplerFromEnv() (Sampler, error) {
+	sampler, ok := os.LookupEnv(tracesSamplerKey)
+	if !ok {
+		return nil, nil
+	}
+
+	sampler = strings.ToLower(strings.TrimSpace(sampler))
+	samplerArg, hasSamplerArg := os.LookupEnv(tracesSamplerArgKey)
+	samplerArg = strings.TrimSpace(samplerArg)
+
+	switch sampler {
+	case samplerAlwaysOn:
+		return AlwaysSample(), nil
+	case samplerAlwaysOff:
+		return NeverSample(), nil
+	case samplerTraceIDRatio:
+		ratio, err := parseTraceIDRatio(samplerArg, hasSamplerArg)
+		return ratio, err
+	case samplerParentBasedAlwaysOn:
+		return ParentBased(AlwaysSample()), nil
+	case samplerParsedBasedAlwaysOff:
+		return ParentBased(NeverSample()), nil
+	case samplerParentBasedTraceIDRatio:
+		ratio, err := parseTraceIDRatio(samplerArg, hasSamplerArg)
+		return ParentBased(ratio), err
+	default:
+		return nil, errUnsupportedSampler(sampler)
+	}
+
+}
+
+func parseTraceIDRatio(arg string, hasSamplerArg bool) (Sampler, error) {
+	if !hasSamplerArg {
+		return TraceIDRatioBased(1.0), nil
+	}
+	v, err := strconv.ParseFloat(arg, 64)
+	if err != nil {
+		return TraceIDRatioBased(1.0), samplerArgParseError{err}
+	}
+	if v < 0.0 {
+		return TraceIDRatioBased(1.0), errNegativeTraceIDRatio
+	}
+	if v > 1.0 {
+		return TraceIDRatioBased(1.0), errGreaterThanOneTraceIDRatio
+	}
+
+	return TraceIDRatioBased(v), nil
+}