tracing: Add trace sampling configuration to the route, to override the route level

api/envoy/api/v2/route/route.proto

@@ -230,6 +230,10 @@ message Route {
   // Specifies a list of HTTP headers that should be removed from each response
   // to requests matching this route.
   repeated string response_headers_to_remove = 11;
+
+  // Presence of the object defines whether the connection manager's tracing configuration
+  // is overridden by this route specific instance.
+  Tracing tracing = 15;
 }
 
 // Compared to the :ref:`cluster <envoy_api_field_route.RouteAction.cluster>` field that specifies a
@@ -1007,6 +1011,34 @@ message Decorator {
   string operation = 1 [(validate.rules).string.min_bytes = 1];
 }
 
+message Tracing {
+
+  // Target percentage of requests managed by this HTTP connection manager that will be force
+  // traced if the :ref:`x-client-trace-id <config_http_conn_man_headers_x-client-trace-id>`
+  // header is set. This field is a direct analog for the runtime variable
+  // 'tracing.client_sampling' in the :ref:`HTTP Connection Manager
+  // <config_http_conn_man_runtime>`.
+  // Default: 100%
+  envoy.type.FractionalPercent client_sampling = 1;
+
+  // Target percentage of requests managed by this HTTP connection manager that will be randomly
+  // selected for trace generation, if not requested by the client or not forced. This field is
+  // a direct analog for the runtime variable 'tracing.random_sampling' in the
+  // :ref:`HTTP Connection Manager <config_http_conn_man_runtime>`.
+  // Default: 100%
+  envoy.type.FractionalPercent random_sampling = 2;
+
+  // Target percentage of requests managed by this HTTP connection manager that will be traced
+  // after all other sampling checks have been applied (client-directed, force tracing, random
+  // sampling). This field functions as an upper limit on the total configured sampling rate. For
+  // instance, setting client_sampling to 100% but overall_sampling to 1% will result in only 1%
+  // of client requests with the appropriate headers to be force traced. This field is a direct
+  // analog for the runtime variable 'tracing.global_enabled' in the
+  // :ref:`HTTP Connection Manager <config_http_conn_man_runtime>`.
+  // Default: 100%
+  envoy.type.FractionalPercent overall_sampling = 3;
+}
+
 // A virtual cluster is a way of specifying a regex matching rule against
 // certain important endpoints such that statistics are generated explicitly for
 // the matched requests. The reason this is useful is that when doing

docs/root/intro/version_history.rst

@@ -45,6 +45,7 @@ Version history
 * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules.
   more info in the `bazel docs <https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#enabling-optional-features>`_.
 * tool: added :repo:`proto <test/tools/router_check/validation.proto>` support for :ref:`router check tool <install_tools_route_table_check_tool>` tests.
+* tracing: add trace sampling configuration to the route, to override the route level.
 * upstream: added :ref:`upstream_cx_pool_overflow <config_cluster_manager_cluster_stats>` for the connection pool circuit breaker.
 * upstream: an EDS management server can now force removal of a host that is still passing active
   health checking by first marking the host as failed via EDS health check and subsequently removing

include/envoy/router/router.h

@@ -768,6 +768,34 @@ class Decorator {
 
 typedef std::unique_ptr<const Decorator> DecoratorConstPtr;
 
+/**
+ * An interface representing the Tracing for the route configuration.
+ */
+class RouteTracing {
+public:
+  virtual ~RouteTracing() {}
+
+  /**
+   * This method returns the client sampling percentage.
+   * @return the client sampling percentage
+   */
+  virtual const envoy::type::FractionalPercent& getClientSampling() const PURE;
+
+  /**
+   * This method returns the random sampling percentage.
+   * @return the random sampling percentage
+   */
+  virtual const envoy::type::FractionalPercent& getRandomSampling() const PURE;
+
+  /**
+   * This method returns the overall sampling percentage.
+   * @return the overall sampling percentage
+   */
+  virtual const envoy::type::FractionalPercent& getOverallSampling() const PURE;
+};
+
+typedef std::unique_ptr<const RouteTracing> RouteTracingConstPtr;
+
 /**
  * An interface that holds a DirectResponseEntry or RouteEntry for a request.
  */
@@ -790,6 +818,11 @@ class Route {
    */
   virtual const Decorator* decorator() const PURE;
 
+  /**
+   * @return the tracing config or nullptr if not defined for the request.
+   */
+  virtual const RouteTracing* tracingConfig() const PURE;
+
   /**
    * @return const RouteSpecificFilterConfig* the per-filter config pre-processed object for
    *  the given filter name. If there is not per-filter config, or the filter factory returns

source/common/http/async_client_impl.h

@@ -270,6 +270,7 @@ class AsyncStreamImpl : public AsyncClient::Stream,
     const Router::DirectResponseEntry* directResponseEntry() const override { return nullptr; }
     const Router::RouteEntry* routeEntry() const override { return &route_entry_; }
     const Router::Decorator* decorator() const override { return nullptr; }
+    const Router::RouteTracing* tracingConfig() const override { return nullptr; }
     const Router::RouteSpecificFilterConfig* perFilterConfig(const std::string&) const override {
       return nullptr;
     }

source/common/http/conn_manager_config.h

@@ -106,9 +106,9 @@ struct ConnectionManagerTracingStats {
 struct TracingConnectionManagerConfig {
   Tracing::OperationName operation_name_;
   std::vector<Http::LowerCaseString> request_headers_for_tags_;
-  uint64_t client_sampling_;
-  uint64_t random_sampling_;
-  uint64_t overall_sampling_;
+  envoy::type::FractionalPercent client_sampling_;
+  envoy::type::FractionalPercent random_sampling_;
+  envoy::type::FractionalPercent overall_sampling_;
   bool verbose_;
 };
 

source/common/http/conn_manager_impl.cc

@@ -708,12 +708,19 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers,
     stream_info_.setDownstreamRemoteAddress(ConnectionManagerUtility::mutateRequestHeaders(
         *request_headers_, connection_manager_.read_callbacks_->connection(),
         connection_manager_.config_, *snapped_route_config_, connection_manager_.random_generator_,
-        connection_manager_.runtime_, connection_manager_.local_info_));
+        connection_manager_.local_info_));
   }
   ASSERT(stream_info_.downstreamRemoteAddress() != nullptr);
 
   ASSERT(!cached_route_);
   refreshCachedRoute();
+
+  if (!state_.is_internally_created_) { // Only mutate tracing headers on first pass.
+    ConnectionManagerUtility::mutateTracingRequestHeader(
+        *request_headers_, connection_manager_.runtime_, connection_manager_.config_,
+        cached_route_.value().get());
+  }
+
   const bool upgrade_rejected = createFilterChain() == false;
 
   // TODO if there are no filters when starting a filter iteration, the connection manager

source/common/http/conn_manager_utility.cc

@@ -52,7 +52,7 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec(
 
 Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequestHeaders(
     HeaderMap& request_headers, Network::Connection& connection, ConnectionManagerConfig& config,
-    const Router::Config& route_config, Runtime::RandomGenerator& random, Runtime::Loader& runtime,
+    const Router::Config& route_config, Runtime::RandomGenerator& random,
     const LocalInfo::LocalInfo& local_info) {
   // If this is a Upgrade request, do not remove the Connection and Upgrade headers,
   // as we forward them verbatim to the upstream hosts.
@@ -213,15 +213,15 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
     request_headers.insertRequestId().value(uuid);
   }
 
-  mutateTracingRequestHeader(request_headers, runtime, config);
   mutateXfccRequestHeader(request_headers, connection, config);
 
   return final_remote_address;
 }
 
 void ConnectionManagerUtility::mutateTracingRequestHeader(HeaderMap& request_headers,
                                                           Runtime::Loader& runtime,
-                                                          ConnectionManagerConfig& config) {
+                                                          ConnectionManagerConfig& config,
+                                                          const Router::Route* route) {
   if (!config.tracingConfig() || !request_headers.RequestId()) {
     return;
   }
@@ -234,23 +234,31 @@ void ConnectionManagerUtility::mutateTracingRequestHeader(HeaderMap& request_hea
     return;
   }
 
+  const envoy::type::FractionalPercent* client_sampling = &config.tracingConfig()->client_sampling_;
+  const envoy::type::FractionalPercent* random_sampling = &config.tracingConfig()->random_sampling_;
+  const envoy::type::FractionalPercent* overall_sampling =
+      &config.tracingConfig()->overall_sampling_;
+
+  if (route && route->tracingConfig()) {
+    client_sampling = &route->tracingConfig()->getClientSampling();
+    random_sampling = &route->tracingConfig()->getRandomSampling();
+    overall_sampling = &route->tracingConfig()->getOverallSampling();
+  }
+
   // Do not apply tracing transformations if we are currently tracing.
   if (UuidTraceStatus::NoTrace == UuidUtils::isTraceableUuid(x_request_id)) {
     if (request_headers.ClientTraceId() &&
-        runtime.snapshot().featureEnabled("tracing.client_enabled",
-                                          config.tracingConfig()->client_sampling_)) {
+        runtime.snapshot().featureEnabled("tracing.client_enabled", *client_sampling)) {
       UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Client);
     } else if (request_headers.EnvoyForceTrace()) {
       UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Forced);
-    } else if (runtime.snapshot().featureEnabled("tracing.random_sampling",
-                                                 config.tracingConfig()->random_sampling_, result,
-                                                 10000)) {
+    } else if (runtime.snapshot().featureEnabled("tracing.random_sampling", *random_sampling,
+                                                 result)) {
       UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::Sampled);
     }
   }
 
-  if (!runtime.snapshot().featureEnabled("tracing.global_enabled",
-                                         config.tracingConfig()->overall_sampling_, result)) {
+  if (!runtime.snapshot().featureEnabled("tracing.global_enabled", *overall_sampling, result)) {
     UuidUtils::setTraceableUuid(x_request_id, UuidTraceStatus::NoTrace);
   }
 

source/common/http/conn_manager_utility.h

@@ -53,8 +53,7 @@ class ConnectionManagerUtility {
   static Network::Address::InstanceConstSharedPtr
   mutateRequestHeaders(HeaderMap& request_headers, Network::Connection& connection,
                        ConnectionManagerConfig& config, const Router::Config& route_config,
-                       Runtime::RandomGenerator& random, Runtime::Loader& runtime,
-                       const LocalInfo::LocalInfo& local_info);
+                       Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info);
 
   static void mutateResponseHeaders(HeaderMap& response_headers, const HeaderMap* request_headers,
                                     const std::string& via);
@@ -64,13 +63,14 @@ class ConnectionManagerUtility {
   // Return false if error happens during the sanitization.
   static bool maybeNormalizePath(HeaderMap& request_headers, const ConnectionManagerConfig& config);
 
-private:
   /**
    * Mutate request headers if request needs to be traced.
    */
   static void mutateTracingRequestHeader(HeaderMap& request_headers, Runtime::Loader& runtime,
-                                         ConnectionManagerConfig& config);
+                                         ConnectionManagerConfig& config,
+                                         const Router::Route* route);
 
+private:
   static void mutateXfccRequestHeader(HeaderMap& request_headers, Network::Connection& connection,
                                       ConnectionManagerConfig& config);
 };

source/common/router/config_impl.cc

@@ -324,6 +324,39 @@ void DecoratorImpl::apply(Tracing::Span& span) const {
 
 const std::string& DecoratorImpl::getOperation() const { return operation_; }
 
+RouteTracingImpl::RouteTracingImpl(const envoy::api::v2::route::Tracing& tracing) {
+  if (!tracing.has_client_sampling()) {
+    client_sampling_.set_numerator(100);
+    client_sampling_.set_denominator(envoy::type::FractionalPercent::HUNDRED);
+  } else {
+    client_sampling_ = tracing.client_sampling();
+  }
+  if (!tracing.has_random_sampling()) {
+    random_sampling_.set_numerator(100);
+    random_sampling_.set_denominator(envoy::type::FractionalPercent::HUNDRED);
+  } else {
+    random_sampling_ = tracing.random_sampling();
+  }
+  if (!tracing.has_overall_sampling()) {
+    overall_sampling_.set_numerator(100);
+    overall_sampling_.set_denominator(envoy::type::FractionalPercent::HUNDRED);
+  } else {
+    overall_sampling_ = tracing.overall_sampling();
+  }
+}
+
+const envoy::type::FractionalPercent& RouteTracingImpl::getClientSampling() const {
+  return client_sampling_;
+}
+
+const envoy::type::FractionalPercent& RouteTracingImpl::getRandomSampling() const {
+  return random_sampling_;
+}
+
+const envoy::type::FractionalPercent& RouteTracingImpl::getOverallSampling() const {
+  return overall_sampling_;
+}
+
 RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost,
                                        const envoy::api::v2::route::Route& route,
                                        Server::Configuration::FactoryContext& factory_context)
@@ -360,7 +393,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost,
                                                        route.response_headers_to_remove())),
       metadata_(route.metadata()), typed_metadata_(route.metadata()),
       match_grpc_(route.match().has_grpc()), opaque_config_(parseOpaqueConfig(route)),
-      decorator_(parseDecorator(route)),
+      decorator_(parseDecorator(route)), route_tracing_(parseRouteTracing(route)),
       direct_response_code_(ConfigUtility::parseDirectResponseCode(route)),
       direct_response_body_(ConfigUtility::parseDirectResponseBody(route, factory_context.api())),
       per_filter_configs_(route.typed_per_filter_config(), route.per_filter_config(),
@@ -667,6 +700,15 @@ DecoratorConstPtr RouteEntryImplBase::parseDecorator(const envoy::api::v2::route
   return ret;
 }
 
+RouteTracingConstPtr
+RouteEntryImplBase::parseRouteTracing(const envoy::api::v2::route::Route& route) {
+  RouteTracingConstPtr ret;
+  if (route.has_tracing()) {
+    ret = RouteTracingConstPtr(new RouteTracingImpl(route.tracing()));
+  }
+  return ret;
+}
+
 const DirectResponseEntry* RouteEntryImplBase::directResponseEntry() const {
   // A route for a request can exclusively be a route entry, a direct response entry,
   // or a redirect entry.

source/common/router/config_impl.h

@@ -87,6 +87,7 @@ class SslRedirectRoute : public Route {
   const DirectResponseEntry* directResponseEntry() const override { return &SSL_REDIRECTOR; }
   const RouteEntry* routeEntry() const override { return nullptr; }
   const Decorator* decorator() const override { return nullptr; }
+  const RouteTracing* tracingConfig() const override { return nullptr; }
   const RouteSpecificFilterConfig* perFilterConfig(const std::string&) const override {
     return nullptr;
   }
@@ -346,6 +347,28 @@ class DecoratorImpl : public Decorator {
   const std::string operation_;
 };
 
+/**
+ * Implementation of RouteTracing that reads from the proto route tracing.
+ */
+class RouteTracingImpl : public RouteTracing {
+public:
+  RouteTracingImpl(const envoy::api::v2::route::Tracing& tracing);
+
+  // Tracing::getClientSampling
+  const envoy::type::FractionalPercent& getClientSampling() const override;
+
+  // Tracing::getRandomSampling
+  const envoy::type::FractionalPercent& getRandomSampling() const override;
+
+  // Tracing::getOverallSampling
+  const envoy::type::FractionalPercent& getOverallSampling() const override;
+
+private:
+  envoy::type::FractionalPercent client_sampling_;
+  envoy::type::FractionalPercent random_sampling_;
+  envoy::type::FractionalPercent overall_sampling_;
+};
+
 /**
  * Base implementation for all route entries.
  */
@@ -436,6 +459,7 @@ class RouteEntryImplBase : public RouteEntry,
   const DirectResponseEntry* directResponseEntry() const override;
   const RouteEntry* routeEntry() const override;
   const Decorator* decorator() const override { return decorator_.get(); }
+  const RouteTracing* tracingConfig() const override { return route_tracing_.get(); }
   const RouteSpecificFilterConfig* perFilterConfig(const std::string&) const override;
 
 protected:
@@ -535,6 +559,7 @@ class RouteEntryImplBase : public RouteEntry,
     const DirectResponseEntry* directResponseEntry() const override { return nullptr; }
     const RouteEntry* routeEntry() const override { return this; }
     const Decorator* decorator() const override { return parent_->decorator(); }
+    const RouteTracing* tracingConfig() const override { return parent_->tracingConfig(); }
 
     const RouteSpecificFilterConfig* perFilterConfig(const std::string& name) const override {
       return parent_->perFilterConfig(name);
@@ -600,6 +625,8 @@ class RouteEntryImplBase : public RouteEntry,
 
   static DecoratorConstPtr parseDecorator(const envoy::api::v2::route::Route& route);
 
+  static RouteTracingConstPtr parseRouteTracing(const envoy::api::v2::route::Route& route);
+
   bool evaluateRuntimeMatch(const uint64_t random_value) const;
 
   HedgePolicyImpl
@@ -656,6 +683,7 @@ class RouteEntryImplBase : public RouteEntry,
   const std::multimap<std::string, std::string> opaque_config_;
 
   const DecoratorConstPtr decorator_;
+  const RouteTracingConstPtr route_tracing_;
   const absl::optional<Http::Code> direct_response_code_;
   std::string direct_response_body_;
   PerFilterConfigs per_filter_configs_;

source/extensions/filters/network/http_connection_manager/config.cc

@@ -231,12 +231,18 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
       request_headers_for_tags.push_back(Http::LowerCaseString(header));
     }
 
-    uint64_t client_sampling{
-        PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(tracing_config, client_sampling, 100, 100)};
-    uint64_t random_sampling{PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(
-        tracing_config, random_sampling, 10000, 10000)};
-    uint64_t overall_sampling{
-        PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(tracing_config, overall_sampling, 100, 100)};
+    envoy::type::FractionalPercent client_sampling;
+    client_sampling.set_numerator(
+        tracing_config.has_client_sampling() ? tracing_config.client_sampling().value() : 100);
+    envoy::type::FractionalPercent random_sampling;
+    // TODO: Random sampling historically was an integer and default to out of 10,000. We should
+    // deprecate that and move to a straight fractional percent config.
+    random_sampling.set_numerator(
+        tracing_config.has_random_sampling() ? tracing_config.random_sampling().value() : 10000);
+    random_sampling.set_denominator(envoy::type::FractionalPercent::TEN_THOUSAND);
+    envoy::type::FractionalPercent overall_sampling;
+    overall_sampling.set_numerator(
+        tracing_config.has_overall_sampling() ? tracing_config.overall_sampling().value() : 100);
 
     tracing_config_ =
         std::make_unique<Http::TracingConnectionManagerConfig>(Http::TracingConnectionManagerConfig{