Merge pull request #74 from nicolas/client-sampling

implement client sampling

docs/GettingStarted.md

@@ -370,6 +370,18 @@ for the first time:
 Remember that the debug mode may affect your application performance and so it must not be used
 in a production environment.
 
+### Sampling
+
+`ddtrace` can perform trace sampling. While the trace agent already samples
+traces to reduce bandwidth usage, client sampling reduces performance
+overhead.
+
+`Datadog::RateSampler` samples a ratio of the traces. For example:
+
+    # Sample rate is between 0 (nothing sampled) to 1 (everything sampled).
+    sampler = Datadog::RateSampler.new(0.5) # sample 50% of the traces
+    Datadog.tracer.configure(sampler: sampler)
+
 ### Supported Versions
 
 #### Ruby interpreters

lib/ddtrace/sampler.rb

@@ -0,0 +1,50 @@
+
+module Datadog
+  # \Sampler performs client-side trace sampling.
+  class Sampler
+    def sample(_span)
+      raise NotImplementedError, 'samplers have to implement the sample() method'
+    end
+  end
+
+  # \AllSampler samples all the traces.
+  class AllSampler < Sampler
+    def sample(span)
+      span.sampled = true
+    end
+  end
+
+  # \RateSampler is based on a sample rate.
+  class RateSampler < Sampler
+    KNUTH_FACTOR = 1111111111111111111
+    SAMPLE_RATE_METRIC_KEY = '_sample_rate'.freeze()
+
+    attr_reader :sample_rate
+
+    # Initialize a \RateSampler.
+    # This sampler keeps a random subset of the traces. Its main purpose is to
+    # reduce the instrumentation footprint.
+    #
+    # * +sample_rate+: the sample rate as a \Float between 0.0 and 1.0. 0.0
+    #   means that no trace will be sampled; 1.0 means that all traces will be
+    #   sampled.
+    def initialize(sample_rate = 1.0)
+      unless sample_rate > 0.0 && sample_rate <= 1.0
+        Datadog::Tracer.log.error('sample rate is not between 0 and 1, disabling the sampler')
+        sample_rate = 1.0
+      end
+
+      self.sample_rate = sample_rate
+    end
+
+    def sample_rate=(sample_rate)
+      @sample_rate = sample_rate
+      @sampling_id_threshold = sample_rate * Span::MAX_ID
+    end
+
+    def sample(span)
+      span.sampled = ((span.trace_id * KNUTH_FACTOR) % Datadog::Span::MAX_ID) <= @sampling_id_threshold
+      span.set_metric(SAMPLE_RATE_METRIC_KEY, @sample_rate)
+    end
+  end
+end

lib/ddtrace/span.rb

@@ -16,7 +16,7 @@ class Span
     attr_accessor :name, :service, :resource, :span_type,
                   :start_time, :end_time,
                   :span_id, :trace_id, :parent_id,
-                  :status, :parent
+                  :status, :parent, :sampled
 
     # Create a new span linked to the given tracer. Call the <tt>finish()</tt> method once the
     # tracer operation is over or use the <tt>finish_at(time)</tt> helper to close the span with the
@@ -40,9 +40,11 @@ def initialize(tracer, name, options = {})
       @trace_id = options.fetch(:trace_id, @span_id)
 
       @meta = {}
+      @metrics = {}
       @status = 0
 
       @parent = nil
+      @sampled = true
 
       @start_time = Time.now.utc
       @end_time = nil
@@ -58,11 +60,22 @@ def set_tag(key, value)
       Datadog::Tracer.log.error("Unable to set the tag #{key}, ignoring it. Caused by: #{e}")
     end
 
-    # Return the tag wth the given key, nil if it doesn't exist.
+    # Return the tag with the given key, nil if it doesn't exist.
     def get_tag(key)
       @meta[key]
     end
 
+    # Set the given key / value metric pair on the span. Keys must be string.
+    # Values must be floating point numbers.
+    def set_metric(key, value)
+      @metrics[key] = value
+    end
+
+    # Return the metric with the given key, nil if it doesn't exist.
+    def get_metric(key)
+      @metrics[key]
+    end
+
     # Mark the span with the given error.
     def set_error(e)
       return if e.nil?
@@ -122,6 +135,7 @@ def to_hash
         resource: @resource,
         type: @span_type,
         meta: @meta,
+        metrics: @metrics,
         error: @status
       }
 
@@ -151,12 +165,18 @@ def pretty_print(q)
         q.text "Start: #{start_time}\n"
         q.text "End: #{end_time}\n"
         q.text "Duration: #{duration}\n"
-        q.group(2, 'Tags: [', ']') do
+        q.group(2, 'Tags: [', "]\n") do
           q.breakable
           q.seplist @meta.each do |key, value|
             q.text "#{key} => #{value}"
           end
         end
+        q.group(2, 'Metrics: [', ']') do
+          q.breakable
+          q.seplist @metrics.each do |key, value|
+            q.text "#{key} => #{value}"
+          end
+        end
       end
     end
   end

lib/ddtrace/tracer.rb

@@ -5,6 +5,7 @@
 require 'ddtrace/span'
 require 'ddtrace/buffer'
 require 'ddtrace/writer'
+require 'ddtrace/sampler'
 
 # \Datadog global namespace that includes all tracing functionality for Tracer and Span classes.
 module Datadog
@@ -13,7 +14,7 @@ module Datadog
   # Even though the request may require multiple resources and machines to handle the request, all
   # of these function calls and sub-requests would be encapsulated within a single trace.
   class Tracer
-    attr_reader :writer, :services
+    attr_reader :writer, :sampler, :services
     attr_accessor :enabled
 
     # Global, memoized, lazy initialized instance of a logger that is used within the the Datadog
@@ -44,6 +45,8 @@ def self.debug_logging
     def initialize(options = {})
       @enabled = options.fetch(:enabled, true)
       @writer = options.fetch(:writer, Datadog::Writer.new)
+      @sampler = options.fetch(:sampler, Datadog::AllSampler.new)
+
       @buffer = Datadog::SpanBuffer.new()
 
       @mutex = Mutex.new
@@ -66,10 +69,12 @@ def configure(options = {})
       enabled = options.fetch(:enabled, nil)
       hostname = options.fetch(:hostname, nil)
       port = options.fetch(:port, nil)
+      sampler = options.fetch(:sampler, nil)
 
       @enabled = enabled unless enabled.nil?
       @writer.transport.hostname = hostname unless hostname.nil?
       @writer.transport.port = port unless port.nil?
+      @sampler = sampler unless sampler.nil?
     end
 
     # Set the information about the given service. A valid example is:
@@ -121,6 +126,13 @@ def trace(name, options = {})
       span.set_parent(parent)
       @buffer.set(span)
 
+      # sampling
+      if parent.nil?
+        @sampler.sample(span)
+      else
+        span.sampled = span.parent.sampled
+      end
+
       # call the finish only if a block is given; this ensures
       # that a call to tracer.trace() without a block, returns
       # a span that should be manually finished.
@@ -155,7 +167,7 @@ def record(span)
         @spans = []
       end
 
-      return if spans.empty?
+      return if spans.empty? || !span.sampled
       write(spans)
     end
 

test/sampler_test.rb

@@ -0,0 +1,90 @@
+
+require 'helper'
+require 'ddtrace/sampler'
+
+class SamplerTest < Minitest::Test
+  def test_all_sampler
+    spans = [Datadog::Span.new(nil, '', trace_id: 1),
+             Datadog::Span.new(nil, '', trace_id: 2),
+             Datadog::Span.new(nil, '', trace_id: 3)]
+
+    sampler = Datadog::AllSampler.new()
+
+    spans.each do |span|
+      sampler.sample(span)
+      assert_equal(true, span.sampled)
+    end
+  end
+
+  def test_rate_sampler_invalid
+    sampler = Datadog::RateSampler.new(-1.0)
+    assert_equal(1.0, sampler.sample_rate)
+
+    sampler = Datadog::RateSampler.new(0.0)
+    assert_equal(1.0, sampler.sample_rate)
+
+    sampler = Datadog::RateSampler.new(1.5)
+    assert_equal(1.0, sampler.sample_rate)
+  end
+
+  def test_rate_sampler_1_0
+    spans = [Datadog::Span.new(nil, '', trace_id: 1),
+             Datadog::Span.new(nil, '', trace_id: 2),
+             Datadog::Span.new(nil, '', trace_id: 3)]
+
+    sampler = Datadog::RateSampler.new(1.0)
+
+    spans.each do |span|
+      sampler.sample(span)
+      assert_equal(true, span.sampled)
+      assert_equal(1.0, span.get_metric(Datadog::RateSampler::SAMPLE_RATE_METRIC_KEY))
+    end
+  end
+
+  def test_rate_sampler
+    prng = Random.new(123)
+
+    [0.1, 0.25, 0.5, 0.9].each do |sample_rate|
+      nb_spans = 1000
+      spans = Array.new(nb_spans) do
+        Datadog::Span.new(nil, '', trace_id: prng.rand(Datadog::Span::MAX_ID))
+      end
+
+      sampler = Datadog::RateSampler.new(sample_rate)
+
+      spans.each do |span|
+        sampler.sample(span)
+
+        if span.sampled
+          assert_equal(sample_rate,
+                       span.get_metric(Datadog::RateSampler::SAMPLE_RATE_METRIC_KEY))
+        end
+      end
+
+      sampled_spans = spans.select(&:sampled)
+      expected = nb_spans * sample_rate
+      assert_in_delta(sampled_spans.length, expected, 0.1 * expected)
+    end
+  end
+
+  def test_tracer_with_rate_sampler
+    prng = Random.new(123)
+
+    tracer = get_test_tracer()
+    tracer.configure(sampler: Datadog::RateSampler.new(0.5))
+
+    nb_spans = 1000
+    nb_spans.times do
+      span = tracer.trace('test', trace_id: prng.rand(Datadog::Span::MAX_ID))
+      span.finish()
+    end
+
+    spans = tracer.writer.spans()
+    expected = nb_spans * 0.5
+    assert_in_delta(spans.length, expected, 0.1 * expected)
+
+    spans.each do |span|
+      assert_equal(0.5, span.get_metric(Datadog::RateSampler::SAMPLE_RATE_METRIC_KEY))
+    end
+  end
+end