Opentracing pd

.gitignore

@@ -9,6 +9,7 @@
 .venv
 .DS_Store
 .rebar/
+.rebar3/
 .erlfdb/
 .eunit/
 log
@@ -54,16 +55,20 @@ src/hyper/
 src/ibrowse/
 src/ioq/
 src/hqueue/
+src/jaeger_passage/
 src/jiffy/
 src/ken/
 src/khash/
+src/local/
 src/meck/
 src/mochiweb/
 src/oauth/
+src/passage/
 src/proper/
 src/rebar/
 src/smoosh/
 src/snappy/
+src/thrift_protocol/
 src/triq/
 tmp/
 

rebar.config.script

@@ -87,6 +87,7 @@ SubDirs = [
     "src/couch_peruser",
     "src/couch_tests",
     "src/couch_views",
+    "src/ctrace",
     "src/ddoc_cache",
     "src/dreyfus",
     "src/fabric",
@@ -119,9 +120,13 @@ DepDescs = [
 {folsom,           "folsom",           {tag, "CouchDB-0.8.3"}},
 {hyper,            "hyper",            {tag, "CouchDB-2.2.0-4"}},
 {ibrowse,          "ibrowse",          {tag, "CouchDB-4.0.1-1"}},
+{jaeger_passage,   "jaeger-passage",   {tag, "CouchDB-0.1.13-1"}},
 {jiffy,            "jiffy",            {tag, "CouchDB-0.14.11-2"}},
+{local,            "local",            {tag, "0.2.1"}},
 {mochiweb,         "mochiweb",         {tag, "v2.19.0"}},
 {meck,             "meck",             {tag, "0.8.8"}},
+{passage,          "passage",          {tag, "0.2.6"}},
+{thrift_protocol,  "thrift-protocol",  {tag, "0.1.3"}},
 
 %% TMP - Until this is moved to a proper Apache repo
 {erlfdb,           "erlfdb",           {branch, "master"}}

rel/overlay/etc/default.ini

@@ -164,8 +164,8 @@ enable_xframe_options = false
 ; CouchDB can optionally enforce a maximum uri length;
 ; max_uri_length = 8000
 ; changes_timeout = 60000
-; config_whitelist = 
-; max_uri_length = 
+; config_whitelist =
+; max_uri_length =
 ; rewrite_limit = 100
 ; x_forwarded_host = X-Forwarded-Host
 ; x_forwarded_proto = X-Forwarded-Proto
@@ -174,7 +174,7 @@ enable_xframe_options = false
 max_http_request_size = 4294967296 ; 4GB
 
 ; [httpd_design_handlers]
-; _view = 
+; _view =
 
 ; [ioq]
 ; concurrency = 10
@@ -188,7 +188,7 @@ port = 6984
 
 ; [chttpd_auth_cache]
 ; max_lifetime = 600000
-; max_objects = 
+; max_objects =
 ; max_size = 104857600
 
 ; [mem3]
@@ -199,7 +199,7 @@ port = 6984
 
 ; [fabric]
 ; all_docs_concurrency = 10
-; changes_duration = 
+; changes_duration =
 ; shard_timeout_factor = 2
 ; uuid_prefix_len = 7
 ; request_timeout = 60000
@@ -242,7 +242,7 @@ iterations = 10 ; iterations for password hashing
 ; proxy_use_secret = false
 ; comma-separated list of public fields, 404 if empty
 ; public_fields =
-; secret = 
+; secret =
 ; users_db_public = false
 ; cookie_domain = example.com
 
@@ -330,7 +330,7 @@ javascript = couch_js
 couch_mrview = true
 
 [feature_flags]
-; This enables any database to be created as a partitioned databases (except system db's). 
+; This enables any database to be created as a partitioned databases (except system db's).
 ; Setting this to false will stop the creation of paritioned databases.
 ; paritioned||allowed* = true will scope the creation of partitioned databases
 ; to databases with 'allowed' prefix.
@@ -530,7 +530,7 @@ min_priority = 2.0
 ; The default number of results returned from a search on a partition
 ; of a database.
 ; limit_partitions = 2000
- 
+
 ; The maximum number of results that can be returned from a global
 ; search query (or any search query on a database without user-defined
 ; partitions). Attempts to set ?limit=N higher than this value will
@@ -564,3 +564,37 @@ min_priority = 2.0
 ;
 ; Jitter applied when checking for new job types.
 ;type_check_max_jitter_msec = 5000
+
+[tracing]
+;
+; Configuration settings for the `ctrace` OpenTracing
+; API.
+;
+; enabled = false ; true | false
+; thrift_format = compact ; compact | binary
+; agent_host = 127.0.0.1
+; agent_port = 6831
+; app_name = couchdb ; value to use for the `location.application` tag
+
+[tracing.filters]
+;
+; Configure tracing for each individual operation. Keys should be set as
+; operation names (i.e., `database-info.read` or `view.build`). Values
+; are essentially an anonymous function that accepts a single argument
+; that is the tags provided to the root span. These definitions
+; should not include a function name or a trailing `.`. Return values
+; must be one of `true`, `false`, or `float()`. A boolean return
+; indicates whether or not to include the trace while a `float()`
+; value between 0 and 1 gives the probability that the trace should
+; be included or not. I.e., if the value is `0.9` then 90% of the
+; traces will be logged. See the `src/ctrace/README.md` for a
+; thorough description of the filter DSL.
+;
+; database-info.read = (#{'http.method' := Method}) when Method == 'GET' -> true
+; view.build = (#{'view.name' := Name}) when Name == "foo" -> 0.25
+;
+; The key `all` is checked for any trace that does not have a
+; corresponding operation name key configured. Thus, users can easily
+; log every generated trace by including the following:
+;
+; all = (#{}) -> true

src/chttpd/src/chttpd.app.src

@@ -26,6 +26,7 @@
         couch_stats,
         config,
         couch,
+        ctrace,
         ets_lru,
         fabric
     ]},

src/chttpd/src/chttpd.erl

@@ -259,13 +259,15 @@ handle_request_int(MochiReq) ->
 
     case after_request(HttpReq2, HttpResp) of
         #httpd_resp{status = ok, response = Resp} ->
+            span_ok(HttpResp),
             {ok, Resp};
         #httpd_resp{status = aborted, reason = Reason} ->
             couch_log:error("Response abnormally terminated: ~p", [Reason]),
             exit(normal)
     end.
 
 before_request(HttpReq) ->
+    ctrace:is_enabled() andalso start_span(HttpReq),
     try
         chttpd_stats:init(),
         chttpd_plugin:before_request(HttpReq)
@@ -316,6 +318,8 @@ process_request(#httpd{mochi_req = MochiReq} = HttpReq) ->
     end.
 
 handle_req_after_auth(HandlerKey, HttpReq) ->
+    #httpd{user_ctx = #user_ctx{name = User}} = HttpReq,
+    ctrace:tag(#{user => User}),
     try
         HandlerFun = chttpd_handlers:url_handler(HandlerKey,
             fun chttpd_db:handle_request/1),
@@ -1050,16 +1054,20 @@ send_error(#httpd{} = Req, Code, ErrorStr, ReasonStr) ->
     send_error(Req, Code, [], ErrorStr, ReasonStr, []).
 
 send_error(Req, Code, Headers, ErrorStr, ReasonStr, []) ->
-    send_json(Req, Code, Headers,
+    Return = send_json(Req, Code, Headers,
         {[{<<"error">>,  ErrorStr},
-        {<<"reason">>, ReasonStr}]});
+        {<<"reason">>, ReasonStr}]}),
+    span_error(Code, ErrorStr, ReasonStr, []),
+    Return;
 send_error(Req, Code, Headers, ErrorStr, ReasonStr, Stack) ->
     log_error_with_stack_trace({ErrorStr, ReasonStr, Stack}),
-    send_json(Req, Code, [stack_trace_id(Stack) | Headers],
+    Return = send_json(Req, Code, [stack_trace_id(Stack) | Headers],
         {[{<<"error">>,  ErrorStr},
         {<<"reason">>, ReasonStr} |
         case Stack of [] -> []; _ -> [{<<"ref">>, stack_hash(Stack)}] end
-    ]}).
+    ]}),
+    span_error(Code, ErrorStr, ReasonStr, Stack),
+    Return.
 
 update_timeout_stats(<<"timeout">>, #httpd{requested_path_parts = PathParts}) ->
     update_timeout_stats(PathParts);
@@ -1228,6 +1236,104 @@ maybe_trace_fdb("true") ->
 maybe_trace_fdb(_) ->
     ok.
 
+start_span(Req) ->
+    #httpd{
+        mochi_req = MochiReq,
+        begin_ts = Begin,
+        peer = Peer,
+        nonce = Nonce,
+        method = Method,
+        path_parts = PathParts
+    } = Req,
+    {OperationName, ExtraTags} = get_action(Req),
+    Tags = maps:merge(#{
+        peer => Peer,
+        'http.method' => Method,
+        nonce => Nonce,
+        'http.url' => MochiReq:get(raw_path),
+        path_parts => PathParts,
+        'span.kind' => <<"server">>,
+        component => <<"couchdb.chttpd">>
+    }, ExtraTags),
+
+    ctrace:start_span(OperationName, [
+        {tags, Tags},
+        {time, Begin}
+    ] ++ maybe_root_span(MochiReq)).
+
+maybe_root_span(MochiReq) ->
+    case get_trace_headers(MochiReq) of
+        [undefined, _, _] ->
+            [];
+        [TraceId, SpanId, ParentSpanId] ->
+            Span = ctrace:external_span(TraceId, SpanId, ParentSpanId),
+            [{root, Span}]
+    end.
+
+parse_trace_id(undefined) ->
+    undefined;
+parse_trace_id(Hex) ->
+    to_int(Hex, 32).
+
+parse_span_id(undefined) ->
+    undefined;
+parse_span_id(Hex) ->
+    to_int(Hex, 16).
+
+to_int(Hex, N) when length(Hex) =:= N ->
+    try
+        list_to_integer(Hex, 16)
+    catch error:badarg ->
+        undefined
+    end.
+
+get_trace_headers(MochiReq) ->
+    case MochiReq:get_header_value("b3") of
+        undefined ->
+            [
+                parse_trace_id(MochiReq:get_header_value("X-B3-TraceId")),
+                parse_span_id(MochiReq:get_header_value("X-B3-SpanId")),
+                parse_span_id(MochiReq:get_header_value("X-B3-ParentSpanId"))
+            ];
+        Value ->
+            case binary:split(Value, <<"-">>, [global]) of
+                [TraceIdStr, SpanIdStr, _SampledStr, ParentSpanIdStr] ->
+                    [
+                        parse_trace_id(TraceIdStr),
+                        parse_span_id(SpanIdStr),
+                        parse_span_id(ParentSpanIdStr)
+                    ];
+                _ ->
+                    [undefined, undefined, undefined]
+             end
+    end.
+
+get_action(#httpd{} = Req) ->
+    try
+        chttpd_handlers:handler_info(Req)
+    catch Tag:Error ->
+        couch_log:error("Cannot set tracing action ~p:~p", [Tag, Error]),
+        {undefind, #{}}
+    end.
+
+span_ok(#httpd_resp{code = Code}) ->
+    ctrace:tag(#{
+        error => false,
+        'http.status_code' => Code
+    }),
+    ctrace:finish_span().
+
+span_error(Code, ErrorStr, ReasonStr, Stack) ->
+    ctrace:tag(#{
+        error => true,
+        'http.status_code' => Code
+    }),
+    ctrace:log(#{
+        'error.kind' => ErrorStr,
+        message => ReasonStr,
+        stack => Stack
+    }),
+    ctrace:finish_span().
 
 -ifdef(TEST).
 

src/chttpd/src/chttpd_handlers.erl

@@ -15,7 +15,8 @@
 -export([
     url_handler/2,
     db_handler/2,
-    design_handler/2
+    design_handler/2,
+    handler_info/1
 ]).
 
 -define(SERVICE_ID, chttpd_handlers).
@@ -35,6 +36,10 @@ db_handler(HandlerKey, DefaultFun) ->
 design_handler(HandlerKey, DefaultFun) ->
     select(collect(design_handler, [HandlerKey]), DefaultFun).
 
+handler_info(HttpReq) ->
+    Default = {'unknown.unknown', #{}},
+    select(collect(handler_info, [HttpReq]), Default).
+
 %% ------------------------------------------------------------------
 %% Internal Function Definitions
 %% ------------------------------------------------------------------

src/chttpd/src/chttpd_httpd_handlers.erl

@@ -12,7 +12,10 @@
 
 -module(chttpd_httpd_handlers).
 
--export([url_handler/1, db_handler/1, design_handler/1]).
+-export([url_handler/1, db_handler/1, design_handler/1, handler_info/1]).
+
+-include_lib("couch/include/couch_db.hrl").
+
 
 url_handler(<<>>)                  -> fun chttpd_misc:handle_welcome_req/1;
 url_handler(<<"favicon.ico">>)     -> fun chttpd_misc:handle_favicon_req/1;
@@ -44,3 +47,28 @@ design_handler(<<"_update">>)  -> fun chttpd_show:handle_doc_update_req/3;
 design_handler(<<"_info">>)    -> fun chttpd_db:handle_design_info_req/3;
 design_handler(<<"_rewrite">>) -> fun chttpd_rewrite:handle_rewrite_req/3;
 design_handler(_) -> no_match.
+
+%% TODO Populate in another PR
+handler_info(#httpd{path_parts=[<<"_all_dbs">>], method=Method})
+        when Method =:= 'HEAD' orelse Method =:= 'GET' ->
+    {'all-dbs.read', #{}};
+
+handler_info(#httpd{path_parts=[<<"_session">>], method=Method})
+        when Method =:= 'HEAD' orelse Method =:= 'GET' ->
+    {'session.read', #{}};
+handler_info(#httpd{path_parts=[<<"_session">>], method='POST'}) ->
+    {'session.write', #{}};
+handler_info(#httpd{path_parts=[<<"_session">>], method='DELETE'}) ->
+    {'session.delete', #{}};
+
+handler_info(#httpd{path_parts=[_Db], method=Method})
+        when Method =:= 'HEAD' orelse Method =:= 'GET' ->
+    {'database-info.read', #{}};
+handler_info(#httpd{path_parts=[_Db], method='POST'}) ->
+    {'document.write', #{}};
+handler_info(#httpd{path_parts=[_Db], method='PUT'}) ->
+    {'database.create', #{}};
+handler_info(#httpd{path_parts=[_Db], method='DELETE'}) ->
+    {'database.delete', #{}};
+
+handler_info(_) -> no_match.