Be able to correlate timeouts in reverse-proxy layer in front of Synapse (tag specific request headers)

changelog.d/13786.feature

@@ -0,0 +1 @@
+Add `opentracing.request_headers_to_tag` config to specify which request headers extract and tag traces with in order to correlate timeouts in reverse-proxy layers in front of Synapse with traces.

docs/usage/configuration/config_documentation.md

@@ -3612,6 +3612,11 @@ Sub-options include:
 * `jaeger_config`: Jaeger can be configured to sample traces at different rates.
    All configuration options provided by Jaeger can be set here. Jaeger's configuration is
    mostly related to trace sampling which is documented [here](https://www.jaegertracing.io/docs/latest/sampling/).
+* `request_headers_to_tag`: A list of headers to extract from the request and
+  add to to the top-level servlet tracing span as tags. Useful when you're using
+  a reverse proxy service like Cloudflare to protect your Synapse instance in
+  order to correlate and match up requests that timed out at the Cloudflare
+  layer to the Synapse traces.
 
 Example configuration:
 ```yaml
@@ -3629,6 +3634,9 @@ opentracing:
         param: 1
       logging:
         false
+
+    request_headers_to_tag:
+      - "cf-ray"
 ```
 ---
 ## Workers ##

synapse/api/auth.py

@@ -173,6 +173,21 @@ async def get_user_by_req(
                     parent_span.set_tag("device_id", requester.device_id)
                 if requester.app_service is not None:
                     parent_span.set_tag("appservice_id", requester.app_service.id)
+
+                # Tag any headers that we need to extract from the request. This
+                # is useful to specify any headers that a reverse-proxy in front
+                # of Synapse may be sending to correlate and match up something
+                # in that layer to a Synapse trace. ex. when Cloudflare times
+                # out it gives a `cf-ray` header which we can also tag here to
+                # find the trace.
+                for header_name in self.hs.config.tracing.request_headers_to_tag:
+                    headers = request.requestHeaders.getRawHeaders(header_name)
+                    if headers is not None:
+                        parent_span.set_tag(
+                            SynapseTags.REQUEST_HEADER_PREFIX + header_name,
+                            str(headers[0]),
+                        )
+
             return requester
 
     @cancellable

synapse/config/tracer.py

@@ -37,6 +37,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
 
         self.force_tracing_for_users: Set[str] = set()
 
+        # A list of headers to extract from the request and add to to the
+        # top-level servlet tracing span as tags. Useful when you're using a
+        # reverse proxy service like Cloudflare to protect your Synapse instance
+        # in order to correlate and match up requests that timed out at the
+        # Cloudflare layer to the Synapse traces.
+        self.request_headers_to_tag = []
+
         if not self.opentracer_enabled:
             return
 
@@ -62,3 +69,13 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
                     ("opentracing", "force_tracing_for_users", f"index {i}"),
                 )
             self.force_tracing_for_users.add(u)
+
+        request_headers_to_tag = opentracing_config.get(
+            "request_headers_to_tag",
+            [],
+        )
+        if not isinstance(request_headers_to_tag, list):
+            raise ConfigError(
+                "Expected a list", ("opentracing", "request_headers_to_tag")
+            )
+        self.request_headers_to_tag = request_headers_to_tag

synapse/logging/opentracing.py

@@ -303,6 +303,10 @@ class SynapseTags:
     # incoming HTTP request ID  (as written in the logs)
     REQUEST_ID = "request_id"
 
+    # Tag a header from the incoming request. The name of the header should be
+    # appended to this prefix.
+    REQUEST_HEADER_PREFIX = "request_header."
+
     # HTTP request tag (used to distinguish full vs incremental syncs, etc)
     REQUEST_TAG = "request_tag"