Trace mutex locks in otel

fix

document otel transforms

fix types

deployment/utils/contour.types.ts

@@ -787,6 +787,7 @@ export interface ContourValues {
       };
       [k: string]: unknown;
     };
+    defaultStorageClass?: string;
     imagePullSecrets?: unknown[];
     imageRegistry?: string;
     storageClass?: string;

deployment/utils/observability.ts

@@ -55,6 +55,8 @@ export class Observability {
         },
       },
       podAnnotations: {
+        // This is done because open-telemetry collector doesn't always update the deployment
+        // when the config file changes. This will force-restart it.
         'pulumi.com/update-timestamp': Date.now().toString(),
       },
       clusterRole: {
@@ -133,21 +135,29 @@ export class Observability {
             limit_mib: 409,
             spike_limit_mib: 128,
           },
+          // Filter OpenTelemetry traces that are not needed for debugging.
           'filter/traces': {
             error_mode: 'ignore',
             traces: {
               span: [
+                // Ignore all HEAD/OPTIONS requests
                 'attributes["component"] == "proxy" and attributes["http.method"] == "HEAD"',
                 'attributes["component"] == "proxy" and attributes["http.method"] == "OPTIONS"',
+                // Ignore health checks
+                'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and attributes["http.url"] == "/_readiness"',
+                'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and attributes["http.url"] == "/_health"',
                 'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["http.url"], ".*/_health") == true',
+                // Ignore Contour/Envoy traces for /usage requests
                 'attributes["component"] == "proxy" and attributes["http.method"] == "POST" and attributes["http.url"] == "/usage"',
+                'attributes["component"] == "proxy" and attributes["http.method"] == "POST" and IsMatch(attributes["upstream_cluster.name"], "default_usage-service-.*") == true',
+                // Ignore metrics scraping
                 'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and attributes["http.url"] == "/metrics"',
-                'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and attributes["http.url"] == "/_readiness"',
-                'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and attributes["http.url"] == "/_health"',
+                // Ignore webapp HTTP calls
                 'attributes["component"] == "proxy" and attributes["http.method"] == "GET" and IsMatch(attributes["upstream_cluster.name"], "default_app-.*") == true',
               ],
             },
           },
+          // Remove raw trace information that we don't really need and exposed by default.
           'attributes/trace_filter': {
             actions: [
               'downstream_cluster',
@@ -156,12 +166,12 @@ export class Observability {
               'zone',
               'upstream_cluster',
               'peer.address',
-              'response_flags',
             ].map(key => ({
               key,
               action: 'delete',
             })),
           },
+          // Remove attributes that are not needed for debugging.
           'resource/trace_cleanup': {
             attributes: [
               'host.arch',
@@ -182,6 +192,10 @@ export class Observability {
               action: 'delete',
             })),
           },
+          // Contour spans are not very human-readable by default, so we are transforming them
+          // into a format that is easier to understand.
+          // First, we modify the span URL to be relative, and remove the hostname and full path,
+          // Then, we rename to be "METHOD /path"
           'transform/patch_envoy_spans': {
             error_mode: 'ignore',
             trace_statements: [

packages/services/api/src/modules/shared/providers/mutex.ts

@@ -1,5 +1,6 @@
 import { Inject, Injectable } from 'graphql-modules';
 import Redlock, { ResourceLockedError } from 'redlock';
+import { traceFn } from '@hive/service-common';
 import { Logger } from './logger';
 import type { Redis } from './redis';
 import { REDIS_INSTANCE } from './redis';
@@ -56,6 +57,18 @@ export class Mutex {
     });
   }
 
+  @traceFn('Mutex.lock', {
+    initAttributes: (id, options) => ({
+      'lock.id': id,
+      'lock.duration': options.duration,
+      'lock.retries': options.retries,
+      'lock.retryDelay': options.retryDelay,
+      'lock.autoExtendThreshold': options.autoExtendThreshold,
+    }),
+    errorAttributes: error => ({
+      'error.message': error.message,
+    }),
+  })
   public lock(
     id: string,
     {

packages/services/service-common/src/tracing.ts

@@ -68,6 +68,7 @@ export class TracingInstance {
       resource: new Resource({
         [SEMRESATTRS_SERVICE_NAME]: this.options.serviceName,
       }),
+      resourceDetectors: [],
       contextManager,
       sampler: {
         shouldSample: (context, traceId, spanName, spanKind, attributes, links) => {
@@ -133,6 +134,7 @@ export class TracingInstance {
     return createSlonikInterceptor({
       shouldExcludeStatement: (_ctx, query) => {
         return (
+          query.sql.includes('Heartbeat') ||
           query.sql === 'SELECT EXISTS(SELECT 1)' ||
           query.sql === 'SELECT 1' ||
           query.sql === '/* Heartbeat */ SELECT 1' ||

packages/web/docs/src/pages/docs/self-hosting/telemetry.mdx

@@ -55,7 +55,7 @@ docker run \
   -p 4318:4318 \
   -p 4317:4317 \
   otel/opentelemetry-collector-contrib \
-  --config otel-local-config.yaml;
+  --config otel-local-config.yaml
 ```
 
 ### Configure Hive to send traces to the OTLP Collector
@@ -64,7 +64,7 @@ To instruct Hive to send traces to the OTLP Collector, you need to set the follo
 variable:
 
 ```
-OPENTELEMETRY_COLLECTOR_ENDPOINT="http://localhost:4318/v1/traces"
+OPENTELEMETRY_COLLECTOR_ENDPOINT="http://localhost:4318"
 ```
 
 </Steps>