Start using open telemetry

.gitignore

@@ -58,3 +58,4 @@ data/
 registration.yaml
 
 storybook-static
+development/tempo-data/

apps/meteor/app/api/server/api.ts

@@ -1,9 +1,9 @@
-import { asyncMethodCallContextStore } from '@rocket.chat/core-services';
 import type { IMethodConnection, IUser, IRoom } from '@rocket.chat/core-typings';
 import { Logger } from '@rocket.chat/logger';
 import { Users } from '@rocket.chat/models';
 import { Random } from '@rocket.chat/random';
 import type { JoinPathPattern, Method } from '@rocket.chat/rest-typings';
+import { trace, context, ROOT_CONTEXT } from '@rocket.chat/tracing';
 import { Accounts } from 'meteor/accounts-base';
 import { DDP } from 'meteor/ddp';
 import { DDPCommon } from 'meteor/ddp-common';
@@ -38,6 +38,8 @@ import type {
 import { getUserInfo } from './helpers/getUserInfo';
 import { parseJsonQuery } from './helpers/parseJsonQuery';
 
+const tracer = trace.getTracer('core');
+
 const logger = new Logger('API');
 
 interface IAPIProperties {
@@ -646,15 +648,23 @@ export class APIClass<TBasePath extends string = ''> extends Restivus {
 							this.queryFields = options.queryFields;
 							this.parseJsonQuery = api.parseJsonQuery.bind(this as PartialThis);
 
-							// TODO: UUID
-							const store = [{ type: 'rest', route: this.request.route, method: this.request.method, userId: this.userId }];
+							const span = tracer.startSpan(`${this.request.method} ${this.request.url}`, {
+								attributes: {
+									url: this.request.url,
+									route: this.request.route,
+									method: this.request.method,
+									userId: this.userId,
+								},
+							});
 
-							result = await asyncMethodCallContextStore.run(store, async () => {
+							result = await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => {
 								return (
 									(await DDP._CurrentInvocation.withValue(invocation as any, async () => originalAction.apply(this))) || API.v1.success()
 								);
 							});
-							console.log('api', this.request.route, { result }, store);
+							console.log('api', this.request.route, { result }, span.spanContext().traceId);
+							this.response.setHeader('X-Trace-Id', span.spanContext().traceId);
+							span.end();
 
 							log.http({
 								status: result.statusCode,

apps/meteor/app/lib/server/lib/debug.js

@@ -1,5 +1,6 @@
 import { InstanceStatus } from '@rocket.chat/instance-status';
 import { Logger } from '@rocket.chat/logger';
+import { context, trace } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 import { WebApp } from 'meteor/webapp';
 import _ from 'underscore';
@@ -50,6 +51,8 @@ const traceConnection = (enable, filter, prefix, name, connection, userId) => {
 	}
 };
 
+const tracer = trace.getTracer('core');
+
 const wrapMethods = function (name, originalHandler, methodsMap) {
 	methodsMap[name] = function (...originalArgs) {
 		traceConnection(Log_Trace_Methods, Log_Trace_Methods_Filter, 'method', name, this.connection, this.userId);
@@ -72,6 +75,23 @@ const wrapMethods = function (name, originalHandler, methodsMap) {
 			...getMethodArgs(name, originalArgs),
 		});
 
+		const currentSpan = trace.getSpan(context.active());
+		if (currentSpan) {
+			const span = tracer.startSpan(`Method ${name}`, {
+				attributes: {
+					method: name,
+					userId: this.userId,
+				},
+			});
+
+			const result = context.with(trace.setSpan(context.active(), span), () => {
+				return originalHandler.apply(this, originalArgs);
+			});
+			end();
+			span.end();
+			return result;
+		}
+
 		const result = originalHandler.apply(this, originalArgs);
 		end();
 		return result;

apps/meteor/app/metrics/server/lib/collectMetrics.ts

@@ -1,6 +1,7 @@
 import http from 'http';
 
 import { Statistics } from '@rocket.chat/models';
+import { trace, ROOT_CONTEXT, context } from '@rocket.chat/tracing';
 import connect from 'connect';
 import { Facts } from 'meteor/facts-base';
 import { Meteor } from 'meteor/meteor';
@@ -16,6 +17,8 @@ import { getAppsStatistics } from '../../../statistics/server/lib/getAppsStatist
 import { Info } from '../../../utils/rocketchat.info';
 import { metrics } from './metrics';
 
+const tracer = trace.getTracer('core');
+
 const { mongo } = MongoInternals.defaultRemoteCollectionDriver();
 
 Facts.incrementServerFact = function (pkg: 'pkg' | 'fact', fact: string | number, increment: number): void {
@@ -169,7 +172,20 @@ const updatePrometheusConfig = async (): Promise<void> => {
 			host: process.env.BIND_IP || '0.0.0.0',
 		});
 
-		timer = setInterval(setPrometheusData, 5000);
+		timer = setInterval(async () => {
+			const span = tracer.startSpan(`setPrometheusData`, {
+				attributes: {
+					port: is.port,
+					host: process.env.BIND_IP || '0.0.0.0',
+				},
+			});
+
+			await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => {
+				void setPrometheusData();
+			});
+
+			span.end();
+		}, 5000);
 	}
 
 	clearInterval(resetTimer);

apps/meteor/app/notification-queue/server/NotificationQueue.ts

@@ -1,11 +1,14 @@
 import type { INotification, INotificationItemPush, INotificationItemEmail, NotificationItem, IUser } from '@rocket.chat/core-typings';
 import { NotificationQueue, Users } from '@rocket.chat/models';
+import { context, ROOT_CONTEXT, trace } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 
 import { SystemLogger } from '../../../server/lib/logger/system';
 import { sendEmailFromData } from '../../lib/server/functions/notifications/email';
 import { PushNotification } from '../../push-notifications/server';
 
+const tracer = trace.getTracer('core');
+
 const {
 	NOTIFICATIONS_WORKER_TIMEOUT = 2000,
 	NOTIFICATIONS_BATCH_SIZE = 100,
@@ -43,25 +46,39 @@ class NotificationClass {
 
 		setTimeout(async () => {
 			try {
-				await this.worker();
+				const span = tracer.startSpan(`NotificationWorker`, {
+					attributes: {
+						a: new Date(),
+					},
+				});
+
+				const continueLater = await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => {
+					return this.worker();
+				});
+
+				span.end();
+
+				if (continueLater) {
+					this.executeWorkerLater();
+				}
 			} catch (err) {
 				SystemLogger.error({ msg: 'Error sending notification', err });
 				this.executeWorkerLater();
 			}
 		}, this.cyclePause);
 	}
 
-	async worker(counter = 0): Promise<void> {
+	async worker(counter = 0): Promise<boolean> {
 		const notification = await this.getNextNotification();
 
 		if (!notification) {
-			return this.executeWorkerLater();
+			return true;
 		}
 
 		// Once we start notifying the user we anticipate all the schedules
 		const flush = await NotificationQueue.clearScheduleByUserId(notification.uid);
 
-		// start worker again it queue flushed
+		// start worker again if queue flushed
 		if (flush.modifiedCount) {
 			await NotificationQueue.unsetSendingById(notification._id);
 			return this.worker(counter);
@@ -86,9 +103,10 @@ class NotificationClass {
 		}
 
 		if (counter >= this.maxBatchSize) {
-			return this.executeWorkerLater();
+			return true;
 		}
-		await this.worker(counter++);
+
+		return this.worker(counter++);
 	}
 
 	getNextNotification(): Promise<INotification | null> {

apps/meteor/package.json

@@ -221,6 +221,9 @@
 		"@nivo/heatmap": "0.84.0",
 		"@nivo/line": "0.84.0",
 		"@nivo/pie": "0.84.0",
+		"@opentelemetry/auto-instrumentations-node": "^0.48.0",
+		"@opentelemetry/exporter-trace-otlp-grpc": "^0.52.1",
+		"@opentelemetry/sdk-node": "^0.52.1",
 		"@react-aria/color": "^3.0.0-beta.15",
 		"@react-aria/toolbar": "^3.0.0-beta.1",
 		"@react-pdf/renderer": "^3.1.14",
@@ -276,6 +279,7 @@
 		"@rocket.chat/sha256": "workspace:^",
 		"@rocket.chat/string-helpers": "~0.31.25",
 		"@rocket.chat/tools": "workspace:^",
+		"@rocket.chat/tracing": "workspace:^",
 		"@rocket.chat/ui-avatar": "workspace:^",
 		"@rocket.chat/ui-client": "workspace:^",
 		"@rocket.chat/ui-composer": "workspace:^",

apps/meteor/server/cron/statistics.ts

@@ -2,6 +2,7 @@ import { cronJobs } from '@rocket.chat/cron';
 import type { Logger } from '@rocket.chat/logger';
 import { Statistics } from '@rocket.chat/models';
 import { serverFetch as fetch } from '@rocket.chat/server-fetch';
+import { context, trace, ROOT_CONTEXT } from '@rocket.chat/tracing';
 import { Meteor } from 'meteor/meteor';
 
 import { getWorkspaceAccessToken } from '../../app/cloud/server';
@@ -34,13 +35,27 @@ async function generateStatistics(logger: Logger): Promise<void> {
 	}
 }
 
+const tracer = trace.getTracer('core');
+
 export async function statsCron(logger: Logger): Promise<void> {
 	const name = 'Generate and save statistics';
-	await generateStatistics(logger);
+	const span = tracer.startSpan(`generateStatistics`);
+
+	await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => {
+		await generateStatistics(logger);
+	});
+
+	span.end();
 
 	const now = new Date();
 
 	await cronJobs.add(name, `12 ${now.getHours()} * * *`, async () => {
-		await generateStatistics(logger);
+		const span = tracer.startSpan(`generateStatistics`);
+
+		await context.with(trace.setSpan(ROOT_CONTEXT, span), async () => {
+			await generateStatistics(logger);
+		});
+
+		span.end();
 	});
 }

apps/meteor/server/main.ts

@@ -1,3 +1,4 @@
+import '@rocket.chat/tracing';
 import './models/startup';
 /**
  * ./settings uses top level await, in theory the settings creation