feat: Replace Epsagon with Lumigo (#1168)

Epsagon is no longer operating. They took down their website on 3
October and backend services have been gradually removed, and since 10
October any service still making requests to Espagon servers faces
delays and Lambda timeouts.

We have now migrated to [Lumigo]. Their platform supports
[auto-tracing], allowing us to begin monitoring without deploying
updates to Lambda Wrapper, however we are missing our metrics and labels
which correspond to [Execution Tags] in Lumigo.

This PR makes the change to wrap handlers with Lumigo's tracer, and
updates our logger to use Execution Tags for metrics and labels. To
enable Lumigo tracing and tags, set `LUMIGO_TRACER_TOKEN` in your Lambda
environment to your [Lumigo token]. Note that if you have enabled
auto-tracing, this will be set automatically and tracing should "just
work".

There are a couple of other little things that need doing (e.g. removing
the `raiseOnEpsagon` flag) that I'll cover in separate PRs.

Jira: [ENG-2764]

[Lumigo]: https://lumigo.io/
[auto-tracing]: https://docs.lumigo.io/docs/serverless-applications#automatic-instrumentation
[Execution Tags]: https://docs.lumigo.io/docs/execution-tags
[Lumigo token]: https://docs.lumigo.io/docs/lumigo-tokens

BREAKING CHANGE: We no longer use Epsagon for monitoring. Lambda Wrapper
now supports [Lumigo](https://lumigo.io/) instead.

Author: seb-cr

README.md

@@ -204,6 +204,14 @@ lambdaWrapper.configure<WithSQSServiceConfig & WithOtherServiceConfig>({
 });
 ```
 
+## Monitoring
+
+At Comic Relief we use [Lumigo](https://lumigo.io/) for monitoring and observability of our deployed services. Lambda Wrapper includes the Lumigo tracer to allow us to tag traces with custom labels and metrics ([execution tags](https://docs.lumigo.io/docs/execution-tags)).
+
+Lumigo integration works out-of-the-box with Lumigo's [auto-trace feature](https://docs.lumigo.io/docs/serverless-applications#automatic-instrumentation). If you prefer manual tracing, enable it by setting `LUMIGO_TRACER_TOKEN` in your Lambda environment variables.
+
+And if you don't use Lumigo, don't worry, their tracer will not be instantiated in your functions and no calls will be made to their servers unless `LUMIGO_TRACER_TOKEN` is set.
+
 ## Notes
 
 Lambda Wrapper's dependency injection relies on class names being preserved. If your build process includes minifying or uglifying your code, you'll need to disable these transformations.

package.json

@@ -47,12 +47,12 @@
     "aws-sdk": "^2.831.0"
   },
   "dependencies": {
+    "@lumigo/tracer": "^1.87.0",
     "@sentry/node": "^6.0.1",
     "@types/aws-lambda": "^8.10.120",
     "alai": "1.0.3",
     "async": "^3.2.4",
     "axios": "^0.27.2",
-    "epsagon": "^1.123.3",
     "useragent": "2.3.0",
     "uuid": "^9.0.1",
     "validate.js": "0.13.1",

src/core/LambdaWrapper.ts

@@ -1,4 +1,4 @@
-import Epsagon from 'epsagon';
+import * as lumigo from '@lumigo/tracer';
 
 import { Context } from '../index';
 import ResponseModel from '../models/ResponseModel';
@@ -86,19 +86,45 @@ export default class LambdaWrapper<TConfig extends LambdaWrapperConfig = LambdaW
       }
     };
 
-    // If Epsagon is enabled, wrap the instance in the Epsagon wrapper
-    if (process.env.EPSAGON_TOKEN && process.env.EPSAGON_SERVICE_NAME) {
-      Epsagon.init({
-        token: process.env.EPSAGON_TOKEN,
-        appName: process.env.EPSAGON_SERVICE_NAME,
-      });
+    // If Lumigo is enabled, wrap the handler in the Lumigo wrapper
+    if (LambdaWrapper.isLumigoEnabled && !LambdaWrapper.isLumigoWrappingUs) {
+      const tracer = lumigo.initTracer({ token: process.env.LUMIGO_TRACER_TOKEN });
 
-      wrapper = Epsagon.lambdaWrapper(wrapper);
+      // Lumigo's wrapper works with both callbacks or promises handlers, and
+      // the returned function behaves the same way as the original. For our
+      // promise-based handler we can safely coerce the type.
+      wrapper = tracer.trace(wrapper) as (event: any, context: Context) => Promise<any>;
     }
 
     return wrapper;
   }
 
+  /**
+   * `true` if we will send traces to Lumigo.
+   *
+   * The `LUMIGO_TRACER_TOKEN` env var is present in both manually traced and
+   * auto-traced functions.
+   */
+  static get isLumigoEnabled(): boolean {
+    return !!process.env.LUMIGO_TRACER_TOKEN;
+  }
+
+  /**
+   * `true` if the Lambda function is already being traced by a higher-level
+   * Lumigo wrapper, in which case we don't need to manually wrap our handlers.
+   *
+   * There are two ways that this can be done, based on the documentation
+   * [here](https://docs.lumigo.io/docs/lambda-layers): using a Lambda runtime
+   * wrapper, or handler redirection. Each method can be detected via its
+   * environment variables. Auto-trace uses the runtime wrapper.
+   */
+  static get isLumigoWrappingUs(): boolean {
+    return this.isLumigoEnabled && (
+      process.env.AWS_LAMBDA_EXEC_WRAPPER === '/opt/lumigo_wrapper'
+      || !!process.env.LUMIGO_ORIGINAL_HANDLER
+    );
+  }
+
   /**
    * Process the result once we have one.
    *
@@ -115,11 +141,10 @@ export default class LambdaWrapper<TConfig extends LambdaWrapperConfig = LambdaW
   }
 
   /**
-   * Gracefully handles an error, logging in Epsagon and generating a response
+   * Gracefully handles an error, logging in Lumigo and generating a response
    * reflecting the `code` of the error, if defined.
    *
-   * Note about Epsagon:
-   * Epsagon generates alerts for logs on level ERROR. This means that
+   * Lumigo generates alerts for logs on level ERROR. This means that
    * `logger.error` will produce an alert. To avoid meaningless notifications,
    * most likely coming from tests, we log INFO unless either:
    *

src/services/LoggerService.ts

@@ -1,10 +1,11 @@
+import * as lumigo from '@lumigo/tracer';
 import * as Sentry from '@sentry/node';
 import { AxiosError } from 'axios';
-import Epsagon from 'epsagon';
 import Winston from 'winston';
 
 import DependencyAwareClass from '../core/DependencyAwareClass';
 import DependencyInjection from '../core/DependencyInjection';
+import LambdaWrapper from '../core/LambdaWrapper';
 
 const sentryIsAvailable = typeof process.env.RAVEN_DSN !== 'undefined' && typeof process.env.RAVEN_DSN === 'string' && process.env.RAVEN_DSN !== 'undefined';
 
@@ -170,14 +171,8 @@ export default class LoggerService extends DependencyAwareClass {
       Sentry.captureException(error);
     }
 
-    if (
-      typeof process.env.EPSAGON_TOKEN === 'string'
-      && process.env.EPSAGON_TOKEN !== 'undefined'
-      && typeof process.env.EPSAGON_SERVICE_NAME === 'string'
-      && process.env.EPSAGON_SERVICE_NAME !== 'undefined'
-      && error instanceof Error
-    ) {
-      Epsagon.setError(error);
+    if (LambdaWrapper.isLumigoEnabled && error instanceof Error) {
+      lumigo.error(message || error.message, { err: error });
     }
 
     this.logger.log('error', message, { error: LoggerService.processMessage(error) });
@@ -221,13 +216,8 @@ export default class LoggerService extends DependencyAwareClass {
    * @param silent If `false`, the label will also be logged. (default: false)
    */
   label(descriptor: string, silent = false) {
-    if (
-      typeof process.env.EPSAGON_TOKEN === 'string'
-      && process.env.EPSAGON_TOKEN !== 'undefined'
-      && typeof process.env.EPSAGON_SERVICE_NAME === 'string'
-      && process.env.EPSAGON_SERVICE_NAME !== 'undefined'
-    ) {
-      Epsagon.label(descriptor, true);
+    if (LambdaWrapper.isLumigoEnabled) {
+      lumigo.addExecutionTag(descriptor, true);
     }
 
     if (!silent) {
@@ -243,13 +233,8 @@ export default class LoggerService extends DependencyAwareClass {
    * @param silent If `false`, the metric will also be logged. (default: false)
    */
   metric(descriptor: string, stat: number | string, silent = false) {
-    if (
-      typeof process.env.EPSAGON_TOKEN === 'string'
-      && process.env.EPSAGON_TOKEN !== 'undefined'
-      && typeof process.env.EPSAGON_SERVICE_NAME === 'string'
-      && process.env.EPSAGON_SERVICE_NAME !== 'undefined'
-    ) {
-      Epsagon.label(descriptor, stat);
+    if (LambdaWrapper.isLumigoEnabled) {
+      lumigo.addExecutionTag(descriptor, stat);
     }
 
     if (silent === false) {

tests/unit/core/LambdaWrapper.spec.ts

@@ -256,6 +256,95 @@ describe('unit.core.LambdaWrapper', () => {
     });
   });
 
+  describe('isLumigoEnabled', () => {
+    describe('when a Lumigo token is present', () => {
+      beforeAll(() => {
+        process.env.LUMIGO_TRACER_TOKEN = 'test';
+      });
+
+      afterAll(() => {
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return true', () => {
+        expect(LambdaWrapper.isLumigoEnabled).toBe(true);
+      });
+    });
+
+    describe('when there is no Lumigo token', () => {
+      beforeAll(() => {
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return false', () => {
+        expect(LambdaWrapper.isLumigoEnabled).toBe(false);
+      });
+    });
+  });
+
+  describe('isLumigoWrappingUs', () => {
+    describe('when using the runtime wrapper (e.g. auto-trace)', () => {
+      beforeAll(() => {
+        process.env.AWS_LAMBDA_EXEC_WRAPPER = '/opt/lumigo_wrapper';
+        delete process.env.LUMIGO_ORIGINAL_HANDLER;
+        process.env.LUMIGO_TRACER_TOKEN = 'test';
+      });
+
+      afterAll(() => {
+        delete process.env.AWS_LAMBDA_EXEC_WRAPPER;
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return true', () => {
+        expect(LambdaWrapper.isLumigoWrappingUs).toBe(true);
+      });
+    });
+
+    describe('when using handler redirection', () => {
+      beforeAll(() => {
+        delete process.env.AWS_LAMBDA_EXEC_WRAPPER;
+        process.env.LUMIGO_ORIGINAL_HANDLER = 'handler.js';
+        process.env.LUMIGO_TRACER_TOKEN = 'test';
+      });
+
+      afterAll(() => {
+        delete process.env.LUMIGO_ORIGINAL_HANDLER;
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return true', () => {
+        expect(LambdaWrapper.isLumigoWrappingUs).toBe(true);
+      });
+    });
+
+    describe('when there is only a Lumigo token', () => {
+      beforeAll(() => {
+        delete process.env.AWS_LAMBDA_EXEC_WRAPPER;
+        delete process.env.LUMIGO_ORIGINAL_HANDLER;
+        process.env.LUMIGO_TRACER_TOKEN = 'test';
+      });
+
+      afterAll(() => {
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return false', () => {
+        expect(LambdaWrapper.isLumigoWrappingUs).toBe(false);
+      });
+    });
+
+    describe('when there is no Lumigo token', () => {
+      beforeAll(() => {
+        delete process.env.AWS_LAMBDA_EXEC_WRAPPER;
+        delete process.env.LUMIGO_TRACER_TOKEN;
+      });
+
+      it('should return false', () => {
+        expect(LambdaWrapper.isLumigoWrappingUs).toBe(false);
+      });
+    });
+  });
+
   describe('handleError', () => {
     ([
       [undefined, 400, 0],