From dc6c2b5121977814f854b674ec3e519f689637c9 Mon Sep 17 00:00:00 2001 From: chrisrichardsevergreen <95699748+chrisrichardsevergreen@users.noreply.github.com> Date: Sun, 13 Mar 2022 08:38:01 +0000 Subject: [PATCH] feat(aws-sdk): lambda client instrumentation (#916) --- .../README.md | 1 + .../doc/lambda.md | 12 + .../package.json | 5 +- .../src/aws-sdk.ts | 5 +- .../src/services/ServicesExtensions.ts | 2 + .../src/services/lambda.ts | 141 +++++++ .../src/types.ts | 1 + .../test/lambda.test.ts | 393 ++++++++++++++++++ 8 files changed, 557 insertions(+), 3 deletions(-) create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts create mode 100644 plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md index b84fb36ee09..53b16b8d8ea 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/README.md @@ -98,6 +98,7 @@ Specific service logic currently implemented for: - [SQS](./docs/sqs.md) - [SNS](./docs/sns.md) +- [Lambda](./docs/lambda.md) - DynamoDb ## Potential Side Effects diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md new file mode 100644 index 00000000000..dbccf3365f6 --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/doc/lambda.md @@ -0,0 +1,12 @@ +# Lambda + +Lambda is Amazon's function-as-a-service (FaaS) platform. This instrumentation follows the [OpenTelemetry specification for FaaS systems](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md). + +## Specific trace semantics + +The following methods are automatically enhanced: + +### Invoke + +- Attributes are added by this instrumentation according to the [spec for Outgoing Invocations of a FaaS from a client](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/faas.md#outgoing-invocations) . +- OpenTelemetry trace context is injected into the `ClientContext` parameter, allowing functions to extract this using the `Custom` property within the function. diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json index 18127234a05..759b899818c 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/package.json @@ -53,10 +53,12 @@ }, "devDependencies": { "@aws-sdk/client-dynamodb": "3.37.0", + "@aws-sdk/client-lambda": "3.37.0", "@aws-sdk/client-s3": "3.37.0", "@aws-sdk/client-sqs": "3.37.0", "@aws-sdk/types": "3.37.0", "@opentelemetry/api": "1.0.1", + "@opentelemetry/contrib-test-utils": "^0.29.0", "@opentelemetry/sdk-trace-base": "1.0.1", "@types/mocha": "8.2.3", "@types/node": "16.11.21", @@ -64,13 +66,12 @@ "aws-sdk": "2.1008.0", "eslint": "8.7.0", "expect": "27.4.2", + "gts": "3.1.0", "mocha": "7.2.0", "nock": "13.2.1", "nyc": "15.1.0", "rimraf": "3.0.2", "sinon": "13.0.1", - "gts": "3.1.0", - "@opentelemetry/contrib-test-utils": "^0.29.0", "test-all-versions": "5.0.1", "ts-mocha": "8.0.0", "typescript": "4.3.4" diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts index 5d9eddaf8af..d32cfad4b70 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/aws-sdk.ts @@ -311,9 +311,11 @@ export class AwsInstrumentation extends InstrumentationBase { } delete v2Request[REQUEST_SPAN_KEY]; + const requestId = response.requestId; const normalizedResponse: NormalizedResponse = { data: response.data, request: normalizedRequest, + requestId: requestId, }; self._callUserResponseHook(span, normalizedResponse); @@ -328,7 +330,7 @@ export class AwsInstrumentation extends InstrumentationBase { ); } - span.setAttribute(AttributeNames.AWS_REQUEST_ID, response.requestId); + span.setAttribute(AttributeNames.AWS_REQUEST_ID, requestId); const httpStatusCode = response.httpResponse?.statusCode; if (httpStatusCode) { @@ -503,6 +505,7 @@ export class AwsInstrumentation extends InstrumentationBase { const normalizedResponse: NormalizedResponse = { data: response.output, request: normalizedRequest, + requestId: requestId, }; self.servicesExtensions.responseHook( normalizedResponse, diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts index 6796783013f..52ab59d8867 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/ServicesExtensions.ts @@ -23,6 +23,7 @@ import { } from '../types'; import { DynamodbServiceExtension } from './dynamodb'; import { SnsServiceExtension } from './sns'; +import { LambdaServiceExtension } from './lambda'; export class ServicesExtensions implements ServiceExtension { services: Map = new Map(); @@ -31,6 +32,7 @@ export class ServicesExtensions implements ServiceExtension { this.services.set('SQS', new SqsServiceExtension()); this.services.set('SNS', new SnsServiceExtension()); this.services.set('DynamoDB', new DynamodbServiceExtension()); + this.services.set('Lambda', new LambdaServiceExtension()); } requestPreSpanHook(request: NormalizedRequest): RequestMetadata { diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts new file mode 100644 index 00000000000..35a9d4a8c57 --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/services/lambda.ts @@ -0,0 +1,141 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + Span, + SpanKind, + Tracer, + diag, + SpanAttributes, +} from '@opentelemetry/api'; +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { + AwsSdkInstrumentationConfig, + NormalizedRequest, + NormalizedResponse, +} from '../types'; +import { RequestMetadata, ServiceExtension } from './ServiceExtension'; +import { context, propagation } from '@opentelemetry/api'; + +class LambdaCommands { + public static readonly Invoke: string = 'Invoke'; +} + +export class LambdaServiceExtension implements ServiceExtension { + requestPreSpanHook(request: NormalizedRequest): RequestMetadata { + const functionName = this.extractFunctionName(request.commandInput); + + let spanAttributes: SpanAttributes = {}; + let spanName: string | undefined; + + switch (request.commandName) { + case 'Invoke': + spanAttributes = { + [SemanticAttributes.FAAS_INVOKED_NAME]: functionName, + [SemanticAttributes.FAAS_INVOKED_PROVIDER]: 'aws', + }; + if (request.region) { + spanAttributes[SemanticAttributes.FAAS_INVOKED_REGION] = + request.region; + } + spanName = `${functionName} ${LambdaCommands.Invoke}`; + break; + } + return { + isIncoming: false, + spanAttributes, + spanKind: SpanKind.CLIENT, + spanName, + }; + } + + requestPostSpanHook = (request: NormalizedRequest) => { + switch (request.commandName) { + case LambdaCommands.Invoke: + { + if (request.commandInput) { + request.commandInput.ClientContext = injectLambdaPropagationContext( + request.commandInput.ClientContext + ); + } + } + break; + } + }; + + responseHook( + response: NormalizedResponse, + span: Span, + tracer: Tracer, + config: AwsSdkInstrumentationConfig + ) { + switch (response.request.commandName) { + case LambdaCommands.Invoke: + { + span.setAttribute( + SemanticAttributes.FAAS_EXECUTION, + response.requestId + ); + } + break; + } + } + + extractFunctionName = (commandInput: Record): string => { + return commandInput?.FunctionName; + }; +} + +const injectLambdaPropagationContext = ( + clientContext: string | undefined +): string | undefined => { + try { + const propagatedContext = {}; + propagation.inject(context.active(), propagatedContext); + + const parsedClientContext = clientContext + ? JSON.parse(Buffer.from(clientContext, 'base64').toString('utf8')) + : {}; + + const updatedClientContext = { + ...parsedClientContext, + Custom: { + ...parsedClientContext.Custom, + ...propagatedContext, + }, + }; + + const encodedClientContext = Buffer.from( + JSON.stringify(updatedClientContext) + ).toString('base64'); + + // The length of client context is capped at 3583 bytes of base64 encoded data + // (https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html#API_Invoke_RequestSyntax) + if (encodedClientContext.length > 3583) { + diag.warn( + 'lambda instrumentation: cannot set context propagation on lambda invoke parameters due to ClientContext length limitations.' + ); + return clientContext; + } + + return encodedClientContext; + } catch (e) { + diag.debug( + 'lambda instrumentation: failed to set context propagation on ClientContext', + e + ); + return clientContext; + } +}; diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts index 340be8deebe..0941018347e 100644 --- a/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/src/types.ts @@ -31,6 +31,7 @@ export interface NormalizedRequest { export interface NormalizedResponse { data: any; request: NormalizedRequest; + requestId: string; } export interface AwsSdkRequestHookInformation { diff --git a/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts new file mode 100644 index 00000000000..372ebc4e674 --- /dev/null +++ b/plugins/node/opentelemetry-instrumentation-aws-sdk/test/lambda.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AwsInstrumentation } from '../src'; +import { + getTestSpans, + registerInstrumentationTesting, +} from '@opentelemetry/contrib-test-utils'; +registerInstrumentationTesting(new AwsInstrumentation()); + +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { SpanKind } from '@opentelemetry/api'; + +import { Lambda, InvocationType } from '@aws-sdk/client-lambda'; +import { ClientRequest } from 'http'; +import * as nock from 'nock'; +import * as expect from 'expect'; + +process.env.AWS_ACCESS_KEY_ID = 'testing'; +process.env.AWS_SECRET_ACCESS_KEY = 'testing'; +const region = 'us-east-2'; + +describe('Lambda', () => { + describe('Invoke', () => { + describe('Request span attributes', () => { + const getInvokedSpan = async (params: any) => { + const lambdaClient = new Lambda({ region }); + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(200, 'null'); + + await lambdaClient.invoke(params); + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + return span; + }; + + it("should set the span name to the ' invoke'", async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.name).toEqual(`${params.FunctionName} Invoke`); + }); + + it('should set the span kind to CLIENT', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.kind).toEqual(SpanKind.CLIENT); + }); + + it('should set the FAAS invoked provider as AWS', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect( + span.attributes[SemanticAttributes.FAAS_INVOKED_PROVIDER] + ).toEqual('aws'); + }); + + it('should add the function name as a semantic attribute', async () => { + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + const span = await getInvokedSpan(params); + + expect(span.attributes[SemanticAttributes.FAAS_INVOKED_NAME]).toEqual( + 'ot-test-function-name' + ); + }); + }); + + describe('Context propagation', () => { + it('should propagate client context onto the ClientContext in the invoke payload', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + await lambdaClient.invoke(params); + + // Context propagation + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + expect(requestHeaders['x-amz-client-context']).toBeDefined(); + const clientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(clientContext.Custom).toHaveProperty('traceparent'); + }); + + it('should skip context propagation in the event it would push the ClientContext over 3583 bytes', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const existingClientContext = Buffer.from( + JSON.stringify({ + Custom: { + text: [...Array(2600)] + .map(x => String.fromCharCode(48 + Math.random() * 74)) + .join(''), + }, + }) + ).toString('base64'); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: existingClientContext, + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + expect(requestHeaders['x-amz-client-context']).toStrictEqual( + existingClientContext + ); + }); + + it('should maintain any existing custom fields in the client context', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from( + JSON.stringify({ + Custom: { + existing: 'data', + }, + }) + ).toString('base64'), + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const clientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(clientContext.Custom).toHaveProperty('existing', 'data'); + expect(clientContext.Custom).toHaveProperty('traceparent'); + }); + + it('should maintain any existing top-level fields in the client context', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const clientContext = { + env: { + locale: 'en-US', + make: 'Nokia', + model: 'N95', + platform: 'Symbian', + platformVersion: '9.2', + }, + Custom: { + existing: 'data', + }, + }; + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from(JSON.stringify(clientContext)).toString( + 'base64' + ), + }; + + await lambdaClient.invoke(params); + + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const updatedClientContext = JSON.parse( + Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString() + ) as Record; + expect(updatedClientContext.env).toStrictEqual(clientContext.env); + expect(updatedClientContext.Custom).toHaveProperty('traceparent'); + }); + + // It probably should be valid JSON, and I'm not sure what the lambda internals make of it if + // it isn't base64 encoded JSON, however there's absolutely nothing stopping an invoker passing + // absolute garbage in + it('should abandon context propagation if the existing client context is not valid JSON', async () => { + const lambdaClient = new Lambda({ region }); + + let request: + | (ClientRequest & { + headers: Record; + }) + | undefined; + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply(function (uri, requestBody, callback) { + request = this.req; + callback(null, [200, 'null']); + }); + + const clientContextContent = [...Array(16)] + .map(x => String.fromCharCode(48 + Math.random() * 74)) + .join(''); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + ClientContext: Buffer.from(clientContextContent).toString('base64'), + }; + + await lambdaClient.invoke(params); + + // Keep whatever was there before + expect(request).toBeDefined(); + const requestHeaders = request!.headers; + const clientContext = Buffer.from( + requestHeaders['x-amz-client-context'], + 'base64' + ).toString(); + expect(clientContext).toStrictEqual(clientContextContent); + + // We still want span attributes though! + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + + expect(span.kind).toEqual(SpanKind.CLIENT); + expect(span.attributes[SemanticAttributes.FAAS_INVOKED_NAME]).toEqual( + 'ot-test-function-name' + ); + expect( + span.attributes[SemanticAttributes.FAAS_INVOKED_PROVIDER] + ).toEqual('aws'); + }); + }); + + it('should add the request ID from the response onto the span', async () => { + const lambdaClient = new Lambda({ region }); + + nock(`https://lambda.${region}.amazonaws.com/`) + .post('/2015-03-31/functions/ot-test-function-name/invocations') + .reply((uri, requestBody, callback) => { + callback(null, [ + 200, + 'null', + { + 'x-amz-executed-version': '$LATEST', + 'x-amzn-requestid': '95882c2b-3fd2-485d-ada3-9fcb1ca65459', + }, + ]); + }); + + const params = { + FunctionName: 'ot-test-function-name', + InvocationType: InvocationType.RequestResponse, + Payload: Buffer.from( + JSON.stringify({ + test: 'payload', + }) + ), + }; + await lambdaClient.invoke(params); + expect(getTestSpans().length).toBe(1); + const [span] = getTestSpans(); + + expect(span.attributes[SemanticAttributes.FAAS_EXECUTION]).toEqual( + '95882c2b-3fd2-485d-ada3-9fcb1ca65459' + ); + }); + }); +});