Serverless V2 (#1450)

* Build new Lambda extension (#1383) * Use new GitHub action for creating Lambda layer zip. * Use new GitHub action for creating zip. * Replace original DSN host/port with localhost:3000 (#1414) * Added script for locally building/release Lambda layer * Added script to attach layer to function Co-authored-by: Neel Shah <neel.shah@sentry.io>
getsentry · Jun 21, 2022 · 0352c79 · 0352c79
1 parent 3d38329
commit 0352c79
Show file tree

Hide file tree

Showing 11 changed files with 295 additions and 180 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: ci
+name: CI
 
 on:
   push:
@@ -11,55 +11,16 @@ on:
 permissions:
   contents: read
 
-jobs:
-  dist:
-    name: distribution packages
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-      - uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-
-      - run: |
-          pip install virtualenv
-          make aws-lambda-layer-build
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ github.sha }}
-          path: |
-            dist/*
-            dist-serverless/*
-
-  docs:
-    timeout-minutes: 10
-    name: build documentation
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-      - uses: actions/setup-python@v3
-        with:
-          python-version: 3.9
-
-      - run: |
-          pip install virtualenv
-          make apidocs
-          cd docs/_build && zip -r gh-pages ./
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ github.sha }}
-          path: docs/_build/gh-pages.zip
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
 
+jobs:
   lint:
-    timeout-minutes: 10
+    name: Lint Sources
     runs-on: ubuntu-latest
+    timeout-minutes: 10
 
     steps:
       - uses: actions/checkout@v3
@@ -72,9 +33,10 @@ jobs:
           tox -e linters
 
   test:
-    continue-on-error: true
-    timeout-minutes: 45
+    name: Run Tests
     runs-on: ${{ matrix.linux-version }}
+    timeout-minutes: 45
+    continue-on-error: true
     strategy:
       matrix:
         linux-version: [ubuntu-latest]
@@ -128,7 +90,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: setup
+      - name: Setup Test Env
         env:
           PGHOST: localhost
           PGPASSWORD: sentry
@@ -137,7 +99,7 @@ jobs:
           psql -c 'create database test_travis_ci_test;' -U postgres
           pip install codecov tox
 
-      - name: run tests
+      - name: Run Tests
         env:
           CI_PYTHON_VERSION: ${{ matrix.python-version }}
         timeout-minutes: 45
@@ -147,3 +109,58 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  build_lambda_layer:
+    name: Build AWS Lambda Layer
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Setup build cache
+        uses: actions/cache@v2
+        id: build_cache
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+      - run: |
+          echo "Creating directory containing Python SDK Lambda Layer"
+          pip install virtualenv
+          make aws-lambda-layer
+
+          echo "Saving SDK_VERSION for later"
+          export SDK_VERSION=$(grep "VERSION = " sentry_sdk/consts.py | cut -f3 -d' ' | tr -d '"')
+          echo "SDK_VERSION=$SDK_VERSION"
+          echo "SDK_VERSION=$SDK_VERSION" >> $GITHUB_ENV
+      - uses: getsentry/action-build-aws-lambda-extension@v1
+        with:
+          artifact_name: ${{ github.sha }}
+          zip_file_name: sentry-python-serverless-${{ env.SDK_VERSION }}.zip
+          build_cache_paths: ${{ env.CACHED_BUILD_PATHS }}
+          build_cache_key: ${{ env.BUILD_CACHE_KEY }}
+
+  docs:
+    name: Build SDK API Doc
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - run: |
+          pip install virtualenv
+          make apidocs
+          cd docs/_build && zip -r gh-pages ./
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.sha }}
+          path: docs/_build/gh-pages.zip
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ pip-log.txt
 /build
 /dist
 /dist-serverless
+sentry-python-serverless*.zip
 .cache
 .idea
 .eggs

diff --git a/CONTRIBUTING-aws-lambda.md b/CONTRIBUTING-aws-lambda.md
@@ -0,0 +1,21 @@
+# Contributing to Sentry AWS Lambda Layer
+
+All the general terms of the [CONTRIBUTING.md](CONTRIBUTING.md) apply.
+
+## Development environment
+
+You need to have a AWS account and AWS CLI installed and setup.
+
+We put together two helper functions that can help you with development:
+
+- `./scripts/aws-deploy-local-layer.sh`
+
+  This script [scripts/aws-deploy-local-layer.sh](scripts/aws-deploy-local-layer.sh) will take the code you have checked out locally, create a Lambda layer out of it and deploy it to the `eu-central-1` region of your configured AWS account using `aws` CLI.
+
+  The Lambda layer will have the name `SentryPythonServerlessSDK-local-dev`
+
+- `./scripts/aws-attach-layer-to-lambda-function.sh`
+
+  You can use this script [scripts/aws-attach-layer-to-lambda-function.sh](scripts/aws-attach-layer-to-lambda-function.sh) to attach the Lambda layer you just deployed (using the first script) onto one of your existing Lambda functions. You will have to give the name of the Lambda function to attach onto as an argument. (See the script for details.)
+
+With this two helper scripts it should be easy to rapidly iterate your development on the Lambda layer.
diff --git a/Makefile b/Makefile
@@ -9,7 +9,7 @@ help:
 	@echo "make test: Run basic tests (not testing most integrations)"
 	@echo "make test-all: Run ALL tests (slow, closest to CI)"
 	@echo "make format: Run code formatters (destructive)"
-	@echo "make aws-lambda-layer-build: Build serverless ZIP dist package"
+	@echo "make aws-lambda-layer: Build AWS Lambda layer directory for serverless integration"
 	@echo
 	@echo "Also make sure to read ./CONTRIBUTING.md"
 	@false
@@ -19,9 +19,8 @@ help:
 	$(VENV_PATH)/bin/pip install tox
 
 dist: .venv
-	rm -rf dist build
+	rm -rf dist dist-serverless build
 	$(VENV_PATH)/bin/python setup.py sdist bdist_wheel
-
 .PHONY: dist
 
 format: .venv
@@ -46,7 +45,6 @@ lint: .venv
 		echo "Bad formatting? Run: make format"; \
 		echo "================================"; \
 		false)
-
 .PHONY: lint
 
 apidocs: .venv
@@ -60,8 +58,8 @@ apidocs-hotfix: apidocs
 	@$(VENV_PATH)/bin/ghp-import -pf docs/_build
 .PHONY: apidocs-hotfix
 
-aws-lambda-layer-build: dist
+aws-lambda-layer: dist
 	$(VENV_PATH)/bin/pip install urllib3
 	$(VENV_PATH)/bin/pip install certifi
-	$(VENV_PATH)/bin/python -m scripts.build_awslambda_layer
-.PHONY: aws-lambda-layer-build
+	$(VENV_PATH)/bin/python -m scripts.build_aws_lambda_layer
+.PHONY: aws-lambda-layer
diff --git a/scripts/aws-attach-layer-to-lambda-function.sh b/scripts/aws-attach-layer-to-lambda-function.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Attaches the layer `SentryPythonServerlessSDK-local-dev` to a given lambda function.
+#
+
+set -euo pipefail
+
+# Check for argument
+if [ $# -eq 0 ]
+  then
+    SCRIPT_NAME=$(basename "$0")
+    echo "ERROR: No argument supplied. Please give the name of a Lambda function!"
+    echo ""
+    echo "Usage: $SCRIPT_NAME <lambda-function-name>"
+    echo ""
+    exit 1
+fi
+
+FUNCTION_NAME=$1
+
+echo "Getting ARN of newest Sentry lambda layer..."
+LAYER_ARN=$(aws lambda list-layer-versions --layer-name SentryPythonServerlessSDK-local-dev --query "LayerVersions[0].LayerVersionArn" | tr -d '"')
+echo "Done getting ARN of newest Sentry lambda layer $LAYER_ARN."
+
+echo "Attaching Lamba layer to function $FUNCTION_NAME..."
+echo "Warning: This remove all other layers!"
+aws lambda update-function-configuration \
+    --function-name "$FUNCTION_NAME" \
+    --layers "$LAYER_ARN" \
+    --no-cli-pager
+echo "Done attaching Lamba layer to function '$FUNCTION_NAME'."
+
+echo "All done. Have a nice day!"
diff --git a/scripts/aws-delete-lamba-layer-versions.sh b/scripts/aws-delete-lamba-layer-versions.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Deletes all versions of the layer specified in LAYER_NAME in one region.
+#
+
+set -euo pipefail
+
+# override default AWS region
+export AWS_REGION=eu-central-1
+
+LAYER_NAME=SentryPythonServerlessSDKLocalDev
+VERSION="0"
+
+while [[ $VERSION != "1" ]]
+do
+  VERSION=$(aws lambda list-layer-versions --layer-name $LAYER_NAME | jq '.LayerVersions[0].Version')
+  aws lambda delete-layer-version --layer-name $LAYER_NAME --version-number $VERSION
+done
diff --git a/scripts/aws-deploy-local-layer.sh b/scripts/aws-deploy-local-layer.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# Builds and deploys the Sentry AWS Lambda layer (including the Sentry SDK and the Sentry Lambda Extension)
+#
+# The currently checked out version of the SDK in your local directory is used.
+# The latest version of the Lambda Extension is fetched from the Sentry Release Registry.
+#
+
+set -euo pipefail
+
+# Creating Lambda layer
+echo "Creating Lambda layer in ./dist-serverless ..."
+make aws-lambda-layer
+echo "Done creating Lambda layer in ./dist-serverless."
+
+# IMPORTANT:
+# Please make sure that this part does the same as the GitHub action that
+# is building the Lambda layer in production!
+# see: https://github.com/getsentry/action-build-aws-lambda-extension/blob/main/action.yml#L23-L40
+
+echo "Downloading relay..."
+mkdir -p dist-serverless/relay
+curl -0 --silent \
+    --output dist-serverless/relay/relay \
+    "$(curl -s https://release-registry.services.sentry.io/apps/relay/latest | jq -r .files.\"relay-Linux-x86_64\".url)"
+chmod +x dist-serverless/relay/relay
+echo "Done downloading relay."
+
+echo "Creating start script..."
+mkdir -p dist-serverless/extensions
+cat > dist-serverless/extensions/sentry-lambda-extension << EOT
+#!/bin/bash
+set -euo pipefail
+exec /opt/relay/relay run \
+    --mode=proxy \
+    --shutdown-timeout=2 \
+    --upstream-dsn="\$SENTRY_DSN" \
+    --aws-runtime-api="\$AWS_LAMBDA_RUNTIME_API"
+EOT
+chmod +x dist-serverless/extensions/sentry-lambda-extension
+echo "Done creating start script."
+
+# Zip Lambda layer and included Lambda extension
+echo "Zipping Lambda layer and included Lambda extension..."
+cd dist-serverless/
+zip -r ../sentry-python-serverless-x.x.x-dev.zip \
+    . \
+    --exclude \*__pycache__\* --exclude \*.yml
+cd ..
+echo "Done Zipping Lambda layer and included Lambda extension to ./sentry-python-serverless-x.x.x-dev.zip."
+
+
+# Deploying zipped Lambda layer to AWS
+echo "Deploying zipped Lambda layer to AWS..."
+
+aws lambda publish-layer-version \
+    --layer-name "SentryPythonServerlessSDK-local-dev" \
+    --region "eu-central-1" \
+    --zip-file "fileb://sentry-python-serverless-x.x.x-dev.zip" \
+    --description "Local test build of SentryPythonServerlessSDK (can be deleted)" \
+    --no-cli-pager
+
+echo "Done deploying zipped Lambda layer to AWS as 'SentryPythonServerlessSDK-local-dev'."
+
+echo "All done. Have a nice day!"
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,6 +12,7 @@ pip-log.txt @@
     /build
     /dist
     /dist-serverless
+    sentry-python-serverless*.zip
     .cache
     .idea
     .eggs
@@ Expand Down @@