Load AWS Lambda secrets in Github CI (#2153)

Make sure our AWS Lambda test setup is correct and the tests work as expected and also in a timely manner.

We run our test in AWS Lambda and then parse the log output to see what events/envelopes where sent. Because Lambda truncates this log output to 4kb, I had to change the tests to make the events/envelopes smaller in size to get the whole event/envelop in the log output.

When the AWS env vars where not set, the tests where skipped but it looked like they where successful. I made them now fail loudly in that case, so we see if they do not run.

Also made the code easier to comprehend.

---------

Co-authored-by: Ivana Kellyerova <ivana.kellyerova@sentry.io>

.craft.yml

@@ -18,6 +18,8 @@ targets:
           # On the other hand, AWS Lambda does not support every Python runtime.
           # The supported runtimes are available in the following link:
           # https://docs.aws.amazon.com/lambda/latest/dg/lambda-python.html
+          - python3.7
+          - python3.8
           - python3.9
           - python3.10
           - python3.11

.github/workflows/test-integration-aws_lambda.yml

@@ -18,6 +18,8 @@ permissions:
   contents: read
 
 env:
+  SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID }}
+  SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY }}
   BUILD_CACHE_KEY: ${{ github.sha }}
   CACHED_BUILD_PATHS: |
     ${{ github.workspace }}/dist-serverless
@@ -31,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7"]
+        python-version: ["3.9"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6

Makefile

@@ -60,7 +60,6 @@ apidocs-hotfix: apidocs
 .PHONY: apidocs-hotfix
 
 aws-lambda-layer: dist
-	$(VENV_PATH)/bin/pip install urllib3
-	$(VENV_PATH)/bin/pip install certifi
+	$(VENV_PATH)/bin/pip install -r aws-lambda-layer-requirements.txt
 	$(VENV_PATH)/bin/python -m scripts.build_aws_lambda_layer
 .PHONY: aws-lambda-layer

aws-lambda-layer-requirements.txt

@@ -0,0 +1,7 @@
+certifi
+
+# In Lambda functions botocore is used, and botocore is not
+# yet supporting urllib3 1.27.0 never mind 2+.
+# So we pin this here to make our Lambda layer work with
+# Lambda Function using Python 3.7+
+urllib3<1.27

scripts/aws-cleanup.sh

@@ -1,11 +1,18 @@
 #!/bin/sh
-# Delete all AWS Lambda functions
+#
+# Helper script to clean up AWS Lambda functions created
+# by the test suite (tests/integrations/aws_lambda/test_aws.py).
+#
+# This will delete all Lambda functions named `test_function_*`.
+#
 
+export AWS_DEFAULT_REGION="us-east-1"
 export AWS_ACCESS_KEY_ID="$SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID"
 export AWS_SECRET_ACCESS_KEY="$SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY"
-export AWS_IAM_ROLE="$SENTRY_PYTHON_TEST_AWS_IAM_ROLE"
 
-for func in $(aws lambda list-functions | jq -r .Functions[].FunctionName); do
+for func in $(aws lambda list-functions --output text --query 'Functions[?starts_with(FunctionName, `test_`) == `true`].FunctionName'); do
     echo "Deleting $func"
-    aws lambda delete-function --function-name $func
+    aws lambda delete-function --function-name "$func"
 done
+
+echo "All done! Have a nice day!"

scripts/aws-deploy-local-layer.sh

@@ -22,7 +22,7 @@ aws lambda publish-layer-version \
     --region "eu-central-1" \
     --zip-file "fileb://dist/$ZIP" \
     --description "Local test build of SentryPythonServerlessSDK (can be deleted)" \
-    --compatible-runtimes python3.6 python3.7 python3.8 python3.9
+    --compatible-runtimes python3.7 python3.8 python3.9 python3.10 python3.11 \
     --no-cli-pager
 
 echo "Done deploying zipped Lambda layer to AWS as 'SentryPythonServerlessSDK-local-dev'."

scripts/build_aws_lambda_layer.py

@@ -1,10 +1,15 @@
 import os
 import shutil
 import subprocess
+import sys
 import tempfile
+from typing import TYPE_CHECKING
 
 from sentry_sdk.consts import VERSION as SDK_VERSION
 
+if TYPE_CHECKING:
+    from typing import Optional
+
 DIST_PATH = "dist"  # created by "make dist" that is called by "make aws-lambda-layer"
 PYTHON_SITE_PACKAGES = "python"  # see https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
 
@@ -13,18 +18,38 @@ class LayerBuilder:
     def __init__(
         self,
         base_dir,  # type: str
+        out_zip_filename=None,  # type: Optional[str]
     ):
         # type: (...) -> None
         self.base_dir = base_dir
         self.python_site_packages = os.path.join(self.base_dir, PYTHON_SITE_PACKAGES)
-        self.out_zip_filename = f"sentry-python-serverless-{SDK_VERSION}.zip"
+        self.out_zip_filename = (
+            f"sentry-python-serverless-{SDK_VERSION}.zip"
+            if out_zip_filename is None
+            else out_zip_filename
+        )
 
     def make_directories(self):
         # type: (...) -> None
         os.makedirs(self.python_site_packages)
 
     def install_python_packages(self):
         # type: (...) -> None
+        # Install requirements for Lambda Layer (these are more limited than the SDK requirements,
+        # because Lambda does not support the newest versions of some packages)
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "install",
+                "-r",
+                "aws-lambda-layer-requirements.txt",
+                "--target",
+                self.python_site_packages,
+            ],
+        )
+
         sentry_python_sdk = os.path.join(
             DIST_PATH,
             f"sentry_sdk-{SDK_VERSION}-py2.py3-none-any.whl",  # this is generated by "make dist" that is called by "make aws-lamber-layer"
@@ -34,6 +59,7 @@ def install_python_packages(self):
                 "pip",
                 "install",
                 "--no-cache-dir",  # always access PyPI
+                "--no-deps",  # the right depencencies have been installed in the call above
                 "--quiet",
                 sentry_python_sdk,
                 "--target",
@@ -80,13 +106,34 @@ def zip(self):
         )
 
 
-def build_packaged_zip():
-    with tempfile.TemporaryDirectory() as base_dir:
-        layer_builder = LayerBuilder(base_dir)
-        layer_builder.make_directories()
-        layer_builder.install_python_packages()
-        layer_builder.create_init_serverless_sdk_package()
-        layer_builder.zip()
+def build_packaged_zip(base_dir=None, make_dist=False, out_zip_filename=None):
+    if base_dir is None:
+        base_dir = tempfile.mkdtemp()
+
+    if make_dist:
+        # Same thing that is done by "make dist"
+        # (which is a dependency of "make aws-lambda-layer")
+        subprocess.check_call(
+            [sys.executable, "setup.py", "sdist", "bdist_wheel", "-d", DIST_PATH],
+        )
+
+    layer_builder = LayerBuilder(base_dir, out_zip_filename=out_zip_filename)
+    layer_builder.make_directories()
+    layer_builder.install_python_packages()
+    layer_builder.create_init_serverless_sdk_package()
+    layer_builder.zip()
+
+    # Just for debugging
+    dist_path = os.path.abspath(DIST_PATH)
+    print("Created Lambda Layer package with this information:")
+    print(" - Base directory for generating package: {}".format(layer_builder.base_dir))
+    print(
+        " - Created Python SDK distribution (in `{}`): {}".format(dist_path, make_dist)
+    )
+    if not make_dist:
+        print("    If 'False' we assume it was already created (by 'make dist')")
+    print(" - Package zip filename: {}".format(layer_builder.out_zip_filename))
+    print(" - Copied package zip to: {}".format(dist_path))
 
 
 if __name__ == "__main__":

scripts/split-tox-gh-actions/ci-yaml-aws-credentials.txt

@@ -0,0 +1,2 @@
+  SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID }}
+  SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY }}

scripts/split-tox-gh-actions/ci-yaml.txt

@@ -18,6 +18,7 @@ permissions:
   contents: read
 
 env:
+{{ aws_credentials }}
   BUILD_CACHE_KEY: ${{ github.sha }}
   CACHED_BUILD_PATHS: |
     ${{ github.workspace }}/dist-serverless

scripts/split-tox-gh-actions/split-tox-gh-actions.py

@@ -28,6 +28,7 @@
 TEMPLATE_FILE = TEMPLATE_DIR / "ci-yaml.txt"
 TEMPLATE_FILE_SERVICES = TEMPLATE_DIR / "ci-yaml-services.txt"
 TEMPLATE_FILE_SETUP_DB = TEMPLATE_DIR / "ci-yaml-setup-db.txt"
+TEMPLATE_FILE_AWS_CREDENTIALS = TEMPLATE_DIR / "ci-yaml-aws-credentials.txt"
 TEMPLATE_SNIPPET_TEST = TEMPLATE_DIR / "ci-yaml-test-snippet.txt"
 TEMPLATE_SNIPPET_TEST_PY27 = TEMPLATE_DIR / "ci-yaml-test-py27-snippet.txt"
 
@@ -40,6 +41,10 @@
     "clickhouse_driver",
 ]
 
+FRAMEWORKS_NEEDING_AWS = [
+    "aws_lambda",
+]
+
 MATRIX_DEFINITION = """
     strategy:
       fail-fast: false
@@ -128,6 +133,11 @@ def write_yaml_file(
                 f = open(TEMPLATE_FILE_SETUP_DB, "r")
                 out += "".join(f.readlines())
 
+        elif template_line.strip() == "{{ aws_credentials }}":
+            if current_framework in FRAMEWORKS_NEEDING_AWS:
+                f = open(TEMPLATE_FILE_AWS_CREDENTIALS, "r")
+                out += "".join(f.readlines())
+
         elif template_line.strip() == "{{ additional_uses }}":
             if current_framework in FRAMEWORKS_NEEDING_CLICKHOUSE:
                 out += ADDITIONAL_USES_CLICKHOUSE