improved stage support in Lambda name

added support for setting the initial value of a new lambda function memory and storage
added support for providing a folder value to the Deploy_Lambda.add_file method
with this new capability, changed how add_file__boto3__lambda so that the boto3__lambda is now placed inside the osbot_aws/aws/lambda_ folder (which means that we don't need to use "# noinspection PyUnresolvedReferences" anymore)

Author: DinisCruz

osbot_aws/aws/lambda_/Lambda.py

@@ -1,5 +1,4 @@
 import json
-
 from osbot_utils.type_safe.Type_Safe                    import Type_Safe
 from osbot_utils.utils.Lists                            import unique
 from osbot_utils.utils.Json                             import json_loads, json_parse
@@ -17,16 +16,20 @@
 from osbot_aws.aws.session.Session__Kwargs__S3          import Session__Kwargs__S3
 
 
-class Lambda(Type_Safe):
+DEFAULT__LAMBDA__MEMORY_SIZE       = 512
+DEFAULT__LAMBDA__EPHEMERAL_STORAGE = 512
+class Lambda(Type_Safe):                                                # todo: refactor to use full value of Type_Safe (also this class is getting quite big)
+
     session_kwargs__lambda: Session__Kwargs__Lambda
     session_kwargs__s3    : Session__Kwargs__S3
-
+    memory_size           : int = DEFAULT__LAMBDA__MEMORY_SIZE              # default to 512Mb max is 10,000 Mb (current AWS limit in Dec 2020)
+    ephemeral_storage     : int = DEFAULT__LAMBDA__EPHEMERAL_STORAGE        # default to 512Mb
+    role                  : str = None
 
     def __init__(self, name='', **kwargs):
         super().__init__(**kwargs)
         self.architecture   = 'x86_64'                          # other option is 'arm64'
         self.runtime        = 'python3.11'
-        self.memory         = 512                               # default to 512Mb max is 10,000 Mb (current AWS limit in Dec 2020)
         self.timeout        = 60                                # default to 60 secs (1m) max is 900 secs (15m)
         self.trace_mode     = 'PassThrough'                     # x-rays disabled
         self.original_name  = name
@@ -122,13 +125,14 @@ def create(self):
             return {'status': 'error', 'error': '{0}'.format(error), 'data': kwargs}
 
     def create_kwargs(self):
-        kwargs = {  'Architectures' : [self.architecture]                                , # no idea why this is an array since we get an exception when there is more than one value
-                    'FunctionName'  : self.name or  random_string(prefix='temp_lambda_') ,
-                    'MemorySize'    : self.memory                                        ,
-                    'Role'          : self.role                                          ,
-                    'Timeout'       : self.timeout                                       ,
-                    'TracingConfig' : { 'Mode': self.trace_mode }                        ,
-                    'Tags'          : self.tags                                          }
+        kwargs = {  'Architectures'    : [self.architecture]                                , # no idea why this is an array since we get an exception when there is more than one value
+                    'FunctionName'     : self.name or  random_string(prefix='temp_lambda_') ,
+                    'MemorySize'       : self.memory_size                                   ,
+                    'EphemeralStorage' : { 'Size': self.ephemeral_storage }                 ,
+                    'Role'             : self.role                                          ,
+                    'Timeout'          : self.timeout                                       ,
+                    'TracingConfig'    : { 'Mode': self.trace_mode }                        ,
+                    'Tags'             : self.tags                                          }
 
         if self.env_variables:
             kwargs['Environment'] = {'Variables': self.env_variables}

osbot_aws/deploy/Deploy_Lambda.py

@@ -1,3 +1,5 @@
+from osbot_aws.aws.lambda_.Lambda               import DEFAULT__LAMBDA__EPHEMERAL_STORAGE, DEFAULT__LAMBDA__MEMORY_SIZE
+from osbot_utils.type_safe.Type_Safe            import Type_Safe
 from osbot_aws.aws.lambda_                      import boto3__lambda
 from osbot_utils.type_safe.decorators.type_safe import type_safe
 from osbot_utils.utils.Env                      import load_dotenv
@@ -10,9 +12,12 @@
 # todo: refactor lambda_name to be a Type_Safe variable, but that will clash with the current lambda_name() function (which is used in other projects)
 #       stage and the other self.* vars set in the __init__ should also be Type_Safe variables
 
-class Deploy_Lambda:
+class Deploy_Lambda(Type_Safe):
+    stage            : str = None
+    ephemeral_storage: int = DEFAULT__LAMBDA__EPHEMERAL_STORAGE
+    memory_size      : int = DEFAULT__LAMBDA__MEMORY_SIZE
 
-    def __init__(self, handler, stage=None, lambda_name=None, **kwargs):
+    def __init__(self, handler, lambda_name=None, **kwargs):
         super().__init__(**kwargs)
         load_dotenv()
         self.osbot_setup          = OSBot_Setup()
@@ -24,11 +29,13 @@ def __init__(self, handler, stage=None, lambda_name=None, **kwargs):
             self.handler          = handler
             self.module_name      = handler.__module__
 
-        self.stage                = stage
         self.role_arn             = Temp_Aws_Roles().for_lambda_invocation__role_arn()
 
         self.package              = self.get_package()
+
         if lambda_name:                                        # if we have provided a name, use it internally
+            if self.stage:                                      # todo: review this logic of setting lambda function name (with stage) and syncing it with others like module_name (which is quite messy at the moment)
+                lambda_name += f'__{self.stage}'
             self.package.lambda_name     = lambda_name
             self.package.aws_lambda.name = lambda_name
             self.module_name             = lambda_name
@@ -44,12 +51,14 @@ def add_function_source_code(self):
         self.package.add_module(root_module_name)
         return self
 
-    def add_file(self, file_path):
-        self.package.add_file(source=file_path)
+    def add_file(self, file_path, folder=None):
+        self.package.add_file(source=file_path, folder=folder)
         return self
 
+
     def add_file__boto3__lambda(self):
-        self.add_file(boto3__lambda.__file__)
+        folder = 'osbot_aws.aws.lambda_'.replace('.','/')
+        self.add_file(file_path = boto3__lambda.__file__, folder=folder)
         return self
 
     def add_folder(self, source, ignore=None):
@@ -109,14 +118,18 @@ def function_url(self):
 
     def get_package(self):
         package = Lambda_Package(self.lambda_name())
-        package.aws_lambda.set_s3_bucket(self.osbot_setup.s3_bucket_lambdas)
-        package.aws_lambda.set_role(self.role_arn)
+        with package.aws_lambda as _:
+            _.set_s3_bucket(self.osbot_setup.s3_bucket_lambdas)
+            _.set_role(self.role_arn)
+            _.ephemeral_storage = self.ephemeral_storage
+            _.memory_size       = self.memory_size
         return package
 
     def lambda_name(self):
-        if self.stage is None:
-            return self.module_name
-        return f"{self.module_name}-{self.stage}"
+        return self.module_name
+        # if self.stage is None:
+        #     return self.module_name
+        # return f"{self.module_name}__{self.stage}"          # breaking change was "-" , and now it is "__" (which is a beter separator)
 
     def lambda_function(self):
         return self.package.aws_lambda

osbot_aws/helpers/Lambda_Package.py

@@ -1,22 +1,21 @@
 import os
 import importlib
 import site
-
+from osbot_utils.type_safe.Type_Safe            import Type_Safe
 from osbot_utils.type_safe.decorators.type_safe import type_safe
-
 from osbot_aws.aws.lambda_.Lambda_Layer         import Lambda_Layer
 from osbot_utils.testing.Temp_Folder            import Temp_Folder
 from osbot_utils.testing.Temp_Zip               import Temp_Zip
 from osbot_utils.utils.Process                  import Process
 from osbot_aws.AWS_Config                       import AWS_Config
 from osbot_aws.apis.test_helpers.Temp_Aws_Roles import Temp_Aws_Roles
-from osbot_utils.utils.Files                    import folder_copy, Files, folder_not_exists, path_combine, folder_name
-
-from osbot_aws.aws.lambda_.Lambda import Lambda
+from osbot_utils.utils.Files                    import folder_copy, Files, folder_not_exists, path_combine, folder_name, create_folder, path_combine_safe
+from osbot_aws.aws.lambda_.Lambda               import Lambda
 
 
-class Lambda_Package:
-    def __init__(self,lambda_name):
+class Lambda_Package(Type_Safe):                            # todo: refactor to use full value of Type_Safe
+    def __init__(self,lambda_name, **kwargs):
+        super().__init__(**kwargs)
         self.lambda_name   = lambda_name
         self.aws_lambda    = Lambda(self.lambda_name)
         self.s3_bucket     = AWS_Config().lambda_s3_bucket()
@@ -54,8 +53,13 @@ def add_layers(self, layers_arn: list):
             self.aws_lambda.add_layer(layer_arn)
         return self
 
-    def add_file(self, source):
-        Files.copy(source, self.tmp_folder)
+    def add_file(self, source, folder=None):
+        if folder is None:
+            target_folder = self.tmp_folder
+        else:                                                                       # if we have a folder path provider
+            target_folder = path_combine_safe(self.tmp_folder, folder)              # merge it with the self.tmp_folder (base folder to capture all files to deploy)
+            create_folder(target_folder)                                            # make sure folder exists
+        Files.copy(source, target_folder)
 
     def add_folder(self, source, ignore=None):
         destination = Files.path_combine(self.tmp_folder, folder_name(source))

tests/integration/aws/lambda_/dependencies/test_Lambda__Dependency__Inside_Lambda.py

@@ -277,8 +277,7 @@ def run(event, context):                        # this runs inside the lambda en
 
     def test_12__using_lambda_deploy__load_dependency__using__only_boto_3_code(self):
         def run(event, context):
-            # noinspection PyUnresolvedReferences
-            from boto3__lambda import load_dependency
+            from osbot_aws.aws.lambda_.boto3__lambda import load_dependency                 # using the lightweight file (which only has the boto3 calls required to call load_dependency
             return load_dependency('colorama==0.4.6')
 
         source_code = function_source_code(run)
@@ -291,7 +290,7 @@ def run(event, context):
                 _.add_file     (lambda_handler)
                 assert _.deploy() is True
                 assert _.invoke() == 'colorama==0.4.6 (loaded from S3)'
-                assert _.info().get('Configuration').get('CodeSize') < 1300                                              # confirm that size of the code uploaded is still very small
+                assert _.info().get('Configuration').get('CodeSize') < 1500                                              # confirm that size of the code uploaded is still very small
                 assert _.delete() is True
 
     def test_dependencies_folder(self):

tests__to-wire-up/unit/apis/test_Lambda.py

@@ -1,16 +1,14 @@
 import pytest
-from time import sleep
-
-from osbot_utils.utils.Lists import list_contains
-
-from osbot_aws.aws.iam.IAM_Role import IAM_Role
-from osbot_aws.aws.iam.IAM_Utils import IAM_Utils
+from time                                        import sleep
+from osbot_utils.utils.Lists                     import list_contains
+from osbot_aws.aws.iam.IAM_Role                  import IAM_Role
+from osbot_aws.aws.iam.IAM_Utils                 import IAM_Utils
 from osbot_utils.utils                           import Misc
 from osbot_utils.utils.Files                     import file_exists, file_contents, Files
 from osbot_aws.AWS_Config                        import AWS_Config
 from osbot_aws.aws.s3.S3                         import S3
 from osbot_aws.helpers.Test_Helper               import Test_Helper
-from osbot_aws.aws.lambda_.Lambda                       import Lambda
+from osbot_aws.aws.lambda_.Lambda                import Lambda
 from osbot_aws.apis.test_helpers.Temp_Lambda     import Temp_Folder_With_Lambda_File, Temp_Lambda
 from osbot_aws.apis.test_helpers.Temp_SQS_Queue  import Temp_SQS_Queue
 from osbot_utils.utils.Assert                    import Assert