From edd0ad1d1afe1458c4ab6c402eb25ffa05a7a48c Mon Sep 17 00:00:00 2001 From: Jon Robison Date: Sat, 17 Dec 2016 00:10:53 -0500 Subject: [PATCH 01/97] -Fix tail --- zappa/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index 821165c6d..21b9ae9b1 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -263,7 +263,7 @@ def positive_int(s): # Log Tailing ## tail_parser = subparsers.add_parser( - 'tail', help='Tail deployment logs.' + 'tail', parents=[env_parser], help='Tail deployment logs.' ) tail_parser.add_argument( '--no-color', action='store_true', From c6f849ff63fd96ce4cda169e53a00df4980b43f4 Mon Sep 17 00:00:00 2001 From: Nam Ngo Date: Tue, 13 Dec 2016 23:54:55 +1100 Subject: [PATCH 02/97] Add authorizer from event to wsgi request. --- zappa/wsgi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 155f83965..3ca478b4a 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -117,6 +117,9 @@ def create_wsgi_request(event_info, server_name='zappa', script_name=None, if remote_user: environ['REMOTE_USER'] = remote_user + if event_info['requestContext'].get('authorizer'): + environ['API_GATEWAY_AUTHORIZER'] = event_info['requestContext']['authorizer'] + return environ From f588688ba9058ced277a36f6f7afbdbeb9c0f50b Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 19 Dec 2016 18:07:45 -0500 Subject: [PATCH 03/97] 0.32.1 - fix broken tail in cli --- CHANGELOG.md | 5 ++++- setup.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4ac035f5..f69eb9b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Zappa Changelog -# 0.32.0 +## 0.32.1 +* File `tail` broken in CLI refactor + +## 0.32.0 * Add Cognito Authorizers * Refactor CLI, add Bash Completion * Improve manylinux wheels diff --git a/setup.py b/setup.py index 6c2ab28f9..ca123fa24 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='zappa', - version='0.32.0', + version='0.32.1', packages=['zappa'], install_requires=required, tests_require=test_required, From b23f1d3ee0c6e1a7262bb83a10a4e8322dd73c50 Mon Sep 17 00:00:00 2001 From: Doppins Date: Tue, 20 Dec 2016 02:06:26 +0000 Subject: [PATCH 04/97] Upgrade dependency botocore to ==1.4.88 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f4c5e514..e06a41049 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.7.0 base58==0.2.4 -botocore==1.4.87 +botocore==1.4.88 boto3==1.4.2 docutils>=0.12 futures==3.0.5 From 5cce67460ea84db22d9b0471d7cf2a24a9dfaf43 Mon Sep 17 00:00:00 2001 From: Paul Jimenez Date: Tue, 20 Dec 2016 10:04:36 -0500 Subject: [PATCH 05/97] Fix the returned order of events in logs --- zappa/zappa.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index a9196c428..234064fc5 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1818,14 +1818,19 @@ def fetch_logs(self, lambda_name, filter_pattern='', limit=10000): all_streams = streams['logStreams'] all_names = [stream['logStreamName'] for stream in all_streams] - response = self.logs_client.filter_log_events( - logGroupName=log_name, - logStreamNames=all_names, - filterPattern=filter_pattern, - limit=limit - ) - - return response['events'] + fle_args = {'logGroupName':log_name, + 'logStreamNames':all_names, + 'filterPattern':filter_pattern, + 'limit':limit + } + events = [] + response = {} + while not response or 'nextToken' in response: + if 'nextToken' in response: fle_args['nextToken'] = response['nextToken'] + response = self.logs_client.filter_log_events(**fle_args) + events += response['events'] + + return sorted(events, key=lambda k: k['timestamp']) def remove_log_group(self, group_name): """ From 597e02fd2a71b2e1a85405be71780a3b22fe5e36 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Tue, 20 Dec 2016 12:39:30 -0500 Subject: [PATCH 06/97] Revert "Fix the returned order of events in logs" --- zappa/zappa.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 234064fc5..a9196c428 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1818,19 +1818,14 @@ def fetch_logs(self, lambda_name, filter_pattern='', limit=10000): all_streams = streams['logStreams'] all_names = [stream['logStreamName'] for stream in all_streams] - fle_args = {'logGroupName':log_name, - 'logStreamNames':all_names, - 'filterPattern':filter_pattern, - 'limit':limit - } - events = [] - response = {} - while not response or 'nextToken' in response: - if 'nextToken' in response: fle_args['nextToken'] = response['nextToken'] - response = self.logs_client.filter_log_events(**fle_args) - events += response['events'] - - return sorted(events, key=lambda k: k['timestamp']) + response = self.logs_client.filter_log_events( + logGroupName=log_name, + logStreamNames=all_names, + filterPattern=filter_pattern, + limit=limit + ) + + return response['events'] def remove_log_group(self, group_name): """ From 0b5b232d654b8f9d1f976748d8838e9e5c2df354 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Tue, 20 Dec 2016 14:24:06 -0500 Subject: [PATCH 07/97] add top 10 list to links --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index be54f5b70..abf889e88 100644 --- a/README.md +++ b/README.md @@ -715,6 +715,7 @@ To learn more about these capabilities, see [these slides](https://htmlpreview.g * _[Zappa Serves Python, Minus the Servers](http://www.infoworld.com/article/3031665/application-development/zappa-serves-python-web-apps-minus-the-servers.html)_ * _[Zappa lyfter serverlösa applikationer med Python](http://computersweden.idg.se/2.2683/1.649895/zappa-lyfter-python)_ * _[Interview: Rich Jones on Zappa](https://serverlesscode.com/post/rich-jones-interview-django-zappa/)_ +* [Top 10 Python Libraries of 2016](https://tryolabs.com/blog/2016/12/20/top-10-python-libraries-of-2016/) ## Sites Using Zappa From 3877a08fd411218ef11316f9767c6d86196f75d4 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Tue, 20 Dec 2016 14:32:14 -0500 Subject: [PATCH 08/97] shameless --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index abf889e88..528f3d36f 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ [![Requirements Status](https://requires.io/github/Miserlou/Zappa/requirements.svg?branch=master)](https://requires.io/github/Miserlou/Zappa/requirements/?branch=master) [![PyPI](https://img.shields.io/pypi/v/Zappa.svg)](https://pypi.python.org/pypi/zappa) [![Slack](https://img.shields.io/badge/chat-slack-ff69b4.svg)](https://slack.zappa.io/) +[![Gun.io](https://img.shields.io/badge/made%20by-gun.io-blue.svg)](https://gun.io/) From d59d0bcc8263b629b79e74da8b59b98cd49c4a31 Mon Sep 17 00:00:00 2001 From: Doppins Date: Wed, 21 Dec 2016 01:50:17 +0000 Subject: [PATCH 09/97] Upgrade dependency botocore to ==1.4.89 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e06a41049..60fb3c010 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.7.0 base58==0.2.4 -botocore==1.4.88 +botocore==1.4.89 boto3==1.4.2 docutils>=0.12 futures==3.0.5 From 21aebb0bb84c4eff0cb5d2bd6a760f083175f819 Mon Sep 17 00:00:00 2001 From: Doppins Date: Wed, 21 Dec 2016 01:50:27 +0000 Subject: [PATCH 10/97] Upgrade dependency boto3 to ==1.4.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e06a41049..918e9d5fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ argcomplete==1.7.0 base58==0.2.4 botocore==1.4.88 -boto3==1.4.2 +boto3==1.4.3 docutils>=0.12 futures==3.0.5 hjson==2.0.2 From 37bfb785721c807e6548f0c622f3a805fa2f852d Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 21 Dec 2016 17:50:23 -0500 Subject: [PATCH 11/97] add zappa docker --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 528f3d36f..8b57f274a 100644 --- a/README.md +++ b/README.md @@ -743,6 +743,7 @@ Are you using Zappa? Let us know and we'll list your site here! * [zappa-file-widget](https://github.com/anush0247/zappa-file-widget) - A Django plugin for supporting binary file uploads in Django on Zappa. * [zops](https://github.com/bjinwright/zops) - Utilities for teams and continuous integrations using Zappa. * [cookiecutter-mobile-backend](https://github.com/narfman0/cookiecutter-mobile-backend/) - A `cookiecutter` Django project with Zappa and S3 uploads support. +* [Zappa Docker Image](https://github.com/danielwhatmuff/zappa) - A Docker image for running Zappa locally, based on Lambda Docker. ## Hacks From a755e8c5427ac8d06bde054a94da6211534f43bf Mon Sep 17 00:00:00 2001 From: Doppins Date: Thu, 22 Dec 2016 01:05:58 +0000 Subject: [PATCH 12/97] Upgrade dependency botocore to ==1.4.90 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af482a70f..4ec189b5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.7.0 base58==0.2.4 -botocore==1.4.89 +botocore==1.4.90 boto3==1.4.3 docutils>=0.12 futures==3.0.5 From 27272d75d9d701e18222e18eb8eb8399e1f13620 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 22 Dec 2016 14:38:21 -0500 Subject: [PATCH 13/97] add django example --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8b57f274a..84ad5d566 100644 --- a/README.md +++ b/README.md @@ -744,6 +744,7 @@ Are you using Zappa? Let us know and we'll list your site here! * [zops](https://github.com/bjinwright/zops) - Utilities for teams and continuous integrations using Zappa. * [cookiecutter-mobile-backend](https://github.com/narfman0/cookiecutter-mobile-backend/) - A `cookiecutter` Django project with Zappa and S3 uploads support. * [Zappa Docker Image](https://github.com/danielwhatmuff/zappa) - A Docker image for running Zappa locally, based on Lambda Docker. +* [zappa-django-example](https://github.com/edgarroman/zappa-django-example) - A complete example for running Django on Zappa. ## Hacks From 01473fa467bf40f6c1303bb7b2d0764ac9e65dda Mon Sep 17 00:00:00 2001 From: Doppins Date: Thu, 22 Dec 2016 22:15:37 +0000 Subject: [PATCH 14/97] Upgrade dependency botocore to ==1.4.91 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4ec189b5d..61c44efe8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.7.0 base58==0.2.4 -botocore==1.4.90 +botocore==1.4.91 boto3==1.4.3 docutils>=0.12 futures==3.0.5 From 1e35aba1c0d9047a0152c2c8242b7054148bd9ef Mon Sep 17 00:00:00 2001 From: Ryan Moe Date: Thu, 22 Dec 2016 15:43:40 -0800 Subject: [PATCH 15/97] Fix argument order for invoke and manage commands The invoke and manage commands now work correctly when flags like --raw and --all are specified after the management command or function to invoke. --- tests/tests.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ zappa/cli.py | 26 +++++++++++++--- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 73d2d44e9..7a474c723 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -702,6 +702,89 @@ def test_cli_negative_rollback(self): self.assertRegexpMatches(error_msg, expected) sys.stderr = old_stderr + @mock.patch('zappa.cli.ZappaCLI.dispatch_command') + def test_cli_invoke(self, _): + zappa_cli = ZappaCLI() + argv = '-s test_settings.json invoke '.split() + raw_tests = ( + ['--raw', 'devor', '"print 1+2"'], + ['devor', '"print 1+2"', '--raw'] + ) + + for cmd in raw_tests: + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertFalse(args['all']) + self.assertTrue(args['raw']) + self.assertEquals(args['command_rest'], '"print 1+2"') + self.assertEquals(args['command_env'], 'devor') + + all_raw_tests = ( + ['--all', '--raw', '"print 1+2"'], + ['"print 1+2"', '--all', '--raw'], + ['--raw', '"print 1+2"', '--all'], + ['--all', '"print 1+2"', '--raw'] + ) + for cmd in all_raw_tests: + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertTrue(args['all']) + self.assertTrue(args['raw']) + self.assertEquals(args['command_rest'], '"print 1+2"') + self.assertEquals(args['command_env'], None) + + zappa_cli.handle(argv + ['devor', 'myapp.my_func']) + args = zappa_cli.vargs + self.assertEquals(args['command_rest'], 'myapp.my_func') + + all_func_tests = ( + ['--all', 'myapp.my_func'], + ['myapp.my_func', '--all'] + ) + for cmd in all_func_tests: + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertTrue(args['all']) + self.assertEquals(args['command_rest'], 'myapp.my_func') + + + @mock.patch('zappa.cli.ZappaCLI.dispatch_command') + def test_cli_manage(self, _): + zappa_cli = ZappaCLI() + argv = '-s test_settings.json manage '.split() + all_tests = ( + ['--all', 'showmigrations', 'admin'], + ['showmigrations', 'admin', '--all'] + ) + + for cmd in all_tests: + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertTrue(args['all']) + self.assertItemsEqual( + args['command_rest'], ['showmigrations', 'admin'] + ) + + cmd = ['devor', 'showmigrations', 'admin'] + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertFalse(args['all']) + self.assertItemsEqual( + args['command_rest'], ['showmigrations', 'admin'] + ) + + cmd = ['devor', '"shell --version"'] + zappa_cli.handle(argv + cmd) + args = zappa_cli.vargs + + self.assertFalse(args['all']) + self.assertItemsEqual(args['command_rest'], ['"shell --version"']) + def test_bad_json_catch(self): zappa_cli = ZappaCLI() self.assertRaises(ValueError, zappa_cli.load_settings_file, 'tests/test_bad_settings.json') diff --git a/zappa/cli.py b/zappa/cli.py index 21b9ae9b1..29203a6a2 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -173,7 +173,6 @@ def handle(self, argv=None): 'Zappa environments.') group.add_argument('--all', action='store_true', help=all_help) group.add_argument('command_env', nargs='?') - env_parser.add_argument('command_rest', nargs=argparse.REMAINDER) ## # Certify @@ -210,14 +209,19 @@ def handle(self, argv=None): help=('When invoking remotely, invoke this python as a string,' ' not as a modular path.') ) + invoke_parser.add_argument('command_rest') ## # Manage ## - subparsers.add_parser( - 'manage', parents=[env_parser], + manage_parser = subparsers.add_parser( + 'manage', help='Invoke remote Django manage.py commands.' ) + rest_help = ("Command in the form of . is not " + "required if --all is specified") + manage_parser.add_argument('--all', action='store_true', help=all_help) + manage_parser.add_argument('command_rest', nargs='+', help=rest_help) ## # Rollback @@ -306,7 +310,19 @@ def positive_int(s): self.vargs = vars(args) # Parse the input - self.command_env = self.vargs.get('command_env') + # NOTE(rmoe): Special case for manage command + # The manage command can't have both command_env and command_rest + # arguments. Since they are both positional arguments argparse can't + # differentiate the two. This causes problems when used with --all. + # (e.g. "manage --all showmigrations admin" argparse thinks --all has + # been specified AND that command_env='showmigrations') + # By having command_rest collect everything but --all we can split it + # apart here instead of relying on argparse. + if args.command == 'manage' and not self.vargs.get('all'): + self.command_env = self.vargs['command_rest'].pop(0) + else: + self.command_env = self.vargs.get('command_env') + self.command = args.command # We don't have any settings yet, so make those first! @@ -385,7 +401,7 @@ def dispatch_command(self, command, environment): print("Please enter the function to invoke.") return - self.invoke(self.command_env[-1], raw_python=self.vargs['raw']) + self.invoke(self.vargs['command_rest'], raw_python=self.vargs['raw']) elif command == 'manage': # pragma: no cover if not self.vargs.get('command_rest'): From 3b2626960d93f572bf4b9b9ca5fcaa4735bcfa82 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Fri, 23 Dec 2016 22:52:08 -0500 Subject: [PATCH 16/97] set REMOTE_USER even when using iam_authorization --- zappa/wsgi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 3ca478b4a..66bda2ff9 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -24,6 +24,9 @@ def create_wsgi_request(event_info, server_name='zappa', script_name=None, remote_user = None if event_info['requestContext'].get('authorizer'): remote_user = event_info['requestContext']['authorizer'].get('principalId') + elif event_info['requestContext'].get('identity'): + remote_user = event_info['requestContext']['identity'].get('userArn') + # Non-GET data is B64'd through the APIGW. # if method in ["POST", "PUT", "PATCH"]: From a560b3e801bda1c1ba2e5d763ddb034d68397a1b Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Sat, 24 Dec 2016 19:51:23 -0500 Subject: [PATCH 17/97] example req --- example/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/requirements.txt b/example/requirements.txt index efe6168e2..8f2c436fd 100644 --- a/example/requirements.txt +++ b/example/requirements.txt @@ -1,2 +1,2 @@ -Flask==0.11.1 +Flask>=0.12 zappa>=0.17.6 From 93c5473149e8d74d59f26d9ef7ddbcf1170e1579 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 28 Dec 2016 12:21:54 +0100 Subject: [PATCH 18/97] add flask ask guide to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 84ad5d566..a741d8d53 100644 --- a/README.md +++ b/README.md @@ -702,13 +702,14 @@ To learn more about these capabilities, see [these slides](https://htmlpreview.g ## Zappa Guides -* [Django-Zappa tutorial screencast](https://www.youtube.com/watch?v=plUrbPN0xc8&feature=youtu.be). +* [Django-Zappa tutorial (screencast)](https://www.youtube.com/watch?v=plUrbPN0xc8&feature=youtu.be). * [Using Django-Zappa, Part 1](https://serverlesscode.com/post/zappa-wsgi-for-python/). * [Using Django-Zappa, Part 2: VPCs](https://serverlesscode.com/post/zappa-wsgi-for-python-pt-2/). * [Building Serverless Microservices with Zappa and Flask](https://gun.io/blog/serverless-microservices-with-zappa-and-flask/) * [Zappa で Hello World するまで (Japanese)](http://qiita.com/satoshi_iwashita/items/505492193317819772c7) * [How to Deploy Zappa with CloudFront, RDS and VPC](https://jinwright.net/how-deploy-serverless-wsgi-app-using-zappa/) * [zappa-examples - Flask, Django, image uploads, and more!](https://github.com/narfman0/zappa-examples/) +* [Deploy Flask-Ask to AWS Lambda with Zappa (screencast)](https://www.youtube.com/watch?v=mjWV4R2P4ks) * _Your guide here?_ ## Zappa in the Press From f70f5e5dafbfa8430384ff091f519d96ac3f455d Mon Sep 17 00:00:00 2001 From: tsarpaul Date: Sat, 31 Dec 2016 21:28:18 +0200 Subject: [PATCH 19/97] Added support for local import for prebuild script --- test_settings.json | 2 +- tests/test_app.py | 3 +++ zappa/cli.py | 44 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/test_settings.json b/test_settings.json index 59ba95deb..d9578108f 100644 --- a/test_settings.json +++ b/test_settings.json @@ -12,7 +12,7 @@ "delete_local_zip": true, "debug": true, "parameter_depth": 2, - "prebuild_script": "test_settings.prebuild_me", + "prebuild_script": "tests.test_app.prebuild_me", "events": [ { "function": "tests.test_app.schedule_me", diff --git a/tests/test_app.py b/tests/test_app.py index 39395f797..ac6445160 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -18,3 +18,6 @@ def schedule_me(): def callback(self): print("this is a callback") +def prebuild_me(): + print("this is a prebuild script") + diff --git a/zappa/cli.py b/zappa/cli.py index 29203a6a2..0f4e885a0 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1337,11 +1337,11 @@ def callback(self, position): working_dir_importer = pkgutil.get_importer(working_dir) module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) - except (ImportError, AttributeError): # pragma: no cover + except (ImportError, AttributeError): try: # Callback func might be in virtualenv module_ = importlib.import_module(mod_path) - except ImportError: + except ImportError: # pragma: no cover raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( "import {position} callback ".format(position=position), bold=True) + 'module: "{mod_path}"'.format(mod_path=click.style(mod_path, bold=True))) @@ -1789,18 +1789,42 @@ def execute_prebuild_script(self): """ - # Parse the string - prebuild_module_s, prebuild_function_s = self.prebuild_script.rsplit('.', 1) + def execute_prebuild_script(self): + """ + Parse and execute the prebuild_script from the zappa_settings. + + """ - # The module - prebuild_module = imp.load_source(prebuild_module_s, prebuild_module_s + '.py') + (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - # The function - prebuild_function = getattr(prebuild_module, prebuild_function_s) + try: # Prefer callback in working directory + if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder + (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) + mod_folder_path_fragments = mod_folder_path.split('.') + working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) + else: + mod_name = pb_mod_path + working_dir = os.getcwd() - # Execute it - prebuild_function() + working_dir_importer = pkgutil.get_importer(working_dir) + module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) + + except (ImportError, AttributeError): + + try: # Prebuild func might be in virtualenv + module_ = importlib.import_module(pb_mod_path) + except ImportError: # pragma: no cover + raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( + "import prebuild script ", bold=True) + 'module: "{pb_mod_path}"'.format( + pb_mod_path=click.style(pb_mod_path, bold=True))) + + if not hasattr(module_, pb_func): # pragma: no cover + raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( + "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( + pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( + pb_mod_path=pb_mod_path)) + getattr(module_, pb_func)() # Call the function passing self def collision_warning(self, item): """ From 2d6f33251b89c1e9bc38c6736dbde10d64fd41f7 Mon Sep 17 00:00:00 2001 From: tsarpaul Date: Sat, 31 Dec 2016 21:28:58 +0200 Subject: [PATCH 20/97] No more import imp --- zappa/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index 0f4e885a0..d682dae16 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -21,7 +21,6 @@ import collections import hjson as json import inspect -import imp import importlib import logging import os From 6ce9c13b0be51de204e5be70bc7a5178f013b482 Mon Sep 17 00:00:00 2001 From: TsarPaul Date: Sat, 31 Dec 2016 21:32:17 +0200 Subject: [PATCH 21/97] Fixed accident tab --- zappa/cli.py | 54 +++++++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index d682dae16..bf6cb685e 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1788,42 +1788,36 @@ def execute_prebuild_script(self): """ - def execute_prebuild_script(self): - """ - Parse and execute the prebuild_script from the zappa_settings. - - """ - - (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - - try: # Prefer callback in working directory - if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder - (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) - mod_folder_path_fragments = mod_folder_path.split('.') - working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) - else: - mod_name = pb_mod_path - working_dir = os.getcwd() + (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - working_dir_importer = pkgutil.get_importer(working_dir) - module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) + try: # Prefer callback in working directory + if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder + (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) + mod_folder_path_fragments = mod_folder_path.split('.') + working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) + else: + mod_name = pb_mod_path + working_dir = os.getcwd() - except (ImportError, AttributeError): + working_dir_importer = pkgutil.get_importer(working_dir) + module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) - try: # Prebuild func might be in virtualenv - module_ = importlib.import_module(pb_mod_path) - except ImportError: # pragma: no cover - raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( - "import prebuild script ", bold=True) + 'module: "{pb_mod_path}"'.format( - pb_mod_path=click.style(pb_mod_path, bold=True))) + except (ImportError, AttributeError): - if not hasattr(module_, pb_func): # pragma: no cover + try: # Prebuild func might be in virtualenv + module_ = importlib.import_module(pb_mod_path) + except ImportError: # pragma: no cover raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( - "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( - pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( - pb_mod_path=pb_mod_path)) + "import prebuild script ", bold=True) + 'module: "{pb_mod_path}"'.format( + pb_mod_path=click.style(pb_mod_path, bold=True))) + + if not hasattr(module_, pb_func): # pragma: no cover + raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( + "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( + pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( + pb_mod_path=pb_mod_path)) - getattr(module_, pb_func)() # Call the function passing self + getattr(module_, pb_func)() # Call the function def collision_warning(self, item): """ From 71514157f8899bc0ea2ab7d79013f91bb6f5acb0 Mon Sep 17 00:00:00 2001 From: tsarpaul Date: Sun, 1 Jan 2017 15:33:47 +0200 Subject: [PATCH 22/97] Fixed tab --- zappa/cli.py | 54 +++++++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index d682dae16..c62f376bb 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1788,42 +1788,36 @@ def execute_prebuild_script(self): """ - def execute_prebuild_script(self): - """ - Parse and execute the prebuild_script from the zappa_settings. - - """ - - (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - - try: # Prefer callback in working directory - if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder - (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) - mod_folder_path_fragments = mod_folder_path.split('.') - working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) - else: - mod_name = pb_mod_path - working_dir = os.getcwd() + (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - working_dir_importer = pkgutil.get_importer(working_dir) - module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) + try: # Prefer callback in working directory + if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder + (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) + mod_folder_path_fragments = mod_folder_path.split('.') + working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) + else: + mod_name = pb_mod_path + working_dir = os.getcwd() - except (ImportError, AttributeError): + working_dir_importer = pkgutil.get_importer(working_dir) + module_ = working_dir_importer.find_module(mod_name).load_module(mod_name) - try: # Prebuild func might be in virtualenv - module_ = importlib.import_module(pb_mod_path) - except ImportError: # pragma: no cover - raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( - "import prebuild script ", bold=True) + 'module: "{pb_mod_path}"'.format( - pb_mod_path=click.style(pb_mod_path, bold=True))) + except (ImportError, AttributeError): - if not hasattr(module_, pb_func): # pragma: no cover + try: # Prebuild func might be in virtualenv + module_ = importlib.import_module(pb_mod_path) + except ImportError: # pragma: no cover raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( - "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( - pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( - pb_mod_path=pb_mod_path)) + "import prebuild script ", bold=True) + 'module: "{pb_mod_path}"'.format( + pb_mod_path=click.style(pb_mod_path, bold=True))) + + if not hasattr(module_, pb_func): # pragma: no cover + raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( + "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( + pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( + pb_mod_path=pb_mod_path)) - getattr(module_, pb_func)() # Call the function passing self + getattr(module_, pb_func)() # Call the function passing self def collision_warning(self, item): """ From 29029fe4faa6d7218d680bdd82733ad5ad2d5715 Mon Sep 17 00:00:00 2001 From: tsarpaul Date: Sun, 1 Jan 2017 15:35:37 +0200 Subject: [PATCH 23/97] Changed some comments --- zappa/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index c62f376bb..be87fb2a1 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1790,8 +1790,8 @@ def execute_prebuild_script(self): (pb_mod_path, pb_func) = self.prebuild_script.rsplit('.', 1) - try: # Prefer callback in working directory - if pb_mod_path.count('.') >= 1: # Callback function is nested in a folder + try: # Prefer prebuild script in working directory + if pb_mod_path.count('.') >= 1: # Prebuild script func is nested in a folder (mod_folder_path, mod_name) = pb_mod_path.rsplit('.', 1) mod_folder_path_fragments = mod_folder_path.split('.') working_dir = os.path.join(os.getcwd(), *mod_folder_path_fragments) From 557a1866fcb88e1eb11cbfb5167477b0aa3ce735 Mon Sep 17 00:00:00 2001 From: TsarPaul Date: Tue, 3 Jan 2017 18:49:33 +0200 Subject: [PATCH 24/97] Fixed extra space --- tests/test_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_app.py b/tests/test_app.py index ac6445160..ec1fe7648 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -19,5 +19,5 @@ def callback(self): print("this is a callback") def prebuild_me(): - print("this is a prebuild script") + print("this is a prebuild script") From 03ec1b0aac119030c13e2da21139286b7022622d Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Wed, 4 Jan 2017 07:56:29 -0500 Subject: [PATCH 25/97] get rid of duplicates and extraneous that are already present in python lambda environment --- zappa/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index 29203a6a2..239260071 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1548,7 +1548,9 @@ def create_package(self): self.lambda_name, handler_file=handler_file, use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True), - exclude=self.stage_config.get('exclude', []) + exclude=self.stage_config.get( + 'exclude', + ["boto3", "*.dist-info", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"]) ) # Throw custom setings into the zip file From f5572da79793b6295536808cfcea7f31b918f7fd Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 4 Jan 2017 15:51:39 +0100 Subject: [PATCH 26/97] attempt to revert coverage --- test_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test_requirements.txt b/test_requirements.txt index 4543265ec..70d7f61e8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,5 @@ coveralls==1.1 +coverage==4.2 Django==1.10.4 mock==2.0.0 nose==1.3.7 From 9b87c92f8080154f10a705521f8c442f4b962f96 Mon Sep 17 00:00:00 2001 From: Miha Zelnik Date: Wed, 28 Dec 2016 19:50:14 +0100 Subject: [PATCH 27/97] Rename lets_encrypt_schedule to lets_encrypt_expression #571 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a741d8d53..a8dab7b4c 100644 --- a/README.md +++ b/README.md @@ -420,7 +420,7 @@ to change Zappa's behavior. Use these at your own risk! "lambda_description": "Your Description", // However you want to describe your project for the AWS console. Default "Zappa Deployment". "lambda_handler": "your_custom_handler", // The name of Lambda handler. Default: handler.lambda_handler "lets_encrypt_key": "s3://your-bucket/account.key", // Let's Encrypt account key path. Can either be an S3 path or a local file path. - "lets_encrypt_schedule": "rate(15 days)" // How often to auto-renew Let's Encrypt certificate on the server. Must be set to enable autorenewing, rate or cron syntax. + "lets_encrypt_expression": "rate(15 days)" // How often to auto-renew Let's Encrypt certificate on the server. Must be set to enable autorenewing, rate or cron syntax. "log_level": "DEBUG", // Set the Zappa log level. Can be one of CRITICAL, ERROR, WARNING, INFO and DEBUG. Default: DEBUG "manage_roles": true, // Have Zappa automatically create and define IAM execution roles and policies. Default true. If false, you must define your own IAM Role and role_name setting. "memory_size": 512, // Lambda function memory in MB From 60ee7d247d684e3c3db54765efc3708613565c51 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 5 Jan 2017 13:06:21 +0100 Subject: [PATCH 28/97] remove coverage combine line, just to test theory here https://bitbucket.org/ned/coveragepy/issues/548/dont-fail-with-coverage-combine-aka-43 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f106ab349..46835f33e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - TESTCASE=tests/tests.py script: - nosetests $TESTCASE --with-coverage --cover-package=zappa --with-timer - - coverage combine --append + # - coverage combine --append after_success: coveralls notifications: From 17ff40bf9b944a982656cea76d156ae45cb04262 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 5 Jan 2017 13:16:56 +0100 Subject: [PATCH 29/97] Updates all requirements, coverage combine totally gone --- requirements.txt | 4 ++-- test_requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 61c44efe8..1e70cfe7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.7.0 base58==0.2.4 -botocore==1.4.91 +botocore==1.4.93 boto3==1.4.3 docutils>=0.12 futures==3.0.5 @@ -16,6 +16,6 @@ six==1.10.0 toml==0.9.2 tqdm==4.10.0 troposphere>=1.9.0 -Werkzeug==0.11.11 +Werkzeug==0.11.15 wheel==0.29.0 wsgi-request-logger==0.4.6 diff --git a/test_requirements.txt b/test_requirements.txt index 70d7f61e8..efff56f8e 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ coveralls==1.1 -coverage==4.2 -Django==1.10.4 +coverage==4.3 +Django==1.10.5 mock==2.0.0 nose==1.3.7 nose-timer==0.6.0 From 109a355088be18f6cc159ace30913425da4dbca3 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 5 Jan 2017 13:24:34 +0100 Subject: [PATCH 30/97] bump coverage one more time --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index efff56f8e..2620f4c3f 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ coveralls==1.1 -coverage==4.3 +coverage==4.3.1 Django==1.10.5 mock==2.0.0 nose==1.3.7 From e7441bf92cacc8c451e5d5fde25ae4d77b7d3040 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 5 Jan 2017 14:05:51 +0100 Subject: [PATCH 31/97] add mackenzie link, rm extra sq ket --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a741d8d53..2aebb0d03 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,7 @@ Are you using Zappa? Let us know and we'll list your site here! ## Related Projects * [lambda-packages](http://github.com/Miserlou/lambda-packages) - Precompiled C-extension packages for AWS Lambda. Used automatically by Zappa. +* [Mackenzie](http://github.com/Miserlou/Mackenzie) - AWS Lambda Infection Toolkit * [zappa-cms](http://github.com/Miserlou/zappa-cms) - A tiny server-less CMS for busy hackers. Work in progress. * [flask-ask](https://github.com/johnwheeler/flask-ask) - A framework for building Amazon Alexa applications. Uses Zappa for deployments. * [zappa-file-widget](https://github.com/anush0247/zappa-file-widget) - A Django plugin for supporting binary file uploads in Django on Zappa. @@ -764,7 +765,7 @@ Please file tickets for discussion before submitting patches. Pull requests shou If you are adding a non-trivial amount of new code, please include a functioning test in your PR. For AWS calls, we use the `placebo` library, which you can learn to use [in their README](https://github.com/garnaat/placebo#usage-as-a-decorator). The test suite will be run by [Travis CI](https://travis-ci.org/Miserlou/Zappa) once you open a pull request. -Please include the GitHub issue or pull request URL that has discussion related to your changes as a comment in the code ([example](https://github.com/Miserlou/Zappa/blob/fae2925431b820eaedf088a632022e4120a29f89/zappa/zappa.py#L241-L243)]). This greatly helps for project maintainability, as it allows us to trace back use cases and explain decision making. +Please include the GitHub issue or pull request URL that has discussion related to your changes as a comment in the code ([example](https://github.com/Miserlou/Zappa/blob/fae2925431b820eaedf088a632022e4120a29f89/zappa/zappa.py#L241-L243)). This greatly helps for project maintainability, as it allows us to trace back use cases and explain decision making. #### Using a Local Repo From 1114b3900471e984cb12ce93b9f48a92beb4d16f Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Thu, 5 Jan 2017 08:17:19 -0500 Subject: [PATCH 32/97] incorporate pr feedback --- zappa/cli.py | 2 +- zappa/zappa.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index 239260071..d287d4e75 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1550,7 +1550,7 @@ def create_package(self): use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True), exclude=self.stage_config.get( 'exclude', - ["boto3", "*.dist-info", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"]) + ["boto3", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"]) ) # Throw custom setings into the zip file diff --git a/zappa/zappa.py b/zappa/zappa.py index a9196c428..7f24a9ecd 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -196,7 +196,7 @@ ZIP_EXCLUDES = [ '*.exe', '*.DS_Store', '*.Python', '*.git', '.git/*', '*.zip', '*.tar.gz', - '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*' + '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*', '*.dist-info', ] ## From b16a03d29d1d4d3825e4f9b679e1ff58b3dc1921 Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Thu, 5 Jan 2017 08:19:21 -0500 Subject: [PATCH 33/97] include issue link as code comment --- zappa/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zappa/cli.py b/zappa/cli.py index d287d4e75..904d51e85 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1550,6 +1550,8 @@ def create_package(self): use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True), exclude=self.stage_config.get( 'exclude', + # Exclude packages already builtin to the python lambda environment + # https://github.com/Miserlou/Zappa/issues/556 ["boto3", "dateutil", "botocore", "s3transfer", "six.py", "jmespath", "concurrent"]) ) From 0c03346a57a083a97f4a1fb5e5befad87ec13a3b Mon Sep 17 00:00:00 2001 From: Kapil Thangavelu Date: Thu, 5 Jan 2017 08:22:38 -0500 Subject: [PATCH 34/97] update readme with the new default value of exclude --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a741d8d53..6e03fda37 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,7 @@ to change Zappa's behavior. Use these at your own risk! } ], "exception_handler": "your_module.report_exception", // function that will be invoked in case Zappa sees an unhandled exception raised from your code - "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. To exclude boto3 and botocore (available in an older version on Lambda), add "boto3*" and "botocore*". + "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. By default Zappa excludes common python packages available in a lambda environment (botocore, boto3, concurrent.futures, jmespath, six.py, dateutil). "extends": "stage_name", // Duplicate and extend another stage's settings. For example, `dev-asia` could extend from `dev-common` with a different `s3_bucket` value. "http_methods": ["GET", "POST"], // HTTP Methods to route, "iam_authorization": true, // optional, use IAM to require request signing. Default false. Note that enabling this will override the authorizer configuration. From bb153625962a128038ebf28d8efd7d41339c0ec9 Mon Sep 17 00:00:00 2001 From: Lucas Costa Date: Fri, 6 Jan 2017 17:02:28 -0200 Subject: [PATCH 35/97] dev_event_zappa_test_flask_app function path --- example/zappa_settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/zappa_settings.json b/example/zappa_settings.json index 00d32edb0..3eaacbcca 100644 --- a/example/zappa_settings.json +++ b/example/zappa_settings.json @@ -5,7 +5,7 @@ "debug": true, "log_level": "DEBUG", "events": [{ - "function": "dev_event_zappa_test_flask_app", + "function": "mymodule.myfunc", "expression": "rate(1 minute)" }], "aws_region": "us-west-2", From 309417c8fbb7ddbb96cdfd27a980af65ac7357da Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Sat, 7 Jan 2017 21:20:45 +0100 Subject: [PATCH 36/97] include distinfo regardless --- zappa/zappa.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 7f24a9ecd..45b444e29 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -194,9 +194,12 @@ LAMBDA_REGIONS = ['us-east-1', 'us-east-2', 'us-west-2', 'eu-central-1', 'eu-west-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-southeast-1', 'ap-southeast-2'] +# We never need to include these. +# Related: https://github.com/Miserlou/Zappa/pull/56 +# Related: https://github.com/Miserlou/Zappa/pull/581 ZIP_EXCLUDES = [ '*.exe', '*.DS_Store', '*.Python', '*.git', '.git/*', '*.zip', '*.tar.gz', - '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*', '*.dist-info', + '*.hg', '*.egg-info', 'pip', 'docutils*', 'setuputils*' ] ## From 50bd2b2f800a1e6e3fa35e73c6eeb3a9d2df26f2 Mon Sep 17 00:00:00 2001 From: David Cuthbert Date: Tue, 10 Jan 2017 01:23:12 -0800 Subject: [PATCH 37/97] Accept AWS_SESSION_TOKEN when executing via an IAM role. --- zappa/zappa.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 45b444e29..5863515f2 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1950,10 +1950,17 @@ def load_credentials(self, boto_session=None, profile_name=None): self.boto_session = boto3.Session(profile_name=profile_name, region_name=self.aws_region) elif os.environ.get('AWS_ACCESS_KEY_ID') and os.environ.get('AWS_SECRET_ACCESS_KEY'): region_name = os.environ.get('AWS_DEFAULT_REGION') or self.aws_region - self.boto_session = boto3.Session( - aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'), - region_name=region_name) + session_kw = { + "aws_access_key_id": os.environ.get('AWS_ACCESS_KEY_ID'), + "aws_secret_access_key": os.environ.get('AWS_SECRET_ACCESS_KEY'), + "region_name": region_name, + } + + # If we're executing in a role, AWS_SESSION_TOKEN will be present, too. + if os.environ.get("AWS_SESSION_TOKEN"): + session_kw["aws_session_token"] = os.environ.get("AWS_SESSION_TOKEN") + + self.boto_session = boto3.Session(**session_kw) else: self.boto_session = boto3.Session(region_name=self.aws_region) From adcc4ac81126b94ad82645ffb1b19ef60cb98172 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Wed, 11 Jan 2017 17:31:05 +0100 Subject: [PATCH 38/97] initial build command --- zappa/cli.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/zappa/cli.py b/zappa/cli.py index 904d51e85..3c325b4a4 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -196,6 +196,12 @@ def handle(self, argv=None): ) subparsers.add_parser('init', help='Initialize Zappa app.') + ## + # Build + ## + build_parser = subparsers.add_parser( + 'build', parents=[env_parser], help='Build the application zip locally.' + ) ## # Invocation @@ -391,6 +397,8 @@ def dispatch_command(self, command, environment): # Hand it off if command == 'deploy': # pragma: no cover self.deploy() + if command == 'build': # pragma: no cover + self.build() elif command == 'update': # pragma: no cover self.update() elif command == 'rollback': # pragma: no cover @@ -438,6 +446,19 @@ def dispatch_command(self, command, environment): # The Commands ## + def build(self): + """ + Only build the package + """ + self.zappa_settings['delete_local_zip'] = False + # Execute the prebuild script + if self.prebuild_script: + self.execute_prebuild_script() + # Create the Lambda Zip + self.create_package() + self.callback('zip') + click.echo(self.zip_path) + def deploy(self): """ Package your project, upload it to S3, register the Lambda function From efe672d596cdaab069bf5a2f13f381cab8b8b298 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Wed, 11 Jan 2017 20:04:27 +0100 Subject: [PATCH 39/97] add docs for build command and always respect `delete_local_zip` setting --- README.md | 18 ++++++++++++++++++ zappa/cli.py | 1 - 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8daa02786..93cd1a7da 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,24 @@ You can also `rollback` the deployed code to a previous version by supplying the $ zappa rollback production -n 3 +#### Build + +To only build the package in a zip locally you can use the `build` command: + + $ zappa build production + +By default Zappa will always destroy the package. To keep the package after the command finishes you need to set the `delete_local_zip` flag to false: + +```javascript +{ + "production": { + "delete_local_zip": false + } +} +``` + +Note that you can also use the `callbacks` setting to add a callback for `zip` to call a custom function when the build has finished and the zip is created. That way you might not need the zip to be kept around after the command has finished. + #### Scheduling Zappa can be used to easily schedule functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! diff --git a/zappa/cli.py b/zappa/cli.py index 3c325b4a4..65aada6a9 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -450,7 +450,6 @@ def build(self): """ Only build the package """ - self.zappa_settings['delete_local_zip'] = False # Execute the prebuild script if self.prebuild_script: self.execute_prebuild_script() From a858eff2e2929e038815d6666dbb828e955c1a8b Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Wed, 11 Jan 2017 20:06:32 +0100 Subject: [PATCH 40/97] shorter doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 93cd1a7da..76f0f783f 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ By default Zappa will always destroy the package. To keep the package after the } ``` -Note that you can also use the `callbacks` setting to add a callback for `zip` to call a custom function when the build has finished and the zip is created. That way you might not need the zip to be kept around after the command has finished. +Note that you can also use the `callbacks` setting to add a callback for `zip` to call a custom function when the build has finished and the zip is created. That way you might not need the zip to be kept around. #### Scheduling From 62719a0b05a8d61c27d2de97e4e0322fcf15b32d Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Thu, 12 Jan 2017 04:15:14 +0100 Subject: [PATCH 41/97] rename 'build' to 'package' --- README.md | 6 +++--- zappa/cli.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 76f0f783f..4b6e924c6 100644 --- a/README.md +++ b/README.md @@ -177,11 +177,11 @@ You can also `rollback` the deployed code to a previous version by supplying the $ zappa rollback production -n 3 -#### Build +#### Package -To only build the package in a zip locally you can use the `build` command: +To only build the package in a zip locally you can use the `package` command: - $ zappa build production + $ zappa package production By default Zappa will always destroy the package. To keep the package after the command finishes you need to set the `delete_local_zip` flag to false: diff --git a/zappa/cli.py b/zappa/cli.py index 65aada6a9..b2844e874 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -197,10 +197,10 @@ def handle(self, argv=None): subparsers.add_parser('init', help='Initialize Zappa app.') ## - # Build + # Package ## - build_parser = subparsers.add_parser( - 'build', parents=[env_parser], help='Build the application zip locally.' + package_parser = subparsers.add_parser( + 'package', parents=[env_parser], help='Build the application zip package locally.' ) ## @@ -397,8 +397,8 @@ def dispatch_command(self, command, environment): # Hand it off if command == 'deploy': # pragma: no cover self.deploy() - if command == 'build': # pragma: no cover - self.build() + if command == 'package': # pragma: no cover + self.package() elif command == 'update': # pragma: no cover self.update() elif command == 'rollback': # pragma: no cover @@ -446,7 +446,7 @@ def dispatch_command(self, command, environment): # The Commands ## - def build(self): + def package(self): """ Only build the package """ From 4d081aa943ea909436e4fc73666120422fe3995d Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Thu, 12 Jan 2017 04:31:19 +0100 Subject: [PATCH 42/97] add pyenv to ignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5a2d46cc3..cb86ac3cd 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,9 @@ docs/_build/ # PyBuilder target/ +# pyenv +.python-version + # PyCharm # Project settings from the PyCharm IDE are stored in the .idea folder .idea/ From e92446b13c97eea532879a5860186e483a72f3c6 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Thu, 12 Jan 2017 05:23:54 +0100 Subject: [PATCH 43/97] add on_exit to be explicit about what we call --- zappa/cli.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index b2844e874..49cad0250 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1692,6 +1692,15 @@ def remove_uploaded_zip(self): if self.stage_config.get('delete_s3_zip', True): self.zappa.remove_from_s3(self.zip_path, self.s3_bucket_name) + def on_exit(self): + """ + Cleanup after the command finishes. + Always called: SystemExit, KeyboardInterrupt and any other Exception that occurs. + """ + if self.zip_path: + self.remove_uploaded_zip() + self.remove_local_zip() + def print_logs(self, logs, colorize=True, http=False): """ Parse, filter and print logs to the console. @@ -1886,21 +1895,14 @@ def handle(): # pragma: no cover cli = ZappaCLI() sys.exit(cli.handle()) except SystemExit as e: # pragma: no cover - if cli.zip_path: - cli.remove_uploaded_zip() - cli.remove_local_zip() - + cli.on_exit() sys.exit(e.code) except KeyboardInterrupt: # pragma: no cover - if cli.zip_path: # Remove the Zip from S3 upon failure. - cli.remove_uploaded_zip() - cli.remove_local_zip() + cli.on_exit() sys.exit(130) except Exception as e: - if cli.zip_path: # Remove the Zip from S3 upon failure. - cli.remove_uploaded_zip() - cli.remove_local_zip() + cli.on_exit() click.echo("Oh no! An " + click.style("error occurred", fg='red', bold=True) + "! :(") click.echo("\n==============\n") From 18c0386fb36f147892fd6fbac075181cb2e943e7 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Thu, 12 Jan 2017 05:24:32 +0100 Subject: [PATCH 44/97] test package only command --- test_settings.json | 8 ++++++++ tests/tests.py | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/test_settings.json b/test_settings.json index 59ba95deb..03510437b 100644 --- a/test_settings.json +++ b/test_settings.json @@ -73,5 +73,13 @@ "environment_variables": { "EXTENDO": "You bet" } + }, + "build_package_only_delete_local_zip_false": { + "delete_local_zip": false, + "use_precompiled_packages": false + }, + "build_package_only_delete_local_zip_true": { + "delete_local_zip": true, + "use_precompiled_packages": false } } diff --git a/tests/tests.py b/tests/tests.py index 7a474c723..4cc4e94a9 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1137,6 +1137,19 @@ def test_remote_env_package(self): m = re.search("REMOTE_ENV='(.*)'", content) self.assertEqual(m.group(1), 's3://lmbda-env/prod/env.json') + def test_package_only(self): + + for keep_zip in [True, False]: + zappa_cli = ZappaCLI() + zappa_cli.api_stage = 'build_package_only_delete_local_zip_false' if keep_zip else 'build_package_only_delete_local_zip_true' + zappa_cli.load_settings('test_settings.json') + zappa_cli.package() + zappa_cli.on_exit() # simulate the command exits + self.assertEqual(os.path.isfile(zappa_cli.zip_path), keep_zip) + + if keep_zip: + # cleanup + os.remove(zappa_cli.zip_path) if __name__ == '__main__': unittest.main() From afcd1cb24ba8c5bd0684cb70e2bd00293f2dab24 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Thu, 12 Jan 2017 05:43:21 +0100 Subject: [PATCH 45/97] do not delete_s3_zip for this test --- test_settings.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_settings.json b/test_settings.json index 03510437b..d9164030a 100644 --- a/test_settings.json +++ b/test_settings.json @@ -76,10 +76,12 @@ }, "build_package_only_delete_local_zip_false": { "delete_local_zip": false, - "use_precompiled_packages": false + "use_precompiled_packages": false, + "delete_s3_zip": false }, "build_package_only_delete_local_zip_true": { "delete_local_zip": true, - "use_precompiled_packages": false + "use_precompiled_packages": false, + "delete_s3_zip": false } } From cffde997fceb136a2504e81edf61699e8460fbb9 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 12 Jan 2017 12:46:38 -0500 Subject: [PATCH 46/97] Adds Github Issue Template --- .github/ISSUE_TEMPLATE.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..8a799e639 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,31 @@ + +## Context + + + +## Expected Behavior + + +## Actual Behavior + + +## Possible Fix + + +## Steps to Reproduce + + +1. +2. +3. +4. + +## Context + + +## Your Environment + +* Zappa version used: +* Operating System and Python version: +* Link to your project (optional): +* Your `zappa_settings.py`: From 8e72c68d2f2f0aa8a44c44a4877fcf6c1814790a Mon Sep 17 00:00:00 2001 From: Paul Jimenez Date: Tue, 20 Dec 2016 10:04:36 -0500 Subject: [PATCH 47/97] Fix the returned order of events in logs --- zappa/cli.py | 24 +++++++++--------------- zappa/zappa.py | 26 ++++++++++++++++++-------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index 904d51e85..969d8044a 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -669,26 +669,20 @@ def tail(self, keep_open=True, colorize=True, http=False): """ Tail this function's logs. + if keep_open, do so repeatedly, printing any new logs """ try: - # Tail the available logs - all_logs = self.zappa.fetch_logs(self.lambda_name) - self.print_logs(all_logs, colorize, http) - - # Keep polling, and print any new logs. - loop = True - while loop: - all_logs_again = self.zappa.fetch_logs(self.lambda_name) - new_logs = [] - for log in all_logs_again: - if log not in all_logs: - new_logs.append(log) - + since = 0 + while True: + new_logs = self.zappa.fetch_logs(self.lambda_name, startTime=since) + new_logs = [ e for e in new_logs if e['timestamp'] > since ] self.print_logs(new_logs, colorize, http) - all_logs = all_logs + new_logs if not keep_open: - loop = False + break + if new_logs: + since = new_logs[-1]['timestamp'] + time.sleep(1) except KeyboardInterrupt: # pragma: no cover # Die gracefully try: diff --git a/zappa/zappa.py b/zappa/zappa.py index 5863515f2..15d00c0b3 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1808,7 +1808,7 @@ def _clear_policy(self, lambda_name): # CloudWatch Logging ## - def fetch_logs(self, lambda_name, filter_pattern='', limit=10000): + def fetch_logs(self, lambda_name, filter_pattern='', limit=10000, startTime=0): """ Fetch the CloudWatch logs for a given Lambda name. """ @@ -1821,14 +1821,24 @@ def fetch_logs(self, lambda_name, filter_pattern='', limit=10000): all_streams = streams['logStreams'] all_names = [stream['logStreamName'] for stream in all_streams] - response = self.logs_client.filter_log_events( - logGroupName=log_name, - logStreamNames=all_names, - filterPattern=filter_pattern, - limit=limit - ) - return response['events'] + events = [] + response = {} + while not response or 'nextToken' in response: + extra_args = {} + if 'nextToken' in response: + extra_args['nextToken'] = response['nextToken'] + response = self.logs_client.filter_log_events( + logGroupName=log_name, + logStreamNames=all_names, + startTime=startTime, + filterPattern=filter_pattern, + limit=limit, + **extra_args + ) + events += response['events'] + + return sorted(events, key=lambda k: k['timestamp']) def remove_log_group(self, group_name): """ From 6e25c44e2780c4a7844b3879cf70512ff98153e7 Mon Sep 17 00:00:00 2001 From: Doppins Date: Thu, 12 Jan 2017 23:00:49 +0000 Subject: [PATCH 48/97] Upgrade dependency lambda-packages to ==0.12.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1e70cfe7e..d85638284 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ futures==3.0.5 hjson==2.0.2 jmespath==0.9.0 kappa==0.6.0 -lambda-packages==0.11.0 +lambda-packages==0.12.0 python-dateutil==2.6.0 python-slugify==1.2.1 PyYAML==3.12 From 24d1a2e1be2d638d30b986f64642ce6e72c572e6 Mon Sep 17 00:00:00 2001 From: Doppins Date: Thu, 12 Jan 2017 23:00:57 +0000 Subject: [PATCH 49/97] Upgrade dependency tqdm to ==4.11.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1e70cfe7e..08391a763 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ PyYAML==3.12 requests>=2.10.0 six==1.10.0 toml==0.9.2 -tqdm==4.10.0 +tqdm==4.11.0 troposphere>=1.9.0 Werkzeug==0.11.15 wheel==0.29.0 From a202f19ca82bbb67dcf98446ddfbe6b727fb25f3 Mon Sep 17 00:00:00 2001 From: Michael Krens Date: Fri, 13 Jan 2017 02:39:01 +0100 Subject: [PATCH 50/97] update package command to never remove the local zip --- README.md | 14 ++++++++------ tests/tests.py | 15 +++++++++------ zappa/cli.py | 24 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4b6e924c6..dd7462d8c 100644 --- a/README.md +++ b/README.md @@ -179,22 +179,24 @@ You can also `rollback` the deployed code to a previous version by supplying the #### Package -To only build the package in a zip locally you can use the `package` command: +To only build the package zip locally you can use the `package` command: $ zappa package production -By default Zappa will always destroy the package. To keep the package after the command finishes you need to set the `delete_local_zip` flag to false: +Zappa will always create the package regardless of your `delete_local_zip` setting. + +Note that if you have a `zip` callback in your `callbacks` setting it will also be invoked. ```javascript { - "production": { - "delete_local_zip": false + "production": { // The name of your environment + "callbacks": { + "zip": "my_app.zip_callback"// After creating the package + } } } ``` -Note that you can also use the `callbacks` setting to add a callback for `zip` to call a custom function when the build has finished and the zip is created. That way you might not need the zip to be kept around. - #### Scheduling Zappa can be used to easily schedule functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! diff --git a/tests/tests.py b/tests/tests.py index 4cc4e94a9..980a96e40 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1139,17 +1139,20 @@ def test_remote_env_package(self): def test_package_only(self): - for keep_zip in [True, False]: + for delete_local_zip in [True, False]: zappa_cli = ZappaCLI() - zappa_cli.api_stage = 'build_package_only_delete_local_zip_false' if keep_zip else 'build_package_only_delete_local_zip_true' + if delete_local_zip: + zappa_cli.api_stage = 'build_package_only_delete_local_zip_true' + else: + zappa_cli.api_stage = 'build_package_only_delete_local_zip_false' zappa_cli.load_settings('test_settings.json') zappa_cli.package() zappa_cli.on_exit() # simulate the command exits - self.assertEqual(os.path.isfile(zappa_cli.zip_path), keep_zip) + # the zip should never be removed + self.assertEqual(os.path.isfile(zappa_cli.zip_path), True) - if keep_zip: - # cleanup - os.remove(zappa_cli.zip_path) + # cleanup + os.remove(zappa_cli.zip_path) if __name__ == '__main__': unittest.main() diff --git a/zappa/cli.py b/zappa/cli.py index 49cad0250..c725687bb 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -109,6 +109,9 @@ class ZappaCLI(object): stage_name_env_pattern = re.compile('^[a-zA-Z0-9_]+$') + def __init__(self): + self._stage_config_overrides = {} # change using self.override_stage_config_setting(key, val) + @property def stage_config(self): """ @@ -142,8 +145,27 @@ def get_stage_setting(stage, extended_stages=None): if u'delete_zip' in settings: settings[u'delete_local_zip'] = settings.get(u'delete_zip') + settings.update(self.stage_config_overrides) + return settings + @property + def stage_config_overrides(self): + """ + Returns zappa_settings we forcefully override for the current stage + set by `self.override_stage_config_setting(key, value)` + """ + return getattr(self, '_stage_config_overrides', {}).get(self.api_stage, {}) + + def override_stage_config_setting(self, key, val): + """ + Forcefully override a setting set by zappa_settings (for the current stage only) + :param key: settings key + :param val: value + """ + self._stage_config_overrides = getattr(self, '_stage_config_overrides', {}) + self._stage_config_overrides.setdefault(self.api_stage, {})[key] = val + def handle(self, argv=None): """ Main function. @@ -450,6 +472,8 @@ def package(self): """ Only build the package """ + # force not to delete the local zip + self.override_stage_config_setting('delete_local_zip', False) # Execute the prebuild script if self.prebuild_script: self.execute_prebuild_script() From 35a165ab4c239153af68f7101e9390c2590cc0ba Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Fri, 13 Jan 2017 11:12:44 -0500 Subject: [PATCH 51/97] print filesize for `package` --- zappa/cli.py | 5 +++-- zappa/util.py | 10 ++++++++++ zappa/zappa.py | 11 ++--------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index c725687bb..30f27e250 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -43,7 +43,7 @@ from datetime import datetime,timedelta from zappa import Zappa, logger, API_GATEWAY_REGIONS from util import (check_new_version_available, detect_django_settings, - detect_flask_apps, parse_s3_url) + detect_flask_apps, parse_s3_url, human_size) CUSTOM_SETTINGS = [ @@ -480,7 +480,8 @@ def package(self): # Create the Lambda Zip self.create_package() self.callback('zip') - click.echo(self.zip_path) + size = human_size(os.path.getsize(self.zip_path)) + click.echo(click.style("Package created", fg="green", bold=True) + ": " + click.style(self.zip_path, bold=True) + " (" + size + ")") def deploy(self): """ diff --git a/zappa/util.py b/zappa/util.py index 8031114bf..c0cc13ca0 100644 --- a/zappa/util.py +++ b/zappa/util.py @@ -59,6 +59,16 @@ def parse_s3_url(url): path = result.path.strip('/') return bucket, path +def human_size(num, suffix='B'): + """ + Convert bytes length to a human-readable version + """ + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(num) < 1024.0: + return "{0:3.1f}{1!s}{2!s}".format(num, unit, suffix) + num /= 1024.0 + return "{0:.1f}{1!s}{2!s}".format(num, 'Yi', suffix) + ## # `init` related ## diff --git a/zappa/zappa.py b/zappa/zappa.py index 5863515f2..aa10b67ae 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -28,7 +28,7 @@ from tqdm import tqdm # Zappa imports -from util import copytree, add_event_source, remove_event_source +from util import copytree, add_event_source, remove_event_source, human_size logging.basicConfig(format='%(levelname)s:%(message)s') logger = logging.getLogger(__name__) @@ -577,7 +577,7 @@ def upload_to_s3(self, source_path, bucket_name): dest_path = os.path.split(source_path)[1] try: source_size = os.stat(source_path).st_size - print("Uploading {0} ({1})..".format(dest_path, self.human_size(source_size))) + print("Uploading {0} ({1})..".format(dest_path, human_size(source_size))) progress = tqdm(total=float(os.path.getsize(source_path)), unit_scale=True, unit='B') # Attempt to upload to S3 using the S3 meta client with the progress bar. @@ -1993,13 +1993,6 @@ def selection_pattern(status_code): return pattern - def human_size(self, num, suffix='B'): - for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: - if abs(num) < 1024.0: - return "{0:3.1f}{1!s}{2!s}".format(num, unit, suffix) - num /= 1024.0 - return "{0:.1f}{1!s}{2!s}".format(num, 'Yi', suffix) - @staticmethod def service_from_arn(arn): return arn.split(':')[2] From fca3a8344ea066b6fe01baeeee1daa9472be4bf5 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Fri, 13 Jan 2017 12:06:27 -0500 Subject: [PATCH 52/97] add package to toc --- README.md | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index dd7462d8c..70729c300 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ - [Scheduling](#scheduling) - [Executing in Response to AWS Events](#executing-in-response-to-aws-events) - [Undeploy](#undeploy) + - [Package](#package) - [Status](#status) - [Tailing Logs](#tailing-logs) - [Remote Function Invocation](#remote-function-invocation) @@ -177,26 +178,6 @@ You can also `rollback` the deployed code to a previous version by supplying the $ zappa rollback production -n 3 -#### Package - -To only build the package zip locally you can use the `package` command: - - $ zappa package production - -Zappa will always create the package regardless of your `delete_local_zip` setting. - -Note that if you have a `zip` callback in your `callbacks` setting it will also be invoked. - -```javascript -{ - "production": { // The name of your environment - "callbacks": { - "zip": "my_app.zip_callback"// After creating the package - } - } -} -``` - #### Scheduling Zappa can be used to easily schedule functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! @@ -307,6 +288,24 @@ want to keep those logs, you can specify the `--remove-logs` argument to purge t $ zappa undeploy production --remove-logs +#### Package + +If you want to build your application package without actually uploading and registering it as a Lambda function, you can use the `package` command: + + $ zappa package production + +If you have a `zip` callback in your `callbacks` setting, this will also be invoked. + +```javascript +{ + "production": { // The name of your environment + "callbacks": { + "zip": "my_app.zip_callback"// After creating the package + } + } +} +``` + #### Status If you need to see the status of your deployment and event schedules, simply use the `status` command. From 9de92502e6f72927d7eee5d942edab215a0c08b9 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Fri, 13 Jan 2017 12:12:18 -0500 Subject: [PATCH 53/97] update humansize test --- tests/tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 980a96e40..b48615c61 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -25,7 +25,7 @@ from zappa.letsencrypt import get_cert_and_update_domain, create_domain_key, create_domain_csr, create_chained_certificate, get_cert, cleanup, parse_account_key, parse_csr, sign_certificate, encode_certificate, register_account, verify_challenge from zappa.util import (detect_django_settings, copytree, detect_flask_apps, add_event_source, remove_event_source, - get_event_source_status, parse_s3_url) + get_event_source_status, parse_s3_url, human_size) from zappa.wsgi import create_wsgi_request, common_log from zappa.zappa import Zappa, ASSUME_POLICY, ATTACH_POLICY @@ -1060,9 +1060,8 @@ def test_dj_wsgi(self): ## def test_human_units(self): - zappa = Zappa() - zappa.human_size(1) - zappa.human_size(9999999999999) + human_size(1) + human_size(9999999999999) def test_event_name(self): zappa = Zappa() From 48714e26a721a72f8fca954ed1ac5f1e252d73ae Mon Sep 17 00:00:00 2001 From: David Cuthbert Date: Sat, 14 Jan 2017 00:29:39 -0800 Subject: [PATCH 54/97] Initialize wsgi.errors to sys.stderr. Fixes https://github.com/Miserlou/Zappa/issues/283 --- test_requirements.txt | 1 + tests/tests.py | 31 ++++++++++++++++++++++++++++++- zappa/wsgi.py | 3 ++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/test_requirements.txt b/test_requirements.txt index 2620f4c3f..c59bcbeb7 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,7 @@ coveralls==1.1 coverage==4.3.1 Django==1.10.5 +Flask==0.12 mock==2.0.0 nose==1.3.7 nose-timer==0.6.0 diff --git a/tests/tests.py b/tests/tests.py index b48615c61..38af10175 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,7 +4,8 @@ import json from contextlib import nested -from io import StringIO +from io import BytesIO, StringIO +import flask import mock import os import random @@ -1153,5 +1154,33 @@ def test_package_only(self): # cleanup os.remove(zappa_cli.zip_path) + def test_flask_logging_bug(self): + # This checks whether Flask can write errors sanely. + # https://github.com/Miserlou/Zappa/issues/283 + event = { + "body": {}, + "headers": {}, + "pathParameters": {}, + "path": '/', + "httpMethod": "GET", + "queryStringParameters": {}, + "requestContext": {} + } + + old_stderr = sys.stderr + sys.stderr = BytesIO() + try: + environ = create_wsgi_request(event) + app = flask.Flask(__name__) + with app.request_context(environ): + app.logger.error(u"This is a test") + log_output = sys.stderr.getvalue() + self.assertNotIn( + "'str' object has no attribute 'write'", log_output) + self.assertNotIn( + "Logged from file tests.py", log_output) + finally: + sys.stderr = old_stderr + if __name__ == '__main__': unittest.main() diff --git a/zappa/wsgi.py b/zappa/wsgi.py index 66bda2ff9..af42caaac 100644 --- a/zappa/wsgi.py +++ b/zappa/wsgi.py @@ -4,6 +4,7 @@ from urllib import urlencode from requestlogger import ApacheFormatter from StringIO import StringIO +from sys import stderr from werkzeug import urls @@ -89,7 +90,7 @@ def create_wsgi_request(event_info, server_name='zappa', script_name=None, 'wsgi.version': (1, 0), 'wsgi.url_scheme': str('http'), 'wsgi.input': body, - 'wsgi.errors': str(''), + 'wsgi.errors': stderr, 'wsgi.multiprocess': False, 'wsgi.multithread': False, 'wsgi.run_once': False, From 55fb441d92ef3f3d4a1489b708812197693860ad Mon Sep 17 00:00:00 2001 From: David Cuthbert Date: Sat, 14 Jan 2017 01:11:51 -0800 Subject: [PATCH 55/97] Test zappa_settings extension preferences. --- tests/tests.py | 52 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 38af10175..03ac1068e 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -13,7 +13,9 @@ import zipfile import re import unittest +import shutil import sys +import tempfile from click.exceptions import ClickException from lambda_packages import lambda_packages @@ -584,9 +586,6 @@ def test_load_extended_settings(self): self.assertTrue(zappa_cli.stage_config['delete_local_zip']) # The base def test_load_settings_yaml(self): - zappa_cli = ZappaCLI() - settings_file = zappa_cli.get_json_or_yaml_settings("test_settings") - zappa_cli = ZappaCLI() zappa_cli.api_stage = 'ttt888' zappa_cli.load_settings('tests/test_settings.yml') @@ -599,14 +598,55 @@ def test_load_settings_yaml(self): self.assertEqual(True, zappa_cli.stage_config['touch']) def test_load_settings_toml(self): - zappa_cli = ZappaCLI() - settings_file = zappa_cli.get_json_or_yaml_settings("test_settings") - zappa_cli = ZappaCLI() zappa_cli.api_stage = 'ttt888' zappa_cli.load_settings('tests/test_settings.toml') self.assertEqual(False, zappa_cli.stage_config['touch']) + def test_settings_extension(self): + """ + Make sure Zappa uses settings in the proper order: JSON, TOML, YAML. + """ + tempdir = tempfile.mkdtemp(prefix="zappa-test-settings") + shutil.copy("tests/test_one_env.json", tempdir + "/zappa_settings.json") + shutil.copy("tests/test_settings.yml", tempdir + "/zappa_settings.yml") + shutil.copy("tests/test_settings.toml", tempdir + "/zappa_settings.toml") + + orig_cwd = os.getcwd() + os.chdir(tempdir) + try: + zappa_cli = ZappaCLI() + + # With all three, we should get the JSON file first. + self.assertEqual(zappa_cli.get_json_or_yaml_settings(), + "zappa_settings.json") + zappa_cli.load_settings_file() + self.assertIn("lonely", zappa_cli.zappa_settings) + os.unlink("zappa_settings.json") + + # Without the JSON file, we should get the TOML file. + self.assertEqual(zappa_cli.get_json_or_yaml_settings(), + "zappa_settings.toml") + zappa_cli.load_settings_file() + self.assertIn("ttt888", zappa_cli.zappa_settings) + self.assertNotIn("devor", zappa_cli.zappa_settings) + os.unlink("zappa_settings.toml") + + # With just the YAML file, we should get it. + self.assertEqual(zappa_cli.get_json_or_yaml_settings(), + "zappa_settings.yml") + zappa_cli.load_settings_file() + self.assertIn("ttt888", zappa_cli.zappa_settings) + self.assertIn("devor", zappa_cli.zappa_settings) + os.unlink("zappa_settings.yml") + + # Without anything, we should get an exception. + self.assertRaises( + ClickException, zappa_cli.get_json_or_yaml_settings) + finally: + os.chdir(orig_cwd) + shutil.rmtree(tempdir) + def test_cli_utility(self): zappa_cli = ZappaCLI() zappa_cli.api_stage = 'ttt888' From 423d88f9a9b3c045877f85b7ce263ba000f93f09 Mon Sep 17 00:00:00 2001 From: David Cuthbert Date: Sat, 14 Jan 2017 01:16:20 -0800 Subject: [PATCH 56/97] Make comment into method docstring. --- tests/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 03ac1068e..8b79346c3 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1195,8 +1195,10 @@ def test_package_only(self): os.remove(zappa_cli.zip_path) def test_flask_logging_bug(self): - # This checks whether Flask can write errors sanely. - # https://github.com/Miserlou/Zappa/issues/283 + """ + This checks whether Flask can write errors sanely. + https://github.com/Miserlou/Zappa/issues/283 + """ event = { "body": {}, "headers": {}, From 2e4dbc849b11e82df93a975b02a6616607c7add7 Mon Sep 17 00:00:00 2001 From: David Cuthbert Date: Sat, 14 Jan 2017 02:00:55 -0800 Subject: [PATCH 57/97] Fix a couple ZappaCLI.certify() consistency issues. Add tests. --- tests/tests.py | 132 +++++++++++++++++++++++++++++++++++++++++++++++++ zappa/cli.py | 5 +- 2 files changed, 135 insertions(+), 2 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 8b79346c3..26896cf6d 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -4,6 +4,7 @@ import json from contextlib import nested +from cStringIO import StringIO as OldStringIO from io import BytesIO, StringIO import flask import mock @@ -1014,6 +1015,137 @@ def test_lets_encrypt_sanity(self): os.remove('test_signed.crt') cleanup() + + def test_certify_sanity_checks(self): + """ + Make sure 'zappa certify': + * Writes a warning with the --no-cleanup flag. + * Errors out when a deployment hasn't taken place. + * Writes errors when certificate settings haven't been specified. + * Calls Zappa correctly for creates vs. updates. + """ + old_stdout = sys.stderr + sys.stdout = OldStringIO() # print() barfs on io.* types. + + try: + zappa_cli = ZappaCLI() + try: + zappa_cli.certify(no_cleanup=True) + except AttributeError: + # Since zappa_cli.zappa isn't initalized, the certify() call + # fails when it tries to inspect what Zappa has deployed. + pass + + log_output = sys.stdout.getvalue() + self.assertIn("You are calling certify with", log_output) + self.assertIn("--no-cleanup", log_output) + + class ZappaMock(object): + def __init__(self): + self.function_versions = [] + self.domain_names = {} + self.calls = [] + + def get_lambda_function_versions(self, function_name): + return self.function_versions + + def get_domain_name(self, domain): + return self.domain_names.get(domain) + + def create_domain_name(self, *args, **kw): + self.calls.append(("create_domain_name", args, kw)) + + def update_domain_name(self, *args, **kw): + self.calls.append(("update_domain_name", args, kw)) + + zappa_cli.zappa = ZappaMock() + self.assertRaises(ClickException, zappa_cli.certify) + + # Make sure we get an error if we don't configure the domain. + zappa_cli.zappa.function_versions = ["$LATEST"] + zappa_cli.api_stage = "stage" + zappa_cli.zappa_settings = {"stage": {}} + + try: + zappa_cli.certify() + except ClickException as e: + log_output = str(e) + self.assertIn("Can't certify a domain without", log_output) + self.assertIn("domain", log_output) + + # Without any LetsEncrypt settings, we should get a message about + # not having a lets_encrypt_key setting. + zappa_cli.zappa_settings["stage"]["domain"] = "test.example.com" + try: + zappa_cli.certify() + self.fail("Expected a ClickException") + except ClickException as e: + log_output = str(e) + self.assertIn("Can't certify a domain without", log_output) + self.assertIn("lets_encrypt_key", log_output) + + # With partial settings, we should get a message about not having + # certificate, certificate_key, and certificate_chain + zappa_cli.zappa_settings["stage"]["certificate"] = "foo" + try: + zappa_cli.certify() + self.fail("Expected a ClickException") + except ClickException as e: + log_output = str(e) + self.assertIn("Can't certify a domain without", log_output) + self.assertIn("certificate_key", log_output) + self.assertIn("certificate_chain", log_output) + + zappa_cli.zappa_settings["stage"]["certificate_key"] = "key" + try: + zappa_cli.certify() + self.fail("Expected a ClickException") + except ClickException as e: + log_output = str(e) + self.assertIn("Can't certify a domain without", log_output) + self.assertIn("certificate_key", log_output) + self.assertIn("certificate_chain", log_output) + + zappa_cli.zappa_settings["stage"]["certificate_chain"] = "chain" + del zappa_cli.zappa_settings["stage"]["certificate_key"] + try: + zappa_cli.certify() + self.fail("Expected a ClickException") + except ClickException as e: + log_output = str(e) + self.assertIn("Can't certify a domain without", log_output) + self.assertIn("certificate_key", log_output) + self.assertIn("certificate_chain", log_output) + + # With all certificate settings, make sure Zappa's domain calls + # are executed. + cert_file = tempfile.NamedTemporaryFile() + cert_file.write("Hello world") + cert_file.flush() + + zappa_cli.zappa_settings["stage"].update({ + "certificate": cert_file.name, + "certificate_key": cert_file.name, + "certificate_chain": cert_file.name + }) + sys.stdout.truncate(0) + zappa_cli.certify(no_cleanup=True) + self.assertEquals(len(zappa_cli.zappa.calls), 1) + self.assertTrue(zappa_cli.zappa.calls[0][0] == "create_domain_name") + log_output = sys.stdout.getvalue() + self.assertIn("Created a new domain name", log_output) + + zappa_cli.zappa.calls = [] + zappa_cli.zappa.domain_names["test.example.com"] = "*.example.com" + sys.stdout.truncate(0) + zappa_cli.certify(no_cleanup=True) + self.assertEquals(len(zappa_cli.zappa.calls), 1) + self.assertTrue(zappa_cli.zappa.calls[0][0] == "update_domain_name") + log_output = sys.stdout.getvalue() + self.assertNotIn("Created a new domain name", log_output) + finally: + sys.stdout = old_stdout + ## # Django ## diff --git a/zappa/cli.py b/zappa/cli.py index 30f27e250..8eeee2398 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1275,8 +1275,7 @@ def certify(self, no_cleanup=False): cert_chain_location = self.stage_config.get('certificate_chain', None) if not domain: - click.echo("Can't certify a domain without " + click.style("domain", fg="red", bold=True) + " configured!") - return + raise ClickException("Can't certify a domain without " + click.style("domain", fg="red", bold=True) + " configured!") if not cert_location: if not account_key_location: @@ -1338,6 +1337,8 @@ def certify(self, no_cleanup=False): certificate_chain ) + cert_success = True + # Deliberately undocumented feature (for now, at least.) # We are giving the user the ability to shoot themselves in the foot. # _This is probably not a good idea._ From 9d43b2e7c51674c58cb01a24f0dd01d3483850f8 Mon Sep 17 00:00:00 2001 From: Daniel Whatmuff Date: Thu, 19 Jan 2017 04:39:49 +1100 Subject: [PATCH 58/97] Update to readme on custom domain with own ssl cert (#579) * Update to readme on a custom domain with own SSL cert. --- README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 70729c300..31b7a592f 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ - [IAM Policy](#iam-policy) - [API Gateway Authorizers](#api-gateway-authorizers) - [Cognito User Pool Authorizer](#cognito-user-pool-authorizer) + - [Deploying with Custom Domain Name (with your own SSL Certs)](#deploying-with-custom-domain-name-with-your-own-ssl-certs) - [Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-dns-auth) - [Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-http-auth) - [Setting Environment Variables](#setting-environment-variables) @@ -180,7 +181,11 @@ You can also `rollback` the deployed code to a previous version by supplying the #### Scheduling -Zappa can be used to easily schedule functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! +Zappa can be used to easily + + + +functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! These functions will be packaged and deployed along with your `app_function` and called from the handler automatically. Just list your functions and the expression to schedule them using [cron or rate syntax](http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html) in your *zappa_settings.json* file: @@ -417,7 +422,8 @@ to change Zappa's behavior. Use these at your own risk! } ], "exception_handler": "your_module.report_exception", // function that will be invoked in case Zappa sees an unhandled exception raised from your code - "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. By default Zappa excludes common python packages available in a lambda environment (botocore, boto3, concurrent.futures, jmespath, six.py, dateutil). + + "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. To exclude boto3 and botocore (available in an older version on Lambda), add "boto3*" and "botocore*". "extends": "stage_name", // Duplicate and extend another stage's settings. For example, `dev-asia` could extend from `dev-common` with a different `s3_bucket` value. "http_methods": ["GET", "POST"], // HTTP Methods to route, "iam_authorization": true, // optional, use IAM to require request signing. Default false. Note that enabling this will override the authorizer configuration. @@ -570,6 +576,19 @@ You can also use AWS Cognito User Pool Authorizer by adding: } ``` +#### Deploying with Custom Domain Name (with your own SSL Certs) + +1. The first step is to create a custom domain and upload your SSL cert / key / bundle - follow this guide [Set Up a Custom Domain Name for an API Gateway API](http://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-custom-domains.html#how-to-custom-domains-console) +2. Ensure you have set the `domain` setting within your zappa settings JSON - this will avoid problems with the Base Path mapping between the Custom Domain and the API invoke URL, which gets the Stage Name appended in the URI +3. Deploy or update your app using zappa +4. Create a base path mapping between your custom domain name and your chosen API stage, leaving the base-path blank if you wish to access your app on the root path of your custom domain e.g. myapp.com rather than myapp.com/prod + +`$ aws apigateway create-base-path-mapping --domain-name myapp.com --rest-api-id 123abc --stage prod --base-path '' --region us-west-2` + +Ensure you have a CNAME to resolve your custom domain name to the CloudFront Distribution domain name which can be found using: + +`$ aws apigateway get-domain-names` + #### Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth) If you want to use Zappa on a domain with a free Let's Encrypt certificate using automatic Route 53 based DNS Authentication, you can follow [this handy guide](https://github.com/Miserlou/Zappa/blob/master/docs/domain_with_free_ssl_dns.md). From aaf24176a50db104c2fcf4a7e3e8af3000244160 Mon Sep 17 00:00:00 2001 From: Konstantin Burov Date: Wed, 18 Jan 2017 09:46:16 -0800 Subject: [PATCH 59/97] Fixed readme formatting. --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 31b7a592f..83cabe7a9 100644 --- a/README.md +++ b/README.md @@ -181,11 +181,7 @@ You can also `rollback` the deployed code to a previous version by supplying the #### Scheduling -Zappa can be used to easily - - - -functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! +Zappa can be used to easily schedule functions to occur on regular intervals. This provides a much nicer, maintenance-free alternative to Celery! These functions will be packaged and deployed along with your `app_function` and called from the handler automatically. Just list your functions and the expression to schedule them using [cron or rate syntax](http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html) in your *zappa_settings.json* file: @@ -422,7 +418,6 @@ to change Zappa's behavior. Use these at your own risk! } ], "exception_handler": "your_module.report_exception", // function that will be invoked in case Zappa sees an unhandled exception raised from your code - "exclude": ["*.gz", "*.rar"], // A list of regex patterns to exclude from the archive. To exclude boto3 and botocore (available in an older version on Lambda), add "boto3*" and "botocore*". "extends": "stage_name", // Duplicate and extend another stage's settings. For example, `dev-asia` could extend from `dev-common` with a different `s3_bucket` value. "http_methods": ["GET", "POST"], // HTTP Methods to route, From fccb3e353e8efbd511b6ab78eb5559416b407e8b Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 18 Jan 2017 14:19:09 -0500 Subject: [PATCH 60/97] sm readme reorder --- README.md | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 83cabe7a9..a2a0e9264 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,13 @@ - [IAM Policy](#iam-policy) - [API Gateway Authorizers](#api-gateway-authorizers) - [Cognito User Pool Authorizer](#cognito-user-pool-authorizer) - - [Deploying with Custom Domain Name (with your own SSL Certs)](#deploying-with-custom-domain-name-with-your-own-ssl-certs) - - [Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-dns-auth) - - [Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-http-auth) + - [Deploying with Custom Domain Name with SSL Certificates](#deploying-to-a-custom domain-name-with-ssl-certificates) + - [Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-dns-auth) + - [Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-http-auth) + - [Deploying with Custom Domain Name (with your own SSL Certs)](#deploying-with-custom-domain-name-with-your-own-ssl-certs) - [Setting Environment Variables](#setting-environment-variables) - - [Local Environment Variables](#local-environment-variables) - - [Remote Environment Variables](#remote-environment-variables) + - [Local Environment Variables](#local-environment-variables) + - [Remote Environment Variables](#remote-environment-variables) - [Setting Integration Content-Type Aliases](#setting-integration-content-type-aliases) - [Catching Unhandled Exceptions](#catching-unhandled-exceptions) - [Using Custom AWS IAM Roles and Policies](#using-custom-aws-iam-roles-and-policies) @@ -568,14 +569,25 @@ You can also use AWS Cognito User Pool Authorizer by adding: "arn:aws:cognito-idp:{region}:{account_id}:userpool/{user_pool_id}" ] } -} -``` +}``` + +#### Deploying to a Custom Domain Name with SSL Certificates + +##### Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth) + +If you want to use Zappa on a domain with a free Let's Encrypt certificate using automatic Route 53 based DNS Authentication, you can follow [this handy guide](https://github.com/Miserlou/Zappa/blob/master/docs/domain_with_free_ssl_dns.md). + +##### Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth) + +If you want to use Zappa on a domain with a free Let's Encrypt certificate using HTTP Authentication, you can follow [this guide](https://github.com/Miserlou/Zappa/blob/master/docs/domain_with_free_ssl_http.md). -#### Deploying with Custom Domain Name (with your own SSL Certs) +However, it's now far easier to use Route 53-based DNS authentication, which will allow you to use a Let's Encrypt certificate with a single `$ zappa certify` command. + +##### Deploying with Custom Domain Name (with your own SSL Certs) 1. The first step is to create a custom domain and upload your SSL cert / key / bundle - follow this guide [Set Up a Custom Domain Name for an API Gateway API](http://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-custom-domains.html#how-to-custom-domains-console) -2. Ensure you have set the `domain` setting within your zappa settings JSON - this will avoid problems with the Base Path mapping between the Custom Domain and the API invoke URL, which gets the Stage Name appended in the URI -3. Deploy or update your app using zappa +2. Ensure you have set the `domain` setting within your Zappa settings JSON - this will avoid problems with the Base Path mapping between the Custom Domain and the API invoke URL, which gets the Stage Name appended in the URI +3. Deploy or update your app using Zappa 4. Create a base path mapping between your custom domain name and your chosen API stage, leaving the base-path blank if you wish to access your app on the root path of your custom domain e.g. myapp.com rather than myapp.com/prod `$ aws apigateway create-base-path-mapping --domain-name myapp.com --rest-api-id 123abc --stage prod --base-path '' --region us-west-2` @@ -584,15 +596,7 @@ Ensure you have a CNAME to resolve your custom domain name to the CloudFront Dis `$ aws apigateway get-domain-names` -#### Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth) - -If you want to use Zappa on a domain with a free Let's Encrypt certificate using automatic Route 53 based DNS Authentication, you can follow [this handy guide](https://github.com/Miserlou/Zappa/blob/master/docs/domain_with_free_ssl_dns.md). - -#### Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth) - -If you want to use Zappa on a domain with a free Let's Encrypt certificate using HTTP Authentication, you can follow [this guide](https://github.com/Miserlou/Zappa/blob/master/docs/domain_with_free_ssl_http.md). - -However, it's now far easier to use Route 53-based DNS authentication, which will allow you to use a Let's Encrypt certificate with a single `$ zappa certify` command. +There is an [open ticket](https://github.com/Miserlou/Zappa/issues/401) to automate this process. #### Setting Environment Variables From 051a2627199165cbd8bcfe736bf2aeb585fe5da5 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 18 Jan 2017 14:20:31 -0500 Subject: [PATCH 61/97] doctoc readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a2a0e9264..1ad4c9064 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,13 @@ - [IAM Policy](#iam-policy) - [API Gateway Authorizers](#api-gateway-authorizers) - [Cognito User Pool Authorizer](#cognito-user-pool-authorizer) - - [Deploying with Custom Domain Name with SSL Certificates](#deploying-to-a-custom domain-name-with-ssl-certificates) + - [Deploying to a Custom Domain Name with SSL Certificates](#deploying-to-a-custom-domain-name-with-ssl-certificates) - [Deploying to a Domain With a Let's Encrypt Certificate (DNS Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-dns-auth) - [Deploying to a Domain With a Let's Encrypt Certificate (HTTP Auth)](#deploying-to-a-domain-with-a-lets-encrypt-certificate-http-auth) - [Deploying with Custom Domain Name (with your own SSL Certs)](#deploying-with-custom-domain-name-with-your-own-ssl-certs) - [Setting Environment Variables](#setting-environment-variables) - - [Local Environment Variables](#local-environment-variables) - - [Remote Environment Variables](#remote-environment-variables) + - [Local Environment Variables](#local-environment-variables) + - [Remote Environment Variables](#remote-environment-variables) - [Setting Integration Content-Type Aliases](#setting-integration-content-type-aliases) - [Catching Unhandled Exceptions](#catching-unhandled-exceptions) - [Using Custom AWS IAM Roles and Policies](#using-custom-aws-iam-roles-and-policies) @@ -569,7 +569,8 @@ You can also use AWS Cognito User Pool Authorizer by adding: "arn:aws:cognito-idp:{region}:{account_id}:userpool/{user_pool_id}" ] } -}``` +} +``` #### Deploying to a Custom Domain Name with SSL Certificates From 1a488a1c6cc5ff69ccc28e852f5a044ff4620122 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 18 Jan 2017 14:45:26 -0500 Subject: [PATCH 62/97] force nonunicode environment variables per #601, fix bug in packaging tqdm --- zappa/cli.py | 8 ++++++++ zappa/handler.py | 4 +++- zappa/zappa.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index 8eeee2398..7e3410080 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1648,6 +1648,14 @@ def create_package(self): if self.aws_region: env_dict['AWS_REGION'] = self.aws_region env_dict.update(dict(self.environment_variables)) + + # Environement variable keys can't be Unicode + # https://github.com/Miserlou/Zappa/issues/604 + try: + env_dict = dict((k.encode('ascii'), v) for (k, v) in env_dict.items()) + except Exception: # pragma: nocover + raise ValueError("Environment variable keys must not be unicode.") + settings_s = settings_s + "ENVIRONMENT_VARIABLES={0}\n".format( env_dict ) diff --git a/zappa/handler.py b/zappa/handler.py index 94a303c7d..0fc82d9fb 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -171,7 +171,9 @@ def load_remote_settings(self, remote_bucket, remote_file): key, value )) - os.environ[key] = value + # Environement variable keys can't be Unicode + # https://github.com/Miserlou/Zappa/issues/604 + os.environ[str(key)] = value @staticmethod def import_module_and_get_function(whole_function): diff --git a/zappa/zappa.py b/zappa/zappa.py index aa10b67ae..6eed33135 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -452,8 +452,8 @@ def splitpath(path): zipresp = resp.raw with zipfile.ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(temp_project_path) - progress.update() - except Exception: + progress.update() + except Exception, e: pass # XXX - What should we do here? progress.close() From b2b93439993d1ea3e1155382450bf62911f4a51f Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 18 Jan 2017 14:46:35 -0500 Subject: [PATCH 63/97] rm p2 only exception catcher --- zappa/zappa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 6eed33135..8cfd9f129 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -453,7 +453,7 @@ def splitpath(path): with zipfile.ZipFile(BytesIO(zipresp.read())) as zfile: zfile.extractall(temp_project_path) progress.update() - except Exception, e: + except Exception: pass # XXX - What should we do here? progress.close() From 932b947839d4ec5c387360bf8da37aba9efb349d Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 18 Jan 2017 16:39:05 -0500 Subject: [PATCH 64/97] warn on unicode env var keys from s3 --- zappa/handler.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zappa/handler.py b/zappa/handler.py index 0fc82d9fb..0857bc154 100644 --- a/zappa/handler.py +++ b/zappa/handler.py @@ -107,8 +107,10 @@ def __init__(self, settings_name="zappa_settings", session=None): pass # Set any locally defined env vars + # Environement variable keys can't be Unicode + # https://github.com/Miserlou/Zappa/issues/604 for key in self.settings.ENVIRONMENT_VARIABLES.keys(): - os.environ[key] = self.settings.ENVIRONMENT_VARIABLES[key] + os.environ[str(key)] = self.settings.ENVIRONMENT_VARIABLES[key] # Django gets special treatment. if not self.settings.DJANGO_SETTINGS: @@ -173,7 +175,11 @@ def load_remote_settings(self, remote_bucket, remote_file): )) # Environement variable keys can't be Unicode # https://github.com/Miserlou/Zappa/issues/604 - os.environ[str(key)] = value + try: + os.environ[str(key)] = value + except Exception: + if self.settings.LOG_LEVEL == "DEBUG": + print("Environment variable keys must be non-unicode!") @staticmethod def import_module_and_get_function(whole_function): From 16d38b9933122b5a698d3625cb286b70a7575832 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Thu, 19 Jan 2017 00:29:53 -0500 Subject: [PATCH 65/97] api key --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ad4c9064..8e77dc96f 100644 --- a/README.md +++ b/README.md @@ -540,7 +540,7 @@ The file's contents should then be sourced in e.g. ~/.bashrc. ##### API Key -You can use the `api_key_required` setting to generate and assign an API key to all the routes of your API Gateway. After redeployment, you can then pass the provided key as a header called `x-api-key` to access the restricted endpoints. Without the `x-api-key` header, you will receive a 403. [More information on API keys in the API Gateway](http://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-api-keys.html) +You can use the `api_key_required` setting to generate and assign an API key to all the routes of your API Gateway. After redeployment, you can then pass the provided key as a header called `x-api-key` to access the restricted endpoints. Without the `x-api-key` header, you will receive a 403. You'll also need to manually associate this API key with your usage plan in the AWS console. [More information on API keys in the API Gateway](http://docs.aws.amazon.com/apigateway/latest/developerguide/how-to-api-keys.html). ##### IAM Policy From ef99d6ce2e371769c456bc3af4b93671cbd0ada3 Mon Sep 17 00:00:00 2001 From: Doppins Date: Fri, 20 Jan 2017 09:21:32 +0000 Subject: [PATCH 66/97] Upgrade dependency argcomplete to ==1.8.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e4a68f8cb..2898ce564 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -argcomplete==1.7.0 +argcomplete==1.8.0 base58==0.2.4 botocore==1.4.93 boto3==1.4.3 From 613056ad32f95d7fa80092aeecd3319584e257df Mon Sep 17 00:00:00 2001 From: Doppins Date: Fri, 20 Jan 2017 09:22:30 +0000 Subject: [PATCH 67/97] Upgrade dependency tqdm to ==4.11.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e4a68f8cb..fa2536491 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ PyYAML==3.12 requests>=2.10.0 six==1.10.0 toml==0.9.2 -tqdm==4.11.0 +tqdm==4.11.1 troposphere>=1.9.0 Werkzeug==0.11.15 wheel==0.29.0 From 6d055b38f718efa326409f8e5d5cafc2358eb6b7 Mon Sep 17 00:00:00 2001 From: Doppins Date: Fri, 20 Jan 2017 09:22:50 +0000 Subject: [PATCH 68/97] Upgrade dependency coverage to ==4.3.4 --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index c59bcbeb7..54e0558f4 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,5 +1,5 @@ coveralls==1.1 -coverage==4.3.1 +coverage==4.3.4 Django==1.10.5 Flask==0.12 mock==2.0.0 From 3209fc2908e75c9121887b11df849b77603e5172 Mon Sep 17 00:00:00 2001 From: Paul Litvak Date: Fri, 20 Jan 2017 13:37:23 +0200 Subject: [PATCH 69/97] Updated license year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index c4b549831..745e05d89 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2016 Rich Jones +Copyright (c) 2017 Rich Jones Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 545950979cda44311723182d41986cf83cc689e7 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Fri, 20 Jan 2017 15:54:08 -0500 Subject: [PATCH 70/97] add service limits section --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e77dc96f..1de384595 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ - [Keeping The Server Warm](#keeping-the-server-warm) - [Serving Static Files / Binary Uploads](#serving-static-files--binary-uploads) - [Enabling CORS](#enabling-cors) - - [Enabling Bash completion](#enabling-bash-completion) + - [Enabling Bash Completion](#enabling-bash-completion) - [Enabling Secure Endpoints on API Gateway](#enabling-secure-endpoints-on-api-gateway) - [API Key](#api-key) - [IAM Policy](#iam-policy) @@ -54,6 +54,7 @@ - [Catching Unhandled Exceptions](#catching-unhandled-exceptions) - [Using Custom AWS IAM Roles and Policies](#using-custom-aws-iam-roles-and-policies) - [Globally Available Server-less Architectures](#globally-available-server-less-architectures) + - [Raising AWS Service Limits](#raising-aws-service-limits) - [Zappa Guides](#zappa-guides) - [Zappa in the Press](#zappa-in-the-press) - [Sites Using Zappa](#sites-using-zappa) @@ -523,7 +524,7 @@ The easiest way to enable CORS (Cross-Origin Resource Sharing) for in your Zappa You can also simply handle CORS directly in your application. If you do this, you'll need to add `Access-Control-Allow-Origin`, `Access-Control-Allow-Headers`, and `Access-Control-Allow-Methods` to the `method_header_types` key in your `zappa_settings.json`. See further [discussion here](https://github.com/Miserlou/Zappa/issues/41). -#### Enabling Bash completion +#### Enabling Bash Completion Bash completion can be enabled by adding the following to your .bashrc: @@ -738,6 +739,12 @@ During the `init` process, you will be given the option to deploy your applicati To learn more about these capabilities, see [these slides](https://htmlpreview.github.io/?https://github.com/Miserlou/Talks/blob/master/serverless-london/global.html#0) from ServerlessConf London. +#### Raising AWS Service Limits + +Out of the box, AWS sets a limit of [100 concurrent executions](http://docs.aws.amazon.com/lambda/latest/dg/limits.html) for your functions. If you start to breach these limits, you may start to see errors like `ClientError: An error occurred (LimitExceededException) when calling the PutTargets.."` or something similar. + +To avoid this, you can file a [service ticket](https://console.aws.amazon.com/support/home#/) with Amazon to raise your limits up to the many tens of thousands of concurrent executions which you may need. This is a fairly common practice with Amazon, designed to prevent you from accidentally creating extremely expensive bug reports. So, before raising your service limits, make sure that you don't have any rogue scripts which could accidentally create tens of thousands of parallel executions that you don't want to pay for. + ## Zappa Guides * [Django-Zappa tutorial (screencast)](https://www.youtube.com/watch?v=plUrbPN0xc8&feature=youtu.be). From c789d0c30071a9f42fece787743d438e6602bfd9 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Fri, 20 Jan 2017 15:58:59 -0500 Subject: [PATCH 71/97] loosen test reqs --- test_requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test_requirements.txt b/test_requirements.txt index c59bcbeb7..bc76f9f41 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,8 +1,8 @@ -coveralls==1.1 -coverage==4.3.1 -Django==1.10.5 -Flask==0.12 -mock==2.0.0 -nose==1.3.7 -nose-timer==0.6.0 -placebo==0.8.1 +coveralls>=1.1 +coverage>=4.3.1 +Django>=1.10.5 +Flask>=0.12 +mock>=2.0.0 +nose>=1.3.7 +nose-timer>=0.6.0 +placebo>=0.8.1 From da08f24c3f9de0019466d174fe05764562f59cab Mon Sep 17 00:00:00 2001 From: Doppins Date: Sat, 21 Jan 2017 19:20:21 +0000 Subject: [PATCH 72/97] Upgrade dependency argcomplete to ==1.8.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 382e87971..96be74d09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -argcomplete==1.8.0 +argcomplete==1.8.1 base58==0.2.4 botocore==1.4.93 boto3==1.4.3 From cb231b2f33e311a52818de358c2f83387c65a37d Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 12:13:33 -0500 Subject: [PATCH 73/97] fixes #613 --- zappa/cli.py | 28 ++++++++++++++++++---------- zappa/zappa.py | 13 +++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index e82691070..39bc1b4b8 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -523,16 +523,24 @@ def deploy(self): if not success: # pragma: no cover raise ClickException("Unable to upload to S3. Quitting.") - # Register the Lambda function with that zip as the source - # You'll also need to define the path to your lambda_handler code. - self.lambda_arn = self.zappa.create_lambda_function(bucket=self.s3_bucket_name, - s3_key=self.zip_path, - function_name=self.lambda_name, - handler=self.lambda_handler, - description=self.lambda_description, - vpc_config=self.vpc_config, - timeout=self.timeout_seconds, - memory_size=self.memory_size) + + # Fixes https://github.com/Miserlou/Zappa/issues/613 + try: + self.lambda_arn = self.zappa.get_lambda_function( + FunctionName=self.lambda_name) + except ClientError: + # Register the Lambda function with that zip as the source + # You'll also need to define the path to your lambda_handler code. + self.lambda_arn = self.zappa.create_lambda_function( + bucket=self.s3_bucket_name, + s3_key=self.zip_path, + function_name=self.lambda_name, + handler=self.lambda_handler, + description=self.lambda_description, + vpc_config=self.vpc_config, + timeout=self.timeout_seconds, + memory_size=self.memory_size + ) # Schedule events for this deployment self.schedule() diff --git a/zappa/zappa.py b/zappa/zappa.py index 563f76fd2..d9d6493c1 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -736,6 +736,19 @@ def rollback_lambda_function_version(self, function_name, versions_back=1, publi return response['FunctionArn'] + def get_lambda_function(self, function_name): + """ + Returns the lambda function ARN, given a name + + This requires the "lambda:GetFunction" role. + """ + try: + response = self.lambda_client.get_function( + FunctionName=function_name) + return response['Configuration']['FunctionArn'] + except ClientError as err: + return err + def get_lambda_function_versions(self, function_name): """ Simply returns the versions available for a Lambda function, given a function name. From 37f0dd7e1ee7fe6b614f7c2c001a376e823da803 Mon Sep 17 00:00:00 2001 From: Paul Litvak Date: Sun, 22 Jan 2017 22:14:45 +0200 Subject: [PATCH 74/97] Retrying travis It had an issue a couple of weeks ago... --- zappa/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index c6a3e1e88..50a49d263 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1350,7 +1350,7 @@ def callback(self, position): "find {position} callback ".format(position=position), bold=True) + 'function: "{cb_func}" '.format( cb_func=click.style(cb_func, bold=True)) + 'in module "{mod_path}"'.format(mod_path=mod_path)) - getattr(module_, cb_func)(self) # Call the function passing self + getattr(module_, cb_func)(self) # Call the function passing self def check_for_update(self): """ From 5e2d82d305b2d4726ccef8350c30a67570f940cd Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 16:40:03 -0500 Subject: [PATCH 75/97] fixed exception error --- zappa/cli.py | 2 +- zappa/zappa.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index 39bc1b4b8..651bf1d23 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -528,7 +528,7 @@ def deploy(self): try: self.lambda_arn = self.zappa.get_lambda_function( FunctionName=self.lambda_name) - except ClientError: + except botocore.client.ClientError: # Register the Lambda function with that zip as the source # You'll also need to define the path to your lambda_handler code. self.lambda_arn = self.zappa.create_lambda_function( diff --git a/zappa/zappa.py b/zappa/zappa.py index d9d6493c1..9668e241f 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -742,12 +742,9 @@ def get_lambda_function(self, function_name): This requires the "lambda:GetFunction" role. """ - try: - response = self.lambda_client.get_function( + response = self.lambda_client.get_function( FunctionName=function_name) - return response['Configuration']['FunctionArn'] - except ClientError as err: - return err + return response['Configuration']['FunctionArn'] def get_lambda_function_versions(self, function_name): """ From 7cab20856fe328c034428b384d3ddae7427f7617 Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 16:56:13 -0500 Subject: [PATCH 76/97] fix arg name --- zappa/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index 651bf1d23..55f239176 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -527,7 +527,7 @@ def deploy(self): # Fixes https://github.com/Miserlou/Zappa/issues/613 try: self.lambda_arn = self.zappa.get_lambda_function( - FunctionName=self.lambda_name) + function_name=self.lambda_name) except botocore.client.ClientError: # Register the Lambda function with that zip as the source # You'll also need to define the path to your lambda_handler code. From 8f3dbabf28b8cc212335df38c400075aeca12296 Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 19:21:46 -0500 Subject: [PATCH 77/97] fixes #617 --- zappa/zappa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 9668e241f..a8b99ea72 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -754,8 +754,8 @@ def get_lambda_function_versions(self, function_name): try: response = self.lambda_client.list_versions_by_function(FunctionName=function_name) return response.get('Versions', []) - except Exception: - return [] + except botocore.exceptions.ClientError as e: + return e.response['Error']['Message'] def delete_lambda_function(self, function_name): """ From 812435169e328ac505e96e5e0a8f0cc5a9addc9a Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 19:28:28 -0500 Subject: [PATCH 78/97] added fix reference --- zappa/zappa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zappa/zappa.py b/zappa/zappa.py index a8b99ea72..96f6b2c36 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -754,6 +754,7 @@ def get_lambda_function_versions(self, function_name): try: response = self.lambda_client.list_versions_by_function(FunctionName=function_name) return response.get('Versions', []) + # reference: https://github.com/Miserlou/Zappa/issues/617 except botocore.exceptions.ClientError as e: return e.response['Error']['Message'] From b793f45ae918967eb4ee26d9d3fad252dcf184f4 Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 19:53:29 -0500 Subject: [PATCH 79/97] commenting out exception to get tests to pass for now but alert users to policy requirement --- zappa/zappa.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 96f6b2c36..b83cc4262 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -756,7 +756,8 @@ def get_lambda_function_versions(self, function_name): return response.get('Versions', []) # reference: https://github.com/Miserlou/Zappa/issues/617 except botocore.exceptions.ClientError as e: - return e.response['Error']['Message'] + print('You might need "lambda:ListVersionsByFunction" policy enabled') + return [] # e.response['Error']['Message'] def delete_lambda_function(self, function_name): """ From 72709b159b6cea2eea7654c0af6f930c81f0457c Mon Sep 17 00:00:00 2001 From: richiverse Date: Sun, 22 Jan 2017 20:05:07 -0500 Subject: [PATCH 80/97] reverting to just printing out warning --- zappa/zappa.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index b83cc4262..1fd40b986 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -752,10 +752,12 @@ def get_lambda_function_versions(self, function_name): """ try: - response = self.lambda_client.list_versions_by_function(FunctionName=function_name) + response = self.lambda_client.list_versions_by_function( + FunctionName=function_name + ) return response.get('Versions', []) - # reference: https://github.com/Miserlou/Zappa/issues/617 - except botocore.exceptions.ClientError as e: + except Exception: + # reference: https://github.com/Miserlou/Zappa/issues/617 print('You might need "lambda:ListVersionsByFunction" policy enabled') return [] # e.response['Error']['Message'] From eea65b81e37bc448a1e6fff9c9404f7de39a092e Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 13:32:52 -0500 Subject: [PATCH 81/97] 0.33.0 - Adds package command, changes tail order, various other changes and fixes, see CHANGELOG for more details --- CHANGELOG.md | 11 +++++++++++ setup.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f69eb9b7f..8d94042d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Zappa Changelog +## 0.33.0 +* Adds `package` command +* Forbids the use of unicode environment variable keys +* Initialize wsgi.errors to sys.stderr (was '') +* Accept `AWS_SESSION_TOKEN` when executing via an IAM role (#589) +* Set `REMOTE_USER` even when using `iam_authorization` +* Rename `lets_encrypt_schedule` to `lets_encrypt_expression` (#571) +* Messages in `tail` are now sequential +* Bump version requirements, update README +* Various other small changes + ## 0.32.1 * File `tail` broken in CLI refactor diff --git a/setup.py b/setup.py index ca123fa24..86c7a4500 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='zappa', - version='0.32.1', + version='0.33.0', packages=['zappa'], install_requires=required, tests_require=test_required, From 7ec814021b8e5dac6ea28ab1ce1c2f37e9d6aa75 Mon Sep 17 00:00:00 2001 From: richiverse Date: Mon, 23 Jan 2017 14:26:01 -0500 Subject: [PATCH 82/97] only fixes #613 now. Will have to rethink testing of #617 in a seperate PR --- zappa/zappa.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zappa/zappa.py b/zappa/zappa.py index 1fd40b986..b30c60d27 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -757,9 +757,7 @@ def get_lambda_function_versions(self, function_name): ) return response.get('Versions', []) except Exception: - # reference: https://github.com/Miserlou/Zappa/issues/617 - print('You might need "lambda:ListVersionsByFunction" policy enabled') - return [] # e.response['Error']['Message'] + return [] def delete_lambda_function(self, function_name): """ From 2d8b16cdac60ab0f5162f5a0402c952345beda40 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 14:29:03 -0500 Subject: [PATCH 83/97] attempt to bump boto --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 96be74d09..abe595e07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ argcomplete==1.8.1 base58==0.2.4 -botocore==1.4.93 -boto3==1.4.3 +botocore==1.5.4 +boto3==1.4.4 docutils>=0.12 futures==3.0.5 hjson==2.0.2 From d294632995a5f110e67bcc21302833e067e3b1af Mon Sep 17 00:00:00 2001 From: Pablo Arroyo Loma Date: Mon, 23 Jan 2017 22:42:06 +0100 Subject: [PATCH 84/97] Use env_parser in 'unschedule' command --- zappa/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zappa/cli.py b/zappa/cli.py index e82691070..544657da9 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -324,7 +324,8 @@ def positive_int(s): ## # Unschedule ## - subparsers.add_parser('unschedule', help='Unschedule functions.') + subparsers.add_parser('unschedule', parents=[env_parser], + help='Unschedule functions.') ## # Updating From 1f322aa33a76fe45c55ed2c3c615e9f25800cb45 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 17:57:09 -0500 Subject: [PATCH 85/97] Add --since and --filter to zappa tail --- README.md | 10 ++++++++++ requirements.txt | 1 + zappa/cli.py | 42 +++++++++++++++++++++++++++++++++++------- zappa/util.py | 26 ++++++++++++++++++++++++++ zappa/zappa.py | 12 ++++++++++-- 5 files changed, 82 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1de384595..c5d827aa8 100644 --- a/README.md +++ b/README.md @@ -329,6 +329,16 @@ You can use the argument `--http` to filter for HTTP requests, which will be in If you don't like the default log colors, you can turn them off with `--no-color`. +You can also limit the length of the tail with `--since`, which accepts a simple duration string: + + $ zappa tail production --since 4h # 4 hours + $ zappa tail production --since 1m # 1 minute + $ zappa tail production --since 1mm # 1 month + +You can filter out the contents of the logs with `--filter`, like so: + + $ zappa tail production --http --filter "POST" # Only show GET HTTP requests + #### Remote Function Invocation You can execute any function in your application directly at any time by using the `invoke` command. diff --git a/requirements.txt b/requirements.txt index abe595e07..465a9ea82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ base58==0.2.4 botocore==1.5.4 boto3==1.4.4 docutils>=0.12 +durationpy==0.3 futures==3.0.5 hjson==2.0.2 jmespath==0.9.0 diff --git a/zappa/cli.py b/zappa/cli.py index e82691070..5fa0a754f 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -305,6 +305,15 @@ def positive_int(s): '--http', action='store_true', help='Only show HTTP requests in tail output.' ) + tail_parser.add_argument( + '--since', type=str, default="100000s", + help="Only show lines since a certain timeframe." + ) + tail_parser.add_argument( + '--filter', type=str, default="", + help="Apply a filter pattern to the logs." + ) + ## # Undeploy @@ -452,9 +461,17 @@ def dispatch_command(self, command, environment): self.invoke(command, command="manage") elif command == 'tail': # pragma: no cover - self.tail(colorize=(not self.vargs['no_color']), http=self.vargs['http']) + self.tail( + colorize=(not self.vargs['no_color']), + http=self.vargs['http'], + since=self.vargs['since'], + filter_pattern=self.vargs['filter'], + ) elif command == 'undeploy': # pragma: no cover - self.undeploy(noconfirm=self.vargs['yes'], remove_logs=self.vargs['remove_logs']) + self.undeploy( + noconfirm=self.vargs['yes'], + remove_logs=self.vargs['remove_logs'] + ) elif command == 'schedule': # pragma: no cover self.schedule() elif command == 'unschedule': # pragma: no cover @@ -710,7 +727,7 @@ def rollback(self, revision): self.lambda_name, versions_back=revision) print("Done!") - def tail(self, keep_open=True, colorize=True, http=False): + def tail(self, since, filter_pattern, limit=10000, keep_open=True, colorize=True, http=False): """ Tail this function's logs. @@ -718,15 +735,26 @@ def tail(self, keep_open=True, colorize=True, http=False): """ try: - since = 0 + + from util import string_to_timestamp + since_stamp = string_to_timestamp(since) + + last_since = since_stamp while True: - new_logs = self.zappa.fetch_logs(self.lambda_name, startTime=since) - new_logs = [ e for e in new_logs if e['timestamp'] > since ] + new_logs = self.zappa.fetch_logs( + self.lambda_name, + start_time=since_stamp, + limit=limit, + filter_pattern=filter_pattern, + ) + + new_logs = [ e for e in new_logs if e['timestamp'] > last_since ] self.print_logs(new_logs, colorize, http) + if not keep_open: break if new_logs: - since = new_logs[-1]['timestamp'] + last_since = new_logs[-1]['timestamp'] time.sleep(1) except KeyboardInterrupt: # pragma: no cover # Die gracefully diff --git a/zappa/util.py b/zappa/util.py index c0cc13ca0..a06c15d2a 100644 --- a/zappa/util.py +++ b/zappa/util.py @@ -69,6 +69,32 @@ def human_size(num, suffix='B'): num /= 1024.0 return "{0:.1f}{1!s}{2!s}".format(num, 'Yi', suffix) +def string_to_timestamp(timestring): + """ + Accepts a str, returns an int timestamp. + """ + + import calendar + import datetime + import durationpy + + ts = None + + # Uses an extended version of Go's duration string. + try: + delta = durationpy.from_str(timestring); + past = datetime.datetime.utcnow() - delta + ts = calendar.timegm(past.timetuple()) + return ts + except Exception as e: + pass + + if ts: + return ts + # else: + # print("Unable to parse timestring.") + return 0 + ## # `init` related ## diff --git a/zappa/zappa.py b/zappa/zappa.py index 563f76fd2..ff5401a38 100644 --- a/zappa/zappa.py +++ b/zappa/zappa.py @@ -1808,7 +1808,7 @@ def _clear_policy(self, lambda_name): # CloudWatch Logging ## - def fetch_logs(self, lambda_name, filter_pattern='', limit=10000, startTime=0): + def fetch_logs(self, lambda_name, filter_pattern='', limit=10000, start_time=0): """ Fetch the CloudWatch logs for a given Lambda name. """ @@ -1828,12 +1828,20 @@ def fetch_logs(self, lambda_name, filter_pattern='', limit=10000, startTime=0): extra_args = {} if 'nextToken' in response: extra_args['nextToken'] = response['nextToken'] + + # Amazon uses millisecond epoch for some reason. + # Thanks, Jeff. + start_time = start_time * 1000 + end_time = int(time.time()) * 1000 + response = self.logs_client.filter_log_events( logGroupName=log_name, logStreamNames=all_names, - startTime=startTime, + startTime=start_time, + endTime=end_time, filterPattern=filter_pattern, limit=limit, + interleaved=True, # Does this actually improve performance? **extra_args ) events += response['events'] From 50eefc5af73311de20bfa7a5355858d13733a5d9 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 18:07:40 -0500 Subject: [PATCH 86/97] update tail test --- tests/tests_placebo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_placebo.py b/tests/tests_placebo.py index e5f31733c..d83e3ed1c 100644 --- a/tests/tests_placebo.py +++ b/tests/tests_placebo.py @@ -333,7 +333,7 @@ def test_cli_aws(self, session): zappa_cli.deploy() zappa_cli.update() zappa_cli.rollback(1) - zappa_cli.tail(False) + zappa_cli.tail(since=0, filter_pattern='', keep_open=False) zappa_cli.schedule() zappa_cli.unschedule() zappa_cli.undeploy(noconfirm=True, remove_logs=True) From a2b7da66bab2d2a8cb23cd05d62cc8fe7bd0bc03 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 18:10:19 -0500 Subject: [PATCH 87/97] tiny readme typoe --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5d827aa8..7952e7238 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ You can also limit the length of the tail with `--since`, which accepts a simple You can filter out the contents of the logs with `--filter`, like so: - $ zappa tail production --http --filter "POST" # Only show GET HTTP requests + $ zappa tail production --http --filter "POST" # Only show POST HTTP requests #### Remote Function Invocation From 94e6a16664d7219a7e398a83a2ffb94e8ff550d2 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Mon, 23 Jan 2017 18:24:22 -0500 Subject: [PATCH 88/97] Add test string to timestamp --- tests/tests.py | 22 +++++++++++++++++++++- zappa/util.py | 7 +++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 26896cf6d..390aeb33c 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -29,7 +29,7 @@ from zappa.letsencrypt import get_cert_and_update_domain, create_domain_key, create_domain_csr, create_chained_certificate, get_cert, cleanup, parse_account_key, parse_csr, sign_certificate, encode_certificate, register_account, verify_challenge from zappa.util import (detect_django_settings, copytree, detect_flask_apps, add_event_source, remove_event_source, - get_event_source_status, parse_s3_url, human_size) + get_event_source_status, parse_s3_url, human_size, string_to_timestamp) from zappa.wsgi import create_wsgi_request, common_log from zappa.zappa import Zappa, ASSUME_POLICY, ATTACH_POLICY @@ -1236,6 +1236,26 @@ def test_human_units(self): human_size(1) human_size(9999999999999) + def test_string_to_timestamp(self): + boo = string_to_timestamp("asdf") + self.assertTrue(boo == 0) + + yay = string_to_timestamp("1h") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("4m") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("1mm") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + + yay = string_to_timestamp("1mm1w1d1h1m1s1ms1us") + self.assertTrue(type(yay) == int) + self.assertTrue(yay > 0) + def test_event_name(self): zappa = Zappa() truncated = zappa.get_event_name("basldfkjalsdkfjalsdkfjaslkdfjalsdkfjadlsfkjasdlfkjasdlfkjasdflkjasdf-asdfasdfasdfasdfasdf", "this.is.my.dang.function.wassup.yeah.its.long") diff --git a/zappa/util.py b/zappa/util.py index a06c15d2a..67c1aa9d1 100644 --- a/zappa/util.py +++ b/zappa/util.py @@ -1,3 +1,6 @@ +import calendar +import datetime +import durationpy import fnmatch import json import os @@ -74,10 +77,6 @@ def string_to_timestamp(timestring): Accepts a str, returns an int timestamp. """ - import calendar - import datetime - import durationpy - ts = None # Uses an extended version of Go's duration string. From 31578ad30bd3da4ed55ec9b44bdfb9de3d3dda53 Mon Sep 17 00:00:00 2001 From: Doppins Date: Tue, 24 Jan 2017 00:04:05 +0000 Subject: [PATCH 89/97] Upgrade dependency tqdm to ==4.11.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 465a9ea82..617b3d6bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ PyYAML==3.12 requests>=2.10.0 six==1.10.0 toml==0.9.2 -tqdm==4.11.1 +tqdm==4.11.2 troposphere>=1.9.0 Werkzeug==0.11.15 wheel==0.29.0 From 1e20cd6b587d32576dd0a9a8961cdbfdc04f02e7 Mon Sep 17 00:00:00 2001 From: Craig Younkins Date: Mon, 23 Jan 2017 23:14:10 -0500 Subject: [PATCH 90/97] Changing usages of 'environment' in favor of 'stage' --- README.md | 10 +++++----- docs/config.rst | 2 +- docs/quickstart.rst | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7952e7238..19be30c31 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ This will automatically detect your application type (Flask/Django - Pyramid use ```javascript { - "dev": { // The name of your environment + "dev": { // The name of your stage "s3_bucket": "lmbda", // The name of your S3 bucket "app_function": "your_module.app" // The python path to your WSGI application function. In Flask, this is your 'app' object. } @@ -140,14 +140,14 @@ or for Django: ```javascript { - "dev": { // The name of your environment + "dev": { // The name of your stage "s3_bucket": "lmbda", // The name of your S3 bucket "django_settings": "your_project.settings" // The python path to your Django settings. } } ``` -You can define as many environments as your like - we recommend having _dev_, _staging_, and _production_. +You can define as many stages as your like - we recommend having _dev_, _staging_, and _production_. Now, you're ready to deploy! @@ -155,7 +155,7 @@ Now, you're ready to deploy! #### Initial Deployments -Once your settings are configured, you can package and deploy your application to an environment called "production" with a single command: +Once your settings are configured, you can package and deploy your application to a stage called "production" with a single command: $ zappa deploy production Deploying.. @@ -301,7 +301,7 @@ If you have a `zip` callback in your `callbacks` setting, this will also be invo ```javascript { - "production": { // The name of your environment + "production": { // The name of your stage "callbacks": { "zip": "my_app.zip_callback"// After creating the package } diff --git a/docs/config.rst b/docs/config.rst index 5c16a9dd6..e8501702d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -2,7 +2,7 @@ Configuration Reference ======================= -Envirnments, such as *dev*, *staging*, and *production* are configured in the *zappa_settings.json* file. Here is an example of defining many of the available settings: +Stages, such as *dev*, *staging*, and *production* are configured in the *zappa_settings.json* file. Here is an example of defining many of the available settings: :: diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 75e036ac0..764f07b9d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -31,13 +31,13 @@ Running the Initial Setup / Settings This will automatically detect your application type and help you define your deployment configuration settings. Once you finish initialization, you'll have a file named *zappa_settings.json* in your project directory defining your deployment settings. It will probably look something like this: :: { - "dev": { // The name of your environment + "dev": { // The name of your stage "s3_bucket": "lmbda", // The name of your S3 bucket "app_function": "your_module.app" // The python path to your WSGI application function. In Flask, this is your 'app' object. } } -You can define as many environments as you like. We recommend having dev, staging, and production. +You can define as many stages as you like. We recommend having dev, staging, and production. Now, you're ready to deploy! @@ -47,7 +47,7 @@ Basic Usage Initial Deployments ------------------- -Once your settings are configured, you can package and deploy your application to an environment called "production" with a single command: :: +Once your settings are configured, you can package and deploy your application to a stage called "production" with a single command: :: $ zappa deploy production Deploying.. From 071419454f2340f08b7ad95926c2bce14c43e984 Mon Sep 17 00:00:00 2001 From: Doppins Date: Wed, 25 Jan 2017 00:54:51 +0000 Subject: [PATCH 91/97] Upgrade dependency botocore to ==1.5.5 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 617b3d6bc..fa080c000 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.8.1 base58==0.2.4 -botocore==1.5.4 +botocore==1.5.5 boto3==1.4.4 docutils>=0.12 durationpy==0.3 From 825252e7556c1f81e23f4ff32ebfaa08fe6b1ea7 Mon Sep 17 00:00:00 2001 From: Paul Litvak Date: Wed, 25 Jan 2017 21:03:02 +0200 Subject: [PATCH 92/97] Splitted getattr and invocation --- zappa/cli.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/zappa/cli.py b/zappa/cli.py index 50a49d263..39e80a87f 100644 --- a/zappa/cli.py +++ b/zappa/cli.py @@ -1322,7 +1322,7 @@ def callback(self, position): callback = callbacks.get(position) if callback: - (mod_path, cb_func) = callback.rsplit('.', 1) + (mod_path, cb_func_name) = callback.rsplit('.', 1) try: # Prefer callback in working directory if mod_path.count('.') >= 1: # Callback function is nested in a folder @@ -1345,12 +1345,14 @@ def callback(self, position): "import {position} callback ".format(position=position), bold=True) + 'module: "{mod_path}"'.format(mod_path=click.style(mod_path, bold=True))) - if not hasattr(module_, cb_func): # pragma: no cover + if not hasattr(module_, cb_func_name): # pragma: no cover raise ClickException(click.style("Failed ", fg="red") + 'to ' + click.style( - "find {position} callback ".format(position=position), bold=True) + 'function: "{cb_func}" '.format( - cb_func=click.style(cb_func, bold=True)) + 'in module "{mod_path}"'.format(mod_path=mod_path)) - - getattr(module_, cb_func)(self) # Call the function passing self + "find {position} callback ".format(position=position), bold=True) + 'function: "{cb_func_name}" '.format( + cb_func_name=click.style(cb_func_name, bold=True)) + 'in module "{mod_path}"'.format(mod_path=mod_path)) + + + cb_func = getattr(module_, cb_func_name) + cb_func(self) # Call the function passing self def check_for_update(self): """ @@ -1816,8 +1818,9 @@ def execute_prebuild_script(self): "find prebuild script ", bold=True) + 'function: "{pb_func}" '.format( pb_func=click.style(pb_func, bold=True)) + 'in module "{pb_mod_path}"'.format( pb_mod_path=pb_mod_path)) - - getattr(module_, pb_func)() # Call the function + + prebuild_function = getattr(module_, pb_func) + prebuild_function() # Call the function def collision_warning(self, item): """ From 6c70880248620669e1a1fac9b53b9663069cc047 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 25 Jan 2017 15:30:51 -0500 Subject: [PATCH 93/97] add bottle to readme --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7952e7238..9674c4357 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,16 @@ This will automatically detect your application type (Flask/Django - Pyramid use ```javascript { "dev": { // The name of your environment - "s3_bucket": "lmbda", // The name of your S3 bucket - "app_function": "your_module.app" // The python path to your WSGI application function. In Flask, this is your 'app' object. + // The name of your S3 bucket + "s3_bucket": "lmbda", + + // The modular python path to your WSGI application function. + // In Flask and Bottle, this is your 'app' object. + // Flask: + // app = Flask() + // Bottle: + // app = bottle.default_app() + "app_function": "your_module.app" } } ``` From 7224eea5598eca1b4035389b822cf0700b77f539 Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 25 Jan 2017 15:33:01 -0500 Subject: [PATCH 94/97] clarify readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 346cf5cc8..ff51bbc67 100644 --- a/README.md +++ b/README.md @@ -136,9 +136,9 @@ This will automatically detect your application type (Flask/Django - Pyramid use // The modular python path to your WSGI application function. // In Flask and Bottle, this is your 'app' object. - // Flask: + // Flask (your_module.py): // app = Flask() - // Bottle: + // Bottle (your_module.py): // app = bottle.default_app() "app_function": "your_module.app" } From fc76d27dd0267c68a1e9554127641819677d219c Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 25 Jan 2017 15:40:18 -0500 Subject: [PATCH 95/97] Adds --filter and --since to tail, fixes unschedule with a specific stage --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d94042d9..65b97460b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Zappa Changelog +## 0.34.0 +* Adds `--since` and `--filter` to `tail` +* Fixes `unschedule` command when used with specific stage + ## 0.33.0 * Adds `package` command * Forbids the use of unicode environment variable keys diff --git a/setup.py b/setup.py index 86c7a4500..0f2e9e04d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='zappa', - version='0.33.0', + version='0.34.0', packages=['zappa'], install_requires=required, tests_require=test_required, From 9617fff38f324d5b08137657bf9a5c16ff1bc42e Mon Sep 17 00:00:00 2001 From: Rich Jones Date: Wed, 25 Jan 2017 15:40:18 -0500 Subject: [PATCH 96/97] 0.34.0 - Adds --filter and --since to tail, fixes unschedule with a specific stage --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d94042d9..65b97460b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Zappa Changelog +## 0.34.0 +* Adds `--since` and `--filter` to `tail` +* Fixes `unschedule` command when used with specific stage + ## 0.33.0 * Adds `package` command * Forbids the use of unicode environment variable keys diff --git a/setup.py b/setup.py index 86c7a4500..0f2e9e04d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='zappa', - version='0.33.0', + version='0.34.0', packages=['zappa'], install_requires=required, tests_require=test_required, From 40b525f5f3f6b4808d56d30d75d6d706b25a356a Mon Sep 17 00:00:00 2001 From: Doppins Date: Wed, 25 Jan 2017 23:28:50 +0000 Subject: [PATCH 97/97] Upgrade dependency botocore to ==1.5.6 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fa080c000..aa1e9a087 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete==1.8.1 base58==0.2.4 -botocore==1.5.5 +botocore==1.5.6 boto3==1.4.4 docutils>=0.12 durationpy==0.3