From 8d0445b1bf5c01f88b920516407a10b0c843c96e Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Fri, 12 Jan 2024 11:49:57 +0100 Subject: [PATCH] Upgrade CDK (v2.119), SAM (v1.107), Jinja2 (v3.1.3), and others to latest compatible version (#676) * Upgrade CDK (v2.113), SAM (v1.103), and others to latest compatible version **Why?** To use the latest versions as made available in the meanwhile. **What?** * Upgrade CDK (v2.113), SAM (v1.103), and others to latest compatible version * Remove Python based awscli v1, rely on default AWS CLI part of CodeBuild instead. * Fix Terraform Enabled Parameter Store test * Hide logs emitted by tests. * Upgrade CDK (v2.115), SAM (v1.105), Python (v3.12) and others to latest compatible **Why?** To use the latest versions as made available in the meanwhile. **What?** * Upgrade CDK (v2.115), SAM (v1.105), and others to latest compatible version * Update to Python v3.12. * Ensure Regular Expressions use raw strings where required. * Upgrade CDK (v2.118.0) and ARM64 **Why?** To use the latest versions as made available in the meanwhile. **What?** * Upgrade CDK (v2.118.0). * Upgrade AWS SAM CLI to v1.107.0. * Switch to the ARM64 architecture for Lambda functions. * Add the new required ImageId property for Cloud9 instances in the service catalog sample. * Undo ARM64 for all Lambda functions Reverting, due to `ruamel-yaml-clib==0.2.8(wheel)` dependency error. * Bump jinja2 Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump jinja2 from 3.1.2 to 3.1.3 in /src/lambda_codebase/initial_commit Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Upgrade ADF Workflow to Python 3.12 * Fix repeated headers blocker in providers guide * Upgrade to CDK v2.119 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/adf.yml | 2 +- Makefile | 2 +- docs/providers-guide.md | 3 + docs/user-guide.md | 4 +- requirements-dev.txt | 14 +- requirements.txt | 7 +- samples/sample-cdk-app/buildspec.yml | 4 +- samples/sample-codebuild-vpc/buildspec.yml | 2 +- samples/sample-codebuild-vpc/testspec.yml | 2 +- .../sample-ec2-with-codedeploy/buildspec.yml | 2 +- samples/sample-ecr-repository/buildspec.yml | 2 +- samples/sample-ecs-cluster/buildspec.yml | 2 +- samples/sample-expunge-vpc/buildspec.yml | 2 +- samples/sample-expunge-vpc/template.yml | 2 +- samples/sample-fargate-node-app/buildspec.yml | 3 +- samples/sample-iam/buildspec.yml | 2 +- .../sample-mono-repo/apps/alpha/buildspec.yml | 2 +- .../sample-mono-repo/apps/beta/buildspec.yml | 2 +- samples/sample-rdk-rules/buildspec.yml | 4 +- .../parameters.json | 2 +- samples/sample-rdk-rules/requirements.txt | 2 +- samples/sample-serverless-app/buildspec.yml | 2 +- samples/sample-serverless-app/template.yml | 2 +- .../buildspec.yml | 2 +- .../productX/template.yml | 1 + samples/sample-terraform/tf_apply.yml | 2 +- samples/sample-terraform/tf_destroy.yml | 2 +- samples/sample-terraform/tf_plan.yml | 2 +- samples/sample-vpc/buildspec.yml | 2 +- src/lambda_codebase/account/main.py | 4 +- .../process_account_files.py | 2 +- .../account_processing/requirements.txt | 3 +- .../cross_region_bucket/main.py | 7 +- .../adf-bootstrap/deployment/global.yml | 12 +- .../determine_default_branch/requirements.txt | 2 +- .../initial_commit/requirements.txt | 4 +- .../pipeline_management/requirements.txt | 3 +- .../deployment/pipeline_management.yml | 8 +- .../bootstrap_repository/adf-build/main.py | 6 +- .../adf-build/requirements-dev.txt | 4 +- .../adf-build/requirements.txt | 15 +- .../adf-build/shared/helpers/requirements.txt | 4 +- .../helpers/retrieve_organization_accounts.py | 4 +- .../shared/helpers/terraform/get_accounts.py | 3 +- .../shared/python/schema_validation.py | 2 +- .../tests/stubs/stub_deployment_map.yml | 2 +- .../shared/python/tests/test_organizations.py | 128 +++++++++++++----- .../adf-build/shared/requirements.txt | 15 +- .../adf-build/tests/test_main.py | 14 +- .../initial_commit/requirements.txt | 4 +- src/lambda_codebase/organization/main.py | 3 +- src/lambda_codebase/organization_unit/main.py | 4 +- src/lambda_codebase/requirements.txt | 2 +- src/template.yml | 35 ++++- tox.ini | 6 +- 55 files changed, 235 insertions(+), 143 deletions(-) diff --git a/.github/workflows/adf.yml b/.github/workflows/adf.yml index f2fbe5857..8698087ce 100644 --- a/.github/workflows/adf.yml +++ b/.github/workflows/adf.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.12"] steps: - name: Checkout Repo diff --git a/Makefile b/Makefile index eb749304f..1185c8d7d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ test: lint: # Linter performs static analysis to catch latent bugs - find src -iname "*.py" -not -path "src/.aws-sam/*" | xargs pylint --rcfile .pylintrc + find src -iname "*.py" -not -path "src/.aws-sam/*" | xargs pylint --verbose --rcfile .pylintrc find src -iname "*.yml" -o -iname "*.yaml" -not -path "src/.aws-sam/*" | xargs yamllint -c .yamllint.yml cfn-lint diff --git a/docs/providers-guide.md b/docs/providers-guide.md index 54009803e..d4e15eacb 100644 --- a/docs/providers-guide.md +++ b/docs/providers-guide.md @@ -1,5 +1,8 @@ # Providers Guide + + + Provider types and their properties can be defined as default config for a pipeline. But also at the stage level of a pipeline to structure the source, build, test, approval, deploy or invoke actions. diff --git a/docs/user-guide.md b/docs/user-guide.md index 1ff7e6254..2ccd8934e 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -954,8 +954,8 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 - nodejs: 18 + python: 3.12 + nodejs: 20 pre_build: commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet diff --git a/requirements-dev.txt b/requirements-dev.txt index babfea67f..af21021c8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ -cfn-lint==0.78.2 -isort==5.12.0 +cfn-lint==0.83.8 +isort==5.13.2 mock==5.1.0 -pylint==2.17.4 -pytest~=7.4.0 -pytest-cov==3.0.0 -tox==3.28.0 -yamllint==1.32.0 +pylint==3.0.3 +pytest~=7.4.4 +pytest-cov==4.1.0 +tox==4.11.4 +yamllint==1.33.0 diff --git a/requirements.txt b/requirements.txt index 4702d9854..16d7241c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -astroid==2.15.6 -boto3==1.28.8 -botocore==1.31.8 +astroid==3.0.2 +aws-sam-cli==1.107.0 +boto3==1.34.17 +botocore==1.34.17 pyyaml~=6.0.1 schema==0.7.5 diff --git a/samples/sample-cdk-app/buildspec.yml b/samples/sample-cdk-app/buildspec.yml index b030ded2e..6fe2b146f 100644 --- a/samples/sample-cdk-app/buildspec.yml +++ b/samples/sample-cdk-app/buildspec.yml @@ -3,8 +3,8 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 - nodejs: 18 + python: 3.12 + nodejs: 20 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-codebuild-vpc/buildspec.yml b/samples/sample-codebuild-vpc/buildspec.yml index 05c5ea3b7..3f6ef2309 100644 --- a/samples/sample-codebuild-vpc/buildspec.yml +++ b/samples/sample-codebuild-vpc/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: # It will connect through the VPC to fetch all the resources. # Make sure the subnets and security groups are configured such that diff --git a/samples/sample-codebuild-vpc/testspec.yml b/samples/sample-codebuild-vpc/testspec.yml index 79ad92022..de0bfb94a 100644 --- a/samples/sample-codebuild-vpc/testspec.yml +++ b/samples/sample-codebuild-vpc/testspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 build: commands: diff --git a/samples/sample-ec2-with-codedeploy/buildspec.yml b/samples/sample-ec2-with-codedeploy/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-ec2-with-codedeploy/buildspec.yml +++ b/samples/sample-ec2-with-codedeploy/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-ecr-repository/buildspec.yml b/samples/sample-ecr-repository/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-ecr-repository/buildspec.yml +++ b/samples/sample-ecr-repository/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-ecs-cluster/buildspec.yml b/samples/sample-ecs-cluster/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-ecs-cluster/buildspec.yml +++ b/samples/sample-ecs-cluster/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-expunge-vpc/buildspec.yml b/samples/sample-expunge-vpc/buildspec.yml index e7c0fba8a..8552f31a2 100644 --- a/samples/sample-expunge-vpc/buildspec.yml +++ b/samples/sample-expunge-vpc/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-expunge-vpc/template.yml b/samples/sample-expunge-vpc/template.yml index d08abf6ba..9ae2acd4b 100644 --- a/samples/sample-expunge-vpc/template.yml +++ b/samples/sample-expunge-vpc/template.yml @@ -34,7 +34,7 @@ Resources: Handler: lambda_function.lambda_handler MemorySize: 128 Role: !GetAtt LambdaVPCPolicyRole.Arn - Runtime: python3.10 + Runtime: python3.12 Timeout: 600 Environment: Variables: diff --git a/samples/sample-fargate-node-app/buildspec.yml b/samples/sample-fargate-node-app/buildspec.yml index 4eb96d3ac..1d52d3e0f 100644 --- a/samples/sample-fargate-node-app/buildspec.yml +++ b/samples/sample-fargate-node-app/buildspec.yml @@ -3,8 +3,7 @@ version: 0.2 phases: install: runtime-versions: - docker: 20 - python: 3.11 + python: 3.12 build: commands: diff --git a/samples/sample-iam/buildspec.yml b/samples/sample-iam/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-iam/buildspec.yml +++ b/samples/sample-iam/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-mono-repo/apps/alpha/buildspec.yml b/samples/sample-mono-repo/apps/alpha/buildspec.yml index 35331b919..40bc977a4 100644 --- a/samples/sample-mono-repo/apps/alpha/buildspec.yml +++ b/samples/sample-mono-repo/apps/alpha/buildspec.yml @@ -7,7 +7,7 @@ env: phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - cd $INFRASTRUCTURE_ROOT_DIR - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet diff --git a/samples/sample-mono-repo/apps/beta/buildspec.yml b/samples/sample-mono-repo/apps/beta/buildspec.yml index 824176f61..1d6fbb406 100644 --- a/samples/sample-mono-repo/apps/beta/buildspec.yml +++ b/samples/sample-mono-repo/apps/beta/buildspec.yml @@ -7,7 +7,7 @@ env: phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - cd $INFRASTRUCTURE_ROOT_DIR - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet diff --git a/samples/sample-rdk-rules/buildspec.yml b/samples/sample-rdk-rules/buildspec.yml index 02d3ef805..cff95a70d 100644 --- a/samples/sample-rdk-rules/buildspec.yml +++ b/samples/sample-rdk-rules/buildspec.yml @@ -2,8 +2,8 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 - nodejs: 18 + python: 3.12 + nodejs: 20 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json b/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json index b2922bf08..1b8ce5dae 100644 --- a/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json +++ b/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json @@ -3,7 +3,7 @@ "Parameters": { "RuleName": "EC2_CHECKS_TERMINATION_PROTECTION_ADF", "Description": "EC2_CHECKS_TERMINATION_PROTECTION_ADF", - "SourceRuntime": "python3.10", + "SourceRuntime": "python3.12", "CodeKey": "EC2_CHECKS_TERMINATION_PROTECTION_ADFeu-central-1.zip", "InputParameters": "{}", "OptionalParameters": "{}", diff --git a/samples/sample-rdk-rules/requirements.txt b/samples/sample-rdk-rules/requirements.txt index 6b3a0c44e..942841010 100644 --- a/samples/sample-rdk-rules/requirements.txt +++ b/samples/sample-rdk-rules/requirements.txt @@ -1,3 +1,3 @@ s3==3.0.0 -boto3==1.28.8 +boto3==1.34.17 argparse==1.4.0 diff --git a/samples/sample-serverless-app/buildspec.yml b/samples/sample-serverless-app/buildspec.yml index 1709ca238..40de04f0b 100644 --- a/samples/sample-serverless-app/buildspec.yml +++ b/samples/sample-serverless-app/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 build: commands: diff --git a/samples/sample-serverless-app/template.yml b/samples/sample-serverless-app/template.yml index 045e3ae10..197834926 100644 --- a/samples/sample-serverless-app/template.yml +++ b/samples/sample-serverless-app/template.yml @@ -17,7 +17,7 @@ Resources: Type: 'AWS::Serverless::Function' Properties: Handler: lambda_function.handler - Runtime: python3.10 + Runtime: python3.12 CodeUri: . Description: Sample Lambda Function MemorySize: 128 diff --git a/samples/sample-service-catalog-product/buildspec.yml b/samples/sample-service-catalog-product/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-service-catalog-product/buildspec.yml +++ b/samples/sample-service-catalog-product/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/samples/sample-service-catalog-product/productX/template.yml b/samples/sample-service-catalog-product/productX/template.yml index fc5a63fcd..04480a4e2 100644 --- a/samples/sample-service-catalog-product/productX/template.yml +++ b/samples/sample-service-catalog-product/productX/template.yml @@ -50,6 +50,7 @@ Resources: AutomaticStopTimeMinutes: !Ref AutomaticStopTimeInMinutes Description: !Ref InstanceDescription InstanceType: !Ref InstanceType + ImageId: 'amazonlinux-2023-x86_64' Name: !Ref InstanceName OwnerArn: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:user/${UserName}" # In this sample case 'sample-developer' from the IAM stack can be used here SubnetId: diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index e7c5b91b0..ba76c5bab 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -10,7 +10,7 @@ env: phases: install: runtime-versions: - python: 3.11 + python: 3.12 build: commands: diff --git a/samples/sample-terraform/tf_destroy.yml b/samples/sample-terraform/tf_destroy.yml index 5dc54f55d..08f1d4a58 100644 --- a/samples/sample-terraform/tf_destroy.yml +++ b/samples/sample-terraform/tf_destroy.yml @@ -10,7 +10,7 @@ env: phases: install: runtime-versions: - python: 3.9 + python: 3.12 build: commands: diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml index ce6e763fc..d492eb111 100644 --- a/samples/sample-terraform/tf_plan.yml +++ b/samples/sample-terraform/tf_plan.yml @@ -10,7 +10,7 @@ env: phases: install: runtime-versions: - python: 3.11 + python: 3.12 build: commands: diff --git a/samples/sample-vpc/buildspec.yml b/samples/sample-vpc/buildspec.yml index e2266266d..6a1fec8e0 100644 --- a/samples/sample-vpc/buildspec.yml +++ b/samples/sample-vpc/buildspec.yml @@ -3,7 +3,7 @@ version: 0.2 phases: install: runtime-versions: - python: 3.11 + python: 3.12 commands: - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q diff --git a/src/lambda_codebase/account/main.py b/src/lambda_codebase/account/main.py index 57f15784a..77ae7ee6c 100644 --- a/src/lambda_codebase/account/main.py +++ b/src/lambda_codebase/account/main.py @@ -5,6 +5,7 @@ deployment account if required. """ +import os from typing import Mapping, Any, Tuple from dataclasses import dataclass, asdict import logging @@ -28,7 +29,8 @@ ORGANIZATION_CLIENT = boto3.client("organizations") SSM_CLIENT = boto3.client("ssm") LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) MAX_RETRIES = 120 # => 120 retries * 5 seconds = 10 minutes diff --git a/src/lambda_codebase/account_processing/process_account_files.py b/src/lambda_codebase/account_processing/process_account_files.py index 8336ca969..03c3b2315 100644 --- a/src/lambda_codebase/account_processing/process_account_files.py +++ b/src/lambda_codebase/account_processing/process_account_files.py @@ -142,7 +142,7 @@ def process_account_list(all_accounts, accounts_in_file): def sanitize_account_name_for_snf(account_name): - return re.sub("[^a-zA-Z0-9_]", "_", account_name[:30]) + return re.sub(r"[^a-zA-Z0-9_]", "_", account_name[:30]) def start_executions( diff --git a/src/lambda_codebase/account_processing/requirements.txt b/src/lambda_codebase/account_processing/requirements.txt index 81d4d5270..8ba2e527e 100644 --- a/src/lambda_codebase/account_processing/requirements.txt +++ b/src/lambda_codebase/account_processing/requirements.txt @@ -1,3 +1,2 @@ -aws-xray-sdk==2.12.0 +aws-xray-sdk==2.12.1 pyyaml~=6.0.1 -wrapt==1.14.1 # https://github.com/aws/aws-lambda-builders/issues/302 diff --git a/src/lambda_codebase/cross_region_bucket/main.py b/src/lambda_codebase/cross_region_bucket/main.py index c86526629..b2cc2e16a 100644 --- a/src/lambda_codebase/cross_region_bucket/main.py +++ b/src/lambda_codebase/cross_region_bucket/main.py @@ -7,12 +7,14 @@ """ +import os from typing import Mapping, Any, Tuple, MutableMapping from dataclasses import dataclass, asdict import logging import json import secrets -import string # pylint: disable=deprecated-module # https://www.logilab.org/ticket/2481 +import string # pylint: disable=deprecated-module +# ^ https://www.logilab.org/ticket/2481 import boto3 from cfn_custom_resource import ( # pylint: disable=unused-import lambda_handler, @@ -34,7 +36,8 @@ # Globals: LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) S3CLIENTS: MutableMapping[Region, S3Client] = {} SSM_CLIENT = boto3.client("ssm") diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 9264881bd..af3a5c840 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -38,9 +38,7 @@ Parameters: ComputeType: Description: The Compute Type to use for AWS CodeBuild Type: String - # BUILD_GENERAL1_LARGE - For threading with large amounts of pipelines - # this is the most effective default: - Default: "BUILD_GENERAL1_LARGE" + Default: "BUILD_GENERAL1_SMALL" AllowedValues: - "BUILD_GENERAL1_SMALL" # 3 GB memory, 2 vCPU - "BUILD_GENERAL1_MEDIUM" # 7 GB memory, 4 vCPU @@ -75,7 +73,7 @@ Conditions: Globals: Function: CodeUri: lambda_codebase - Runtime: python3.10 + Runtime: python3.12 Resources: LambdaLayerVersion: @@ -83,7 +81,7 @@ Resources: Properties: ContentUri: "../../adf-build/shared/" CompatibleRuntimes: - - python3.10 + - python3.12 Description: "Shared Lambda Layer between master and deployment account" LayerName: shared_layer @@ -689,8 +687,8 @@ Resources: phases: install: runtime-versions: - python: 3.11 - nodejs: 18 + python: 3.12 + nodejs: 20 commands: - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -r adf-build/helpers/requirements.txt -q -t ./adf-build diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt index b2109c785..790ada691 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt @@ -1,2 +1,2 @@ -boto3==1.28.8 +boto3==1.34.17 cfn-custom-resource~=1.0.1 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt index 279e145cf..6cd53216a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt @@ -1,4 +1,4 @@ -Jinja2==3.1.2 -boto3==1.28.8 +Jinja2==3.1.3 +boto3==1.34.17 cfn-custom-resource~=1.0.1 markupsafe==2.1.3 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt index 188d33645..07f5f2004 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt @@ -1,4 +1,3 @@ pyyaml~=6.0.1 schema==0.7.5 -tenacity==8.2.2 -wrapt==1.14.1 # https://github.com/aws/aws-lambda-builders/issues/302 +tenacity==8.2.3 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 6bcae811b..9fcf18b84 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -56,7 +56,7 @@ Globals: Architectures: - arm64 CodeUri: lambda_codebase/pipeline_management - Runtime: python3.10 + Runtime: python3.12 Timeout: 300 Tracing: Active Layers: @@ -671,10 +671,10 @@ Resources: phases: install: runtime-versions: - python: 3.11 - nodejs: 18 + python: 3.12 + nodejs: 20 commands: - - npm install aws-cdk@2.79 -g -y --quiet --no-progress + - npm install aws-cdk@2.119.0 -g -y --quiet --no-progress - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q -t ./adf-build - chmod 755 adf-build/cdk/execute_pipeline_stacks.py adf-build/cdk/generate_pipeline_stacks.py diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index 5b1cd26e3..25ce08606 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -10,7 +10,7 @@ import sys import time from math import floor -from datetime import datetime +from datetime import datetime, timezone from thread import PropagatingThread import boto3 @@ -45,7 +45,7 @@ "CODEBUILD_START_TIME", # Fall back to 10 minutes ago + convert Python timestamp from # seconds to milliseconds: - floor(datetime.now().timestamp() - (10 * 60)) * 1000, + floor(datetime.now(timezone.utc).timestamp() - (10 * 60)) * 1000, ) ) / 1000.0 # Convert milliseconds to seconds ) @@ -331,7 +331,7 @@ def await_sfn_executions(sfn_client): ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, filter_lambda=lambda item: ( ( - item.get('startDate', datetime.now()).timestamp() + item.get('startDate', datetime.now(timezone.utc)).timestamp() >= CODEBUILD_START_TIME_UNIXTS ) and item.get('status') in ['FAILED', 'TIMED_OUT', 'ABORTED'] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements-dev.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements-dev.txt index 4edac7e10..3e91a948e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements-dev.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements-dev.txt @@ -1,3 +1,3 @@ mock~=5.1.0 -pylint~=2.17.4 -pytest~=7.4.0 +pylint~=3.0.3 +pytest~=7.4.3 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt index 13cbc5b7e..87f7a3a9e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt @@ -1,12 +1,11 @@ # Install libs here that you might want in AWS CodeBuild (On the management # account) -astroid==2.15.6 -aws-sam-cli==1.93.0 -awscli==1.29.8 -boto3==1.28.8 -botocore==1.31.8 -pip~=23.2 +astroid==3.0.2 +aws-sam-cli==1.107.0 +boto3==1.34.17 +botocore==1.34.17 +pip~=23.3 pyyaml~=6.0.1 six~=1.16.0 -tenacity==8.2.2 -urllib3~=1.26.16 +tenacity==8.2.3 +urllib3~=2.0.7 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt index 328198a76..6107c7292 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt @@ -1,4 +1,4 @@ -boto3==1.28.8 -botocore==1.31.8 +boto3==1.34.17 +botocore==1.34.17 docopt~=0.6.2 schema==0.7.5 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py index 7080131f3..e96c7aec1 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py @@ -66,6 +66,7 @@ retrieve_organization_accounts.py -v -f Id -f Email -o src/lambda/dat.json """ +import os import sys import logging import json @@ -79,7 +80,8 @@ # Configure logging logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) def main(): diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index b8fda6e3f..91fab866f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -15,7 +15,8 @@ # Configure logging logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] TARGET_OUS = os.environ.get("TARGET_OUS") diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py index aff0c3e32..c31063420 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py @@ -390,7 +390,7 @@ # Allow any top level key starting with "x-" or "x_". # ADF will ignore these, but users can use them to define anchors # in one place. - Optional(Regex('^[x][-_].*')): object + Optional(Regex(r"^[x][-_].*")): object } diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_deployment_map.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_deployment_map.yml index 07e860443..ab0d69d0c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_deployment_map.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_deployment_map.yml @@ -113,7 +113,7 @@ pipelines: build: provider: codebuild properties: - # Use a specific docker image (to use Python 3.11) for the build stage + # Use a specific docker image (to use Python 3.12) for the build stage # in this pipeline -> https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-codebuild.LinuxBuildImage.html image: "STANDARD_7_0" deploy: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py index 2be8ff764..5b5c5d4cc 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py @@ -3,7 +3,7 @@ # pylint: skip-file -from datetime import datetime +from datetime import datetime, timezone import os import boto3 @@ -118,7 +118,8 @@ def test_get_accounts_with_suspended(paginator_mock, cls): { "Id": account_id, "Status": ( - "SUSPENDED" if account_id in suspended_account_ids else "ACTIVE" + "SUSPENDED" if account_id in suspended_account_ids + else "ACTIVE" ), } ), @@ -341,16 +342,21 @@ def test_describe_ou_name(cls): def test_describe_account_name(cls): cls.client = Mock() - cls.client.describe_account.return_value = stub_organizations.describe_account + cls.client.describe_account.return_value = ( + stub_organizations.describe_account + ) assert cls.describe_account_name("some_account_id") == "some_account_name" def test_determine_ou_path(cls): - assert ( - cls.determine_ou_path("some_path", "some_ou_name") == "some_path/some_ou_name" + assert cls.determine_ou_path("some_path", "some_ou_name") == ( + "some_path/some_ou_name" ) assert ( - cls.determine_ou_path("some_path/longer_path/plus_more", "some_ou_name") + cls.determine_ou_path( + "some_path/longer_path/plus_more", + "some_ou_name", + ) == "some_path/longer_path/plus_more/some_ou_name" ) @@ -401,16 +407,20 @@ def test_original_ou_paths(self): } list_organizational_units_for_production_response = { - "OrganizationalUnits": [{"Id": "ou-080922", "Arn": "", "Name": "banking"}] + "OrganizationalUnits": [ + {"Id": "ou-080922", "Arn": "", "Name": "banking"}, + ], } list_organizational_units_for_banking_response = { "OrganizationalUnits": [ {"Id": "ou-09092022", "Arn": "", "Name": "investment"} - ] + ], } - list_organizational_units_for_investment_response = {"OrganizationalUnits": []} + list_organizational_units_for_investment_response = { + "OrganizationalUnits": [], + } list_accounts_for_banking_response_page_0 = { "Accounts": [ @@ -420,7 +430,9 @@ def test_original_ou_paths(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ], "NextToken": "PAGE1", @@ -433,7 +445,9 @@ def test_original_ou_paths(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -446,7 +460,9 @@ def test_original_ou_paths(self): "Email": "account+3@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -458,7 +474,9 @@ def test_original_ou_paths(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, { "Id": "22222222222", @@ -466,7 +484,9 @@ def test_original_ou_paths(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, ] @@ -540,21 +560,25 @@ def test_original_nested_paths(self): list_organizational_units_for_root_response = { "OrganizationalUnits": [ - {"Id": "ou-123456", "Arn": "", "Name": "production"} - ] + {"Id": "ou-123456", "Arn": "", "Name": "production"}, + ], } list_organizational_units_for_production_response = { - "OrganizationalUnits": [{"Id": "ou-080922", "Arn": "", "Name": "banking"}] + "OrganizationalUnits": [ + {"Id": "ou-080922", "Arn": "", "Name": "banking"}, + ], } list_organizational_units_for_banking_response = { "OrganizationalUnits": [ - {"Id": "ou-09092022", "Arn": "", "Name": "investment"} - ] + {"Id": "ou-09092022", "Arn": "", "Name": "investment"}, + ], } - list_organizational_units_for_investment_response = {"OrganizationalUnits": []} + list_organizational_units_for_investment_response = { + "OrganizationalUnits": [], + } list_accounts_for_banking_response_page_0 = { "Accounts": [ @@ -564,7 +588,9 @@ def test_original_nested_paths(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ], "NextToken": "PAGE1", @@ -577,7 +603,9 @@ def test_original_nested_paths(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -590,7 +618,9 @@ def test_original_nested_paths(self): "Email": "account+3@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -602,7 +632,9 @@ def test_original_nested_paths(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, { "Id": "22222222222", @@ -610,7 +642,9 @@ def test_original_nested_paths(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, { "Id": "3333333333", @@ -618,7 +652,9 @@ def test_original_nested_paths(self): "Email": "account+3@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, ] @@ -699,7 +735,9 @@ def test_nested_paths_with_exclusions(self): } list_organizational_units_for_production_response = { - "OrganizationalUnits": [{"Id": "ou-080922", "Arn": "", "Name": "banking"}] + "OrganizationalUnits": [ + {"Id": "ou-080922", "Arn": "", "Name": "banking"}, + ], } list_organizational_units_for_banking_response = { @@ -709,9 +747,13 @@ def test_nested_paths_with_exclusions(self): ] } - list_organizational_units_for_investment_response = {"OrganizationalUnits": []} + list_organizational_units_for_investment_response = { + "OrganizationalUnits": [], + } - list_organizational_units_for_commercial_response = {"OrganizationalUnits": []} + list_organizational_units_for_commercial_response = { + "OrganizationalUnits": [], + } list_accounts_for_banking_response_page_0 = { "Accounts": [ @@ -721,7 +763,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ], "NextToken": "PAGE1", @@ -734,7 +778,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -747,7 +793,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+3@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), } ] } @@ -760,7 +808,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+4@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 9, 26), + "JoinedTimestamp": ( + datetime(2022, 9, 26, tzinfo=timezone.utc) + ), } ] } @@ -772,7 +822,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+1@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, { "Id": "22222222222", @@ -780,7 +832,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+2@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 8, 9), + "JoinedTimestamp": ( + datetime(2022, 8, 9, tzinfo=timezone.utc) + ), }, { "Id": "444444444", @@ -788,7 +842,9 @@ def test_nested_paths_with_exclusions(self): "Email": "account+4@example.com", "Status": "ACTIVE", "JoinedMethod": "Invited", - "JoinedTimestamp": datetime(2022, 9, 26), + "JoinedTimestamp": ( + datetime(2022, 9, 26, tzinfo=timezone.utc) + ), }, ] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt index 018ce260e..158e727c4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt @@ -1,12 +1,11 @@ # Install libs here that you might want in AWS CodeBuild -aws-cdk-lib~=2.88.0 +aws-cdk-lib~=2.119.0 constructs>=10.0.0,<11.0.0 -aws-sam-cli==1.93.0 -awscli==1.29.8 -boto3==1.28.8 -botocore==1.31.8 +aws-sam-cli==1.107.0 +boto3==1.34.17 +botocore==1.34.17 pyyaml~=6.0.1 schema~=0.7.5 -tenacity==8.2.2 -typing-extensions~=4.7.1 -urllib3~=1.26.16 +tenacity==8.2.3 +typing-extensions~=4.9.0 +urllib3~=2.0.7 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py index 9ddfb0f3c..c7aaac85f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py @@ -145,7 +145,7 @@ def test_prepare_deployment_account_defaults(param_store_cls, cls, sts): deploy_param_store.put_parameter.assert_has_calls( [ call('adf_version', '1.0.0'), - call('adf_log_level', 'INFO'), + call('adf_log_level', 'CRITICAL'), call('deployment_account_bucket', 'some_deployment_account_bucket'), call('default_scm_branch', 'master'), call('/adf/org/stage', 'none'), @@ -187,10 +187,8 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): 'auto-create-repositories': 'disabled', 'default-scm-branch': 'main', } - cls.config['extensions'] = { - 'terraform': { - 'enabled': 'True', - }, + cls.extensions['terraform'] = { + 'enabled': 'True', } cls.config['org'] = { 'stage': 'test-stage', @@ -232,14 +230,14 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): param_store.put_parameter.assert_has_calls( [ call('organization_id', 'o-123456789'), - call('/adf/extensions/terraform/enabled', 'False'), + call('/adf/extensions/terraform/enabled', 'True'), ], any_order=False, ) deploy_param_store.put_parameter.assert_has_calls( [ call('adf_version', '1.0.0'), - call('adf_log_level', 'INFO'), + call('adf_log_level', 'CRITICAL'), call('deployment_account_bucket', 'some_deployment_account_bucket'), call('default_scm_branch', 'main'), call('/adf/org/stage', 'test-stage'), @@ -252,7 +250,7 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): f"{deployment_account_id}:function:SendSlackNotification", ), call('/notification_endpoint/main', 'slack-channel'), - call('/adf/extensions/terraform/enabled', 'False'), + call('/adf/extensions/terraform/enabled', 'True'), call('/adf/deployment-maps/allow-empty-target', 'False'), ], any_order=True, diff --git a/src/lambda_codebase/initial_commit/requirements.txt b/src/lambda_codebase/initial_commit/requirements.txt index 279e145cf..6cd53216a 100644 --- a/src/lambda_codebase/initial_commit/requirements.txt +++ b/src/lambda_codebase/initial_commit/requirements.txt @@ -1,4 +1,4 @@ -Jinja2==3.1.2 -boto3==1.28.8 +Jinja2==3.1.3 +boto3==1.34.17 cfn-custom-resource~=1.0.1 markupsafe==2.1.3 diff --git a/src/lambda_codebase/organization/main.py b/src/lambda_codebase/organization/main.py index cdc510212..a02f632c2 100644 --- a/src/lambda_codebase/organization/main.py +++ b/src/lambda_codebase/organization/main.py @@ -29,7 +29,8 @@ # Globals: ORGANIZATION_CLIENT = boto3.client("organizations") LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) class InvalidPhysicalResourceId(Exception): diff --git a/src/lambda_codebase/organization_unit/main.py b/src/lambda_codebase/organization_unit/main.py index 779825e1b..fdb3ce318 100644 --- a/src/lambda_codebase/organization_unit/main.py +++ b/src/lambda_codebase/organization_unit/main.py @@ -6,6 +6,7 @@ deployment OU. """ +import os from typing import Mapping, Any, Tuple from dataclasses import dataclass, asdict import logging @@ -28,7 +29,8 @@ # Globals: ORGANIZATION_CLIENT = boto3.client("organizations") LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +logging.basicConfig(level=logging.INFO) class InvalidPhysicalResourceId(Exception): diff --git a/src/lambda_codebase/requirements.txt b/src/lambda_codebase/requirements.txt index 364b7e83c..942537cc7 100644 --- a/src/lambda_codebase/requirements.txt +++ b/src/lambda_codebase/requirements.txt @@ -1 +1 @@ -tenacity==8.2.2 +tenacity==8.2.3 diff --git a/src/template.yml b/src/template.yml index 5f1a9e76d..b372f9626 100644 --- a/src/template.yml +++ b/src/template.yml @@ -114,7 +114,7 @@ Parameters: Globals: Function: CodeUri: lambda_codebase - Runtime: python3.10 + Runtime: python3.12 Timeout: 300 Resources: @@ -970,10 +970,15 @@ Resources: Type: "AWS::Serverless::LayerVersion" Properties: ContentUri: "./lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/" + CompatibleArchitectures: + - x86_64 CompatibleRuntimes: - - python3.10 + - python3.12 Description: "Shared Lambda Layer between master and deployment account" LayerName: shared_layer + Metadata: + BuildMethod: python3.12 + BuildArchitecture: x86_64 LambdaLayerVersionPermission: Type: "AWS::Lambda::LayerVersionPermission" @@ -1037,6 +1042,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: wait_until_complete.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - StackWaiterFunction" @@ -1055,6 +1062,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: determine_event.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - DetermineEvent" @@ -1074,6 +1083,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: account_bootstrap.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - CrossAccountExecuteFunction" @@ -1094,6 +1105,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: deployment_account_config.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - RoleStackDeploymentFunction" @@ -1111,6 +1124,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: moved_to_root.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - MovedToRootActionFunction" @@ -1129,6 +1144,8 @@ Resources: Type: "AWS::Serverless::Function" Properties: Handler: generic_account_config.lambda_handler + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - UpdateResourcePoliciesFunction" @@ -1355,7 +1372,7 @@ Resources: phases: install: runtime-versions: - python: 3.11 + python: 3.12 pre_build: commands: - >- @@ -1773,6 +1790,8 @@ Resources: Type: AWS::Serverless::Function Properties: Handler: handler.lambda_handler + Architectures: + - x86_64 CodeUri: lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch Description: "ADF Lambda Function - BootstrapDetermineDefaultBranchName" Policies: @@ -1806,6 +1825,8 @@ Resources: Type: AWS::Serverless::Function Properties: Handler: handler.lambda_handler + Architectures: + - x86_64 CodeUri: lambda_codebase/initial_commit Description: "ADF Lambda Function - BootstrapCreateInitialCommitFunction" Policies: @@ -1864,6 +1885,8 @@ Resources: Properties: Handler: handler.lambda_handler CodeUri: lambda_codebase/cross_region_bucket + Architectures: + - x86_64 Layers: - !Ref LambdaLayerVersion Description: "ADF Lambda Function - Create Deployment Bucket in Main Deployment Region" @@ -1900,6 +1923,8 @@ Resources: Properties: Handler: handler.lambda_handler CodeUri: lambda_codebase/organization + Architectures: + - x86_64 Description: "ADF Lambda Function - Enable AWS Organizations" Policies: - Version: "2012-10-17" @@ -1928,6 +1953,8 @@ Resources: Properties: Handler: handler.lambda_handler CodeUri: lambda_codebase/organization_unit + Architectures: + - x86_64 Description: "ADF Lambda Function - Create Organization Unit" Policies: - Version: "2012-10-17" @@ -1955,6 +1982,8 @@ Resources: Properties: Handler: handler.lambda_handler CodeUri: lambda_codebase/account + Architectures: + - x86_64 Description: "ADF Lambda Function - Create Account" Policies: - Version: "2012-10-17" diff --git a/tox.ini b/tox.ini index 22dd1bfb1..7340d7866 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py310 +envlist = py312 skipsdist = True @@ -21,13 +21,13 @@ setenv= DEPLOYMENT_ACCOUNT_BUCKET=some_deployment_account_bucket MASTER_ACCOUNT_ID=123 ADF_VERSION=1.0.0 - ADF_LOG_LEVEL=INFO + ADF_LOG_LEVEL=CRITICAL ADF_PROJECT_NAME=whatever ACCOUNT_ID=111111111111 allowlist_externals = make - python +# python deps = -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-dev.txt