awslabs · sbkok · Jan 12, 2024 · Dec 5, 2023 · Dec 20, 2023 · Dec 27, 2023
diff --git a/.github/workflows/adf.yml b/.github/workflows/adf.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.12"]
 
     steps:
       - name: Checkout Repo

diff --git a/Makefile b/Makefile
@@ -17,7 +17,7 @@ test:
 
 lint:
 	# Linter performs static analysis to catch latent bugs
-	find src -iname "*.py" -not -path "src/.aws-sam/*" | xargs pylint --rcfile .pylintrc
+	find src -iname "*.py" -not -path "src/.aws-sam/*" | xargs pylint --verbose --rcfile .pylintrc
 	find src -iname "*.yml" -o -iname "*.yaml" -not -path "src/.aws-sam/*" | xargs yamllint -c .yamllint.yml
 	cfn-lint
 

diff --git a/docs/providers-guide.md b/docs/providers-guide.md
@@ -1,5 +1,8 @@
 # Providers Guide
 
+<!-- markdownlint-disable MD024 -->
+<!-- ^ Allow repeated headers to be used in this file -->
+
 Provider types and their properties can be defined as default config for a
 pipeline. But also at the stage level of a pipeline to structure the source,
 build, test, approval, deploy or invoke actions.

diff --git a/docs/user-guide.md b/docs/user-guide.md
@@ -954,8 +954,8 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
-      nodejs: 18
+      python: 3.12
+      nodejs: 20
   pre_build:
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,8 +1,8 @@
-cfn-lint==0.78.2
-isort==5.12.0
+cfn-lint==0.83.8
+isort==5.13.2
 mock==5.1.0
-pylint==2.17.4
-pytest~=7.4.0
-pytest-cov==3.0.0
-tox==3.28.0
-yamllint==1.32.0
+pylint==3.0.3
+pytest~=7.4.4
+pytest-cov==4.1.0
+tox==4.11.4
+yamllint==1.33.0
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,6 @@
-astroid==2.15.6
-boto3==1.28.8
-botocore==1.31.8
+astroid==3.0.2
+aws-sam-cli==1.107.0
+boto3==1.34.17
+botocore==1.34.17
 pyyaml~=6.0.1
 schema==0.7.5
diff --git a/samples/sample-cdk-app/buildspec.yml b/samples/sample-cdk-app/buildspec.yml
@@ -3,8 +3,8 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
-      nodejs: 18
+      python: 3.12
+      nodejs: 20
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-codebuild-vpc/buildspec.yml b/samples/sample-codebuild-vpc/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       # It will connect through the VPC to fetch all the resources.
       # Make sure the subnets and security groups are configured such that

diff --git a/samples/sample-codebuild-vpc/testspec.yml b/samples/sample-codebuild-vpc/testspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-ec2-with-codedeploy/buildspec.yml b/samples/sample-ec2-with-codedeploy/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-ecr-repository/buildspec.yml b/samples/sample-ecr-repository/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-ecs-cluster/buildspec.yml b/samples/sample-ecs-cluster/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-expunge-vpc/buildspec.yml b/samples/sample-expunge-vpc/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-expunge-vpc/template.yml b/samples/sample-expunge-vpc/template.yml
@@ -34,7 +34,7 @@ Resources:
       Handler: lambda_function.lambda_handler
       MemorySize: 128
       Role: !GetAtt LambdaVPCPolicyRole.Arn
-      Runtime: python3.10
+      Runtime: python3.12
       Timeout: 600
       Environment:
         Variables:

diff --git a/samples/sample-fargate-node-app/buildspec.yml b/samples/sample-fargate-node-app/buildspec.yml
@@ -3,8 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      docker: 20
-      python: 3.11
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-iam/buildspec.yml b/samples/sample-iam/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-mono-repo/apps/alpha/buildspec.yml b/samples/sample-mono-repo/apps/alpha/buildspec.yml
@@ -7,7 +7,7 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - cd $INFRASTRUCTURE_ROOT_DIR
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet

diff --git a/samples/sample-mono-repo/apps/beta/buildspec.yml b/samples/sample-mono-repo/apps/beta/buildspec.yml
@@ -7,7 +7,7 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - cd $INFRASTRUCTURE_ROOT_DIR
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet

diff --git a/samples/sample-rdk-rules/buildspec.yml b/samples/sample-rdk-rules/buildspec.yml
@@ -2,8 +2,8 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
-      nodejs: 18
+      python: 3.12
+      nodejs: 20
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json b/samples/sample-rdk-rules/config-rules/EC2_CHECKS_TERMINATION_PROTECTION_ADF/parameters.json
@@ -3,7 +3,7 @@
     "Parameters": {
         "RuleName": "EC2_CHECKS_TERMINATION_PROTECTION_ADF",
         "Description": "EC2_CHECKS_TERMINATION_PROTECTION_ADF",
-        "SourceRuntime": "python3.10",
+        "SourceRuntime": "python3.12",
         "CodeKey": "EC2_CHECKS_TERMINATION_PROTECTION_ADFeu-central-1.zip",
         "InputParameters": "{}",
         "OptionalParameters": "{}",

diff --git a/samples/sample-rdk-rules/requirements.txt b/samples/sample-rdk-rules/requirements.txt
@@ -1,3 +1,3 @@
 s3==3.0.0
-boto3==1.28.8
+boto3==1.34.17
 argparse==1.4.0
diff --git a/samples/sample-serverless-app/buildspec.yml b/samples/sample-serverless-app/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-serverless-app/template.yml b/samples/sample-serverless-app/template.yml
@@ -17,7 +17,7 @@ Resources:
     Type: 'AWS::Serverless::Function'
     Properties:
       Handler: lambda_function.handler
-      Runtime: python3.10
+      Runtime: python3.12
       CodeUri: .
       Description: Sample Lambda Function
       MemorySize: 128

diff --git a/samples/sample-service-catalog-product/buildspec.yml b/samples/sample-service-catalog-product/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/samples/sample-service-catalog-product/productX/template.yml b/samples/sample-service-catalog-product/productX/template.yml
@@ -50,6 +50,7 @@ Resources:
       AutomaticStopTimeMinutes: !Ref AutomaticStopTimeInMinutes
       Description: !Ref InstanceDescription
       InstanceType: !Ref InstanceType
+      ImageId: 'amazonlinux-2023-x86_64'
       Name: !Ref InstanceName
       OwnerArn: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:user/${UserName}"  # In this sample case 'sample-developer' from the IAM stack can be used here
       SubnetId:

diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml
@@ -10,7 +10,7 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-terraform/tf_destroy.yml b/samples/sample-terraform/tf_destroy.yml
@@ -10,7 +10,7 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.9
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml
@@ -10,7 +10,7 @@ env:
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
 
   build:
     commands:

diff --git a/samples/sample-vpc/buildspec.yml b/samples/sample-vpc/buildspec.yml
@@ -3,7 +3,7 @@ version: 0.2
 phases:
   install:
     runtime-versions:
-      python: 3.11
+      python: 3.12
     commands:
       - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet
       - pip install -r adf-build/requirements.txt -q

diff --git a/src/lambda_codebase/account/main.py b/src/lambda_codebase/account/main.py
@@ -5,6 +5,7 @@
 deployment account if required.
 """
 
+import os
 from typing import Mapping, Any, Tuple
 from dataclasses import dataclass, asdict
 import logging
@@ -28,7 +29,8 @@
 ORGANIZATION_CLIENT = boto3.client("organizations")
 SSM_CLIENT = boto3.client("ssm")
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.INFO)
+LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO))
+logging.basicConfig(level=logging.INFO)
 MAX_RETRIES = 120  # => 120 retries * 5 seconds = 10 minutes
 
 

diff --git a/src/lambda_codebase/account_processing/process_account_files.py b/src/lambda_codebase/account_processing/process_account_files.py
@@ -142,7 +142,7 @@ def process_account_list(all_accounts, accounts_in_file):
 
 
 def sanitize_account_name_for_snf(account_name):
-    return re.sub("[^a-zA-Z0-9_]", "_", account_name[:30])
+    return re.sub(r"[^a-zA-Z0-9_]", "_", account_name[:30])
 
 
 def start_executions(

diff --git a/src/lambda_codebase/account_processing/requirements.txt b/src/lambda_codebase/account_processing/requirements.txt
@@ -1,3 +1,2 @@
-aws-xray-sdk==2.12.0
+aws-xray-sdk==2.12.1
 pyyaml~=6.0.1
-wrapt==1.14.1 # https://github.com/aws/aws-lambda-builders/issues/302
diff --git a/src/lambda_codebase/cross_region_bucket/main.py b/src/lambda_codebase/cross_region_bucket/main.py
@@ -7,12 +7,14 @@
 """
 
 
+import os
 from typing import Mapping, Any, Tuple, MutableMapping
 from dataclasses import dataclass, asdict
 import logging
 import json
 import secrets
-import string # pylint: disable=deprecated-module # https://www.logilab.org/ticket/2481
+import string  # pylint: disable=deprecated-module
+# ^ https://www.logilab.org/ticket/2481
 import boto3
 from cfn_custom_resource import ( # pylint: disable=unused-import
     lambda_handler,
@@ -34,7 +36,8 @@
 
 # Globals:
 LOGGER = logging.getLogger(__name__)
-LOGGER.setLevel(logging.INFO)
+LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO))
+logging.basicConfig(level=logging.INFO)
 S3CLIENTS: MutableMapping[Region, S3Client] = {}
 SSM_CLIENT = boto3.client("ssm")
 

diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml
@@ -38,9 +38,7 @@ Parameters:
   ComputeType:
     Description: The Compute Type to use for AWS CodeBuild
     Type: String
-    # BUILD_GENERAL1_LARGE - For threading with large amounts of pipelines
-    # this is the most effective default:
-    Default: "BUILD_GENERAL1_LARGE"
+    Default: "BUILD_GENERAL1_SMALL"
     AllowedValues:
       - "BUILD_GENERAL1_SMALL"  # 3 GB memory, 2 vCPU
       - "BUILD_GENERAL1_MEDIUM"  # 7 GB memory, 4 vCPU
@@ -75,15 +73,15 @@ Conditions:
 Globals:
   Function:
     CodeUri: lambda_codebase
-    Runtime: python3.10
+    Runtime: python3.12
 
 Resources:
   LambdaLayerVersion:
     Type: "AWS::Serverless::LayerVersion"
     Properties:
       ContentUri: "../../adf-build/shared/"
       CompatibleRuntimes:
-        - python3.10
+        - python3.12
       Description: "Shared Lambda Layer between master and deployment account"
       LayerName: shared_layer
 
@@ -689,8 +687,8 @@ Resources:
           phases:
             install:
               runtime-versions:
-                python: 3.11
-                nodejs: 18
+                python: 3.12
+                nodejs: 20
               commands:
                 - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet
                 - pip install -r adf-build/requirements.txt -r adf-build/helpers/requirements.txt -q -t ./adf-build

diff --git a/...sitory/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt b/...sitory/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt
@@ -1,2 +1,2 @@
-boto3==1.28.8
+boto3==1.34.17
 cfn-custom-resource~=1.0.1
diff --git a/...strap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt b/...strap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt
@@ -1,4 +1,4 @@
-Jinja2==3.1.2
-boto3==1.28.8
+Jinja2==3.1.3
+boto3==1.34.17
 cfn-custom-resource~=1.0.1
 markupsafe==2.1.3
diff --git a/..._repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt b/..._repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt
@@ -1,4 +1,3 @@
 pyyaml~=6.0.1
 schema==0.7.5
-tenacity==8.2.2
-wrapt==1.14.1 # https://github.com/aws/aws-lambda-builders/issues/302
+tenacity==8.2.3