diff --git a/CHANGELOG.md b/CHANGELOG.md index b08d73f..bcb40af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,22 @@ -## [0.2.10](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.9...v0.2.10) (2023-12-22) +## [0.2.11](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.10...v0.2.11) (2023-12-23) ### Bug Fixes -* add policy attchments to roles. add a config to apigateway session w longer timeout ([ef2ffb7](https://github.com/FullStackWithLawrence/aws-rekognition/commit/ef2ffb7ee6ad76205d95ea5d1e2e7bba9ef3ab3d)) +* add Services class to control which services are enabled and should be tested ([c217965](https://github.com/FullStackWithLawrence/aws-rekognition/commit/c21796533a2b7a4c37bc61a0b42a0a82241d84d1)) +* raise error if a disabled client setter is called ([8de3c2c](https://github.com/FullStackWithLawrence/aws-rekognition/commit/8de3c2c7cd70676c80314f8bab810eb55c24552e)) -## [0.2.9](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.8...v0.2.9) (2023-12-22) +## [0.2.10](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.9...v0.2.10) (2023-12-22) +### Bug Fixes + +- add policy attachments to roles. add a config to apigateway session w longer timeout ([ef2ffb7](https://github.com/FullStackWithLawrence/aws-rekognition/commit/ef2ffb7ee6ad76205d95ea5d1e2e7bba9ef3ab3d)) + +## [0.2.9](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.8...v0.2.9) (2023-12-22) ### Bug Fixes -* switch aws_s3_client to resource. lint lambda policy. ([d85bfae](https://github.com/FullStackWithLawrence/aws-rekognition/commit/d85bfae995076755d9b4e09e5c3cdfc952b50279)) +- switch aws_s3_client to resource. lint lambda policy. ([d85bfae](https://github.com/FullStackWithLawrence/aws-rekognition/commit/d85bfae995076755d9b4e09e5c3cdfc952b50279)) ## [0.2.8](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.7...v0.2.8) (2023-12-22) diff --git a/README.md b/README.md index 84435b1..7080d18 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) [![hack.d Lawrence McDaniel](https://img.shields.io/badge/hack.d-Lawrence%20McDaniel-orange.svg)](https://lawrencemcdaniel.com) -A facial recognition microservice built with AWS Rekognition, DynamoDB, S3, API Gateway and Lambda. +A facial recognition microservice built with AWS Rekognition, DynamoDB, S3, IAM, CloudWatch, API Gateway and Lambda. See this [json dump](./doc/json/info_endpoint.json) for configuration options. ## Usage @@ -55,12 +55,13 @@ terraform plan terraform apply ``` -## API Key features +## API Features - Highly secure. This project follows best practices for handling AWS credentials. The API runs over https using [AWS managed SSL/TLS](https://aws.amazon.com/certificate-manager/) encryption certificates. The API uses an api key. User data is persisted to a non-public AWS S3 bucket. This api fully implements [CORS (Cross-origin resource sharing)](https://en.wikipedia.org/wiki/Cross-origin_resource_sharing). Backend services run privately, inside an [AWS VPC](https://aws.amazon.com/vpc/), with no public access. - Cost effective. In most cases the running cost of this API remains within AWS' free usage tier for most/all services. - [CloudWatch](https://aws.amazon.com/cloudwatch/) logs for Lambda as well as API Gateway. - [AWS serverless](https://aws.amazon.com/serverless/) implementation using [AWS API Gateway](https://aws.amazon.com/api-gateway/), [AWS DynamoDB](https://aws.amazon.com/dynamodb/), and [AWS Lambda](https://aws.amazon.com/lambda/). +- Meta data endpoint [/info](./doc/json/info_endpoint.json) that returns a JSON dict of the entire platform configuration. - Robust, performant and infinitely scalable. - AWS API Gateway usage policy and managed api key. - Preconfigured [Postman](https://www.postman.com/) files for testing. diff --git a/doc/json/info_endpoint.json b/doc/json/info_endpoint.json new file mode 100644 index 0000000..3a67450 --- /dev/null +++ b/doc/json/info_endpoint.json @@ -0,0 +1,388 @@ +{ + "aws": { + "apigateway": { + "api_id": "287c9j8dcf", + "domains": [ + { + "domainName": "api.rekognition.example.com", + "certificateArn": "arn:aws:acm:us-east-1:012345678901:certificate/613c0965-4495-4ce3-8e3e-63538eb40fd0", + "certificateUploadDate": "2023-12-13", + "distributionDomainName": "d1b9nzmfj7j3jr.cloudfront.net", + "distributionHostedZoneId": "Z2FDTNDATAQYW2", + "endpointConfiguration": { + "types": ["EDGE"] + }, + "domainNameStatus": "AVAILABLE", + "securityPolicy": "TLS_1_2", + "tags": { + "contact": "YOUR CONTACT INFORMATION", + "project": "Facial Recognition microservice", + "shared_resource_identifier": "rekognition", + "terraform": "true" + } + } + ], + "stage": "v1" + }, + "dynamodb": { + "table_name": "arn:aws:dynamodb:us-east-1:012345678901:table/rekognition" + }, + "iam": { + "policies": { + "rekognition-lambda": { + "Arn": "arn:aws:iam::012345678901:policy/rekognition-lambda", + "Policy": { + "Statement": [ + { + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + }, + { + "Action": ["s3:GetObject"], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::012345678901-rekognition-f799c26b853b1d12e9092e66413f4492/*" + ] + }, + { + "Action": ["dynamodb:PutItem"], + "Effect": "Allow", + "Resource": [ + "arn:aws:dynamodb:us-east-1:012345678901:table/rekognition/*" + ] + }, + { + "Action": [ + "apigateway:GET", + "iam:ListPolicies", + "iam:GetPolicy", + "iam:GetPolicyVersion", + "iam:ListAttachedRolePolicies", + "s3:ListAllMyBuckets", + "rekognition:IndexFaces", + "rekognition:DescribeCollection", + "ec2:DescribeRegions", + "route53:ListHostedZones", + "route53:ListResourceRecordSets" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + } + } + }, + "roles": { + "rekognition-apigateway": { + "Arn": "arn:aws:iam::012345678901:role/rekognition-apigateway", + "AttachedPolicies": [ + { + "PolicyName": "AmazonAPIGatewayPushToCloudWatchLogs", + "PolicyArn": "arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs" + } + ], + "Role": { + "Arn": "arn:aws:iam::012345678901:role/rekognition-apigateway", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "apigateway.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ], + "Version": "2012-10-17" + }, + "CreateDate": "2023-12-13", + "Description": "rekognition: Allows API Gateway to push files to an S3 bucket", + "MaxSessionDuration": 3600, + "Path": "/", + "RoleId": "AROARKEXDU3E64TIYPAYH", + "RoleName": "rekognition-apigateway" + } + }, + "rekognition-lambda": { + "Arn": "arn:aws:iam::012345678901:role/rekognition-lambda", + "AttachedPolicies": [ + { + "PolicyName": "AWSLambdaExecute", + "PolicyArn": "arn:aws:iam::aws:policy/AWSLambdaExecute" + }, + { + "PolicyName": "AmazonRekognitionFullAccess", + "PolicyArn": "arn:aws:iam::aws:policy/AmazonRekognitionFullAccess" + }, + { + "PolicyName": "AmazonDynamoDBFullAccess", + "PolicyArn": "arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess" + }, + { + "PolicyName": "AmazonS3ObjectLambdaExecutionRolePolicy", + "PolicyArn": "arn:aws:iam::aws:policy/service-role/AmazonS3ObjectLambdaExecutionRolePolicy" + }, + { + "PolicyName": "rekognition-lambda", + "PolicyArn": "arn:aws:iam::012345678901:policy/rekognition-lambda" + }, + { + "PolicyName": "lambda_logging", + "PolicyArn": "arn:aws:iam::012345678901:policy/lambda_logging" + } + ], + "Role": { + "Arn": "arn:aws:iam::012345678901:role/rekognition-lambda", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ], + "Version": "2012-10-17" + }, + "CreateDate": "2023-12-13", + "MaxSessionDuration": 3600, + "Path": "/", + "RoleId": "AROARKEXDU3E7FDUUVX4I", + "RoleName": "rekognition-lambda" + } + } + } + }, + "lambda": { + "rekognition_index": "arn:aws:lambda:us-east-1:012345678901:function:rekognition_index", + "rekognition_info": "arn:aws:lambda:us-east-1:012345678901:function:rekognition_info", + "rekognition_search": "arn:aws:lambda:us-east-1:012345678901:function:rekognition_search" + }, + "rekognition": { + "collection_id": "rekognition-collection" + }, + "route53": { + "AliasTarget": { + "DNSName": "d1b9nzmfj7j3jr.cloudfront.net.", + "EvaluateTargetHealth": false, + "HostedZoneId": "Z2FDTNDATAQYW2" + }, + "Name": "api.rekognition.example.com.", + "Type": "A" + }, + "s3": { + "bucket_name": "arn:aws:s3:::012345678901-rekognition-f799c26b853b1d12e9092e66413f4492" + } + }, + "settings": { + "aws_apigateway": { + "aws_apigateway_create_custom_domaim": true, + "aws_apigateway_domain_name": "api.rekognition.example.com", + "aws_apigateway_name": "rekognition-api", + "aws_apigateway_root_domain": "example.com" + }, + "aws_auth": { + "aws_access_key_id_source": "overridden by IAM role-based security", + "aws_profile": "lawrence", + "aws_region": "us-east-1", + "aws_secret_access_key_source": "overridden by IAM role-based security" + }, + "aws_dynamodb": { + "aws_dynamodb_table_id": "rekognition" + }, + "aws_lambda": {}, + "aws_rekognition": { + "aws_rekognition_collection_id": "rekognition-collection", + "aws_rekognition_face_detect_attributes": "DEFAULT", + "aws_rekognition_face_detect_max_faces_count": 10, + "aws_rekognition_face_detect_quality_filter": "AUTO", + "aws_rekognition_face_detect_threshold": 10 + }, + "aws_s3": { + "aws_s3_bucket_prefix": "012345678901-rekognition" + }, + "environment": { + "boto3": "1.27.1", + "debug_mode": true, + "dump_defaults": true, + "is_using_aws_dynamodb": true, + "is_using_aws_rekognition": true, + "is_using_dotenv_file": false, + "is_using_tfvars_file": true, + "os": "posix", + "python_build": ["main", "Oct 2 2023 18:13:32"], + "python_compiler": "GCC 7.3.1 20180712 (Red Hat 7.3.1-17)", + "python_implementation": "CPython", + "python_installed_packages": [ + { + "name": "certifi", + "version": "2023.11.17" + }, + { + "name": "typing-extensions", + "version": "4.9.0" + }, + { + "name": "python-hcl2", + "version": "4.3.2" + }, + { + "name": "idna", + "version": "3.6" + }, + { + "name": "charset-normalizer", + "version": "3.3.2" + }, + { + "name": "requests", + "version": "2.31.0" + }, + { + "name": "pydantic-core", + "version": "2.14.5" + }, + { + "name": "pydantic", + "version": "2.5.2" + }, + { + "name": "pydantic-settings", + "version": "2.1.0" + }, + { + "name": "urllib3", + "version": "2.1.0" + }, + { + "name": "lark", + "version": "1.1.8" + }, + { + "name": "python-dotenv", + "version": "1.0.0" + }, + { + "name": "annotated-types", + "version": "0.6.0" + }, + { + "name": "setuptools", + "version": "65.5.0" + }, + { + "name": "pip", + "version": "23.2.1" + }, + { + "name": "simplejson", + "version": "3.17.2" + }, + { + "name": "python-dateutil", + "version": "2.8.2" + }, + { + "name": "awslambdaric", + "version": "2.0.7" + }, + { + "name": "botocore", + "version": "1.30.1" + }, + { + "name": "boto3", + "version": "1.27.1" + }, + { + "name": "six", + "version": "1.16.0" + }, + { + "name": "jmespath", + "version": "1.0.1" + }, + { + "name": "s3transfer", + "version": "0.6.1" + } + ], + "python_version": "3.11.6", + "release": "5.10.201-213.748.amzn2.x86_64", + "shared_resource_identifier": "rekognition", + "system": "Linux", + "tfvars": { + "aws_account_id": "****", + "aws_apigateway_create_custom_domaim": true, + "aws_apigateway_root_domain": "example.com", + "aws_profile": "lawrence", + "aws_region": "us-east-1", + "aws_rekognition_face_detect_attributes": "DEFAULT", + "aws_rekognition_face_detect_quality_filter": "AUTO", + "aws_rekognition_face_detect_threshold": 10, + "aws_rekognition_max_faces_count": 10, + "debug_mode": true, + "lambda_memory_size": 256, + "lambda_python_runtime": "python3.11", + "lambda_timeout": 60, + "log_retention_days": 3, + "logging_level": "INFO", + "quota_settings_limit": 500, + "quota_settings_offset": 0, + "quota_settings_period": "DAY", + "shared_resource_identifier": "rekognition", + "stage": "v1", + "tags": { + "contact": "YOUR CONTACT INFORMATION", + "project": "Facial Recognition microservice", + "shared_resource_identifier": "rekognition", + "terraform": "true" + }, + "throttle_settings_burst_limit": 25, + "throttle_settings_rate_limit": 10 + }, + "version": "0.2.11" + }, + "services": [ + "apigateway", + "aws-cli", + "cloudwatch", + "dynamodb", + "ec2", + "iam", + "lambda", + "rekognition", + "route53", + "s3" + ], + "settings_defaults": { + "AWS_ACCESS_KEY_ID": "***MASKED***", + "AWS_APIGATEWAY_CONNECT_TIMEOUT": 70, + "AWS_APIGATEWAY_CREATE_CUSTOM_DOMAIN": true, + "AWS_APIGATEWAY_MAX_ATTEMPTS": 10, + "AWS_APIGATEWAY_READ_TIMEOUT": 70, + "AWS_APIGATEWAY_ROOT_DOMAIN": "example.com", + "AWS_DYNAMODB_TABLE_ID": "rekognition", + "AWS_PROFILE": "lawrence", + "AWS_REGION": "us-east-1", + "AWS_REKOGNITION_COLLECTION_ID": "rekognition-collection", + "AWS_REKOGNITION_FACE_DETECT_ATTRIBUTES": "DEFAULT", + "AWS_REKOGNITION_FACE_DETECT_MAX_FACES_COUNT": 10, + "AWS_REKOGNITION_FACE_DETECT_QUALITY_FILTER": "AUTO", + "AWS_REKOGNITION_FACE_DETECT_THRESHOLD": 10, + "AWS_SECRET_ACCESS_KEY": "***MASKED***", + "DEBUG_MODE": true, + "DUMP_DEFAULTS": true, + "SHARED_RESOURCE_IDENTIFIER": "rekognition" + } + } +} diff --git a/terraform/python/rekognition_api/__version__.py b/terraform/python/rekognition_api/__version__.py index e145ef0..a736d2d 100644 --- a/terraform/python/rekognition_api/__version__.py +++ b/terraform/python/rekognition_api/__version__.py @@ -1,3 +1,4 @@ # -*- coding: utf-8 -*- +# DO NOT EDIT. # Managed via automated CI/CD in .github/workflows/semanticVersionBump.yml. -__version__ = "0.2.9-next.1" +__version__ = "0.2.11-next.1" diff --git a/terraform/python/rekognition_api/aws.py b/terraform/python/rekognition_api/aws.py index e5877e5..e7cd4c3 100644 --- a/terraform/python/rekognition_api/aws.py +++ b/terraform/python/rekognition_api/aws.py @@ -5,7 +5,7 @@ import socket # our stuff -from rekognition_api.conf import settings +from rekognition_api.conf import Services, settings from rekognition_api.utils import recursive_sort_dict @@ -16,32 +16,28 @@ class AWSInfrastructureConfig: @property def dump(self): """Return a dict of the AWS infrastructure config.""" - api = self.get_api(settings.aws_apigateway_name) - - retval = { - "apigateway": { + retval = {} + if Services.enabled(Services.AWS_APIGATEWAY): + api = self.get_api(settings.aws_apigateway_name) + retval["apigateway"] = { "api_id": api.get("id"), "stage": self.get_api_stage(), "domains": self.get_api_custom_domains(), - }, - "dynamodb": { - "table_name": self.get_dyanmodb_table_by_name( - settings.aws_dynamodb_table_id, - ) - }, - "s3": { - "bucket_name": self.get_bucket_by_prefix(settings.aws_s3_bucket_name), - }, - "rekognition": { - "collection_id": self.get_rekognition_collection_by_id(settings.aws_rekognition_collection_id), - }, - "iam": { - "policies": self.get_iam_policies(), - "roles": self.get_iam_roles(), - }, - "lambda": self.get_lambdas(), - "route53": self.get_dns_record_from_hosted_zone(), - } + } + if Services.enabled(Services.AWS_S3): + retval["s3"] = {"bucket_name": self.get_bucket_by_prefix(settings.aws_s3_bucket_name)} + if Services.enabled(Services.AWS_DYNAMODB): + retval["dynamodb"] = {"table_name": self.get_dyanmodb_table_by_name(settings.aws_dynamodb_table_id)} + if Services.enabled(Services.AWS_REKOGNITION): + retval["rekognition"] = { + "collection_id": self.get_rekognition_collection_by_id(settings.aws_rekognition_collection_id) + } + if Services.enabled(Services.AWS_IAM): + retval["iam"] = {"policies": self.get_iam_policies(), "roles": self.get_iam_roles()} + if Services.enabled(Services.AWS_LAMBDA): + retval["lambda"] = self.get_lambdas() + if Services.enabled(Services.AWS_ROUTE53): + retval["route53"] = self.get_dns_record_from_hosted_zone() return recursive_sort_dict(retval) def get_lambdas(self): diff --git a/terraform/python/rekognition_api/conf.py b/terraform/python/rekognition_api/conf.py index 95e85ca..4e989a2 100644 --- a/terraform/python/rekognition_api/conf.py +++ b/terraform/python/rekognition_api/conf.py @@ -4,26 +4,25 @@ Configuration for Lambda functions. This module is used to configure the Lambda functions. It uses the pydantic_settings -library to validate the configuration values. The configuration values are read from -environment variables, or alternatively these can be set when instantiating Settings(). - -The configuration values are validated using pydantic. If the configuration values are -invalid, then a RekognitionConfigurationError is raised. - -The configuration values are dumped to a dict using the dump property. This is used -to display the configuration values in the /info endpoint. - +library to validate the configuration values. The configuration values are initialized +according to the following prioritization sequence: + 1. constructor + 2. environment variables + 3. dotenv file + 4. tfvars file + 5. defaults + +The Settings class also provides a dump property that returns a dictionary of all +configuration values. This is useful for debugging and logging. """ - -import importlib.util - # python stuff +import importlib.util import logging import os # library for interacting with the operating system import platform # library to view information about the server host this Lambda runs on import re -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple, Union # 3rd party stuff import boto3 # AWS SDK for Python https://boto3.amazonaws.com/v1/documentation/api/latest/index.html @@ -33,9 +32,9 @@ from dotenv import load_dotenv from pydantic import Field, SecretStr, ValidationError, ValidationInfo, field_validator from pydantic_settings import BaseSettings -from rekognition_api.const import HERE, IS_USING_TFVARS, TFVARS # our stuff +from rekognition_api.const import HERE, IS_USING_TFVARS, TFVARS from rekognition_api.exceptions import ( RekognitionConfigurationError, RekognitionValueError, @@ -51,6 +50,8 @@ def load_version() -> Dict[str, str]: """Stringify the __version__ module.""" version_file_path = os.path.join(HERE, "__version__.py") + if not os.path.exists(version_file_path): + return {} spec = importlib.util.spec_from_file_location("__version__", version_file_path) version_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(version_module) @@ -78,29 +79,92 @@ def get_semantic_version() -> str: - pypi does not allow semantic version numbers to contain a 'v' prefix. - pypi does not allow semantic version numbers to contain a 'next' suffix. """ - version = VERSION["__version__"] + if not isinstance(VERSION, dict): + return "unknown" + + version = VERSION.get("__version__") + if not version: + return "unknown" version = re.sub(r"-next\.\d+", "", version) return re.sub(r"-next-major\.\d+", "", version) +class Services: + """Services enabled for this solution. This is intended to be permanently read-only""" + + AWS_CLI = ("aws-cli", True) + AWS_APIGATEWAY = ("apigateway", True) + AWS_CLOUDWATCH = ("cloudwatch", True) + AWS_DYNAMODB = ("dynamodb", True) + AWS_EC2 = ("ec2", True) + AWS_IAM = ("iam", True) + AWS_LAMBDA = ("lambda", True) + AWS_REKOGNITION = ("rekognition", True) + AWS_ROUTE53 = ("route53", True) + AWS_S3 = ("s3", True) + AWS_RDS = ("rds", False) + + @classmethod + def enabled(cls, service: Union[str, Tuple[str, bool]]) -> bool: + """Is the service enabled?""" + if isinstance(service, tuple): + service = service[0] + return service in cls.enabled_services() + + @classmethod + def raise_error_on_disabled(cls, service: Union[str, Tuple[str, bool]]) -> None: + """Raise an error if the service is disabled""" + if not cls.enabled(service): + raise RekognitionConfigurationError(f"{service} is not enabled. See conf.Services") + + @classmethod + def to_dict(cls): + """Convert Services to dict""" + return { + key: value + for key, value in Services.__dict__.items() + if not key.startswith("__") and not callable(key) and key != "to_dict" + } + + @classmethod + def enabled_services(cls) -> List[str]: + """Return a list of enabled services""" + return [ + getattr(cls, key)[0] + for key in dir(cls) + if not key.startswith("__") + and not callable(getattr(cls, key)) + and key != "to_dict" + and getattr(cls, key)[1] is True + ] + + class SettingsDefaults: """Default values for Settings""" + # defaults for this Python package + SHARED_RESOURCE_IDENTIFIER = TFVARS.get("shared_resource_identifier", "rekognition") + DEBUG_MODE: bool = bool(TFVARS.get("debug_mode", False)) + DUMP_DEFAULTS: bool = bool(TFVARS.get("dump_defaults", True)) + + # aws auth AWS_PROFILE = TFVARS.get("aws_profile", None) AWS_ACCESS_KEY_ID = SecretStr(None) AWS_SECRET_ACCESS_KEY = SecretStr(None) AWS_REGION = TFVARS.get("aws_region", "us-east-1") - DUMP_DEFAULTS = TFVARS.get("dump_defaults", False) - DEBUG_MODE: bool = bool(TFVARS.get("debug_mode", False)) - SHARED_RESOURCE_IDENTIFIER = TFVARS.get("shared_resource_identifier", "rekognition_api") - + # aws api gateway defaults AWS_APIGATEWAY_CREATE_CUSTOM_DOMAIN = TFVARS.get("aws_apigateway_create_custom_domaim", False) AWS_APIGATEWAY_ROOT_DOMAIN = TFVARS.get("aws_apigateway_root_domain", None) + AWS_APIGATEWAY_READ_TIMEOUT: int = TFVARS.get("aws_apigateway_read_timeout", 70) + AWS_APIGATEWAY_CONNECT_TIMEOUT: int = TFVARS.get("aws_apigateway_connect_timeout", 70) + AWS_APIGATEWAY_MAX_ATTEMPTS: int = TFVARS.get("aws_apigateway_max_attempts", 10) - AWS_DYNAMODB_TABLE_ID = "rekognition" + # aws dynamodb defaults + AWS_DYNAMODB_TABLE_ID = SHARED_RESOURCE_IDENTIFIER - AWS_REKOGNITION_COLLECTION_ID = AWS_DYNAMODB_TABLE_ID + "-collection" + # aws rekognition defaults + AWS_REKOGNITION_COLLECTION_ID = SHARED_RESOURCE_IDENTIFIER + "-collection" AWS_REKOGNITION_FACE_DETECT_MAX_FACES_COUNT: int = int(TFVARS.get("aws_rekognition_max_faces_count", 10)) AWS_REKOGNITION_FACE_DETECT_THRESHOLD: int = int(TFVARS.get("aws_rekognition_face_detect_threshold", 10)) AWS_REKOGNITION_FACE_DETECT_ATTRIBUTES = TFVARS.get("aws_rekognition_face_detect_attributes", "DEFAULT") @@ -110,15 +174,17 @@ class SettingsDefaults: def to_dict(cls): """Convert SettingsDefaults to dict""" return { - key: value + key: "***MASKED***" if key in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] else value for key, value in SettingsDefaults.__dict__.items() if not key.startswith("__") and not callable(key) and key != "to_dict" } -ec2 = boto3.Session().client("ec2") -regions = ec2.describe_regions() -AWS_REGIONS = [region["RegionName"] for region in regions["Regions"]] +AWS_REGIONS = [] +if Services.enabled(Services.AWS_EC2): + ec2 = boto3.Session().client("ec2") + regions = ec2.describe_regions() + AWS_REGIONS = [region["RegionName"] for region in regions["Regions"]] def empty_str_to_bool_default(v: str, default: bool) -> bool: @@ -156,6 +222,10 @@ class Settings(BaseSettings): # pylint: disable=too-many-branches def __init__(self, **data: Any): super().__init__(**data) + if not Services.enabled(Services.AWS_CLI): + self._initialized = True + return + if bool(os.environ.get("AWS_DEPLOYED", False)): # If we're running inside AWS Lambda, then we don't need to set the AWS credentials. self._aws_access_key_id_source: str = "overridden by IAM role-based security" @@ -163,7 +233,7 @@ def __init__(self, **data: Any): self._aws_session = boto3.Session() self._initialized = True - if not self._initialized and bool(os.environ.get("GITHUB_ACTIONS", False)): + if not self.initialized and bool(os.environ.get("GITHUB_ACTIONS", False)): try: self._aws_session = boto3.Session( region_name=os.environ.get("AWS_REGION", "us-east-1"), @@ -181,13 +251,13 @@ def __init__(self, **data: Any): self._aws_secret_access_key_source = "environ" self._initialized = True - if not self._initialized: + if not self.initialized: if self.aws_profile: self._aws_access_key_id_source = "aws_profile" self._aws_secret_access_key_source = "aws_profile" self._initialized = True - if not self._initialized: + if not self.initialized: if "aws_access_key_id" in data or "aws_secret_access_key" in data: if "aws_access_key_id" in data: self._aws_access_key_id_source = "constructor" @@ -195,7 +265,7 @@ def __init__(self, **data: Any): self._aws_secret_access_key_source = "constructor" self._initialized = True - if not self._initialized: + if not self.initialized: if "AWS_ACCESS_KEY_ID" in os.environ: self._aws_access_key_id_source = "environ" if "AWS_SECRET_ACCESS_KEY" in os.environ: @@ -282,10 +352,24 @@ def __init__(self, **data: Any): getter=lambda v: empty_str_to_int_default(v, SettingsDefaults.AWS_REKOGNITION_FACE_DETECT_THRESHOLD), ) + @property + def initialized(self): + """Is settings initialized?""" + return self._initialized + @property def aws_account_id(self): """AWS account id""" - return self.aws_session.client("sts").get_caller_identity()["Account"] + Services.raise_error_on_disabled(Services.AWS_CLI) + sts_client = self.aws_session.client("sts") + if not sts_client: + logger.warning("could not initialize sts_client") + return None + retval = sts_client.get_caller_identity() + if not isinstance(retval, dict): + logger.warning("sts_client.get_caller_identity() did not return a dict") + return None + return retval.get("Account", None) @property def aws_access_key_id_source(self): @@ -310,6 +394,7 @@ def aws_auth(self) -> dict: @property def aws_session(self): """AWS session""" + Services.raise_error_on_disabled(Services.AWS_CLI) if not self._aws_session: if self.aws_profile: logger.debug("creating new aws_session with aws_profile: %s", self.aws_profile) @@ -334,19 +419,26 @@ def aws_session(self): @property def aws_route53_client(self): """Route53 client""" + Services.raise_error_on_disabled(Services.AWS_ROUTE53) return self.aws_session.client("route53") @property def aws_apigateway_client(self): """API Gateway client""" + Services.raise_error_on_disabled(Services.AWS_APIGATEWAY) if not self._aws_apigateway_client: - config = Config(read_timeout=70, connect_timeout=70, retries={"max_attempts": 10}) + config = Config( + read_timeout=SettingsDefaults.AWS_APIGATEWAY_READ_TIMEOUT, + connect_timeout=SettingsDefaults.AWS_APIGATEWAY_CONNECT_TIMEOUT, + retries={"max_attempts": SettingsDefaults.AWS_APIGATEWAY_MAX_ATTEMPTS}, + ) self._aws_apigateway_client = self.aws_session.client("apigateway", config=config) return self._aws_apigateway_client @property def aws_s3_client(self): """S3 client""" + Services.raise_error_on_disabled(Services.AWS_S3) if not self._aws_s3_client: self._aws_s3_client = self.aws_session.resource("s3") return self._aws_s3_client @@ -354,6 +446,7 @@ def aws_s3_client(self): @property def aws_dynamodb_client(self): """DynamoDB client""" + Services.raise_error_on_disabled(Services.AWS_DYNAMODB) if not self._aws_dynamodb_client: self._aws_dynamodb_client = self.aws_session.client("dynamodb") return self._aws_dynamodb_client @@ -361,6 +454,7 @@ def aws_dynamodb_client(self): @property def aws_rekognition_client(self): """Rekognition client""" + Services.raise_error_on_disabled(Services.AWS_REKOGNITION) if not self._aws_rekognition_client: self._aws_rekognition_client = self.aws_session.client("rekognition") return self._aws_rekognition_client @@ -368,6 +462,7 @@ def aws_rekognition_client(self): @property def dynamodb_table(self): """DynamoDB table""" + Services.raise_error_on_disabled(Services.AWS_DYNAMODB) dynamodb_resource = boto3.resource("dynamodb") return dynamodb_resource.Table(self.aws_dynamodb_table_id) @@ -442,16 +537,17 @@ def get_installed_packages(): package_list = [(d.project_name, d.version) for d in installed_packages] return package_list - if self._dump and self._initialized: + if self._dump and self.initialized: return self._dump - if not self._initialized: + if not self.initialized: return {} packages = get_installed_packages() packages_dict = [{"name": name, "version": version} for name, version in packages] self._dump = { + "services": Services.enabled_services(), "environment": { "is_using_tfvars_file": self.is_using_tfvars_file, "is_using_dotenv_file": self.is_using_dotenv_file, diff --git a/terraform/python/rekognition_api/tests/test_aws.py b/terraform/python/rekognition_api/tests/test_aws.py index 043e808..e1e0715 100644 --- a/terraform/python/rekognition_api/tests/test_aws.py +++ b/terraform/python/rekognition_api/tests/test_aws.py @@ -2,6 +2,8 @@ # pylint: disable=wrong-import-position """Test configuration Settings class.""" +import inspect + # python stuff import os import sys @@ -19,7 +21,7 @@ # our stuff from rekognition_api.aws import aws_infrastructure_config as aws_config # noqa: E402 -from rekognition_api.conf import settings # noqa: E402 +from rekognition_api.conf import Services, settings # noqa: E402 from rekognition_api.tests.test_setup import get_test_image # noqa: E402 @@ -41,6 +43,9 @@ def setUp(self): def test_rekognition_collection_exists(self): """Test that the Rekognition collection exists.""" + if not Services.enabled(Services.AWS_REKOGNITION): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.rekognition_collection_exists(), f"Rekognition collection {settings.aws_rekognition_collection_id} does not exist.", @@ -48,6 +53,9 @@ def test_rekognition_collection_exists(self): def test_aws_connection_works(self): """Test that the AWS connection works.""" + if not Services.enabled(Services.AWS_CLI): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue(aws_config.aws_connection_works(), "AWS connection failed.") def test_domain_exists(self): @@ -56,11 +64,17 @@ def test_domain_exists(self): def test_bucket_exists(self): """Test that the S3 bucket exists.""" + if not Services.enabled(Services.AWS_S3): + print("skipping: ", inspect.currentframe().f_code.co_name) + return bucket_prefix = settings.aws_s3_bucket_name self.assertTrue(aws_config.bucket_exists(bucket_prefix), f"S3 bucket {bucket_prefix} does not exist.") def test_dynamodb_table_exists(self): """Test that the DynamoDB table exists.""" + if not Services.enabled(Services.AWS_DYNAMODB): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.dynamodb_table_exists(settings.shared_resource_identifier), f"DynamoDB table {settings.shared_resource_identifier} does not exist.", @@ -68,11 +82,16 @@ def test_dynamodb_table_exists(self): def test_api_exists(self): """Test that the API Gateway exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return api = aws_config.get_api(settings.aws_apigateway_name) self.assertIsInstance(api, dict, "API Gateway does not exist.") def test_api_resource_index_exists(self): """Test that the API Gateway index resource exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + return self.assertTrue( aws_config.api_resource_and_method_exists("/index/{filename}", "PUT"), "API Gateway index (PUT) resource does not exist.", @@ -80,6 +99,9 @@ def test_api_resource_index_exists(self): def test_api_resource_search_exists(self): """Test that the API Gateway index resource exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.api_resource_and_method_exists("/search", "ANY"), "API Gateway search (ANY) resource does not exist.", @@ -87,12 +109,18 @@ def test_api_resource_search_exists(self): def test_api_key_exists(self): """Test that an API key exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return api_key = aws_config.get_api_keys() self.assertIsInstance(api_key, str, "API key does not exist.") self.assertGreaterEqual(len(api_key), 15, "API key is too short.") def test_index_endpoint(self): """Test that the index endpoint works.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return filename = "Keanu-Reeves.jpg" api_key = aws_config.get_api_keys() url = aws_config.get_url(f"/index/{filename}") @@ -106,6 +134,9 @@ def test_index_endpoint(self): def test_search_endpoint(self): """Test that the search endpoint works.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return filename = "Keanu-Avril-Mike.jpg" api_key = aws_config.get_api_keys() url = aws_config.get_url("/search")