diff --git a/CHANGELOG.md b/CHANGELOG.md index 805e239..b622ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,8 @@ ## [0.2.10](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.9...v0.2.10) (2023-12-22) - ### Bug Fixes -* add policy attchments to roles. add a config to apigateway session w longer timeout ([ef2ffb7](https://github.com/FullStackWithLawrence/aws-rekognition/commit/ef2ffb7ee6ad76205d95ea5d1e2e7bba9ef3ab3d)) +- add policy attachments to roles. add a config to apigateway session w longer timeout ([ef2ffb7](https://github.com/FullStackWithLawrence/aws-rekognition/commit/ef2ffb7ee6ad76205d95ea5d1e2e7bba9ef3ab3d)) ## [0.2.9](https://github.com/FullStackWithLawrence/aws-rekognition/compare/v0.2.8...v0.2.9) (2023-12-22) diff --git a/README.md b/README.md index 51c9047..7080d18 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) [![hack.d Lawrence McDaniel](https://img.shields.io/badge/hack.d-Lawrence%20McDaniel-orange.svg)](https://lawrencemcdaniel.com) -A facial recognition microservice built with AWS Rekognition, DynamoDB, S3, API Gateway and Lambda. +A facial recognition microservice built with AWS Rekognition, DynamoDB, S3, IAM, CloudWatch, API Gateway and Lambda. See this [json dump](./doc/json/info_endpoint.json) for configuration options. ## Usage diff --git a/doc/json/info_endpoint.json b/doc/json/info_endpoint.json index b732693..3a67450 100644 --- a/doc/json/info_endpoint.json +++ b/doc/json/info_endpoint.json @@ -1,21 +1,21 @@ { "aws": { "apigateway": { - "api_id": "287c8j8dcf", + "api_id": "287c9j8dcf", "domains": [ { "domainName": "api.rekognition.example.com", "certificateArn": "arn:aws:acm:us-east-1:012345678901:certificate/613c0965-4495-4ce3-8e3e-63538eb40fd0", "certificateUploadDate": "2023-12-13", - "distributionDomainName": "d1b9nzmXj7j3jr.cloudfront.net", - "distributionHostedZoneId": "ABCDEFG123HIJKL", + "distributionDomainName": "d1b9nzmfj7j3jr.cloudfront.net", + "distributionHostedZoneId": "Z2FDTNDATAQYW2", "endpointConfiguration": { "types": ["EDGE"] }, "domainNameStatus": "AVAILABLE", "securityPolicy": "TLS_1_2", "tags": { - "contact": "YOUR CONTACT INFO HERE", + "contact": "YOUR CONTACT INFORMATION", "project": "Facial Recognition microservice", "shared_resource_identifier": "rekognition", "terraform": "true" @@ -172,9 +172,9 @@ }, "route53": { "AliasTarget": { - "DNSName": "d1b9nzmXj7j3jr.cloudfront.net.", + "DNSName": "d1b9nzmfj7j3jr.cloudfront.net.", "EvaluateTargetHealth": false, - "HostedZoneId": "ABCDEFG123HIJKL" + "HostedZoneId": "Z2FDTNDATAQYW2" }, "Name": "api.rekognition.example.com.", "Type": "A" @@ -192,7 +192,7 @@ }, "aws_auth": { "aws_access_key_id_source": "overridden by IAM role-based security", - "aws_profile": "default", + "aws_profile": "lawrence", "aws_region": "us-east-1", "aws_secret_access_key_source": "overridden by IAM role-based security" }, @@ -324,7 +324,7 @@ "aws_account_id": "****", "aws_apigateway_create_custom_domaim": true, "aws_apigateway_root_domain": "example.com", - "aws_profile": "default", + "aws_profile": "lawrence", "aws_region": "us-east-1", "aws_rekognition_face_detect_attributes": "DEFAULT", "aws_rekognition_face_detect_quality_filter": "AUTO", @@ -342,7 +342,7 @@ "shared_resource_identifier": "rekognition", "stage": "v1", "tags": { - "contact": "YOUR CONTACT INFO HERE", + "contact": "YOUR CONTACT INFORMATION", "project": "Facial Recognition microservice", "shared_resource_identifier": "rekognition", "terraform": "true" @@ -350,8 +350,20 @@ "throttle_settings_burst_limit": 25, "throttle_settings_rate_limit": 10 }, - "version": "0.2.10" + "version": "0.2.11" }, + "services": [ + "apigateway", + "aws-cli", + "cloudwatch", + "dynamodb", + "ec2", + "iam", + "lambda", + "rekognition", + "route53", + "s3" + ], "settings_defaults": { "AWS_ACCESS_KEY_ID": "***MASKED***", "AWS_APIGATEWAY_CONNECT_TIMEOUT": 70, @@ -360,7 +372,7 @@ "AWS_APIGATEWAY_READ_TIMEOUT": 70, "AWS_APIGATEWAY_ROOT_DOMAIN": "example.com", "AWS_DYNAMODB_TABLE_ID": "rekognition", - "AWS_PROFILE": null, + "AWS_PROFILE": "lawrence", "AWS_REGION": "us-east-1", "AWS_REKOGNITION_COLLECTION_ID": "rekognition-collection", "AWS_REKOGNITION_FACE_DETECT_ATTRIBUTES": "DEFAULT", diff --git a/terraform/python/rekognition_api/__version__.py b/terraform/python/rekognition_api/__version__.py index 537f905..a736d2d 100644 --- a/terraform/python/rekognition_api/__version__.py +++ b/terraform/python/rekognition_api/__version__.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- # DO NOT EDIT. # Managed via automated CI/CD in .github/workflows/semanticVersionBump.yml. -__version__ = "0.2.10-next.1" +__version__ = "0.2.11-next.1" diff --git a/terraform/python/rekognition_api/aws.py b/terraform/python/rekognition_api/aws.py index e5877e5..e7cd4c3 100644 --- a/terraform/python/rekognition_api/aws.py +++ b/terraform/python/rekognition_api/aws.py @@ -5,7 +5,7 @@ import socket # our stuff -from rekognition_api.conf import settings +from rekognition_api.conf import Services, settings from rekognition_api.utils import recursive_sort_dict @@ -16,32 +16,28 @@ class AWSInfrastructureConfig: @property def dump(self): """Return a dict of the AWS infrastructure config.""" - api = self.get_api(settings.aws_apigateway_name) - - retval = { - "apigateway": { + retval = {} + if Services.enabled(Services.AWS_APIGATEWAY): + api = self.get_api(settings.aws_apigateway_name) + retval["apigateway"] = { "api_id": api.get("id"), "stage": self.get_api_stage(), "domains": self.get_api_custom_domains(), - }, - "dynamodb": { - "table_name": self.get_dyanmodb_table_by_name( - settings.aws_dynamodb_table_id, - ) - }, - "s3": { - "bucket_name": self.get_bucket_by_prefix(settings.aws_s3_bucket_name), - }, - "rekognition": { - "collection_id": self.get_rekognition_collection_by_id(settings.aws_rekognition_collection_id), - }, - "iam": { - "policies": self.get_iam_policies(), - "roles": self.get_iam_roles(), - }, - "lambda": self.get_lambdas(), - "route53": self.get_dns_record_from_hosted_zone(), - } + } + if Services.enabled(Services.AWS_S3): + retval["s3"] = {"bucket_name": self.get_bucket_by_prefix(settings.aws_s3_bucket_name)} + if Services.enabled(Services.AWS_DYNAMODB): + retval["dynamodb"] = {"table_name": self.get_dyanmodb_table_by_name(settings.aws_dynamodb_table_id)} + if Services.enabled(Services.AWS_REKOGNITION): + retval["rekognition"] = { + "collection_id": self.get_rekognition_collection_by_id(settings.aws_rekognition_collection_id) + } + if Services.enabled(Services.AWS_IAM): + retval["iam"] = {"policies": self.get_iam_policies(), "roles": self.get_iam_roles()} + if Services.enabled(Services.AWS_LAMBDA): + retval["lambda"] = self.get_lambdas() + if Services.enabled(Services.AWS_ROUTE53): + retval["route53"] = self.get_dns_record_from_hosted_zone() return recursive_sort_dict(retval) def get_lambdas(self): diff --git a/terraform/python/rekognition_api/conf.py b/terraform/python/rekognition_api/conf.py index f70a3e7..4e989a2 100644 --- a/terraform/python/rekognition_api/conf.py +++ b/terraform/python/rekognition_api/conf.py @@ -4,26 +4,25 @@ Configuration for Lambda functions. This module is used to configure the Lambda functions. It uses the pydantic_settings -library to validate the configuration values. The configuration values are read from -environment variables, or alternatively these can be set when instantiating Settings(). - -The configuration values are validated using pydantic. If the configuration values are -invalid, then a RekognitionConfigurationError is raised. - -The configuration values are dumped to a dict using the dump property. This is used -to display the configuration values in the /info endpoint. - +library to validate the configuration values. The configuration values are initialized +according to the following prioritization sequence: + 1. constructor + 2. environment variables + 3. dotenv file + 4. tfvars file + 5. defaults + +The Settings class also provides a dump property that returns a dictionary of all +configuration values. This is useful for debugging and logging. """ - -import importlib.util - # python stuff +import importlib.util import logging import os # library for interacting with the operating system import platform # library to view information about the server host this Lambda runs on import re -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple, Union # 3rd party stuff import boto3 # AWS SDK for Python https://boto3.amazonaws.com/v1/documentation/api/latest/index.html @@ -33,9 +32,9 @@ from dotenv import load_dotenv from pydantic import Field, SecretStr, ValidationError, ValidationInfo, field_validator from pydantic_settings import BaseSettings -from rekognition_api.const import HERE, IS_USING_TFVARS, TFVARS # our stuff +from rekognition_api.const import HERE, IS_USING_TFVARS, TFVARS from rekognition_api.exceptions import ( RekognitionConfigurationError, RekognitionValueError, @@ -51,6 +50,8 @@ def load_version() -> Dict[str, str]: """Stringify the __version__ module.""" version_file_path = os.path.join(HERE, "__version__.py") + if not os.path.exists(version_file_path): + return {} spec = importlib.util.spec_from_file_location("__version__", version_file_path) version_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(version_module) @@ -78,18 +79,73 @@ def get_semantic_version() -> str: - pypi does not allow semantic version numbers to contain a 'v' prefix. - pypi does not allow semantic version numbers to contain a 'next' suffix. """ - version = VERSION["__version__"] + if not isinstance(VERSION, dict): + return "unknown" + + version = VERSION.get("__version__") + if not version: + return "unknown" version = re.sub(r"-next\.\d+", "", version) return re.sub(r"-next-major\.\d+", "", version) +class Services: + """Services enabled for this solution. This is intended to be permanently read-only""" + + AWS_CLI = ("aws-cli", True) + AWS_APIGATEWAY = ("apigateway", True) + AWS_CLOUDWATCH = ("cloudwatch", True) + AWS_DYNAMODB = ("dynamodb", True) + AWS_EC2 = ("ec2", True) + AWS_IAM = ("iam", True) + AWS_LAMBDA = ("lambda", True) + AWS_REKOGNITION = ("rekognition", True) + AWS_ROUTE53 = ("route53", True) + AWS_S3 = ("s3", True) + AWS_RDS = ("rds", False) + + @classmethod + def enabled(cls, service: Union[str, Tuple[str, bool]]) -> bool: + """Is the service enabled?""" + if isinstance(service, tuple): + service = service[0] + return service in cls.enabled_services() + + @classmethod + def raise_error_on_disabled(cls, service: Union[str, Tuple[str, bool]]) -> None: + """Raise an error if the service is disabled""" + if not cls.enabled(service): + raise RekognitionConfigurationError(f"{service} is not enabled. See conf.Services") + + @classmethod + def to_dict(cls): + """Convert Services to dict""" + return { + key: value + for key, value in Services.__dict__.items() + if not key.startswith("__") and not callable(key) and key != "to_dict" + } + + @classmethod + def enabled_services(cls) -> List[str]: + """Return a list of enabled services""" + return [ + getattr(cls, key)[0] + for key in dir(cls) + if not key.startswith("__") + and not callable(getattr(cls, key)) + and key != "to_dict" + and getattr(cls, key)[1] is True + ] + + class SettingsDefaults: """Default values for Settings""" # defaults for this Python package SHARED_RESOURCE_IDENTIFIER = TFVARS.get("shared_resource_identifier", "rekognition") DEBUG_MODE: bool = bool(TFVARS.get("debug_mode", False)) - DUMP_DEFAULTS = TFVARS.get("dump_defaults", True) + DUMP_DEFAULTS: bool = bool(TFVARS.get("dump_defaults", True)) # aws auth AWS_PROFILE = TFVARS.get("aws_profile", None) @@ -100,9 +156,9 @@ class SettingsDefaults: # aws api gateway defaults AWS_APIGATEWAY_CREATE_CUSTOM_DOMAIN = TFVARS.get("aws_apigateway_create_custom_domaim", False) AWS_APIGATEWAY_ROOT_DOMAIN = TFVARS.get("aws_apigateway_root_domain", None) - AWS_APIGATEWAY_READ_TIMEOUT = TFVARS.get("aws_apigateway_read_timeout", 70) - AWS_APIGATEWAY_CONNECT_TIMEOUT = TFVARS.get("aws_apigateway_connect_timeout", 70) - AWS_APIGATEWAY_MAX_ATTEMPTS = TFVARS.get("aws_apigateway_max_attempts", 10) + AWS_APIGATEWAY_READ_TIMEOUT: int = TFVARS.get("aws_apigateway_read_timeout", 70) + AWS_APIGATEWAY_CONNECT_TIMEOUT: int = TFVARS.get("aws_apigateway_connect_timeout", 70) + AWS_APIGATEWAY_MAX_ATTEMPTS: int = TFVARS.get("aws_apigateway_max_attempts", 10) # aws dynamodb defaults AWS_DYNAMODB_TABLE_ID = SHARED_RESOURCE_IDENTIFIER @@ -124,9 +180,11 @@ def to_dict(cls): } -ec2 = boto3.Session().client("ec2") -regions = ec2.describe_regions() -AWS_REGIONS = [region["RegionName"] for region in regions["Regions"]] +AWS_REGIONS = [] +if Services.enabled(Services.AWS_EC2): + ec2 = boto3.Session().client("ec2") + regions = ec2.describe_regions() + AWS_REGIONS = [region["RegionName"] for region in regions["Regions"]] def empty_str_to_bool_default(v: str, default: bool) -> bool: @@ -164,6 +222,10 @@ class Settings(BaseSettings): # pylint: disable=too-many-branches def __init__(self, **data: Any): super().__init__(**data) + if not Services.enabled(Services.AWS_CLI): + self._initialized = True + return + if bool(os.environ.get("AWS_DEPLOYED", False)): # If we're running inside AWS Lambda, then we don't need to set the AWS credentials. self._aws_access_key_id_source: str = "overridden by IAM role-based security" @@ -171,7 +233,7 @@ def __init__(self, **data: Any): self._aws_session = boto3.Session() self._initialized = True - if not self._initialized and bool(os.environ.get("GITHUB_ACTIONS", False)): + if not self.initialized and bool(os.environ.get("GITHUB_ACTIONS", False)): try: self._aws_session = boto3.Session( region_name=os.environ.get("AWS_REGION", "us-east-1"), @@ -189,13 +251,13 @@ def __init__(self, **data: Any): self._aws_secret_access_key_source = "environ" self._initialized = True - if not self._initialized: + if not self.initialized: if self.aws_profile: self._aws_access_key_id_source = "aws_profile" self._aws_secret_access_key_source = "aws_profile" self._initialized = True - if not self._initialized: + if not self.initialized: if "aws_access_key_id" in data or "aws_secret_access_key" in data: if "aws_access_key_id" in data: self._aws_access_key_id_source = "constructor" @@ -203,7 +265,7 @@ def __init__(self, **data: Any): self._aws_secret_access_key_source = "constructor" self._initialized = True - if not self._initialized: + if not self.initialized: if "AWS_ACCESS_KEY_ID" in os.environ: self._aws_access_key_id_source = "environ" if "AWS_SECRET_ACCESS_KEY" in os.environ: @@ -290,10 +352,24 @@ def __init__(self, **data: Any): getter=lambda v: empty_str_to_int_default(v, SettingsDefaults.AWS_REKOGNITION_FACE_DETECT_THRESHOLD), ) + @property + def initialized(self): + """Is settings initialized?""" + return self._initialized + @property def aws_account_id(self): """AWS account id""" - return self.aws_session.client("sts").get_caller_identity()["Account"] + Services.raise_error_on_disabled(Services.AWS_CLI) + sts_client = self.aws_session.client("sts") + if not sts_client: + logger.warning("could not initialize sts_client") + return None + retval = sts_client.get_caller_identity() + if not isinstance(retval, dict): + logger.warning("sts_client.get_caller_identity() did not return a dict") + return None + return retval.get("Account", None) @property def aws_access_key_id_source(self): @@ -318,6 +394,7 @@ def aws_auth(self) -> dict: @property def aws_session(self): """AWS session""" + Services.raise_error_on_disabled(Services.AWS_CLI) if not self._aws_session: if self.aws_profile: logger.debug("creating new aws_session with aws_profile: %s", self.aws_profile) @@ -342,11 +419,13 @@ def aws_session(self): @property def aws_route53_client(self): """Route53 client""" + Services.raise_error_on_disabled(Services.AWS_ROUTE53) return self.aws_session.client("route53") @property def aws_apigateway_client(self): """API Gateway client""" + Services.raise_error_on_disabled(Services.AWS_APIGATEWAY) if not self._aws_apigateway_client: config = Config( read_timeout=SettingsDefaults.AWS_APIGATEWAY_READ_TIMEOUT, @@ -359,6 +438,7 @@ def aws_apigateway_client(self): @property def aws_s3_client(self): """S3 client""" + Services.raise_error_on_disabled(Services.AWS_S3) if not self._aws_s3_client: self._aws_s3_client = self.aws_session.resource("s3") return self._aws_s3_client @@ -366,6 +446,7 @@ def aws_s3_client(self): @property def aws_dynamodb_client(self): """DynamoDB client""" + Services.raise_error_on_disabled(Services.AWS_DYNAMODB) if not self._aws_dynamodb_client: self._aws_dynamodb_client = self.aws_session.client("dynamodb") return self._aws_dynamodb_client @@ -373,6 +454,7 @@ def aws_dynamodb_client(self): @property def aws_rekognition_client(self): """Rekognition client""" + Services.raise_error_on_disabled(Services.AWS_REKOGNITION) if not self._aws_rekognition_client: self._aws_rekognition_client = self.aws_session.client("rekognition") return self._aws_rekognition_client @@ -380,6 +462,7 @@ def aws_rekognition_client(self): @property def dynamodb_table(self): """DynamoDB table""" + Services.raise_error_on_disabled(Services.AWS_DYNAMODB) dynamodb_resource = boto3.resource("dynamodb") return dynamodb_resource.Table(self.aws_dynamodb_table_id) @@ -454,16 +537,17 @@ def get_installed_packages(): package_list = [(d.project_name, d.version) for d in installed_packages] return package_list - if self._dump and self._initialized: + if self._dump and self.initialized: return self._dump - if not self._initialized: + if not self.initialized: return {} packages = get_installed_packages() packages_dict = [{"name": name, "version": version} for name, version in packages] self._dump = { + "services": Services.enabled_services(), "environment": { "is_using_tfvars_file": self.is_using_tfvars_file, "is_using_dotenv_file": self.is_using_dotenv_file, diff --git a/terraform/python/rekognition_api/tests/test_aws.py b/terraform/python/rekognition_api/tests/test_aws.py index 043e808..e1e0715 100644 --- a/terraform/python/rekognition_api/tests/test_aws.py +++ b/terraform/python/rekognition_api/tests/test_aws.py @@ -2,6 +2,8 @@ # pylint: disable=wrong-import-position """Test configuration Settings class.""" +import inspect + # python stuff import os import sys @@ -19,7 +21,7 @@ # our stuff from rekognition_api.aws import aws_infrastructure_config as aws_config # noqa: E402 -from rekognition_api.conf import settings # noqa: E402 +from rekognition_api.conf import Services, settings # noqa: E402 from rekognition_api.tests.test_setup import get_test_image # noqa: E402 @@ -41,6 +43,9 @@ def setUp(self): def test_rekognition_collection_exists(self): """Test that the Rekognition collection exists.""" + if not Services.enabled(Services.AWS_REKOGNITION): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.rekognition_collection_exists(), f"Rekognition collection {settings.aws_rekognition_collection_id} does not exist.", @@ -48,6 +53,9 @@ def test_rekognition_collection_exists(self): def test_aws_connection_works(self): """Test that the AWS connection works.""" + if not Services.enabled(Services.AWS_CLI): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue(aws_config.aws_connection_works(), "AWS connection failed.") def test_domain_exists(self): @@ -56,11 +64,17 @@ def test_domain_exists(self): def test_bucket_exists(self): """Test that the S3 bucket exists.""" + if not Services.enabled(Services.AWS_S3): + print("skipping: ", inspect.currentframe().f_code.co_name) + return bucket_prefix = settings.aws_s3_bucket_name self.assertTrue(aws_config.bucket_exists(bucket_prefix), f"S3 bucket {bucket_prefix} does not exist.") def test_dynamodb_table_exists(self): """Test that the DynamoDB table exists.""" + if not Services.enabled(Services.AWS_DYNAMODB): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.dynamodb_table_exists(settings.shared_resource_identifier), f"DynamoDB table {settings.shared_resource_identifier} does not exist.", @@ -68,11 +82,16 @@ def test_dynamodb_table_exists(self): def test_api_exists(self): """Test that the API Gateway exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return api = aws_config.get_api(settings.aws_apigateway_name) self.assertIsInstance(api, dict, "API Gateway does not exist.") def test_api_resource_index_exists(self): """Test that the API Gateway index resource exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + return self.assertTrue( aws_config.api_resource_and_method_exists("/index/{filename}", "PUT"), "API Gateway index (PUT) resource does not exist.", @@ -80,6 +99,9 @@ def test_api_resource_index_exists(self): def test_api_resource_search_exists(self): """Test that the API Gateway index resource exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return self.assertTrue( aws_config.api_resource_and_method_exists("/search", "ANY"), "API Gateway search (ANY) resource does not exist.", @@ -87,12 +109,18 @@ def test_api_resource_search_exists(self): def test_api_key_exists(self): """Test that an API key exists.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return api_key = aws_config.get_api_keys() self.assertIsInstance(api_key, str, "API key does not exist.") self.assertGreaterEqual(len(api_key), 15, "API key is too short.") def test_index_endpoint(self): """Test that the index endpoint works.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return filename = "Keanu-Reeves.jpg" api_key = aws_config.get_api_keys() url = aws_config.get_url(f"/index/{filename}") @@ -106,6 +134,9 @@ def test_index_endpoint(self): def test_search_endpoint(self): """Test that the search endpoint works.""" + if not Services.enabled(Services.AWS_APIGATEWAY): + print("skipping: ", inspect.currentframe().f_code.co_name) + return filename = "Keanu-Avril-Mike.jpg" api_key = aws_config.get_api_keys() url = aws_config.get_url("/search")