From 2482a06c586eb3d9098476debcfd9673a6cfa6ee Mon Sep 17 00:00:00 2001 From: "Eric Z. Beard" Date: Fri, 26 May 2023 13:19:07 -0700 Subject: [PATCH] Pipeline for ApplicationAutoscaling (#212) * Requirements and config change for app autoscaling * Add App autoscaling to readme * Fixing linter errors * Change run order * Fix build policy * Add application-autoscaling:DescribeScalableTargets * Add all autoscaling actions to policy * Change runorder to avoid throttling errors * Fix roles for KMS hook --- .gitignore | 4 +- README.md | 1 + release/awscommunity/cicd.yml | 127 ++++++++++++++--- .../.gitignore | 133 ++++++++++++++++++ .../.rpdk-config | 2 +- .../requirements-dev.txt | 8 ++ .../requirements.txt | 15 +- ...prod-role.yaml => resource-role-prod.yaml} | 0 .../run-test.sh | 16 +++ .../handler_test.py | 6 + .../handlers.py | 24 +++- 11 files changed, 297 insertions(+), 39 deletions(-) create mode 100644 resources/ApplicationAutoscaling_ScheduledAction/.gitignore create mode 100644 resources/ApplicationAutoscaling_ScheduledAction/requirements-dev.txt rename resources/ApplicationAutoscaling_ScheduledAction/{resource-prod-role.yaml => resource-role-prod.yaml} (100%) create mode 100755 resources/ApplicationAutoscaling_ScheduledAction/run-test.sh create mode 100644 resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handler_test.py diff --git a/.gitignore b/.gitignore index ea06e33b..5f704ca8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ cfn-submit-output.json local/ rpdk.log -.idea/ \ No newline at end of file +.idea/ +*.zip + diff --git a/README.md b/README.md index 0fdef36d..9b455b1b 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ first be activated using the instructions |Name|Type|Version|Description| |----|----|-------|-----------| |[AwsCommunity::Account::AlternateContact](./resources/Account_AlternateContact)|Resource|Prod|An alternate contact attached to an Amazon Web Services account| +|[AwsCommunity::ApplicationAutoscaling::ScheduledAction](./resources/ApplicationAutoscaling_ScheduledAction)|Resource|Prod|Application Autoscaling Scheduled Action| |[AwsCommunity::CloudFront::LoggingEnabled](./hooks/CloudFront_LoggingEnabled)|Hook|Alpha|Validate that a CloudFront distribution has logging enabled| |[AwsCommunity::CloudFront::S3Website::MODULE](./modules/CloudFront_S3Website/)|Module|Prod|CloudFront backed by an S3 bucket with Route53 integration| |[AwsCommunity::DynamoDB::Item](./resources/DynamoDB_Item)|Resource|Prod|Manage the lifecycle of items in a DynamoDB table| diff --git a/release/awscommunity/cicd.yml b/release/awscommunity/cicd.yml index 49ab245f..cf882661 100644 --- a/release/awscommunity/cicd.yml +++ b/release/awscommunity/cicd.yml @@ -283,6 +283,19 @@ Resources: ManagedPolicyArns: - Fn::ImportValue: !Sub "cep-${Env}-common-build-project-policy" + ApplicationAutoscalingScheduledActionBuildProjectRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Version: '2012-10-17' + ManagedPolicyArns: + - Fn::ImportValue: !Sub "cep-${Env}-common-build-project-policy" + S3BucketNotificationBuildProjectPolicy: Type: AWS::IAM::Policy Properties: @@ -812,11 +825,50 @@ Resources: - iam:UpdateRole - iam:UpdateRoleDescription Resource: !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/TrailS3Cleanup-integ-*-awscommunity-kms-encryptionsettings' + - Effect: Allow + Action: + - iam:CreateServiceLinkedRole + - iam:DeleteServiceLinkedRole + - iam:GetServiceLinkedRoleDeletionStatus + Resource: "*" Version: '2012-10-17' PolicyName: kms-encryptionsettings-build-project-policy Roles: - !Ref KMSEncryptionSettingsBuildProjectRole + ApplicationAutoscalingScheduledActionBuildProjectRolePolicy: + Type: AWS::IAM::Policy + Properties: + PolicyDocument: + Statement: + - Action: + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - codebuild:StopBuild + - codebuild:RetryBuild + - codebuild:StartBuildBatch + - codebuild:RetryBuildBatch + - codebuild:StopBuildBatch + Effect: Allow + Resource: + - !GetAtt ApplicationAutoscalingScheduledActionBuildProject.Arn + - Action: + - application-autoscaling:* + Effect: Allow + Resource: "*" + - Action: + - dynamodb:* + Effect: Allow + Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/awscommunityscheduledactiontesttable" + - Action: + - iam:CreateServiceLinkedRole + Effect: Allow + Resource: "*" + Version: '2012-10-17' + PolicyName: application-autoscaling-scheduledaction-build-project-policy + Roles: + - !Ref ApplicationAutoscalingScheduledActionBuildProjectRole + S3BucketNotificationBuildProject: Type: AWS::CodeBuild::Project Properties: @@ -1240,6 +1292,28 @@ Resources: BuildSpec: !Sub "hooks/${Env}-buildspec-java.yml" TimeoutInMinutes: 480 + ApplicationAutoscalingScheduledActionBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Name: !Sub "${PrefixLower}-${Env}-app-autosc-sched" + Artifacts: + Type: CODEPIPELINE + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/cep-cicd:latest" + ImagePullCredentialsType: SERVICE_ROLE + PrivilegedMode: true + Type: LINUX_CONTAINER + EnvironmentVariables: + - Name: RESOURCE_PATH + Type: PLAINTEXT + Value: "placeholder-for-path-to-resource" + ServiceRole: !GetAtt ApplicationAutoscalingScheduledActionBuildProjectRole.Arn + Source: + Type: CODEPIPELINE + BuildSpec: !Sub "resources/${Env}-buildspec-python.yml" + TimeoutInMinutes: 480 + SourceBucket: Type: AWS::S3::Bucket Metadata: @@ -1318,6 +1392,7 @@ Resources: - !GetAtt CloudFrontS3WebsiteModuleBuildProject.Arn - !GetAtt AlternateContactBuildProject.Arn - !GetAtt KMSEncryptionSettingsBuildProject.Arn + - !GetAtt ApplicationAutoscalingScheduledActionBuildProject.Arn - Action: - kms:* Effect: Allow @@ -1387,6 +1462,7 @@ Resources: - !GetAtt S3BucketModuleBuildProjectRole.Arn - !GetAtt CloudFrontS3WebsiteModuleBuildProjectRole.Arn - !GetAtt KMSEncryptionSettingsBuildProjectRole.Arn + - !GetAtt ApplicationAutoscalingScheduledActionBuildProjectRole.Arn Resource: "*" MultiRegion: true @@ -1452,7 +1528,7 @@ Resources: "value": "resources/S3_BucketNotification" } ] - RunOrder: 1 + RunOrder: 2 - !Ref AWS::NoValue # - Name: CloudFrontWebAclAssociation # InputArtifacts: @@ -1472,7 +1548,7 @@ Resources: # "value": "resources/CloudFront_WebACLAssociation" # } # ] - # RunOrder: 1 + # RunOrder: 2 - Name: S3DeleteBucketContents InputArtifacts: - Name: extensions-source @@ -1491,7 +1567,7 @@ Resources: "value": "resources/S3_DeleteBucketContents" } ] - RunOrder: 1 + RunOrder: 2 - Name: ResourceLookup InputArtifacts: - Name: extensions-source @@ -1510,7 +1586,7 @@ Resources: "value": "resources/Resource_Lookup" } ] - RunOrder: 1 + RunOrder: 2 - Name: DynamoDBItem InputArtifacts: - Name: extensions-source @@ -1534,7 +1610,7 @@ Resources: "value": "resources/DynamoDB_Item" } ] - RunOrder: 1 + RunOrder: 2 - Name: TimeStatic InputArtifacts: - Name: extensions-source @@ -1558,7 +1634,7 @@ Resources: "value": "resources/Time_Static" } ] - RunOrder: 1 + RunOrder: 2 - Name: TimeSleep InputArtifacts: - Name: extensions-source @@ -1582,7 +1658,7 @@ Resources: "value": "resources/Time_Sleep" } ] - RunOrder: 1 + RunOrder: 2 - Name: TimeOffset InputArtifacts: - Name: extensions-source @@ -1606,7 +1682,7 @@ Resources: "value": "resources/Time_Offset" } ] - RunOrder: 1 + RunOrder: 2 - Name: Account_AlternateContact InputArtifacts: - Name: extensions-source @@ -1625,7 +1701,7 @@ Resources: "value": "resources/Account_AlternateContact" } ] - RunOrder: 1 + RunOrder: 2 - Name: S3BucketModule InputArtifacts: - Name: extensions-source @@ -1644,7 +1720,7 @@ Resources: "value": "modules/S3_Bucket" } ] - RunOrder: 1 + RunOrder: 2 - Name: CloudFrontS3Website InputArtifacts: - Name: extensions-source @@ -1663,7 +1739,7 @@ Resources: "value": "modules/CloudFront_S3Website" } ] - RunOrder: 1 + RunOrder: 2 - Name: S3BucketVersioningEnabled InputArtifacts: - Name: extensions-source @@ -1682,7 +1758,7 @@ Resources: "value": "hooks/S3_BucketVersioningEnabled" } ] - RunOrder: 2 + RunOrder: 3 - Name: HookEC2SecurityGroupRestrictedSSH InputArtifacts: - Name: extensions-source @@ -1701,7 +1777,7 @@ Resources: "value": "hooks/EC2_SecurityGroupRestrictedSSH" } ] - RunOrder: 2 + RunOrder: 3 - Name: HookS3PublicAccessControlsRestricted InputArtifacts: - Name: extensions-source @@ -1720,7 +1796,7 @@ Resources: "value": "hooks/S3_PublicAccessControlsRestricted" } ] - RunOrder: 3 + RunOrder: 4 - Name: KMSEncryptionSettings InputArtifacts: - Name: extensions-source @@ -1739,7 +1815,26 @@ Resources: "value": "hooks/KMS_EncryptionSettings" } ] - RunOrder: 3 + RunOrder: 4 + - Name: ApplicationAutoscalingScheduledAction + InputArtifacts: + - Name: extensions-source + ActionTypeId: + Category: Build + Owner: AWS + Provider: CodeBuild + Version: 1 + Configuration: + ProjectName: !Ref ApplicationAutoscalingScheduledActionBuildProject + EnvironmentVariables: |- + [ + { + "name": "RESOURCE_PATH", + "type": "PLAINTEXT", + "value": "resources/ApplicationAutoscaling_ScheduledAction" + } + ] + RunOrder: 1 - !If - IsBeta - Name: CopyBuildToProd @@ -1778,7 +1873,7 @@ Resources: # "value": "hooks/CloudFront_LoggingEnabled" # } # ] - # RunOrder: 1 + # RunOrder: 2 PublishBuildBucketRole: Type: AWS::IAM::Role diff --git a/resources/ApplicationAutoscaling_ScheduledAction/.gitignore b/resources/ApplicationAutoscaling_ScheduledAction/.gitignore new file mode 100644 index 00000000..a83a6e48 --- /dev/null +++ b/resources/ApplicationAutoscaling_ScheduledAction/.gitignore @@ -0,0 +1,133 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# contains credentials +sam-tests/ + +rpdk.log* +*.zip +.DS_Store +publish.yml + diff --git a/resources/ApplicationAutoscaling_ScheduledAction/.rpdk-config b/resources/ApplicationAutoscaling_ScheduledAction/.rpdk-config index 82b2d66b..86f40d57 100644 --- a/resources/ApplicationAutoscaling_ScheduledAction/.rpdk-config +++ b/resources/ApplicationAutoscaling_ScheduledAction/.rpdk-config @@ -15,7 +15,7 @@ "endpoint_url": null, "region": null, "target_schemas": [], - "use_docker": false, + "use_docker": true, "protocolVersion": "2.0.0" } } diff --git a/resources/ApplicationAutoscaling_ScheduledAction/requirements-dev.txt b/resources/ApplicationAutoscaling_ScheduledAction/requirements-dev.txt new file mode 100644 index 00000000..1daeb533 --- /dev/null +++ b/resources/ApplicationAutoscaling_ScheduledAction/requirements-dev.txt @@ -0,0 +1,8 @@ +importlib_metadata==4.7.1 +cfn-lint==0.64.1 +cloudformation-cli-python-lib==2.1.16 +git+https://github.com/aws-cloudformation/cloudformation-cli.git@master +git+https://github.com/aws-cloudformation/cloudformation-cli-python-plugin.git@master +pylint==2.15.2 +pytest==7.2.0 +bandit==1.7.4 diff --git a/resources/ApplicationAutoscaling_ScheduledAction/requirements.txt b/resources/ApplicationAutoscaling_ScheduledAction/requirements.txt index 2f45490f..de9b258f 100644 --- a/resources/ApplicationAutoscaling_ScheduledAction/requirements.txt +++ b/resources/ApplicationAutoscaling_ScheduledAction/requirements.txt @@ -1,14 +1 @@ -attrs==23.1.0 ; python_version >= "3.8" and python_version < "4.0" -aws-encryption-sdk==3.1.1 ; python_version >= "3.8" and python_version < "4.0" -boto3==1.26.131 ; python_version >= "3.8" and python_version < "4.0" -botocore==1.29.131 ; python_version >= "3.8" and python_version < "4.0" -cffi==1.15.1 ; python_version >= "3.8" and python_version < "4.0" -cloudformation-cli-python-lib==2.1.16 ; python_version >= "3.8" and python_version < "4.0" -cryptography==40.0.2 ; python_version >= "3.8" and python_version < "4.0" -jmespath==1.0.1 ; python_version >= "3.8" and python_version < "4.0" -pycparser==2.21 ; python_version >= "3.8" and python_version < "4.0" -python-dateutil==2.8.2 ; python_version >= "3.8" and python_version < "4.0" -s3transfer==0.6.1 ; python_version >= "3.8" and python_version < "4.0" -six==1.16.0 ; python_version >= "3.8" and python_version < "4.0" -urllib3==1.26.15 ; python_version >= "3.8" and python_version < "4.0" -wrapt==1.15.0 ; python_version >= "3.8" and python_version < "4.0" +cloudformation-cli-python-lib>=2.1.16 diff --git a/resources/ApplicationAutoscaling_ScheduledAction/resource-prod-role.yaml b/resources/ApplicationAutoscaling_ScheduledAction/resource-role-prod.yaml similarity index 100% rename from resources/ApplicationAutoscaling_ScheduledAction/resource-prod-role.yaml rename to resources/ApplicationAutoscaling_ScheduledAction/resource-role-prod.yaml diff --git a/resources/ApplicationAutoscaling_ScheduledAction/run-test.sh b/resources/ApplicationAutoscaling_ScheduledAction/run-test.sh new file mode 100755 index 00000000..bdf50110 --- /dev/null +++ b/resources/ApplicationAutoscaling_ScheduledAction/run-test.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -eou pipefail + +echo "Linting..." +pylint --rcfile ../../config/.pylintrc src/awscommunity_applicationautoscaling_scheduledaction/*.py + +echo "Testing..." +pytest src + +echo "Bandit..." +bandit -c ../../config/.banditrc -r src + +echo "About to run cfn test..." +cfn validate && cfn generate && cfn submit --dry-run && cfn test -v + + diff --git a/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handler_test.py b/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handler_test.py new file mode 100644 index 00000000..6fcb67fc --- /dev/null +++ b/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handler_test.py @@ -0,0 +1,6 @@ +"Placeholder for unit tests" + +def test(): + "Placeholder" + assert True + diff --git a/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handlers.py b/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handlers.py index dc66c065..38adfb02 100644 --- a/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handlers.py +++ b/resources/ApplicationAutoscaling_ScheduledAction/src/awscommunity_applicationautoscaling_scheduledaction/handlers.py @@ -1,7 +1,8 @@ +"Handlers for the Application Autoscaling Scheduled Action extension" +#pylint:disable=W0613 from __future__ import annotations import logging -import sys from typing import Any, MutableMapping, Optional, Union from cloudformation_cli_python_lib import ( @@ -11,8 +12,6 @@ ProgressEvent, Resource, SessionProxy, - exceptions, - identifier_utils, ) from dateutil import parser from dateutil.parser import ParserError @@ -28,6 +27,7 @@ def rule_exists(session: SessionProxy, model) -> Union[dict, None]: + "Check if a rule exists" if not model: return None exists_query_r = session.client( @@ -45,6 +45,7 @@ def rule_exists(session: SessionProxy, model) -> Union[dict, None]: def define_scalable_target_action(model) -> dict: + "Configure the scalable target action" action: dict = { "ScheduledActionName": model.ScheduledActionName, "ServiceNamespace": model.ServiceNamespace, @@ -78,6 +79,7 @@ def define_scalable_target_action(model) -> dict: def must_exist( session: SessionProxy, model, progress: ProgressEvent, context: str ) -> Union[ProgressEvent, None]: + "Check to make sure a rule exists" if not model: return None try: @@ -90,7 +92,7 @@ def must_exist( ) return progress except Exception as error: - LOG.error(f"{context} pre-validation error") + LOG.error("%s pre-validation error", context) LOG.exception(error) return ProgressEvent( status=OperationStatus.FAILED, @@ -106,6 +108,7 @@ def create_handler( request: ResourceHandlerRequest, callback_context: MutableMapping[str, Any], ) -> ProgressEvent: + "Create handler" model = request.desiredResourceState progress: ProgressEvent = ProgressEvent( status=OperationStatus.IN_PROGRESS, @@ -120,7 +123,8 @@ def create_handler( f"Scheduled Action {model.ScheduledActionName} already exists for" f" {model.ServiceNamespace}|{model.ResourceId}" ) - LOG.debug("Rule {} already exists".format(model.ScheduledActionName)) + msg = f"Rule {model.ScheduledActionName} already exists" + LOG.debug(msg) return progress except Exception as error: LOG.error("pre-create validation error") @@ -132,7 +136,8 @@ def create_handler( ) try: kwargs = define_scalable_target_action(model) - LOG.debug("API Args: {}".format(kwargs)) + msg = f"API Args: {kwargs}" + LOG.debug(msg) application_autoscaling_client.put_scheduled_action(**kwargs) resource_r = rule_exists(session, model) primary_identifier = resource_r["ScheduledActionARN"] @@ -148,7 +153,8 @@ def create_handler( status=OperationStatus.FAILED, errorCode=HandlerErrorCode.NotFound, message=( - f"Scalable target {model.ResourceId}|{model.ScalableDimension}|{model.ServiceNamespace} not found" + f"Scalable target {model.ResourceId}|{model.ScalableDimension}"+\ + f"|{model.ServiceNamespace} not found" ), ) except Exception as error: @@ -167,6 +173,7 @@ def update_handler( request: ResourceHandlerRequest, callback_context: MutableMapping[str, Any], ) -> ProgressEvent: + "Update handler" model = request.desiredResourceState progress: ProgressEvent = ProgressEvent( status=OperationStatus.IN_PROGRESS, @@ -198,6 +205,7 @@ def delete_handler( request: ResourceHandlerRequest, callback_context: MutableMapping[str, Any], ) -> ProgressEvent: + "Delete handler" model = request.desiredResourceState progress: ProgressEvent = ProgressEvent( status=OperationStatus.IN_PROGRESS, @@ -233,6 +241,7 @@ def read_handler( request: ResourceHandlerRequest, callback_context: MutableMapping[str, Any], ) -> ProgressEvent: + "Read handler" model = request.desiredResourceState progress = ProgressEvent(status=OperationStatus.IN_PROGRESS, resourceModel=model) cannot_proceed = must_exist(session, model, progress, "Read") @@ -250,6 +259,7 @@ def list_handler( request: ResourceHandlerRequest, callback_context: MutableMapping[str, Any], ) -> ProgressEvent: + "List handler" return ProgressEvent( status=OperationStatus.SUCCESS, resourceModels=[],