From a9100b06fe09f8fd8458642393e7bc6836f6adcf Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Thu, 13 Jun 2024 16:22:32 +0100 Subject: [PATCH] Feat/policy refactor (#3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Starting refactor of policy application * initial testing * Unit tests for OrganisationPolicy class * Linting * wip * Merging * sìos leis a' Bheurla * Resetting generate params * Fixing spelling mistakes * Updating documentation * Apply suggestions from code review Co-authored-by: Simon Kok * Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py Co-authored-by: Simon Kok * Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py Co-authored-by: Simon Kok * Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py Co-authored-by: Simon Kok * fixing tests * fixing linting * linting again * temp remove assertion * updating logging * running black with ll 80 * linting * Tox no longer complaining --------- Co-authored-by: Simon Kok Co-authored-by: Simon Kok --- docs/admin-guide.md | 61 + .../bootstrap_repository/adf-build/main.py | 303 ++--- .../adf-build/organization_policy.py | 29 +- .../adf-build/organization_policy_schema.py | 42 + .../adf-build/organization_policy_v2.py | 229 ++++ .../adf-build/requirements.txt | 1 + .../adf-build/shared/generate_params.py | 108 +- .../python/organization_policy_campaign.py | 629 +++++++++ .../adf-build/shared/python/organizations.py | 4 +- .../test_organizational_policy_campaigns.py | 1165 +++++++++++++++++ .../adf-build/tests/adf-policies/scp/scp.json | 32 + .../adf-build/tests/test_main.py | 47 +- .../adf-build/tests/test_org_policy_schema.py | 66 + .../tests/test_organization_policy_v2.py | 95 ++ src/template.yml | 13 + 15 files changed, 2590 insertions(+), 234 deletions(-) create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_schema.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_v2.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizational_policy_campaigns.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/adf-policies/scp/scp.json create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_org_policy_schema.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_organization_policy_v2.py diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 731eda722..f2b68241c 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -876,6 +876,67 @@ Once you have enabled all features within your Organization, ADF can manage and automate the application and updating process of the Tag Policies. For more information, see [here](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_tag-policies.html). +## Policies V2 + +### What is Policies V2 + +A new feature of ADF, that gives you the ability to define a policy in a +single location, and apply it to multiple targets. + +### Enabling the new version + +Because of the difference in this approach to applying policies, it is not +currently the default method and will have to be enabled. In order to enable it, +you have to update your serverlessrepo stack in the organizational root account +and set the parameter `EnablePolicyV2' to "TRUE". Once the stack has redeployed, +it will be enabled. + +### Using the new version + +Inside your adf-bootstrap folder, create a directory named `adf-policies`, +Inside the `adf-policies` directory you then create subdirectories per policy type. +Currently, only `scp` and `tagging-policy` are supported in the AWS partition. +Inside this directory you can create a JSON file that defines your policy. +So in the following example, if you wanted to create an scp policy it would be in +`adf-policies/scp/.json` +Using the following Schema: + +```json +{ + "Targets": [ + "YourOrg", "YourOtherOrg", + ], + "Version": "2022-10-14", + "PolicyName": "Example", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "cloudtrail:Stop*", + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + }, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + "config:Stop*" + ], + "Resource": "*" + } + ] + } +} + +``` + ## Integrating Slack ### Integrating with Slack using Lambda diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index a22c9375f..48e1161a0 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -14,6 +14,7 @@ from thread import PropagatingThread import boto3 +from botocore.exceptions import ClientError from logger import configure_logger from cache import Cache @@ -26,6 +27,7 @@ from s3 import S3 from partition import get_partition from config import Config +from organization_policy_v2 import OrganizationPolicy as OrgPolicyV2 from organization_policy import OrganizationPolicy @@ -46,7 +48,8 @@ # seconds to milliseconds: floor(datetime.now(timezone.utc).timestamp() - (10 * 60)) * 1000, ) - ) / 1000.0 # Convert milliseconds to seconds + ) + / 1000.0 # Convert milliseconds to seconds ) ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN = os.environ.get( "ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN", @@ -54,9 +57,10 @@ ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN = os.environ.get( "ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN" ) -ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'main' -ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET = 'disabled' +ADF_DEFAULT_SCM_FALLBACK_BRANCH = "main" +ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET = "disabled" ADF_DEFAULT_ORG_STAGE = "none" + LOGGER = configure_logger(__name__) @@ -64,21 +68,25 @@ def ensure_generic_account_can_be_setup(sts, config, account_id): """ If the target account has been configured returns the role to assume """ - return sts.assume_bootstrap_deployment_role( - PARTITION, - MANAGEMENT_ACCOUNT_ID, - account_id, - config.cross_account_access_role, - 'base_update', - ) + try: + return sts.assume_bootstrap_deployment_role( + PARTITION, + MANAGEMENT_ACCOUNT_ID, + account_id, + config.cross_account_access_role, + "base_update", + ) + except ClientError as error: + raise GenericAccountConfigureError from error def update_deployment_account_output_parameters( - deployment_account_region, - region, - kms_and_bucket_dict, - deployment_account_role, - cloudformation): + deployment_account_region, + region, + kms_and_bucket_dict, + deployment_account_role, + cloudformation, +): """ Update parameters on the deployment account across target regions based on the output of CloudFormation base stacks @@ -87,24 +95,17 @@ def update_deployment_account_output_parameters( deployment_account_parameter_store = ParameterStore( deployment_account_region, deployment_account_role ) - parameter_store = ParameterStore( - region, deployment_account_role - ) + parameter_store = ParameterStore(region, deployment_account_role) outputs = cloudformation.get_stack_regional_outputs() kms_and_bucket_dict[region] = {} - kms_and_bucket_dict[region]['kms'] = outputs['kms_arn'] - kms_and_bucket_dict[region]['s3_regional_bucket'] = ( - outputs['s3_regional_bucket'] - ) + kms_and_bucket_dict[region]["kms"] = outputs["kms_arn"] + kms_and_bucket_dict[region]["s3_regional_bucket"] = outputs["s3_regional_bucket"] for key, value in outputs.items(): deployment_account_parameter_store.put_parameter( - f"cross_region/{key}/{region}", - value - ) - parameter_store.put_parameter( - f"cross_region/{key}/{region}", - value + f"cross_region/{key}/{region}", value ) + parameter_store.put_parameter(f"cross_region/{key}/{region}", value) + parameter_store.put_parameter(f"/cross_region/{key}/{region}", value) return kms_and_bucket_dict @@ -120,114 +121,117 @@ def prepare_deployment_account(sts, deployment_account_id, config): MANAGEMENT_ACCOUNT_ID, deployment_account_id, config.cross_account_access_role, - 'management', + "management", ) for region in config.sorted_regions(): deployment_account_parameter_store = ParameterStore( - region, - deployment_account_role + region, deployment_account_role ) deployment_account_parameter_store.put_parameter( - 'adf_version', + "adf_version", ADF_VERSION, ) deployment_account_parameter_store.put_parameter( - 'adf_log_level', + "adf_log_level", ADF_LOG_LEVEL, ) deployment_account_parameter_store.put_parameter( - 'cross_account_access_role', + "cross_account_access_role", config.cross_account_access_role, ) deployment_account_parameter_store.put_parameter( - 'shared_modules_bucket', + "shared_modules_bucket", SHARED_MODULES_BUCKET_NAME, ) deployment_account_parameter_store.put_parameter( - 'bootstrap_templates_bucket', + "bootstrap_templates_bucket", S3_BUCKET_NAME, ) deployment_account_parameter_store.put_parameter( - 'deployment_account_id', + "deployment_account_id", deployment_account_id, ) deployment_account_parameter_store.put_parameter( - 'management_account_id', + "management_account_id", MANAGEMENT_ACCOUNT_ID, ) deployment_account_parameter_store.put_parameter( - 'organization_id', + "organization_id", os.environ["ORGANIZATION_ID"], ) _store_extension_parameters(deployment_account_parameter_store, config) # In main deployment region only: deployment_account_parameter_store = ParameterStore( - config.deployment_account_region, - deployment_account_role + config.deployment_account_region, deployment_account_role + ) + auto_create_repositories = config.config.get("scm", {}).get( + "auto-create-repositories" ) - auto_create_repositories = config.config.get( - 'scm', {}).get('auto-create-repositories') if auto_create_repositories is not None: deployment_account_parameter_store.put_parameter( - 'scm/auto_create_repositories', str(auto_create_repositories) + "scm/auto_create_repositories", str(auto_create_repositories) ) deployment_account_parameter_store.put_parameter( - 'scm/default_scm_branch', + "scm/default_scm_branch", ( - config.config - .get('scm', {}) - .get('default-scm-branch', ADF_DEFAULT_SCM_FALLBACK_BRANCH) - ) + config.config.get("scm", {}).get( + "default-scm-branch", ADF_DEFAULT_SCM_FALLBACK_BRANCH + ) + ), ) deployment_account_parameter_store.put_parameter( - 'scm/default_scm_codecommit_account_id', + "scm/default_scm_codecommit_account_id", ( - config.config - .get('scm', {}) - .get('default-scm-codecommit-account-id', deployment_account_id) - ) + config.config.get("scm", {}).get( + "default-scm-codecommit-account-id", deployment_account_id + ) + ), ) deployment_account_parameter_store.put_parameter( - 'deployment_maps/allow_empty_target', - config.config.get('deployment-maps', {}).get( - 'allow-empty-target', + "deployment_maps/allow_empty_target", + config.config.get("deployment-maps", {}).get( + "allow-empty-target", ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET, - ) + ), ) deployment_account_parameter_store.put_parameter( - 'org/stage', - config.config.get('org', {}).get( - 'stage', + "org/stage", + config.config.get("org", {}).get( + "stage", ADF_DEFAULT_ORG_STAGE, - ) + ), + ) + auto_create_repositories = config.config.get("scm", {}).get( + "auto-create-repositories" ) - if '@' not in config.notification_endpoint: + + if auto_create_repositories is not None: + deployment_account_parameter_store.put_parameter( + "auto_create_repositories", str(auto_create_repositories) + ) + if "@" not in config.notification_endpoint: config.notification_channel = config.notification_endpoint config.notification_endpoint = ( f"arn:{PARTITION}:lambda:{config.deployment_account_region}:" f"{deployment_account_id}:function:SendSlackNotification" ) - for item in ( - 'notification_type', - 'notification_endpoint', - 'notification_channel' - ): + for item in ("notification_type", "notification_endpoint", "notification_channel"): if getattr(config, item) is not None: deployment_account_parameter_store.put_parameter( ( - 'notification_endpoint/main' - if item == 'notification_channel' + "notification_endpoint/main" + if item == "notification_channel" else item ), - str(getattr(config, item)) + str(getattr(config, item)), ) return deployment_account_role def _store_extension_parameters(parameter_store, config): - if not hasattr(config, 'extensions'): + if not hasattr(config, "extensions"): return for extension, attributes in config.extensions.items(): @@ -254,23 +258,14 @@ def worker_thread( """ LOGGER.debug("%s - Starting new worker thread", account_id) - organizations = Organizations( - role=boto3, - account_id=account_id - ) + organizations = Organizations(role=boto3, account_id=account_id) ou_id = organizations.get_parent_info().get("ou_parent_id") account_path = organizations.build_account_path( - ou_id, - [], # Initial empty array to hold OU Path, - cache + ou_id, [], cache # Initial empty array to hold OU Path, ) try: - role = ensure_generic_account_can_be_setup( - sts, - config, - account_id - ) + role = ensure_generic_account_can_be_setup(sts, config, account_id) # Regional base stacks can be updated after global for region in config.sorted_regions(): @@ -278,34 +273,34 @@ def worker_thread( # are available on the target account. parameter_store = ParameterStore(region, role) parameter_store.put_parameter( - 'deployment_account_id', + "deployment_account_id", deployment_account_id, ) parameter_store.put_parameter( - 'kms_arn', - updated_kms_bucket_dict[region]['kms'], + "kms_arn", + updated_kms_bucket_dict[region]["kms"], ) parameter_store.put_parameter( - 'bucket_name', - updated_kms_bucket_dict[region]['s3_regional_bucket'], + "bucket_name", + updated_kms_bucket_dict[region]["s3_regional_bucket"], ) if region == config.deployment_account_region: parameter_store.put_parameter( - 'management_account_id', + "management_account_id", MANAGEMENT_ACCOUNT_ID, ) parameter_store.put_parameter( - 'bootstrap_templates_bucket', + "bootstrap_templates_bucket", S3_BUCKET_NAME, ) # Ensuring the stage parameter on the target account is up-to-date parameter_store.put_parameter( - 'org/stage', - config.config.get('org', {}).get( - 'stage', + "org/stage", + config.config.get("org", {}).get( + "stage", ADF_DEFAULT_ORG_STAGE, - ) + ), ) cloudformation = CloudFormation( region=region, @@ -315,7 +310,7 @@ def worker_thread( stack_name=None, s3=s3, s3_key_path="adf-bootstrap/" + account_path, - account_id=account_id + account_id=account_id, ) try: cloudformation.delete_deprecated_base_stacks() @@ -323,13 +318,13 @@ def worker_thread( if region == config.deployment_account_region: cloudformation.create_iam_stack() except GenericAccountConfigureError as error: - if 'Unable to fetch parameters' in str(error): + if "Unable to fetch parameters" in str(error): LOGGER.error( - '%s - Failed to update its base stack due to missing ' - 'parameters (deployment_account_id or kms_arn), ' - 'ensure this account has been bootstrapped correctly ' - 'by being moved from the root into an Organizational ' - 'Unit within AWS Organizations.', + "%s - Failed to update its base stack due to missing " + "parameters (deployment_account_id or kms_arn), " + "ensure this account has been bootstrapped correctly " + "by being moved from the root into an Organizational " + "Unit within AWS Organizations.", account_id, ) raise LookupError from error @@ -346,22 +341,22 @@ def await_sfn_executions(sfn_client): sfn_client, ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN, filter_lambda=lambda item: ( - item.get('name', '').find(CODEPIPELINE_EXECUTION_ID) > 0 + item.get("name", "").find(CODEPIPELINE_EXECUTION_ID) > 0 ), - status_filter='RUNNING', + status_filter="RUNNING", ) _await_running_sfn_executions( sfn_client, ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, filter_lambda=None, - status_filter='RUNNING', + status_filter="RUNNING", ) if _sfn_execution_exists_with( sfn_client, ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN, filter_lambda=lambda item: ( - item.get('name', '').find(CODEPIPELINE_EXECUTION_ID) > 0 - and item.get('status') in ['FAILED', 'TIMED_OUT', 'ABORTED'] + item.get("name", "").find(CODEPIPELINE_EXECUTION_ID) > 0 + and item.get("status") in ["FAILED", "TIMED_OUT", "ABORTED"] ), status_filter=None, ): @@ -378,7 +373,7 @@ def await_sfn_executions(sfn_client): LOGGER.warning( "Please note: If you resolved the error, but still run into this " "warning, make sure you release a change on the pipeline (by " - "clicking the orange \"Release Change\" button. " + 'clicking the orange "Release Change" button. ' "The pipeline checks for failed executions of the state machine " "that were triggered by this pipeline execution. Only a new " "pipeline execution updates the identified that it uses to track " @@ -390,10 +385,10 @@ def await_sfn_executions(sfn_client): ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, filter_lambda=lambda item: ( ( - item.get('startDate', datetime.now(timezone.utc)).timestamp() + item.get("startDate", datetime.now(timezone.utc)).timestamp() >= CODEBUILD_START_TIME_UNIXTS ) - and item.get('status') in ['FAILED', 'TIMED_OUT', 'ABORTED'] + and item.get("status") in ["FAILED", "TIMED_OUT", "ABORTED"] ), status_filter=None, ): @@ -417,12 +412,7 @@ def _await_running_sfn_executions( filter_lambda, status_filter, ): - while _sfn_execution_exists_with( - sfn_client, - sfn_arn, - filter_lambda, - status_filter - ): + while _sfn_execution_exists_with(sfn_client, sfn_arn, filter_lambda, status_filter): LOGGER.info( "Waiting for 30 seconds for the executions of %s to finish.", sfn_arn, @@ -442,10 +432,11 @@ def _sfn_execution_exists_with( if status_filter: request_params["statusFilter"] = status_filter - paginator = sfn_client.get_paginator('list_executions') + paginator = sfn_client.get_paginator("list_executions") for page in paginator.paginate(**request_params): filtered = ( - list(filter(filter_lambda, page["executions"])) if filter_lambda + list(filter(filter_lambda, page["executions"])) + if filter_lambda else page["executions"] ) if filtered: @@ -464,39 +455,31 @@ def main(): # pylint: disable=R0915 LOGGER.info("ADF Version %s", ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) - await_sfn_executions(boto3.client('stepfunctions')) + await_sfn_executions(boto3.client("stepfunctions")) - policies = OrganizationPolicy() + if os.getenv("ENABLED_V2_ORG_POLICY", "False").lower() == "true": + LOGGER.info("Using new organization policy") + policies = OrgPolicyV2() + else: + policies = OrganizationPolicy() config = Config() try: parameter_store = ParameterStore(REGION_DEFAULT, boto3) - deployment_account_id = parameter_store.fetch_parameter( - 'deployment_account_id' - ) - organizations = Organizations( - role=boto3, - account_id=deployment_account_id - ) + deployment_account_id = parameter_store.fetch_parameter("deployment_account_id") + organizations = Organizations(role=boto3, account_id=deployment_account_id) policies.apply(organizations, parameter_store, config.config) sts = STS() deployment_account_role = prepare_deployment_account( - sts=sts, - deployment_account_id=deployment_account_id, - config=config + sts=sts, deployment_account_id=deployment_account_id, config=config ) cache = Cache() ou_id = organizations.get_parent_info().get("ou_parent_id") account_path = organizations.build_account_path( - ou_id=ou_id, - account_path=[], - cache=cache - ) - s3 = S3( - region=REGION_DEFAULT, - bucket=S3_BUCKET_NAME + ou_id=ou_id, account_path=[], cache=cache ) + s3 = S3(region=REGION_DEFAULT, bucket=S3_BUCKET_NAME) kms_and_bucket_dict = {} # First Setup/Update the Deployment Account in all regions (KMS Key and @@ -510,7 +493,7 @@ def main(): # pylint: disable=R0915 stack_name=None, s3=s3, s3_key_path="adf-bootstrap/" + account_path, - account_id=deployment_account_id + account_id=deployment_account_id, ) cloudformation.delete_deprecated_base_stacks() cloudformation.create_stack() @@ -519,7 +502,7 @@ def main(): # pylint: disable=R0915 region=region, kms_and_bucket_dict=kms_and_bucket_dict, deployment_account_role=deployment_account_role, - cloudformation=cloudformation + cloudformation=cloudformation, ) if region == config.deployment_account_region: cloudformation.create_iam_stack() @@ -528,24 +511,26 @@ def main(): # pylint: disable=R0915 account_ids = [ account_id["Id"] for account_id in organizations.get_accounts( - protected_ou_ids=config.config.get('protected'), + protected_ou_ids=config.config.get("protected"), include_root=False, ) ] - non_deployment_account_ids = sorted([ - account for account in account_ids - if account != deployment_account_id - ]) + non_deployment_account_ids = sorted( + [account for account in account_ids if account != deployment_account_id] + ) for account_id in non_deployment_account_ids: - thread = PropagatingThread(target=worker_thread, args=( - account_id, - deployment_account_id, - sts, - config, - s3, - cache, - kms_and_bucket_dict - )) + thread = PropagatingThread( + target=worker_thread, + args=( + account_id, + deployment_account_id, + sts, + config, + s3, + cache, + kms_and_bucket_dict, + ), + ) thread.start() threads.append(thread) @@ -559,19 +544,19 @@ def main(): # pylint: disable=R0915 deployment_account_region=config.deployment_account_region, regions=config.target_regions, account_ids=account_ids, - update_pipelines_only=0 + update_pipelines_only=0, ) step_functions.execute_statemachine() except ParameterNotFoundError: LOGGER.info( - 'A Deployment Account is ready to be bootstrapped! ' - 'The Account provisioner will now kick into action, ' - 'be sure to check out its progress in AWS Step Functions ' - 'in this account.' + "A Deployment Account is ready to be bootstrapped! " + "The Account provisioner will now kick into action, " + "be sure to check out its progress in AWS Step Functions " + "in this account." ) return -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py index bab013907..c74007d4d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py @@ -31,6 +31,16 @@ def _find_all(policy): ) return [f.replace("./adf-bootstrap", ".") for f in _files] + @staticmethod + def _find_all_polices_v2(policy): + _files = list( + glob.iglob( + f"./adf-bootstrap/{policy}/*.json", + recursive=True, + ) + ) + return [f.replace("./adf-bootstrap", ".") for f in _files] + def _compare_ordered_policy(self, obj): if isinstance(obj, dict): return sorted((k, self._compare_ordered_policy(v)) for k, v in obj.items()) @@ -98,7 +108,7 @@ def clean_and_remove_policy_attachment( organization_mapping[path], ) except organizations.client.exceptions.DuplicatePolicyAttachmentException: - pass + LOGGER.info("Policy already attached") organizations.detach_policy(policy_id, organization_mapping[path]) organizations.delete_policy(policy_id) LOGGER.info( @@ -135,6 +145,23 @@ def apply( if self._is_govcloud(REGION_DEFAULT): supported_policies = ["scp"] + self.apply_policies_v1( + organizations, + parameter_store, + config, + organization_mapping, + supported_policies, + ) + + # pylint: disable=too-many-locals + def apply_policies_v1( + self, + organizations, + parameter_store, + config, + organization_mapping, + supported_policies, + ): for policy in supported_policies: _type = "SERVICE_CONTROL_POLICY" if policy == "scp" else "TAG_POLICY" organizations.enable_organization_policies(_type) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_schema.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_schema.py new file mode 100644 index 000000000..e974f7497 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_schema.py @@ -0,0 +1,42 @@ +# Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Schema Validation for Organization Policy Files +""" + +from schema import Schema, Or, Optional +from logger import configure_logger + +LOGGER = configure_logger(__name__) + +V2022_10_14_POLICY_TARGET_SCHEMA = Or([str], str) + + +V2022_10_14Schema = { + "Targets": V2022_10_14_POLICY_TARGET_SCHEMA, + Optional("Version", default="2022-10-14"): "2022-10-14", + "PolicyName": str, + "Policy": dict, +} + +GENERIC_SCHEMA = { + Optional("Version", default="2022-10-14"): "2022-10-14", + object: object, +} + +SCHEMA_MAP = { + "2022-10-14": V2022_10_14Schema, +} + + +class OrgPolicySchema: + def __init__(self, schema_to_validate: dict): + LOGGER.info("Validating Policy Schema: %s", schema_to_validate) + versioned_schema = Schema(GENERIC_SCHEMA).validate(schema_to_validate) + LOGGER.info("Versioned Schema: %s", versioned_schema) + self.schema = Schema(SCHEMA_MAP[versioned_schema["Version"]]).validate( + versioned_schema + ) + if isinstance(self.schema.get("Targets"), str): + self.schema["Targets"] = [self.schema["Targets"]] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_v2.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_v2.py new file mode 100644 index 000000000..f963aad40 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy_v2.py @@ -0,0 +1,229 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Organizations Policy module used throughout the ADF. +""" + +import glob +import os +import ast +import json +import boto3 + +from logger import configure_logger +from organization_policy_campaign import ( + OrganizationPolicyApplicationCampaign, +) +from errors import ParameterNotFoundError + + +from organizations import Organizations + +from organization_policy_schema import OrgPolicySchema + +LOGGER = configure_logger(__name__) +REGION_DEFAULT = os.getenv("AWS_REGION") +DEFAULT_ADF_POLICIES_DIR = "./adf-policies" +DEFAULT_LEGACY_POLICY_DIR = "./adf-bootstrap" + + +# pylint: disable=C0209,W1202 +class OrganizationPolicy: + adf_policies_dir: str + + def __init__(self, adf_policies_dir=None, legacy_policy_dir=None): + self.adf_policies_dir = adf_policies_dir or DEFAULT_ADF_POLICIES_DIR + self.legacy_policy_dir = legacy_policy_dir or DEFAULT_LEGACY_POLICY_DIR + + # pylint: disable-next=logging-not-lazy + LOGGER.info( + "OrgPolicy dirs: %s and %s " + % (self.adf_policies_dir, self.legacy_policy_dir) + ) + + def _find_all_legacy_policies(self, policy): + _files = list( + glob.iglob( + f"{self.legacy_policy_dir}/**/{policy}.json", + recursive=True, + ) + ) + return [f.replace(self.legacy_policy_dir, ".") for f in _files] + + def _find_all_polices(self, policy): + _files = list( + glob.iglob( + f"{self.adf_policies_dir}/{policy}/*.json", + recursive=True, + ) + ) + return [f.replace(f"{self.adf_policies_dir}", ".") for f in _files] + + @staticmethod + def _trim_scp_file_name(policy): + """ + returns the name of the scp policy target. + for example if the policy path is "./deployment/scp.json" this will strip + "./" (two chars) from the front and "/scp.json"(9 chars) from the path. + + If it is ./scp.json, then the target will be /. Stripping the "." and + "scp.json" from the path but leaving the solitary "/" + """ + # pylint: disable-next=logging-not-lazy + LOGGER.info("Policy is: %s" % policy) + return policy[1:][:-8] if policy[1:][:-8] == "/" else policy[2:][:-9] + + @staticmethod + def _trim_tagging_policy_file_name(policy): + """ + returns the name of the scp policy target. + for example if the policy path is "./deployment/tagging-policy.json" this will strip + "./" (two chars) from the front and "/tagging-policy.json"(20 chars) from the path. + + If it is ./tagging-policy.json, then the target will be /. Stripping the "." (1 char) and + "tagging-policy.json" (19 chars) from the path but leaving the solitary "/" + """ + return policy[1:][:-19] if policy[1:][:-19] == "/" else policy[2:][:-20] + + @staticmethod + def _is_govcloud(region: str) -> bool: + """ + Evaluates the region to determine if it is part of GovCloud. + + :param region: a region (us-east-1, us-gov-west-1) + :return: Returns True if the region is GovCloud, False otherwise. + """ + return region.startswith("us-gov") + + def get_policy_body(self, path): + with open( + f"{self.adf_policies_dir}/{path}", mode="r", encoding="utf-8" + ) as policy: + return json.dumps(json.load(policy)) + + def apply( + self, organizations, parameter_store, config + ): # pylint: disable=R0912, R0915 + status = organizations.get_organization_info() + if status.get("feature_set") != "ALL": + LOGGER.info( + "All Features are currently NOT enabled for this Organization, " + "this is required to apply SCPs or Tagging Policies", + ) + return + + supported_policies = { + "scp": "SERVICE_CONTROL_POLICY", + "tagging-policy": "TAG_POLICY", + } + + if self._is_govcloud(REGION_DEFAULT): + supported_policies = { + "scp": "SERVICE_CONTROL_POLICY", + } + + LOGGER.info("Currently supported policy types: %s", supported_policies.values()) + for _, policy_type in supported_policies.items(): + organizations.enable_organization_policies(policy_type) + + LOGGER.info("Building organization map") + organization_mapping = organizations.get_organization_map( + { + "/": organizations.get_ou_root_id(), + } + ) + LOGGER.info("Organization map built!") + + self.apply_policies( + boto3.client("organizations"), + parameter_store, + config, + organization_mapping, + supported_policies, + ) + + # pylint: disable=R0914 + def apply_policies( + self, + organizations, + parameter_store, + config, + organization_mapping, + supported_policies, + ): + LOGGER.info("V2 Method for applying policies") + for policy, policy_type in supported_policies.items(): + _type = policy_type + campaign = OrganizationPolicyApplicationCampaign( + _type, + organization_mapping, + config.get("scp"), + organizations, + ) + legacy_policies = self._find_all_legacy_policies(policy) + # pylint: disable-next=logging-not-lazy + LOGGER.info("Discovered the following legacy policies: %s" % legacy_policies) + try: + current_stored_policy = ast.literal_eval( + parameter_store.fetch_parameter(policy) + ) + for stored_policy in current_stored_policy: + path = ( + OrganizationPolicy._trim_scp_file_name(stored_policy) + if policy == "scp" + else OrganizationPolicy._trim_tagging_policy_file_name( + stored_policy + ) + ) + if stored_policy not in legacy_policies: + # Schedule Policy deletion + LOGGER.info( + "Scheduling policy: %s for deletion", + stored_policy, + ) + campaign.delete_policy(f"adf-{policy}-{path}") + except ParameterNotFoundError: + LOGGER.debug( + "Parameter %s was not found in Parameter Store, continuing.", + policy, + ) + for legacy_policy in legacy_policies: + LOGGER.info("Loading policy: %s", legacy_policy) + proposed_policy = json.loads( + Organizations.get_policy_body(legacy_policy) + ) + + path = ( + OrganizationPolicy._trim_scp_file_name(legacy_policy) + if policy == "scp" + else OrganizationPolicy._trim_tagging_policy_file_name( + legacy_policy + ) + ) + proposed_policy_name = f"adf-{policy}-{path}" + LOGGER.debug(proposed_policy) + policy_instance = campaign.get_policy( + proposed_policy_name, proposed_policy + ) + target = campaign.get_target(path) + policy_instance.set_targets([target]) + + LOGGER.info(target) + + v2_policies = self._find_all_polices(policy) + LOGGER.info("Discovered the following policies: %s", v2_policies) + for v2_policy in v2_policies: + raw_policy_definition = json.loads(self.get_policy_body(v2_policy)) + policy_definition = OrgPolicySchema(raw_policy_definition).schema + # pylint: disable-next=logging-not-lazy + LOGGER.debug("Proposed policy: %s" % policy_definition) + proposed_policy = policy_definition.get("Policy") + proposed_policy_name = policy_definition.get("PolicyName") + campaign_policy = campaign.get_policy( + proposed_policy_name, proposed_policy + ) + targets = policy_definition.get("Targets", []) + campaign_policy.set_targets([campaign.get_target(t) for t in targets]) + campaign.apply() + parameter_store.put_parameter(policy, str(legacy_policies)) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt index 484813f7a..ac07631bc 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt @@ -6,6 +6,7 @@ boto3==1.34.80 botocore==1.34.80 pip~=24.0 pyyaml~=6.0.1 +schema==0.7.5 six~=1.16.0 tenacity==8.2.3 urllib3~=2.2.1 diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py index 878fdfc85..c2048e290 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py @@ -11,6 +11,7 @@ from copy import deepcopy import json import secrets + # Not all string functions are deprecated, the ones we use are not. # Hence disabling the lint finding: from string import ascii_lowercase, digits # pylint: disable=deprecated-module @@ -31,6 +32,7 @@ class ParametersAndTags(TypedDict): """ The param files will have Parameters and Tags, where these are """ + Parameters: Dict[str, str] Tags: Dict[str, str] @@ -50,6 +52,7 @@ class ParamGenWaveTarget(TypedDict): Optimized parameter generation wave target with clearly identified fields as used in the generate parameters process. """ + id: str account_name: str path: WaveTargetPath @@ -73,6 +76,7 @@ class InputPipelineWaveTarget(TypedDict): Each wave target in a pipeline will have the following fields to point to the target account. """ + id: str name: str path: WaveTargetPath @@ -84,9 +88,7 @@ class InputPipelineWaveTarget(TypedDict): # make sure that referencing 100 accounts for example will be broken down # into two waves of 50 accounts each as max supported by CodePipeline. TargetWavesWithNestedWaveTargets = List[ # Waves - List[ # Wave Targets - InputPipelineWaveTarget - ] + List[InputPipelineWaveTarget] # Wave Targets ] @@ -95,6 +97,7 @@ class InputEnvironmentDefinition(TypedDict): Inside the pipeline input environment, the list of targets is defined as a list of waves that each contain a list of wave targets. """ + targets: TargetWavesWithNestedWaveTargets @@ -103,6 +106,7 @@ class InputDefinition(TypedDict): The input of the pipeline definition holds the environment with all the targets defined inside. """ + environment: InputEnvironmentDefinition @@ -111,6 +115,7 @@ class PipelineDefinition(TypedDict): Bare minimum input pipeline definition as required for traversal in this generation of parameters. """ + input: InputDefinition @@ -125,6 +130,7 @@ class Parameters: """ Parameter generation class. """ + def __init__( self, build_name: str, @@ -138,8 +144,7 @@ def __init__( self.build_name = build_name self.definition_s3 = definition_s3 self.file_name = "".join( - secrets.choice(ascii_lowercase + digits) - for _ in range(6) + secrets.choice(ascii_lowercase + digits) for _ in range(6) ) def _retrieve_pipeline_definition(self) -> PipelineDefinition: @@ -155,43 +160,47 @@ def _retrieve_pipeline_targets(self) -> PipelineTargets: pipeline_input_key = ( # Support to fallback to 'input' definition key. # This is scheduled to be deprecated in v4.0 - "pipeline_input" if "pipeline_input" in pipeline_definition + "pipeline_input" + if "pipeline_input" in pipeline_definition else "input" ) - input_targets: TargetWavesWithNestedWaveTargets = ( - pipeline_definition[pipeline_input_key]['environments']['targets'] - ) + input_targets: TargetWavesWithNestedWaveTargets = pipeline_definition[ + pipeline_input_key + ]["environments"]["targets"] # Since the input_targets returns a list of waves that each contain # a list of wave_targets, we need to flatten them to iterate: wave_targets: Iterator[ParamGenWaveTarget] = map( lambda wt: { # Change wt: InputPipelineWaveTarget to ParamGenWaveTarget - 'id': wt['id'], - 'account_name': wt['name'], - 'path': wt['path'], - 'regions': wt['regions'], + "id": wt["id"], + "account_name": wt["name"], + "path": wt["path"], + "regions": wt["regions"], }, filter( - lambda wt: wt['id'] != 'approval', + lambda wt: wt["id"] != "approval", # Flatten the three levels of nested arrays to one iterable: chain.from_iterable( chain.from_iterable( input_targets, ), ), - ) # Returns an Iterator[InputPipelineWaveTarget] + ), # Returns an Iterator[InputPipelineWaveTarget] ) for wave_target in wave_targets: - if wave_target['id'] in pipeline_targets: + if wave_target["id"] in pipeline_targets: # Lets merge the regions to show what regions it deploys # to - stored_target = pipeline_targets[wave_target['id']] - stored_target['regions'] = sorted(list(set( - stored_target['regions'] - + wave_target['regions'], - ))) + stored_target = pipeline_targets[wave_target["id"]] + stored_target["regions"] = sorted( + list( + set( + stored_target["regions"] + wave_target["regions"], + ) + ) + ) else: - pipeline_targets[wave_target['id']] = wave_target + pipeline_targets[wave_target["id"]] = wave_target # Returns a list of targets: # [ # { @@ -210,7 +219,7 @@ def _retrieve_pipeline_targets(self) -> PipelineTargets: def _create_params_folder(self) -> None: try: - dir_path = f'{self.cwd}/params' + dir_path = f"{self.cwd}/params" os.mkdir(dir_path) LOGGER.debug("Created directory: %s", dir_path) except FileExistsError: @@ -245,10 +254,10 @@ def create_parameter_files(self) -> None: this case. """ for target in self._retrieve_pipeline_targets().values(): - for region in target['regions']: + for region in target["regions"]: LOGGER.debug( "Generating parameters for the %s account in %s", - target['account_name'], + target["account_name"], region, ) current_params = deepcopy(EMPTY_PARAMS_DICT) @@ -262,18 +271,17 @@ def create_parameter_files(self) -> None: current_params = self._merge_params( Parameters._parse( params_root_path=self.cwd, - params_filename=target['account_name'], + params_filename=target["account_name"], ), current_params, ) - path_references_ou = ( - isinstance(target['path'], str) - and not Parameters._is_account_id(target['path']) - ) + path_references_ou = isinstance( + target["path"], str + ) and not Parameters._is_account_id(target["path"]) if path_references_ou: # Compare account_region final to ou_region - ou_id_or_path = target['path'] - if ou_id_or_path.startswith('/'): + ou_id_or_path = target["path"] + if ou_id_or_path.startswith("/"): # Skip the first slash ou_id_or_path = ou_id_or_path[1:] # Cleanup the ou name to include only alphanumeric, dashes, @@ -283,7 +291,7 @@ def create_parameter_files(self) -> None: params_root_path=self.cwd, params_filename=f"{ou_id_or_path}_{region}", ), - current_params + current_params, ) # Compare account_region final to ou current_params = self._merge_params( @@ -291,7 +299,7 @@ def create_parameter_files(self) -> None: params_root_path=self.cwd, params_filename=ou_id_or_path, ), - current_params + current_params, ) # Compare account_region final to deployment_account_region current_params = self._merge_params( @@ -299,7 +307,7 @@ def create_parameter_files(self) -> None: params_root_path=self.cwd, params_filename=f"global_{region}", ), - current_params + current_params, ) # Compare account_region final to global_stage if ADF_ORG_STAGE: @@ -316,7 +324,7 @@ def create_parameter_files(self) -> None: params_root_path=self.cwd, params_filename="global", ), - current_params + current_params, ) if current_params: self._write_params( @@ -332,7 +340,7 @@ def _is_account_id(wave_target_path: WaveTargetPath) -> bool: def _clean_params_filename(params_filename: str) -> str: # Cleanup the params_filename to include only alphanumeric, dashes, # slashes, and underscores: - return re.sub(r'[^0-9a-zA-Z_\-/]+', '_', params_filename) + return re.sub(r"[^0-9a-zA-Z_\-/]+", "_", params_filename) @staticmethod def _parse( @@ -363,7 +371,7 @@ def _parse( ) file_path = f"{params_root_path}/params/{clean_file_name}" try: - with open(f"{file_path}.json", encoding='utf-8') as file: + with open(f"{file_path}.json", encoding="utf-8") as file: json_content = json.load(file) LOGGER.debug( "Read %s.yml: %s", @@ -382,14 +390,14 @@ def _parse( ) return yaml_content except yaml.scanner.ScannerError: - LOGGER.exception('Invalid Yaml for %s.yml', file_path) + LOGGER.exception("Invalid Yaml for %s.yml", file_path) raise except FileNotFoundError: LOGGER.debug( "File not found for %s.{json or yml}, defaulting to empty", file_path, ) - return {'Parameters': {}, 'Tags': {}} + return {"Parameters": {}, "Tags": {}} def _write_params( self, @@ -411,13 +419,11 @@ def _write_params( filepath, new_params, ) - with open(filepath, mode='w', encoding='utf-8') as outfile: + with open(filepath, mode="w", encoding="utf-8") as outfile: json.dump(new_params, outfile) def _merge_params( - self, - new_params: ParametersAndTags, - current_params: ParametersAndTags + self, new_params: ParametersAndTags, current_params: ParametersAndTags ) -> ParametersAndTags: """ Merge the new_params Parameters and Tags found into a clone of the @@ -442,12 +448,12 @@ def _merge_params( if root_key not in merged_params: merged_params[root_key] = {} for key in new_params[root_key]: - if merged_params[root_key].get(key, '') == '': - merged_params[root_key][key] = ( - self.resolver.apply_intrinsic_function_if_any( - new_params[root_key][key], - self.file_name, - ) + if merged_params[root_key].get(key, "") == "": + merged_params[root_key][ + key + ] = self.resolver.apply_intrinsic_function_if_any( + new_params[root_key][key], + self.file_name, ) LOGGER.debug( "Merged result %s", @@ -473,5 +479,5 @@ def main() -> None: parameters.create_parameter_files() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py new file mode 100644 index 000000000..66d50d33f --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organization_policy_campaign.py @@ -0,0 +1,629 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Organizations Policy module used throughout the ADF. +""" + +import os +import json +from typing import List +import boto3 + +from logger import configure_logger + + +LOGGER = configure_logger(__name__) +REGION_DEFAULT = os.getenv("AWS_REGION") +ENABLE_V2 = os.getenv("ENABLE_V2") +DEFAULT_POLICY_ID = "p-FullAWSAccess" + +# pylint: disable=W1508. R1735, W0235, R1734, W1201 + + +class OrganizationPolicyException(Exception): + def __init__(self, *args: object) -> None: + super().__init__(*args) + + +class PolicyTargetNotFoundException(OrganizationPolicyException): + def __init__(self, *args: object) -> None: + super().__init__(*args) + +# pylint: disable=W1508. R1735, W0235, R1734, W1201, C0209 +class OrganizationPolicyTarget: + existing_policy_ids: dict() + path: str + type: str + id: str + config: dict() + + def __repr__(self) -> str: + return f"{self.path} ({self.id}) ({self.type})" + + def __init__( + self, + target_path, + policy_type, + policy_id, + config, + organizations_client=None, + ) -> None: + + self.path = target_path + self.type = policy_type + self.id = policy_id + self.config = config + self.organizations_client = ( + organizations_client + if organizations_client + else boto3.client("organizations") + ) + self.existing_policy_ids = self.get_existing_policies() + + def get_existing_policies(self): + existing_policy_ids = { + p["Id"]: p["Name"] + for p in self.organizations_client.list_policies_for_target( + TargetId=self.id, Filter=self.type + ).get("Policies") + } + return existing_policy_ids + + def attach_policy(self, policy_id, policy_name): + LOGGER.debug("Existing Policy Ids: %s", self.existing_policy_ids) + if policy_id not in self.existing_policy_ids: + self.organizations_client.attach_policy( + PolicyId=policy_id, TargetId=self.id + ) + self.existing_policy_ids[policy_id] = policy_name + else: + LOGGER.info( + "Policy %s (%s) already attached to %s" % + (policy_name, policy_id, self), + ) + + if ( + DEFAULT_POLICY_ID in self.existing_policy_ids.keys() + and self.config.get("keep-default-scp", "enabled") == "disabled" + ): + self.organizations_client.detach_policy( + PolicyId=DEFAULT_POLICY_ID, TargetId=self.id + ) + + +# pylint: disable=W1508. R1735, W0235, R1734, W1201, W1203 +class OrganizationalPolicyCampaignPolicy: + name: str + body: str + id: str + type: str + campaign_config: dict() + current_targets: list() + targets_requiring_attachment: dict() + policy_has_changed: bool + targets_not_scheduled_for_deletion: list() + + def __repr__(self) -> str: + return f"{self.name} ({self.id}) ({self.type})" + + def __init__( + self, + policy_name, + policy_body, + policy_type, + config, + policy_id=None, + policy_has_changed=False, + organizations_client=None, + ): + self.name = policy_name + self.body = policy_body + self.id = policy_id + self.type = policy_type + self.campaign_config = config + self.organizations_client = ( + organizations_client + if organizations_client + else boto3.client("organizations") + ) + self.current_targets = self.get_current_targets_for_policy() + self.targets_requiring_attachment = {} + self.policy_has_changed = policy_has_changed + self.targets_not_scheduled_for_deletion = [] + + def get_current_targets_for_policy(self): + if self.id: + try: + current_targets = [ + t.get("TargetId") + for t in self.organizations_client.list_targets_for_policy( + PolicyId=self.id + ).get("Targets") + ] + return current_targets + except ( + self.organizations_client.exceptions.AccessDeniedException + ) as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Access Denied" + % (self.name.self.id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error fetching targets for policy {self.name} {self.id}: Access Denied" + ) from e + except ( + self.organizations_client.exceptions.AWSOrganizationsNotInUseException + ) as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Organizations not in use" + % (self.name.self.id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + "Organizations not in use" + ) from e + except ( + self.organizations_client.exceptions.InvalidInputException + ) as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Invalid Input" + % (self.name.self.id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error fetching targets for policy {self.name} {self.id}: Invalid Input" + ) from e + except self.organizations_client.exceptions.ServiceException as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Service Exception" % + (self.name.self.id), + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error fetching targets for policy {self.name} {self.id}: Service Exception " + ) from e + except ( + self.organizations_client.exceptions.TooManyRequestsException + ) as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Access Denied" + % (self.name.self.id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + "Too Many Requests to Organizations API" + ) from e + except Exception as e: + LOGGER.critical( + "Error fetching targets for policy %s %s: Unexpected exception", + self.name, + self.id, + ) + LOGGER.error(e) + raise e + else: + return [] + + def set_targets(self, targets): + target: OrganizationPolicyTarget + LOGGER.info("Current targets: %s", self.current_targets) + for target in targets: + if target.id in self.current_targets: + LOGGER.info( + "%s already exists as a target for policy: %s", + target.id, + self.name, + ) + self.targets_not_scheduled_for_deletion.append(target.id) + continue + LOGGER.info( + "%s is not a target for: %s, marking it for attachment", + target.id, + self.name, + ) + self.targets_requiring_attachment[target.id] = target + + def update_targets(self): + if self.targets_requiring_attachment.values(): + LOGGER.info( + "Attaching the policy (%s) to the following targets: %s", + self.name, + self.targets_requiring_attachment, + ) + for target in self.targets_requiring_attachment.values(): + target.attach_policy(self.id, self.name) + + targets_to_detach = set(self.current_targets) - set( + self.targets_not_scheduled_for_deletion + ) + + if targets_to_detach: + LOGGER.info( + "Removing the policy (%s) from the following targets: %s", + self.name, + targets_to_detach, + ) + + for target_id in targets_to_detach: + LOGGER.info( + "Detaching policy (%s) from target (%s)", self.name, target_id + ) + try: + self.organizations_client.detach_policy( + PolicyId=self.id, TargetId=target_id + ) + except ( + self.organizations_client.exceptions.AccessDeniedException + ) as e: + LOGGER.critical( + "Error detaching policy %s (%s) from target %s: Access Denied" % + (self.name, self.id, target_id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error detaching policy: {self.name} {self.id} from {target_id}: Access Denied" + ) from e + except ( + self.organizations_client.exceptions.PolicyNotAttachedException + ) as e: + LOGGER.warning( + "Error detaching policy %s (%s) from target %s: Policy Not Attached" % + (self.name, self.id, target_id) + ) + LOGGER.info(e) + return + except ( + self.organizations_client.exceptions.TargetNotFoundException + ) as e: + LOGGER.critical( + "Error detaching policy %s (%s) from target %s: Target Not Found" % + (self.name, self.id, target_id) + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error detaching policy {self.name} {self.id}: Target {target_id} Not Found" + ) from e + except Exception as e: + LOGGER.critical( + "Error detaching policy %s %s: Unexpected Exception" % + (self.name, self.id) + ) + LOGGER.error(e) + raise e + + def create(self): + policy_type_name = ( + "scp" if self.type == "SERVICE_CONTROL_POLICY" else "tagging-policy" + ) + try: + self.id = ( + self.organizations_client.create_policy( + Content=json.dumps(self.body), + Description=f"ADF Managed {policy_type_name}", + Name=self.name, + Type=self.type, + ) + .get("Policy") + .get("PolicySummary") + .get("Id") + ) + except self.organizations_client.exceptions.AccessDeniedException as e: + LOGGER.critical(f"Error creating policy {self.name}: Access Denied") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Access Denied" + ) from e + except ( + self.organizations_client.exceptions.ConcurrentModificationException + ) as e: + LOGGER.critical( + f"Error creating policy {self.name}: Concurrent Modification Ongoing" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Concurrent Modification Ongoing" + ) from e + except ( + self.organizations_client.exceptions.ConstraintViolationException + ) as e: + LOGGER.critical( + f"Error creating policy {self.name}: Constraint Violation" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Constraint Violation" + ) from e + except ( + self.organizations_client.exceptions.DuplicatePolicyException + ) as e: + LOGGER.warning( + f"Error creating policy {self.name}: Duplicate Policy" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Duplicate Policy" + ) from e + except self.organizations_client.exceptions.InvalidInputException as e: + LOGGER.warning(f"Error creating policy {self.name}: Invalid Input") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Invalid Input" + ) from e + except ( + self.organizations_client.exceptions.MalformedPolicyDocumentException + ) as e: + LOGGER.warning( + f"Error creating policy {self.name}: Policy Content Malformed" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + f"Error creating policy {self.name}: Policy Content Malformed" + ) from e + except Exception as e: + LOGGER.critical( + f"Error creating policy {self.name}: Unexpected Exception" + ) + LOGGER.error(e) + raise e + LOGGER.info("Policy %s created with id: %s", self.name, self.id) + self.update_targets() + + def update(self): + if self.policy_has_changed: + LOGGER.info( + "Policy %s has changed. Updating the policy with new content", + self.name, + ) + self.organizations_client.update_policy( + PolicyId=self.id, Content=json.dumps(self.body) + ) + self.update_targets() + + def delete(self): + self.update_targets() + self.organizations_client.delete_policy(PolicyId=self.id) + + +# pylint: disable=W1508. R1735, W0235, R1734, W1201, W1203 +class OrganizationPolicyApplicationCampaign: + targets: dict() + campaign_config: dict() + type: str + organizational_mapping: dict + policies_to_be_created: List[OrganizationalPolicyCampaignPolicy] + policies_to_be_updated: List[OrganizationalPolicyCampaignPolicy] + policies_to_be_deleted: List[OrganizationalPolicyCampaignPolicy] + + def __init__( + self, + policy_type, + organizational_mapping, + campaign_config, + organizations_client, + ) -> None: + self.targets = {} + self.type = policy_type + self.organizational_mapping = organizational_mapping + self.organizations = organizations_client + self.policies_to_be_created = [] + self.policies_to_be_updated = [] + self.policies_to_be_deleted = [] + self.existing_policy_lookup = self.get_existing_policies() + self.campaign_config = campaign_config + + def get_existing_policies(self): + try: + # TODO: Implement paginator here + response = self.organizations.list_policies(Filter=self.type) + except self.organizations.exceptions.AccessDeniedException as e: + LOGGER.critical("Error fetching existing policies: Access Denied") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Access Denied when fetching existing policies ({self.type})" + ) from e + except ( + self.organizations.exceptions.AWSOrganizationsNotInUseException + ) as e: + LOGGER.critical( + "Error fetching existing policies: AWS Orgs not in use" + ) + LOGGER.error(e) + raise OrganizationPolicyException("Organizations not in use") from e + except self.organizations.exceptions.InvalidInputException as e: + LOGGER.critical("Error fetching existing policies: Invalid Input") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Invalid input fetching existing policies: {self.type}" + ) from e + except self.organizations.exceptions.ServiceException as e: + LOGGER.critical( + "Error fetching existing policies: Service Exception" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + "Service Error when fetching existing Org Policies" + ) from e + except self.organizations.exceptions.TooManyRequestsException as e: + LOGGER.critical( + "Error fetching existing policies: Too Many Requests" + ) + LOGGER.error(e) + raise OrganizationPolicyException( + "Too Many Requests to Organizations API" + ) from e + except Exception as e: + LOGGER.critical( + "Unexpected exception when fetching existing policies" + ) + LOGGER.error(e) + raise e + + policy_type_name = ( + "scp" if self.type == "SERVICE_CONTROL_POLICY" else "tagging-policy" + ) + return { + p["Name"]: p["Id"] + for p in response["Policies"] + if f"ADF Managed {policy_type_name}" in p["Description"] + } + + def get_target(self, target: str) -> OrganizationPolicyTarget: + if target not in self.targets: + try: + self.targets[target] = OrganizationPolicyTarget( + target_path=target, + policy_type=self.type, + policy_id=self.organizational_mapping[target], + config=self.campaign_config, + organizations_client=self.organizations, + ) + except KeyError as e: + LOGGER.critical( + f"The target {e} was not found in the OU target Map" + ) + LOGGER.info("Current OU map: %s", self.organizational_mapping) + raise PolicyTargetNotFoundException( + f"The target {e} was not found in the OU target Map" + ) from e + return self.targets[target] + + def get_policy(self, policy_name, policy_body): + if policy_name not in self.existing_policy_lookup: + return self.create_policy(policy_name, policy_body) + return self.update_policy(policy_name, policy_body) + + def create_policy(self, policy_name, policy_body): + policy = OrganizationalPolicyCampaignPolicy( + policy_name, + policy_body, + self.type, + self.campaign_config, + None, + True, + self.organizations, + ) + self.policies_to_be_created.append(policy) + return policy + + def update_policy(self, policy_name, policy_body): + current_policy = {} + try: + current_policy = json.loads( + self.organizations.describe_policy( + PolicyId=self.existing_policy_lookup[policy_name] + ) + .get("Policy") + .get("Content") + ) + except self.organizations.exceptions.AccessDeniedException as e: + LOGGER.critical("Error describing existing policy: Access Denied") + LOGGER.error(e) + policy_id = self.existing_policy_lookup[policy_name] + raise OrganizationPolicyException( + f"Access Denied when fetching policy : {policy_name}{policy_id} ({self.type})" + ) from e + except ( + self.organizations.exceptions.AWSOrganizationsNotInUseException + ) as e: + LOGGER.critical( + "Error describing existing policy: AWS Orgs not in use" + ) + LOGGER.error(e) + raise OrganizationPolicyException("Organizations not in use") from e + except self.organizations.exceptions.InvalidInputException as e: + LOGGER.critical("Error fetching existing policies: Invalid Input") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Invalid input fetching existing policy: {self.type}" + ) from e + except self.organizations.exceptions.ServiceException as e: + LOGGER.critical("Error fetching existing policy: Service Exception") + LOGGER.error(e) + raise OrganizationPolicyException( + f"Service Error when fetching existing policy {policy_name}" + ) from e + except self.organizations.exceptions.TooManyRequestsException as e: + LOGGER.critical("Error describing policy: Too Many Requests") + LOGGER.error(e) + raise OrganizationPolicyException( + "Too Many Requests to Organizations API" + ) from e + except Exception as e: # pylint: disable=W0703 + LOGGER.critical( + "Unexpected exception when describing existing policy" + ) + LOGGER.error(e) + + policy_has_changed = current_policy != policy_body + policy = OrganizationalPolicyCampaignPolicy( + policy_name, + policy_body, + self.type, + self.campaign_config, + self.existing_policy_lookup[policy_name], + policy_has_changed, + self.organizations, + ) + self.policies_to_be_updated.append(policy) + + return policy + + def delete_policy(self, policy_name): + if policy_name in self.existing_policy_lookup: + policy = OrganizationalPolicyCampaignPolicy( + policy_name, + {}, + self.type, + self.campaign_config, + self.existing_policy_lookup[policy_name], + False, + self.organizations, + ) + self.policies_to_be_deleted.append(policy) + + def apply(self): + if self.policies_to_be_created: + LOGGER.info( + "The following policies need to be created: %s", + [policy.name for policy in self.policies_to_be_created], + ) + for policy in self.policies_to_be_created: + policy.create() + + if self.policies_to_be_updated: + LOGGER.info( + "The following policies (may) need to be updated: %s", + [policy.name for policy in self.policies_to_be_updated], + ) + for policy in self.policies_to_be_updated: + policy.update() + + policies_defined_from_files = { + policy.name for policy in self.policies_to_be_updated + } + + adf_managed_policy_names = set(self.existing_policy_lookup.keys()) + self.policies_to_be_deleted.extend( + [ + OrganizationalPolicyCampaignPolicy( + p, + {}, + self.type, + self.campaign_config, + self.existing_policy_lookup[p], + False, + self.organizations, + ) + for p in adf_managed_policy_names - policies_defined_from_files + ] + ) + if self.policies_to_be_deleted: + LOGGER.info( + "The following will be deleted as they are no longer defined in a file: %s", + self.policies_to_be_deleted, + ) + + for policy in self.policies_to_be_deleted: + policy.delete() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py index afdaa72a3..60dfc9384 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py @@ -222,9 +222,7 @@ def describe_policy_id_for_target( return [] def describe_policy(self, policy_id): - response = self.client.describe_policy( - PolicyId=policy_id, - ) + response = self.client.describe_policy(PolicyId=policy_id) return response.get("Policy") def attach_policy(self, policy_id, target_id): diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizational_policy_campaigns.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizational_policy_campaigns.py new file mode 100644 index 000000000..ba49fdf13 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizational_policy_campaigns.py @@ -0,0 +1,1165 @@ +""" +Tests creation and execution of organizational policy campaigns +""" +import unittest +import json + +import boto3 +from botocore.stub import Stubber, ANY + + +from organization_policy_campaign import ( + OrganizationPolicyApplicationCampaign, + OrganizationPolicyException, +) + +POLICY_DEFINITIONS = [ + { + "Targets": ["MyFirstOrg"], + "Version": "2022-10-14", + "PolicyName": "MyFirstPolicy", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Deny", "Action": "cloudtrail:Stop*", "Resource": "*"}, + {"Effect": "Allow", "Action": "*", "Resource": "*"}, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + "config:Stop*", + ], + "Resource": "*", + }, + ], + }, + }, + { + "Targets": ["MySecondOrg"], + "Version": "2022-10-14", + "PolicyName": "MySecondPolicy", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Deny", "Action": "cloudtrail:Stop*", "Resource": "*"}, + {"Effect": "Allow", "Action": "*", "Resource": "*"}, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + ], + "Resource": "*", + }, + ], + }, + }, +] + +# pylint: disable=too-many-lines + + +class HappyTestCases(unittest.TestCase): + def test_scp_campaign_creation_no_existing_policies(self): + """ + Test case that covers the creation of two new SCPs that have one target + each. + """ + + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # No existing ADF managed policies. + stubber.add_response("list_policies", {"Policies": []}) + + # When the target object is created, + # it will query to see if it has any policies already + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Create Policy API Call + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id", + "Arn": "arn:aws:organizations:policy/fake-policy-id", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MyFirstPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + + # Once created, the policy is attached to the target + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id", "TargetId": "123456789012"}, + ) + + # Creation and attachment of second policy + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id-2", + "Arn": "arn:aws:organizations:policy/fake-policy-id-2", + "Name": "MySecondPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MySecondPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id-2", "TargetId": "09876543210"}, + ) + + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in POLICY_DEFINITIONS: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(1, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_one_existing_policy_different_content(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + # When a preexisting policy is loaded - describe policy is used to get + # the existing policy content. + stubber.add_response( + "describe_policy", + { + "Policy": { + "PolicySummary": {}, + "Content": json.dumps({"old-policy": "content"}), + } + }, + {"PolicyId": "fake-policy-1"}, + ) + + # When loading a policy object that exists already, this API call is + # used to populate the list of existing targets. + stubber.add_response( + "list_targets_for_policy", + {"Targets": []}, + {"PolicyId": "fake-policy-1"}, + ) + + # Creation of target object - Query for existing policies + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Creates the second policy and attaches the policy to the target + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id-2", + "Arn": "arn:aws:organizations:policy/fake-policy-id-2", + "Name": "MySecondPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MySecondPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id-2", "TargetId": "09876543210"}, + ) + + # Update the content of the existing policy + stubber.add_response( + "update_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "PolicyId": "fake-policy-1", + "Content": json.dumps(POLICY_DEFINITIONS[0].get("Policy")), + }, + ) + + # Attach 1st policy to the target as part of the update process. + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "123456789012"}, + ) + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in POLICY_DEFINITIONS: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(1, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_one_existing_policy(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + # When a preexisting policy is loaded - describe policy is used to get + # the existing policy content. + stubber.add_response( + "describe_policy", + { + "Policy": { + "PolicySummary": {}, + "Content": json.dumps(POLICY_DEFINITIONS[0].get("Policy")), + } + }, + {"PolicyId": "fake-policy-1"}, + ) + + # When loading a policy object that exists already, this API call is + # used to populate the list of existing targets. + stubber.add_response( + "list_targets_for_policy", + {"Targets": []}, + {"PolicyId": "fake-policy-1"}, + ) + + # Creation of target object - Query for existing policies + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Creates the second policy and attaches the policy to the target + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id-2", + "Arn": "arn:aws:organizations:policy/fake-policy-id-2", + "Name": "MySecondPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MySecondPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id-2", "TargetId": "09876543210"}, + ) + + # Attach 1st policy to the target as part of the update process. + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "123456789012"}, + ) + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in POLICY_DEFINITIONS: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(1, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_one_existing_policy_with_existing_target(self): + policy_definitions = [ + { + "Targets": ["MyFirstOrg", "MySecondOrg", "MyThirdOrg"], + "Version": "2022-10-14", + "PolicyName": "MyFirstPolicy", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "cloudtrail:Stop*", + "Resource": "*", + }, + {"Effect": "Allow", "Action": "*", "Resource": "*"}, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + "config:Stop*", + ], + "Resource": "*", + }, + ], + }, + } + ] + org_client = boto3.client("organizations") + org_mapping = { + "MyFirstOrg": "123456789012", + "MySecondOrg": "09876543210", + "MyThirdOrg": "11223344556", + } + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + # When a preexisting policy is loaded - describe policy is used to get + # the existing policy content. + stubber.add_response( + "describe_policy", + { + "Policy": { + "PolicySummary": {}, + "Content": json.dumps(POLICY_DEFINITIONS[0].get("Policy")), + } + }, + {"PolicyId": "fake-policy-1"}, + ) + + # When loading a policy object that exists already, this API call is + # used to populate the list of existing targets. + stubber.add_response( + "list_targets_for_policy", + { + "Targets": [ + { + "TargetId": "11223344556", + "Arn": "arn:aws:organizations:account11223344556", + "Name": "MyThirdOrg", + "Type": "ORGANIZATIONAL_UNIT", + } + ] + }, + {"PolicyId": "fake-policy-1"}, + ) + + # Creation of target object - Query for existing policies (three targets) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "11223344556", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Attach the policy to two OUs - MyFirstOrg / MySecondOrg + + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "123456789012"}, + ) + + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "09876543210"}, + ) + + stubber.activate() + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in policy_definitions: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(2, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_one_existing_policy_with_existing_target_deletion( + self, + ): + policy_definitions = [ + { + "Targets": ["MyFirstOrg", "MySecondOrg"], + "Version": "2022-10-14", + "PolicyName": "MyFirstPolicy", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "cloudtrail:Stop*", + "Resource": "*", + }, + {"Effect": "Allow", "Action": "*", "Resource": "*"}, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + "config:Stop*", + ], + "Resource": "*", + }, + ], + }, + } + ] + org_client = boto3.client("organizations") + org_mapping = { + "MyFirstOrg": "123456789012", + "MySecondOrg": "09876543210", + "MyThirdOrg": "11223344556", + } + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + # When a preexisting policy is loaded - describe policy is used to get + # the existing policy content. + stubber.add_response( + "describe_policy", + { + "Policy": { + "PolicySummary": {}, + "Content": json.dumps(POLICY_DEFINITIONS[0].get("Policy")), + } + }, + {"PolicyId": "fake-policy-1"}, + ) + + # When loading a policy object that exists already, this API call is + # used to populate the list of existing targets. + stubber.add_response( + "list_targets_for_policy", + { + "Targets": [ + { + "TargetId": "11223344556", + "Arn": "arn:aws:organizations:account11223344556", + "Name": "MyThirdOrg", + "Type": "ORGANIZATIONAL_UNIT", + } + ] + }, + {"PolicyId": "fake-policy-1"}, + ) + + # Creation of target object - Query for existing policies (two targets) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Attach the policy to two OUs - MyFirstOrg / MySecondOrg + + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "123456789012"}, + ) + + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "09876543210"}, + ) + + stubber.add_response( + "detach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "11223344556"}, + ) + + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in policy_definitions: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(2, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_no_existing_policies_targets_have_default_policy( + self, + ): + """ + Test case that covers the creation of two new SCPs that have one target with the default SCP + each. + """ + + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # No existing ADF managed policies. + stubber.add_response("list_policies", {"Policies": []}) + + # When the target object is created, + # it will query to see if it has any policies already + stubber.add_response( + "list_policies_for_target", + { + "Policies": [ + { + "Id": "p-FullAWSAccess", + "Arn": "arn:aws:organization:policy/p-FullAWSAccess", + "Name": "FullAWSAccess", + "Description": "FullAWSAccess", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": True, + } + ] + }, + {"TargetId": "123456789012", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Create Policy API Call + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id", + "Arn": "arn:aws:organizations:policy/fake-policy-id", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MyFirstPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + + # Once created, the policy is attached to the target + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id", "TargetId": "123456789012"}, + ) + # Once the other policy is attached, we detach the default policy + stubber.add_response( + "detach_policy", + {}, + {"PolicyId": "p-FullAWSAccess", "TargetId": "123456789012"}, + ) + + # Creation and attachment of second policy + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id-2", + "Arn": "arn:aws:organizations:policy/fake-policy-id-2", + "Name": "MySecondPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MySecondPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id-2", "TargetId": "09876543210"}, + ) + + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", + org_mapping, + {"keep-default-scp": "disabled"}, + org_client, + ) + + for _policy in POLICY_DEFINITIONS: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(1, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + + def test_scp_campaign_creation_one_existing_policy_not_in_definitions(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "09876543210", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Creates the second policy and attaches the policy to the target + stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id-2", + "Arn": "arn:aws:organizations:policy/fake-policy-id-2", + "Name": "MySecondPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "MySecondPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id-2", "TargetId": "09876543210"}, + ) + + stubber.add_response( + "list_targets_for_policy", + { + "Targets": [ + { + "TargetId": "11223344556", + "Arn": "arn:aws:organizations:account11223344556", + "Name": "MyThirdOrg", + "Type": "ORGANIZATIONAL_UNIT", + } + ] + }, + {"PolicyId": "fake-policy-1"}, + ) + + stubber.add_response( + "detach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "11223344556"}, + ) + stubber.add_response( + "delete_policy", + {}, + {"PolicyId": "fake-policy-1"}, + ) + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + for _policy in POLICY_DEFINITIONS[1:]: + policy = policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + self.assertEqual(0, len(policy.targets_requiring_attachment)) + policy.set_targets( + [policy_campaign.get_target(t) for t in _policy.get("Targets")] + ) + self.assertEqual(1, len(policy.targets_requiring_attachment)) + + policy_campaign.apply() + stubber.assert_no_pending_responses() + + +class SadTestCases(unittest.TestCase): + def test_scp_campaign_creation_access_denied_error_fetching_policies(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + stubber.add_client_error( + method="list_policies", + service_error_code="AccessDeniedException", + service_message="Access Denied", + ) + + stubber.activate() + with self.assertRaises(OrganizationPolicyException): + OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + def test_scp_campaign_creation_orgs_not_in_use_fetching_policies(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + stubber.add_client_error( + method="list_policies", + service_error_code="AWSOrganizationsNotInUseException", + service_message="AWSOrganizationsNotInUseException", + ) + + stubber.activate() + with self.assertRaises(OrganizationPolicyException): + OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + def test_scp_campaign_creation_invalid_input_fetching_policies(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + stubber.add_client_error( + method="list_policies", + service_error_code="InvalidInputException", + service_message="InvalidInputException", + ) + + stubber.activate() + with self.assertRaises(OrganizationPolicyException): + OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + def test_scp_campaign_creations_service_exception_fetching_policies(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + stubber.add_client_error( + method="list_policies", + service_error_code="ServiceException", + service_message="ServiceException", + ) + + stubber.activate() + with self.assertRaises(OrganizationPolicyException): + OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + def test_scp_campaign_creations_too_many_requests_fetching_policies(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + stubber.add_client_error( + method="list_policies", + service_error_code="TooManyRequestsException", + service_message="TooManyRequestsException", + ) + + stubber.activate() + with self.assertRaises(OrganizationPolicyException): + OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + def test_scp_campaign_creation_load_policy_access_denied(self): + org_client = boto3.client("organizations") + org_mapping = {"MyFirstOrg": "123456789012", "MySecondOrg": "09876543210"} + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-1", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + # When a preexisting policy is loaded - describe policy is used to get + # the existing policy content. + stubber.add_client_error( + method="describe_policy", + service_error_code="AccessDeniedException", + service_message="Access Denied", + ) + + # Attach 1st policy to the target as part of the update process. + stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-1", "TargetId": "123456789012"}, + ) + stubber.activate() + + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + + with self.assertRaises(OrganizationPolicyException): + for _policy in POLICY_DEFINITIONS: + policy_campaign.get_policy( + _policy.get("PolicyName"), _policy.get("Policy") + ) + + def test_policy_detachment_error_handling_access_denied(self): + org_client = boto3.client("organizations") + org_mapping = { + "MyFirstOrg": "11223344556", + } + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-3", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + stubber.add_response( + "list_targets_for_policy", + { + "Targets": [ + { + "TargetId": "11223344556", + "Arn": "arn:aws:organizations:account11223344556", + "Name": "MyFirstOrg", + "Type": "ORGANIZATIONAL_UNIT", + } + ] + }, + {"PolicyId": "fake-policy-3"}, + ) + + stubber.add_client_error( + method="detach_policy", + service_error_code="AccessDeniedException", + service_message="Access Denied", + expected_params={"PolicyId": "fake-policy-3", "TargetId": "11223344556"}, + ) + + stubber.activate() + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + with self.assertRaises(OrganizationPolicyException): + policy_campaign.apply() + + stubber.assert_no_pending_responses() + + def test_policy_detachment_error_handling_policy_not_attached(self): + org_client = boto3.client("organizations") + org_mapping = { + "MyFirstOrg": "11223344556", + } + stubber = Stubber(org_client) + + # One pre-existing ADF managed policy + stubber.add_response( + "list_policies", + { + "Policies": [ + { + "Id": "fake-policy-3", + "Arn": "arn:aws:organizations:policy/fake-policy-1", + "Name": "MyFirstPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + } + ] + }, + ) + + stubber.add_response( + "list_targets_for_policy", + { + "Targets": [ + { + "TargetId": "11223344556", + "Arn": "arn:aws:organizations:account11223344556", + "Name": "MyFirstOrg", + "Type": "ORGANIZATIONAL_UNIT", + } + ] + }, + {"PolicyId": "fake-policy-3"}, + ) + + stubber.add_client_error( + method="detach_policy", + service_error_code="PolicyNotAttachedException", + service_message="Policy Not Attached", + expected_params={"PolicyId": "fake-policy-3", "TargetId": "11223344556"}, + ) + + stubber.add_response( + "delete_policy", + {}, + {"PolicyId": "fake-policy-3"}, + ) + + stubber.activate() + policy_campaign = OrganizationPolicyApplicationCampaign( + "SERVICE_CONTROL_POLICY", org_mapping, {}, org_client + ) + ex_r = ( + "WARNING:organization_policy_campaign:Error detaching policy" + " MyFirstPolicy (fake-policy-3) from target 11223344556: " + "Policy Not Attached" + ) + + with self.assertLogs("organization_policy_campaign", "WARNING") as log: + policy_campaign.apply() + self.assertIn( + ex_r, + log.output, + ) + + stubber.assert_no_pending_responses() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/adf-policies/scp/scp.json b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/adf-policies/scp/scp.json new file mode 100644 index 000000000..39963a7e0 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/adf-policies/scp/scp.json @@ -0,0 +1,32 @@ +{ + "Targets": [ + "TestOrg" + ], + "Version": "2022-10-14", + "PolicyName": "TestPolicy", + "Policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": "cloudtrail:Stop*", + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": "*", + "Resource": "*" + }, + { + "Effect": "Deny", + "Action": [ + "config:DeleteConfigRule", + "config:DeleteConfigurationRecorder", + "config:DeleteDeliveryChannel", + "config:Stop*" + ], + "Resource": "*" + } + ] + } +} diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py index 8d66e5b08..f70df7534 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py @@ -81,7 +81,7 @@ def test_update_deployment_account_output_parameters(cls, sts): kms_and_bucket_dict={}, cloudformation=cloudformation ) - assert 4 == mock.call_count + assert 6 == mock.call_count mock.assert_has_calls(expected_calls, any_order=True) @@ -155,17 +155,21 @@ def test_prepare_deployment_account_defaults(param_store_cls, cls, sts): ) deploy_param_store.put_parameter.assert_has_calls( [ + call('adf_version', '1.0.0'), + call('adf_log_level', 'CRITICAL'), + call('cross_account_access_role', 'some_role'), + call('shared_modules_bucket', 'some_shared_modules_bucket'), + call('bootstrap_templates_bucket', 'some_bucket'), + call('deployment_account_id', '111122223333'), + call('management_account_id', '123'), + call('organization_id', 'o-123456789'), + call('extensions/terraform/enabled', 'False'), call('scm/default_scm_branch', 'main'), - call( - 'scm/default_scm_codecommit_account_id', - deployment_account_id, - ), + call('scm/default_scm_codecommit_account_id', '111122223333'), call('deployment_maps/allow_empty_target', 'disabled'), call('org/stage', 'none'), call('notification_type', 'email'), - call('notification_endpoint', 'john@example.com'), - call('extensions/terraform/enabled', 'False'), - ], + call('notification_endpoint', 'john@example.com')], any_order=True, ) @@ -236,7 +240,7 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): ) for param_store in parameter_store_list: assert param_store.put_parameter.call_count == ( - 17 if param_store == deploy_param_store else 9 + 18 if param_store == deploy_param_store else 9 ) param_store.put_parameter.assert_has_calls( [ @@ -257,21 +261,24 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): ) deploy_param_store.put_parameter.assert_has_calls( [ + call('adf_version', '1.0.0'), + call('adf_log_level', 'CRITICAL'), + call('cross_account_access_role', 'some_role'), + call('shared_modules_bucket', 'some_shared_modules_bucket'), + call('bootstrap_templates_bucket', 'some_bucket'), + call('deployment_account_id', '111122223333'), + call('management_account_id', '123'), + call('organization_id', 'o-123456789'), + call('extensions/terraform/enabled', 'True'), call('scm/auto_create_repositories', 'disabled'), call('scm/default_scm_branch', 'main'), - call( - 'scm/default_scm_codecommit_account_id', - deployment_account_id, - ), + call('scm/default_scm_codecommit_account_id', '111122223333'), call('deployment_maps/allow_empty_target', 'disabled'), call('org/stage', 'test-stage'), + call('auto_create_repositories', 'disabled'), call('notification_type', 'slack'), - call( - 'notification_endpoint', - "arn:aws:lambda:eu-central-1:" - f"{deployment_account_id}:function:SendSlackNotification", - ), - call('notification_endpoint/main', 'slack-channel'), - ], + call('notification_endpoint', 'arn:aws:lambda:eu-central-1:111122223333:function:SendSlackNotification'), + call('notification_endpoint/main', 'slack-channel') + ], any_order=False, ) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_org_policy_schema.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_org_policy_schema.py new file mode 100644 index 000000000..affeb4a59 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_org_policy_schema.py @@ -0,0 +1,66 @@ +""" +Tests organization policy schema. +""" + +import unittest +from schema import SchemaError +import organization_policy_schema + + +class HappyTestCases(unittest.TestCase): + def test_basic_schema_2022_10_14(self): + schema = { + "Targets": ["target1", "target2"], + "PolicyName": "policy1", + "Policy": {}, + } + expected_schema = { + "Targets": ["target1", "target2"], + "PolicyName": "policy1", + "Policy": {}, + "Version": "2022-10-14", + } + + validated_policy = organization_policy_schema.OrgPolicySchema(schema).schema + self.assertDictEqual(validated_policy, expected_schema) + + def test_basic_schema_2022_10_14_target_should_be_list(self): + schema = { + "Targets": "target1", + "PolicyName": "policy1", + "Policy": {}, + "Version": "2022-10-14", + } + expected_schema = { + "Targets": ["target1"], + "PolicyName": "policy1", + "Policy": {}, + "Version": "2022-10-14", + } + + validated_policy = organization_policy_schema.OrgPolicySchema(schema).schema + self.assertDictEqual(validated_policy, expected_schema) + + +class SadTestCases(unittest.TestCase): + def test_invalid_version(self): + schema = { + "Targets": ["target1", "target2"], + "PolicyName": "policy1", + "Policy": {}, + "Version": "2022-11-15", + } + + with self.assertRaises(SchemaError): + organization_policy_schema.OrgPolicySchema(schema) + + def test_invalid_field(self): + schema = { + "targets": ["target1", "target2"], + "PolicyName": "policy1", + "Policy": {}, + "Version": "2022-11-14", + } + + with self.assertRaises(SchemaError): + organization_policy_schema.OrgPolicySchema(schema) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_organization_policy_v2.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_organization_policy_v2.py new file mode 100644 index 000000000..dd8633634 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_organization_policy_v2.py @@ -0,0 +1,95 @@ +""" +Tests for organizational policy v2 +""" +import unittest +import os +import boto3 +from botocore.stub import Stubber, ANY + +from organization_policy_v2 import OrganizationPolicy + +SCP_ONLY = {"scp": "SERVICE_CONTROL_POLICY"} +# This hurts me. +TOX_PATH_PREFIX = "/src/lambda_codebase/initial_commit/bootstrap_repository" +TEST_POLICY_PATH = "/adf-build/tests/adf-policies" +POLICY_PATH = ( + f"{TOX_PATH_PREFIX}{TEST_POLICY_PATH}" + if not os.getenv("CODEPIPELINE_EXECUTION_ID") + else TEST_POLICY_PATH +) + + +class FakeParamStore: + params: dict() # pylint: disable=R1735 + + def __init__(self) -> None: + self.params = {} + + def put_parameter(self, key, value): + self.params[key] = value + + def fetch_parameter(self, key): + return self.params[key] + + +class HappyTestCases(unittest.TestCase): + def test_org_policy_campaign_creates_a_new_policy(self): + org_client = boto3.client("organizations") + org_stubber = Stubber(org_client) + + # No existing policy to look up + org_stubber.add_response("list_policies", {"Policies": []}) + + # loads up a target "TestOrg" and needs to get all policies it + # this test case has no existing policies returned. + org_stubber.add_response( + "list_policies_for_target", + {"Policies": []}, + {"TargetId": "ou-123456789", "Filter": "SERVICE_CONTROL_POLICY"}, + ) + + # Creates a policy + org_stubber.add_response( + "create_policy", + { + "Policy": { + "PolicySummary": { + "Id": "fake-policy-id", + "Arn": "arn:aws:organizations:policy/fake-policy-id", + "Name": "TestPolicy", + "Description": "ADF Managed scp", + "Type": "SERVICE_CONTROL_POLICY", + "AwsManaged": False, + }, + "Content": "fake-policy-content", + } + }, + { + "Content": ANY, + "Description": "ADF Managed scp", + "Name": "TestPolicy", + "Type": "SERVICE_CONTROL_POLICY", + }, + ) + + # Once created, the policy is then attached to the target + org_stubber.add_response( + "attach_policy", + {}, + {"PolicyId": "fake-policy-id", "TargetId": "ou-123456789"}, + ) + + org_stubber.activate() + + param_store = FakeParamStore() + # No existing (legacy) SCPs have been put + param_store.put_parameter("scp", "[]") + + policy_dir = f"{os.getcwd()}{POLICY_PATH}" + + org_policies_client = OrganizationPolicy(policy_dir) + with self.assertLogs("organization_policy_v2") as log: + org_policies_client.apply_policies( + org_client, param_store, {}, {"TestOrg": "ou-123456789"}, SCP_ONLY + ) + self.assertGreaterEqual(len(log.records), 0) diff --git a/src/template.yml b/src/template.yml index f1118633c..cbeeb4295 100644 --- a/src/template.yml +++ b/src/template.yml @@ -111,6 +111,16 @@ Parameters: - ERROR - CRITICAL + EnablePolicyV2: + Description: >- + Enable the second generation of ADF Policies. + See the documentation for more details. + Type: String + Default: "FALSE" + AllowedValues: + - "TRUE" + - "FALSE" + AllowBootstrappingOfManagementAccount: Description: >- Would ADF need to bootstrap the Management Account of your AWS @@ -1528,6 +1538,7 @@ Resources: - "organizations:ListParents" - "organizations:ListPolicies" - "organizations:ListPoliciesForTarget" + - "organizations:ListTargetsForPolicy" - "organizations:ListRoots" - "organizations:UpdatePolicy" - "sts:GetCallerIdentity" @@ -1651,6 +1662,8 @@ Resources: Value: !Ref DeploymentAccountMainRegion - Name: ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN Value: !Ref AccountBootstrappingStateMachine + - Name: ENABLED_V2_ORG_POLICY + Value: !Ref EnablePolicyV2 Type: LINUX_CONTAINER Name: "adf-bootstrap-pipeline-build" ServiceRole: !GetAtt BootstrapCodeBuildRole.Arn