From c2048efda0d7ec3e9829b1cde8324215ce33e926 Mon Sep 17 00:00:00 2001 From: Stefan Gordon Date: Wed, 21 Aug 2019 14:33:09 -0700 Subject: [PATCH] azure - cosmosdb firewall action (#4627) --- .../firewall-cosmosaction-networking.rst | 45 ++++ .../firewall-rulesfiltering-networking.rst | 6 +- tools/c7n_azure/c7n_azure/actions/base.py | 15 +- tools/c7n_azure/c7n_azure/actions/firewall.py | 82 +++++++ .../c7n_azure/resources/cosmos_db.py | 181 ++++++++++++++- .../c7n_azure/c7n_azure/resources/storage.py | 190 +++++++--------- ...sDBFirewallActionTest.firewall_action.json | 104 +++++++++ tools/c7n_azure/tests/templates/cosmosdb.json | 2 +- tools/c7n_azure/tests/test_cosmos_db.py | 209 +++++++++++++++++- .../c7n_azure/tests/test_firewall_actions.py | 62 ++---- 10 files changed, 722 insertions(+), 174 deletions(-) create mode 100644 docs/source/azure/examples/firewall-cosmosaction-networking.rst create mode 100644 tools/c7n_azure/c7n_azure/actions/firewall.py create mode 100644 tools/c7n_azure/tests/cassettes/CosmosDBFirewallActionTest.firewall_action.json diff --git a/docs/source/azure/examples/firewall-cosmosaction-networking.rst b/docs/source/azure/examples/firewall-cosmosaction-networking.rst new file mode 100644 index 00000000000..ec2f983231b --- /dev/null +++ b/docs/source/azure/examples/firewall-cosmosaction-networking.rst @@ -0,0 +1,45 @@ +Firewall - Update CosmosDB Rules +============================================ + +In this example we identify Cosmos DB accounts that either have no firewall +configured or which have one configured which is allowing access outside of +expected ranges. + +We then reconfigure that firewall to known-safe defaults which include a bypass for +all of the Azure Cloud as well as additional space in our data center. + +Virtual network rules are not specified so they will not be modified. + +.. code-block:: yaml + + policies: + - name: cosmos-firewall-enable + description: | + Find all incorrect firewalls and enable + with a set of defaults + resource: azure.cosmosdb + filters: + - or: + - type: value + key: properties.ipRangeFilter + value: empty # The firewall is disabled + + - not: + - type: firewall-rules + only: # Should *only* allow access within the specified maximums here + - 19.0.0.0/16 + - 20.0.1.2 + - ServiceTags.AzureCloud + + actions: + - type: set-firewall-rules + append: False + bypass-rules: # Enable firewall and allow all Azure Cloud + - AzureCloud + - Portal + ip-rules: # and some external IP space + - 19.0.0.0/16 + - 20.0.1.2 + + + diff --git a/docs/source/azure/examples/firewall-rulesfiltering-networking.rst b/docs/source/azure/examples/firewall-rulesfiltering-networking.rst index 7511e9ab0f3..89b1ef6f748 100644 --- a/docs/source/azure/examples/firewall-rulesfiltering-networking.rst +++ b/docs/source/azure/examples/firewall-rulesfiltering-networking.rst @@ -1,5 +1,5 @@ -Firewall - Rules Filtering -============================================== +Firewall - Filter Storage Accounts By Rules +============================================ This example demonstrates a common filtering scenario where we would like to ensure all firewalls are configured to only allow access from @@ -40,3 +40,5 @@ to remediate the non-compliant resources. - '8.8.8.8' - '10.0.0.0/16' - '20.0.0.0 - 20.10.0.0' + + diff --git a/tools/c7n_azure/c7n_azure/actions/base.py b/tools/c7n_azure/c7n_azure/actions/base.py index eb3b6063d76..dcec21cc7f6 100644 --- a/tools/c7n_azure/c7n_azure/actions/base.py +++ b/tools/c7n_azure/c7n_azure/actions/base.py @@ -64,10 +64,17 @@ def _process_resources(self, resources, event): try: self._process_resource(r) except Exception as e: - self.log.error("Failed to process resource.\n" - "Type: {0}.\n" - "Name: {1}.\n" - "Error: {2}".format(r['type'], r['name'], e)) + if isinstance(e, CloudError): + self.log.error("Failed to process resource.\n" + "Type: {0}.\n" + "Name: {1}.\n" + "Error: {2}\n" + "Message: {3}".format(r['type'], r['name'], e, e.message)) + else: + self.log.error("Failed to process resource.\n" + "Type: {0}.\n" + "Name: {1}.\n" + "Error: {2}".format(r['type'], r['name'], e)) def _prepare_processing(self): pass diff --git a/tools/c7n_azure/c7n_azure/actions/firewall.py b/tools/c7n_azure/c7n_azure/actions/firewall.py new file mode 100644 index 00000000000..b8d3c5c193a --- /dev/null +++ b/tools/c7n_azure/c7n_azure/actions/firewall.py @@ -0,0 +1,82 @@ +# Copyright 2019 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import abstractmethod + +from c7n_azure.actions.base import AzureBaseAction +from c7n_azure.utils import resolve_service_tag_alias +from netaddr import IPAddress + +from c7n.filters.core import type_schema + + +class SetFirewallAction(AzureBaseAction): + + schema = type_schema( + 'set-firewall-rules', + required=[], + **{ + 'append': {'type': 'boolean', 'default': True}, + 'bypass-rules': {'type': 'array'}, + 'ip-rules': {'type': 'array', 'items': {'type': 'string'}}, + 'virtual-network-rules': {'type': 'array', 'items': {'type': 'string'}} + } + ) + + @abstractmethod + def __init__(self, data, manager=None): + super(SetFirewallAction, self).__init__(data, manager) + + def _prepare_processing(self): + self.client = self.manager.get_client() + self.append = self.data.get('append', True) + + @abstractmethod + def _process_resource(self, resource): + pass + + def _build_bypass_rules(self, existing_bypass, new_rules): + if self.append: + without_duplicates = [r for r in existing_bypass if r not in new_rules] + new_rules.extend(without_duplicates) + return ','.join(new_rules or ['None']) + + def _build_vnet_rules(self, existing_vnet, new_rules): + if self.append: + without_duplicates = [r for r in existing_vnet if r not in new_rules] + new_rules.extend(without_duplicates) + return new_rules + + def _build_ip_rules(self, existing_ip, new_rules): + rules = [] + for rule in new_rules: + # attempt to resolve this rule as a service tag alias + # if it isn't a valid alias then we'll get `None` back. + resolved_set = resolve_service_tag_alias(rule) + if resolved_set: + # this is a service tag alias, so we need to insert the whole + # aliased array into the ruleset + ranges = list(resolved_set.iter_cidrs()) + for r in range(len(ranges)): + if len(ranges[r]) == 1: + ranges[r] = IPAddress(ranges[r].first) + rules.extend(map(str, ranges)) + else: + # just a normal rule, append + rules.append(rule) + + if self.append: + without_duplicates = [r for r in existing_ip if r not in rules] + rules.extend(without_duplicates) + return rules diff --git a/tools/c7n_azure/c7n_azure/resources/cosmos_db.py b/tools/c7n_azure/c7n_azure/resources/cosmos_db.py index e7b437a1e0d..b089d0b0770 100644 --- a/tools/c7n_azure/c7n_azure/resources/cosmos_db.py +++ b/tools/c7n_azure/c7n_azure/resources/cosmos_db.py @@ -15,17 +15,20 @@ import logging from itertools import groupby -import azure.mgmt.cosmosdb from azure.cosmos.cosmos_client import CosmosClient from azure.cosmos.errors import HTTPFailure +from azure.mgmt.cosmosdb.models import VirtualNetworkRule + from c7n_azure import constants from c7n_azure.actions.base import AzureBaseAction +from c7n_azure.actions.firewall import SetFirewallAction from c7n_azure.filters import FirewallRulesFilter from c7n_azure.provider import resources from c7n_azure.query import ChildResourceManager, ChildTypeInfo from c7n_azure.resources.arm import ArmResourceManager from c7n_azure.tags import TagHelper from c7n_azure.utils import ResourceIdParser + from concurrent.futures import as_completed from netaddr import IPSet @@ -69,7 +72,7 @@ class resource_type(ArmResourceManager.resource_type): doc_groups = ['Databases'] service = 'azure.mgmt.cosmosdb' - client = 'CosmosDB' # type: azure.mgmt.cosmosdb.CosmosDB + client = 'CosmosDB' enum_spec = ('database_accounts', 'list', None) default_report_fields = ( 'name', @@ -92,10 +95,15 @@ def log(self): return self._log def _query_rules(self, resource): - ip_range_string = resource['properties']['ipRangeFilter'] - resource_rules = IPSet(ip_range_string.split(',')) + parts = ip_range_string.split(',') + + # We need to remove the 'magic string' they use for AzureCloud bypass + if '0.0.0.0' in parts: + parts.remove('0.0.0.0') + + resource_rules = IPSet(filter(None, parts)) return resource_rules @@ -491,7 +499,6 @@ def get_cosmos_data_client_for_account(account_id, account_endpoint, manager, re manager.get_client(), readonly ) - data_client = CosmosClient(url_connection=account_endpoint, auth={'masterKey': key}) return data_client @@ -531,3 +538,167 @@ def execute_in_parallel_grouped_by_account( results.extend(f.result()) return results + + +@CosmosDB.action_registry.register('set-firewall-rules') +class CosmosSetFirewallAction(SetFirewallAction): + """ Set Firewall Rules Action + + Updates CosmosDB Firewall settings. Learn about the firewall at: + https://docs.microsoft.com/en-us/azure/cosmos-db/firewall-support + + By default the firewall rules are appended with the new values. The ``append: False`` + flag can be used to replace the old rules with the new ones on + the resource. + + You may also reference azure public cloud Service Tags by name in place of + an IP address. Use ``ServiceTags.`` followed by the ``name`` of any group + from https://www.microsoft.com/en-us/download/details.aspx?id=56519. + + Note that there are firewall rule number limits. The limit for CosmosDB is + 1000 rules (maximum tested rule count). + + .. code-block:: yaml + + - type: set-firewall-rules + ip-rules: + - 11.12.13.0/16 + - ServiceTags.AppService.CentralUS + + + :example: + + Find CosmosDB accounts without any firewall rules. + + Enable the firewall and allow: + - All Azure Cloud IP space + - All Portal UI IP space + - Two additional external IP ranges + + ``append: True`` (default) ensures we only add to the existing configuration. + + .. code-block:: yaml + + policies: + - name: cosmos-firewall + resource: azure.cosmosdb + filters: + # The firewall is disabled + - type: value + key: properties.ipRangeFilter + value: empty + actions: + - type: set-firewall-rules + append: True + bypass-rules: + - AzureCloud + - Portal + ip-rules: + - 19.0.0.0/16 + - 20.0.1.2 + + + Cosmos firewalls are disabled by simply configuring them with empty values. + We can do this by passing an empty array with ``append: False`` + + .. code-block:: yaml + + policies: + - name: cosmos-firewall-clear + resource: azure.cosmosdb + filters: + # The firewall is enabled + - not: + - type: value + key: properties.ipRangeFilter + value: empty + actions: + - type: set-firewall-rules + append: False + ip-rules: [] + + + """ + + schema = type_schema( + 'set-firewall-rules', + rinherit=SetFirewallAction.schema, + **{ + 'bypass-rules': {'type': 'array', 'items': { + 'enum': ['Portal', 'AzureCloud']}}, + } + ) + + def __init__(self, data, manager=None): + super(CosmosSetFirewallAction, self).__init__(data, manager) + self._log = logging.getLogger('custodian.azure.cosmosdb') + self.rule_limit = 1000 + self.portal = ['104.42.195.92', + '40.76.54.131', + '52.176.6.30', + '52.169.50.45', + '52.187.184.26'] + self.azure_cloud = ['0.0.0.0'] + + def _process_resource(self, resource): + + # IP rules + existing_ip = list(filter(None, resource['properties'].get('ipRangeFilter', '').split(','))) + if self.data.get('ip-rules') is not None: + ip_rules = self._build_ip_rules(existing_ip, self.data.get('ip-rules', [])) + else: + ip_rules = existing_ip + + # Bypass rules + # Cosmos DB does not have real bypass + # instead the portal UI adds values to your + # rules filter when you check the bypass box. + existing_bypass = [] + if set(self.azure_cloud).issubset(existing_ip): + existing_bypass.append('AzureCloud') + + if set(self.portal).issubset(existing_ip): + existing_bypass.append('Portal') + + # If unset, then we put the old values back in to emulate patch behavior + bypass_rules = self.data.get('bypass-rules', existing_bypass) + + if 'Portal' in bypass_rules: + ip_rules.extend(set(self.portal).difference(ip_rules)) + if 'AzureCloud' in bypass_rules: + ip_rules.extend(set(self.azure_cloud).difference(ip_rules)) + + # If the user has too many rules raise exception + if len(ip_rules) > self.rule_limit: + raise ValueError("Skipped updating firewall for %s. " + "%s exceeds maximum rule count of %s." % + (resource['name'], len(ip_rules), self.rule_limit)) + + # Add VNET rules + existing_vnet = \ + [r['id'] for r in resource['properties'].get('virtualNetworkRules', [])] + + if self.data.get('virtual-network-rules') is not None: + vnet_rules = self._build_vnet_rules(existing_vnet, + self.data.get('virtual-network-rules', [])) + else: + vnet_rules = existing_vnet + + # Workaround for bug https://git.io/fjFLY + resource['properties']['locations'] = [] + for loc in resource['properties'].get('readLocations'): + resource['properties']['locations'].append( + {'location_name': loc['locationName'], + 'failover_priority': loc['failoverPriority'], + 'is_zone_redundant': loc.get('isZoneRedundant', False)}) + + resource['properties']['ipRangeFilter'] = ','.join(ip_rules) + resource['properties']['virtualNetworkRules'] = \ + [VirtualNetworkRule(id=r) for r in vnet_rules] + + # Update resource + self.client.database_accounts.create_or_update( + resource['resourceGroup'], + resource['name'], + create_update_parameters=resource + ) diff --git a/tools/c7n_azure/c7n_azure/resources/storage.py b/tools/c7n_azure/c7n_azure/resources/storage.py index 1880a383f3e..18f658cacf9 100644 --- a/tools/c7n_azure/c7n_azure/resources/storage.py +++ b/tools/c7n_azure/c7n_azure/resources/storage.py @@ -24,13 +24,14 @@ from azure.storage.file import FileService from azure.storage.queue import QueueService from c7n_azure.actions.base import AzureBaseAction +from c7n_azure.actions.firewall import SetFirewallAction from c7n_azure.constants import BLOB_TYPE, FILE_TYPE, QUEUE_TYPE, TABLE_TYPE from c7n_azure.filters import FirewallRulesFilter, ValueFilter from c7n_azure.provider import resources from c7n_azure.resources.arm import ArmResourceManager from c7n_azure.storage_utils import StorageUtilities -from c7n_azure.utils import ThreadHelper, resolve_service_tag_alias -from netaddr import IPSet, IPAddress +from c7n_azure.utils import ThreadHelper +from netaddr import IPSet from c7n.exceptions import PolicyValidationError from c7n.filters.core import type_schema @@ -64,115 +65,118 @@ class resource_type(ArmResourceManager.resource_type): @Storage.action_registry.register('set-firewall-rules') -class StorageSetNetworkRulesAction(AzureBaseAction): - """ Set Network Rules Action +class StorageSetFirewallAction(SetFirewallAction): + """ Set Firewall Rules Action - Updates Azure Storage Firewalls and Virtual Networks settings. + Updates Azure Storage Firewalls and Virtual Networks settings. - By default the firewall rules are replaced with the new values. The ``append`` - flag can be used to force merging the new rules with the existing ones on - the resource. + By default the firewall rules are appended with the new values. The ``append: False`` + flag can be used to replace the old rules with the new ones on + the resource. - You may also reference azure public cloud Service Tags by name in place of - an IP address. Use ``ServiceTags.`` followed by the ``name`` of any group - from https://www.microsoft.com/en-us/download/details.aspx?id=56519. + You may also reference azure public cloud Service Tags by name in place of + an IP address. Use ``ServiceTags.`` followed by the ``name`` of any group + from https://www.microsoft.com/en-us/download/details.aspx?id=56519. - Note that there are firewall rule number limits and that you will likely need to - use a regional block to fit within the limit. The limit for storage accounts is - 200 rules. - - .. code-block:: yaml - - - type: set-firewall-rules - bypass-rules: - - Logging - - Metrics - ip-rules: - - 11.12.13.0/16 - - ServiceTags.AppService.CentralUS + Note that there are firewall rule number limits and that you will likely need to + use a regional block to fit within the limit. The limit for storage accounts is + 200 rules. + .. code-block:: yaml - :example: + - type: set-firewall-rules + bypass-rules: + - Logging + - Metrics + ip-rules: + - 11.12.13.0/16 + - ServiceTags.AppService.CentralUS - Find storage accounts without any firewall rules. - Configure default-action to ``Deny`` and then allow: - - Azure Logging and Metrics services - - Two specific IPs - - Two subnets + :example: - .. code-block:: yaml + Find storage accounts without any firewall rules. - policies: - - name: add-storage-firewall - resource: azure.storage + Configure default-action to ``Deny`` and then allow: + - Azure Logging and Metrics services + - Two specific IPs + - Two subnets - filters: - - type: value - key: properties.networkAcls.ipRules - value_type: size - op: eq - value: 0 - - actions: - - type: set-firewall-rules - bypass-rules: - - Logging - - Metrics - ip-rules: - - 11.12.13.0/16 - - 21.22.23.24 - virtual-network-rules: - - - - + .. code-block:: yaml - """ + policies: + - name: add-storage-firewall + resource: azure.storage + + filters: + - type: value + key: properties.networkAcls.ipRules + value_type: size + op: eq + value: 0 + + actions: + - type: set-firewall-rules + append: False + bypass-rules: + - Logging + - Metrics + ip-rules: + - 11.12.13.0/16 + - 21.22.23.24 + virtual-network-rules: + - + - + + """ schema = type_schema( 'set-firewall-rules', - required=[], + rinherit=SetFirewallAction.schema, **{ 'default-action': {'enum': ['Allow', 'Deny'], "default": 'Deny'}, - 'append': {'type': 'boolean', "default": False}, 'bypass-rules': {'type': 'array', 'items': { 'enum': ['AzureServices', 'Logging', 'Metrics']}}, - 'ip-rules': {'type': 'array', 'items': {'type': 'string'}}, - 'virtual-network-rules': {'type': 'array', 'items': {'type': 'string'}} } ) def __init__(self, data, manager=None): - super(StorageSetNetworkRulesAction, self).__init__(data, manager) + super(StorageSetFirewallAction, self).__init__(data, manager) self._log = logging.getLogger('custodian.azure.storage') self.rule_limit = 200 - def _prepare_processing(self): - self.client = self.manager.get_client() - self.append = self.data.get('append', False) - def _process_resource(self, resource): - rules = self._build_ip_rules(resource, self.data.get('ip-rules', [])) - # Build out the ruleset model to update the resource rule_set = NetworkRuleSet(default_action=self.data.get('default-action', 'Deny')) - # If the user has too many rules log and skip - if len(rules) > self.rule_limit: - self._log.error("Skipped updating firewall for %s. " - "%s exceeds maximum rule count of %s." % - (resource['name'], len(rules), self.rule_limit)) - return - # Add IP rules - rule_set.ip_rules = [IPRule(ip_address_or_range=r) for r in rules] + if self.data.get('ip-rules') is not None: + existing_ip = resource['properties']['networkAcls'].get('ipRules', []) + ip_rules = self._build_ip_rules(existing_ip, self.data.get('ip-rules', [])) + + # If the user has too many rules raise exception + if len(ip_rules) > self.rule_limit: + raise ValueError("Skipped updating firewall for %s. " + "%s exceeds maximum rule count of %s." % + (resource['name'], len(ip_rules), self.rule_limit)) + + rule_set.ip_rules = [IPRule(ip_address_or_range=r) for r in ip_rules] # Add VNET rules - vnet_rules = self._build_vnet_rules(resource, self.data.get('virtual-network-rules', [])) - rule_set.virtual_network_rules = [ - VirtualNetworkRule(virtual_network_resource_id=r) for r in vnet_rules] + if self.data.get('virtual-network-rules') is not None: + existing_vnet = \ + [r['id'] for r in + resource['properties']['networkAcls'].get('virtualNetworkRules', [])] + vnet_rules = \ + self._build_vnet_rules(existing_vnet, self.data.get('virtual-network-rules', [])) + rule_set.virtual_network_rules = \ + [VirtualNetworkRule(virtual_network_resource_id=r) for r in vnet_rules] # Configure BYPASS - rule_set.bypass = self._build_bypass_rules(resource, self.data.get('bypass', [])) + if self.data.get('bypass-rules') is not None: + existing_bypass = resource['properties']['networkAcls'].get('bypass', '').split(',') + rule_set.bypass = self._build_bypass_rules( + existing_bypass, self.data.get('bypass-rules', [])) # Update resource self.client.storage_accounts.update( @@ -180,40 +184,6 @@ def _process_resource(self, resource): resource['name'], StorageAccountUpdateParameters(network_rule_set=rule_set)) - def _build_bypass_rules(self, resource, new_rules): - if self.append: - existing_bypass = resource['properties']['networkAcls'].get('bypass', '').split(',') - without_duplicates = [r for r in existing_bypass if r not in new_rules] - new_rules.extend(without_duplicates) - return ','.join(new_rules or ['None']) - - def _build_vnet_rules(self, resource, new_rules): - if self.append: - existing_rules = [r['id'] for r in - resource['properties']['networkAcls'].get('virtualNetworkRules', [])] - without_duplicates = [r for r in existing_rules if r not in new_rules] - new_rules.extend(without_duplicates) - return new_rules - - def _build_ip_rules(self, resource, new_rules): - rules = [] - for rule in new_rules: - resolved_set = resolve_service_tag_alias(rule) - if resolved_set: - ranges = list(resolved_set.iter_cidrs()) - for r in range(len(ranges)): - if len(ranges[r]) == 1: - ranges[r] = IPAddress(ranges[r].first) - rules.extend(map(str, ranges)) - else: - rules.append(rule) - - if self.append: - existing_rules = resource['properties']['networkAcls'].get('ipRules', []) - without_duplicates = [r['value'] for r in existing_rules if r['value'] not in rules] - rules.extend(without_duplicates) - return rules - @Storage.filter_registry.register('firewall-rules') class StorageFirewallRulesFilter(FirewallRulesFilter): diff --git a/tools/c7n_azure/tests/cassettes/CosmosDBFirewallActionTest.firewall_action.json b/tools/c7n_azure/tests/cassettes/CosmosDBFirewallActionTest.firewall_action.json new file mode 100644 index 00000000000..3f83f1e3ded --- /dev/null +++ b/tools/c7n_azure/tests/cassettes/CosmosDBFirewallActionTest.firewall_action.json @@ -0,0 +1,104 @@ +{ + "version": 1, + "interactions": [ + { + "request": { + "method": "GET", + "uri": "https://management.azure.com/subscriptions/ea42f556-5106-4743-99b0-c129bfa71a47/providers/Microsoft.DocumentDB/databaseAccounts?api-version=2015-04-08", + "body": null, + "headers": {} + }, + "response": { + "status": { + "code": 200, + "message": "Ok" + }, + "headers": { + "cache-control": [ + "no-store, no-cache" + ], + "date": [ + "Wed, 21 Aug 2019 15:19:19 GMT" + ], + "content-type": [ + "application/json" + ], + "content-length": [ + "1783" + ] + }, + "body": { + "data": { + "value": [ + { + "id": "/subscriptions/ea42f556-5106-4743-99b0-c129bfa71a47/resourceGroups/test_cosmosdb/providers/Microsoft.DocumentDB/databaseAccounts/cctestcosmosdb", + "name": "cctestcosmosdb", + "location": "South Central US", + "type": "Microsoft.DocumentDB/databaseAccounts", + "kind": "GlobalDocumentDB", + "tags": {}, + "properties": { + "provisioningState": "Succeeded", + "documentEndpoint": "https://cctestcosmosdb.documents.azure.com:443/", + "ipRangeFilter": "0.0.0.0/1,128.0.0.0/1,104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26", + "enableAutomaticFailover": false, + "enableMultipleWriteLocations": false, + "enablePartitionKeyMonitor": false, + "isVirtualNetworkFilterEnabled": false, + "virtualNetworkRules": [], + "EnabledApiTypes": "Sql", + "databaseAccountOfferType": "Standard", + "consistencyPolicy": { + "defaultConsistencyLevel": "Session", + "maxIntervalInSeconds": 5, + "maxStalenessPrefix": 100 + }, + "configurationOverrides": {}, + "writeLocations": [ + { + "id": "cctestcosmosdb-southcentralus", + "locationName": "South Central US", + "documentEndpoint": "https://cctestcosmosdb-southcentralus.documents.azure.com:443/", + "provisioningState": "Succeeded", + "failoverPriority": 0, + "isZoneRedundant": false + } + ], + "readLocations": [ + { + "id": "cctestcosmosdb-southcentralus", + "locationName": "South Central US", + "documentEndpoint": "https://cctestcosmosdb-southcentralus.documents.azure.com:443/", + "provisioningState": "Succeeded", + "failoverPriority": 0, + "isZoneRedundant": false + } + ], + "locations": [ + { + "id": "cctestcosmosdb-southcentralus", + "locationName": "South Central US", + "documentEndpoint": "https://cctestcosmosdb-southcentralus.documents.azure.com:443/", + "provisioningState": "Succeeded", + "failoverPriority": 0, + "isZoneRedundant": false + } + ], + "failoverPolicies": [ + { + "id": "cctestcosmosdb-southcentralus", + "locationName": "South Central US", + "failoverPriority": 0 + } + ], + "cors": [], + "capabilities": [] + } + } + ] + } + } + } + } + ] +} \ No newline at end of file diff --git a/tools/c7n_azure/tests/templates/cosmosdb.json b/tools/c7n_azure/tests/templates/cosmosdb.json index fd72ea24cc0..209b7b9c8d5 100755 --- a/tools/c7n_azure/tests/templates/cosmosdb.json +++ b/tools/c7n_azure/tests/templates/cosmosdb.json @@ -12,7 +12,7 @@ "location": "South Central US", "scale": null, "properties": { - "ipRangeFilter": "0.0.0.0/1,128.0.0.0/1", + "ipRangeFilter": "0.0.0.0/1,128.0.0.0/1,104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26", "databaseAccountOfferType": "Standard", "consistencyPolicy": { "defaultConsistencyLevel": "Session", diff --git a/tools/c7n_azure/tests/test_cosmos_db.py b/tools/c7n_azure/tests/test_cosmos_db.py index 9311558ed5b..c3eb905ec1c 100644 --- a/tools/c7n_azure/tests/test_cosmos_db.py +++ b/tools/c7n_azure/tests/test_cosmos_db.py @@ -15,11 +15,12 @@ unicode_literals) from azure.cosmos.cosmos_client import CosmosClient - from azure_common import BaseTest, arm_template, cassette_name -from c7n.utils import local_session from c7n_azure.resources.cosmos_db import CosmosDBChildResource from c7n_azure.session import Session +from mock import patch + +from c7n.utils import local_session class CosmosDBTest(BaseTest): @@ -47,6 +48,24 @@ def test_cosmos_db_schema_validate(self): }, validate=True) self.assertTrue(p) + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'bypass-rules': ['Portal'], + 'ip-rules': ['0.0.0.0/1', '11.12.13.14', '21.22.23.24'] + } + ] + }, validate=True) + self.assertTrue(p) + @arm_template('cosmosdb.json') def test_find_by_name(self): p = self.load_policy({ @@ -182,8 +201,192 @@ def test_store_throughput_state_collection_action(self): self.assertEqual(expected_tag_value, tag_value) -class CosmosDBThroughputActionsTest(BaseTest): +class CosmosDBFirewallActionTest(BaseTest): + + @patch('azure.mgmt.cosmosdb.operations.database_accounts_operations.' + 'DatabaseAccountsOperations.create_or_update') + @cassette_name('firewall_action') + @arm_template('cosmosdb.json') + def test_set_ip_range_filter_append(self, update_mock): + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'ip-rules': ['0.0.0.0/1', '11.12.13.14', '21.22.23.24'] + } + ] + }) + resources = p.run() + self.assertEqual(len(resources), 1) + self.assertEqual(1, len(update_mock.mock_calls)) + name, args, kwargs = update_mock.mock_calls[0] + + self.assertEqual(resources[0]['resourceGroup'], args[0]) + self.assertEqual(resources[0]['name'], args[1]) + self.assertEqual( + set('0.0.0.0/1,128.0.0.0/1,11.12.13.14,21.22.23.24,' + '104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26'.split(',')), + set(kwargs['create_update_parameters']['properties']['ipRangeFilter'].split(','))) + + @patch('azure.mgmt.cosmosdb.operations.database_accounts_operations.' + 'DatabaseAccountsOperations.create_or_update') + @cassette_name('firewall_action') + @arm_template('cosmosdb.json') + def test_set_ip_range_filter_replace(self, update_mock): + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'append': False, + 'ip-rules': ['0.0.0.0/1', '11.12.13.14', '21.22.23.24'] + } + ] + }) + resources = p.run() + self.assertEqual(len(resources), 1) + + self.assertEqual(1, len(update_mock.mock_calls)) + name, args, kwargs = update_mock.mock_calls[0] + + self.assertEqual(resources[0]['resourceGroup'], args[0]) + self.assertEqual(resources[0]['name'], args[1]) + self.assertEqual( + set('0.0.0.0/1,11.12.13.14,21.22.23.24,104.42.195.92,40.76.54.131,' + '52.176.6.30,52.169.50.45,52.187.184.26'.split(',')), + set(kwargs['create_update_parameters']['properties']['ipRangeFilter'].split(','))) + + @patch('azure.mgmt.cosmosdb.operations.database_accounts_operations.' + 'DatabaseAccountsOperations.create_or_update') + @cassette_name('firewall_action') + @arm_template('cosmosdb.json') + def test_set_ip_range_filter_replace_bypass(self, update_mock): + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'append': False, + 'bypass-rules': ['Portal', 'AzureCloud'], + 'ip-rules': ['0.0.0.0/1', '11.12.13.14', '21.22.23.24'] + } + ] + }) + resources = p.run() + self.assertEqual(len(resources), 1) + + self.assertEqual(1, len(update_mock.mock_calls)) + name, args, kwargs = update_mock.mock_calls[0] + + self.assertEqual(resources[0]['resourceGroup'], args[0]) + self.assertEqual(resources[0]['name'], args[1]) + self.assertEqual( + {'0.0.0.0/1', + '104.42.195.92', + '11.12.13.14', + '21.22.23.24', + '40.76.54.131', + '52.169.50.45', + '52.176.6.30', + '52.187.184.26', + '0.0.0.0' + }, + set(kwargs['create_update_parameters']['properties']['ipRangeFilter'].split(','))) + + @patch('azure.mgmt.cosmosdb.operations.database_accounts_operations.' + 'DatabaseAccountsOperations.create_or_update') + @cassette_name('firewall_action') + @arm_template('cosmosdb.json') + def test_set_ip_range_filter_remove_bypass(self, update_mock): + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'append': False, + 'bypass-rules': [], + 'ip-rules': ['21.22.23.24'] + } + ] + }) + resources = p.run() + self.assertEqual(len(resources), 1) + + self.assertEqual(1, len(update_mock.mock_calls)) + name, args, kwargs = update_mock.mock_calls[0] + + self.assertEqual(resources[0]['resourceGroup'], args[0]) + self.assertEqual(resources[0]['name'], args[1]) + self.assertEqual( + {'21.22.23.24'}, + set(kwargs['create_update_parameters']['properties']['ipRangeFilter'].split(','))) + + @patch('azure.mgmt.cosmosdb.operations.database_accounts_operations.' + 'DatabaseAccountsOperations.create_or_update') + @cassette_name('firewall_action') + @arm_template('cosmosdb.json') + def test_set_vnet_append(self, update_mock): + p = self.load_policy({ + 'name': 'test-azure-cosmosdb', + 'resource': 'azure.cosmosdb', + 'filters': [ + {'type': 'value', + 'key': 'name', + 'op': 'eq', + 'value_type': 'normalize', + 'value': 'cctestcosmosdb'}], + 'actions': [ + {'type': 'set-firewall-rules', + 'append': True, + 'virtual-network-rules': ['id1', 'id2'], + 'ip-rules': ['0.0.0.0/1', '11.12.13.14', '21.22.23.24'] + } + ] + }) + resources = p.run() + self.assertEqual(len(resources), 1) + + name, args, kwargs = update_mock.mock_calls[0] + + self.assertEqual(resources[0]['resourceGroup'], args[0]) + self.assertEqual(resources[0]['name'], args[1]) + self.assertEqual( + set('0.0.0.0/1,128.0.0.0/1,11.12.13.14,21.22.23.24,' + '104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26'.split(',')), + set(kwargs['create_update_parameters']['properties']['ipRangeFilter'].split(','))) + self.assertEqual( + {'id1', 'id2'}, + set([r.id for r in + kwargs['create_update_parameters']['properties']['virtualNetworkRules']])) + + +class CosmosDBThroughputActionsTest(BaseTest): def setUp(self, *args, **kwargs): super(CosmosDBThroughputActionsTest, self).setUp(*args, **kwargs) self.client = local_session(Session).client('azure.mgmt.cosmosdb.CosmosDB') diff --git a/tools/c7n_azure/tests/test_firewall_actions.py b/tools/c7n_azure/tests/test_firewall_actions.py index 4a59355187b..8c95ebb6ebc 100644 --- a/tools/c7n_azure/tests/test_firewall_actions.py +++ b/tools/c7n_azure/tests/test_firewall_actions.py @@ -14,7 +14,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals from azure_common import BaseTest -from c7n_azure.resources.storage import StorageSetNetworkRulesAction +from c7n_azure.resources.storage import StorageSetFirewallAction class TestFirewallActions(BaseTest): @@ -25,21 +25,13 @@ def test_build_bypass_rules(self): 'bypass-rules': ['Logging', 'Metrics'], } - resource = { - 'properties': { - 'networkAcls': { - 'bypass': 'Hello,World' - } - } - } - - action = StorageSetNetworkRulesAction(data) + action = StorageSetFirewallAction(data) action.append = False - rules = action._build_bypass_rules(resource, data['bypass-rules']) + rules = action._build_bypass_rules(['Hello', 'World'], data['bypass-rules']) self.assertEqual('Logging,Metrics', rules) action.append = True - rules = action._build_bypass_rules(resource, data['bypass-rules']) + rules = action._build_bypass_rules(['Hello', 'World'], data['bypass-rules']) self.assertEqual('Logging,Metrics,Hello,World', rules) def test_build_vnet_rules(self): @@ -47,23 +39,13 @@ def test_build_vnet_rules(self): 'virtual-network-rules': ['id1', 'id2'] } - resource = { - 'properties': { - 'networkAcls': { - 'virtualNetworkRules': [ - {'id': 'Hello'}, {'id': 'World'} - ] - } - } - } - - action = StorageSetNetworkRulesAction(data) + action = StorageSetFirewallAction(data) action.append = False - rules = action._build_vnet_rules(resource, data['virtual-network-rules']) + rules = action._build_vnet_rules(['Hello', 'World'], data['virtual-network-rules']) self.assertEqual(sorted(['id1', 'id2']), sorted(rules)) action.append = True - rules = action._build_vnet_rules(resource, data['virtual-network-rules']) + rules = action._build_vnet_rules(['Hello', 'World'], data['virtual-network-rules']) self.assertEqual(sorted(['id1', 'id2', 'Hello', 'World']), sorted(rules)) def test_build_ip_rules(self): @@ -71,22 +53,13 @@ def test_build_ip_rules(self): 'ip-rules': ['1.1.1.1', '6.0.0.0/16'] } - resource = { - 'properties': { - 'networkAcls': { - 'ipRules': [ - {'value': '1.1.1.1'}, {'value': '8.0.0.0/12'}] - } - } - } - - action = StorageSetNetworkRulesAction(data) + action = StorageSetFirewallAction(data) action.append = False - rules = action._build_ip_rules(resource, data['ip-rules']) + rules = action._build_ip_rules(['1.1.1.1', '8.0.0.0/12'], data['ip-rules']) self.assertEqual(sorted(['1.1.1.1', '6.0.0.0/16']), sorted(rules)) action.append = True - rules = action._build_ip_rules(resource, data['ip-rules']) + rules = action._build_ip_rules(['1.1.1.1', '8.0.0.0/12'], data['ip-rules']) self.assertEqual(sorted(['1.1.1.1', '6.0.0.0/16', '8.0.0.0/12']), sorted(rules)) def test_build_ip_rules_alias(self): @@ -94,23 +67,14 @@ def test_build_ip_rules_alias(self): 'ip-rules': ['ServiceTags.ApiManagement.WestUS', '6.0.0.0/16'] } - resource = { - 'properties': { - 'networkAcls': { - 'ipRules': [ - {'value': '1.1.1.1'}, {'value': '8.0.0.0/12'}] - } - } - } - - action = StorageSetNetworkRulesAction(data) + action = StorageSetFirewallAction(data) action.append = False - rules = action._build_ip_rules(resource, data['ip-rules']) + rules = action._build_ip_rules(['1.1.1.1', '8.0.0.0/12'], data['ip-rules']) self.assertIn('6.0.0.0/16', rules) self.assertEqual(4, len(rules)) # With append we expect all our specified values + others from the service tag. action.append = True - rules = action._build_ip_rules(resource, data['ip-rules']) + rules = action._build_ip_rules(['1.1.1.1', '8.0.0.0/12'], data['ip-rules']) self.assertTrue({'6.0.0.0/16', '1.1.1.1', '8.0.0.0/12'} <= set(rules)) self.assertEqual(6, len(rules))