From b0d315877524340b97f22044e85f2a70fad38996 Mon Sep 17 00:00:00 2001 From: Shahaf Ben Yakir <44666568+ShahafBenYakir@users.noreply.github.com> Date: Sat, 31 Aug 2024 19:19:35 +0300 Subject: [PATCH 1/5] updating corealerts in core (#36070) * updating corealerts in core * revert xpanse --- Config/corepacks_override.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Config/corepacks_override.json b/Config/corepacks_override.json index 2f5736e0a9ca..256d34e9ee82 100644 --- a/Config/corepacks_override.json +++ b/Config/corepacks_override.json @@ -1,6 +1,6 @@ { "server_version": "8.7.0", - "file_version": "1", + "file_version": "2", "xsoar_saas": { "updated_corepacks_content": { "corePacks": [ @@ -69,7 +69,7 @@ "FeedMitreAttackv2/1.1.38/FeedMitreAttackv2.zip", "ThreatIntelligenceManagement/1.1.10/ThreatIntelligenceManagement.zip", "FiltersAndTransformers/1.2.72/FiltersAndTransformers.zip", - "CoreAlertFields/1.0.34/CoreAlertFields.zip", + "CoreAlertFields/1.0.37/CoreAlertFields.zip", "FeedUnit42v2/1.0.54/FeedUnit42v2.zip", "AutoFocus/2.2.1/AutoFocus.zip", "EDL/3.3.1/EDL.zip", @@ -86,7 +86,7 @@ "Unit42Intel/1.0.23/Unit42Intel.zip", "CommonTypes/3.5.4/CommonTypes.zip", "CortexAttackSurfaceManagement/1.7.39/CortexAttackSurfaceManagement.zip", - "Base/1.34.12/Base.zip", + "Base/1.34.35/Base.zip", "DemistoRESTAPI/1.3.55/DemistoRESTAPI.zip", "rasterize/2.0.13/rasterize.zip" ], From a36d589d46e86f7f83ee77e3e4514d2b3e6f02af Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Sat, 31 Aug 2024 21:44:40 +0300 Subject: [PATCH 2/5] update candidate image to demisto/py3-native:8.8.0.108569 (#36083) --- Tests/docker_native_image_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/docker_native_image_config.json b/Tests/docker_native_image_config.json index 3c634bfe6bc4..27ee05f51889 100644 --- a/Tests/docker_native_image_config.json +++ b/Tests/docker_native_image_config.json @@ -128,7 +128,7 @@ "netutils", "auth-utils" ], - "docker_ref": "demisto/py3-native:8.8.0.108239" + "docker_ref": "demisto/py3-native:8.8.0.108569" } }, "ignored_content_items": [ From efe0f4ea78a5ab9621be22c7054a6ccd3e78ff6b Mon Sep 17 00:00:00 2001 From: Sapir Malka <44067957+itssapir@users.noreply.github.com> Date: Sun, 1 Sep 2024 13:31:39 +0300 Subject: [PATCH 3/5] CIAC-11004 Feeds Enrichment Excluded field (#36004) * Added excludeEnrichment parameter to relevant feed integrations * precommit pass * Added excludeEnrichment field to feeds - Added excludeEnrichment to all indicators created by fetch_indicators in the following feeds: - All feeds using HTTP/JSON Feed API Modules - Azure - Cisco WebEx - Office365 - Zoom - Public DNS * Docker image updates * Only add exclude parameter to indicator objects when required * Bugfix for Zoom Feed changes * Added unit testing * Updated pack versions and release notes for non-deprecated integrations * Changed feed param description and info as defined in figma * Changed integration params to new Connect-Collect layout and advanced settings * Modified release notes to indicate exclude enrichment is hidden for on prem * updated README files * Apply suggestions from doc review Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> * Formatter run and docker image updates * updated docker versions in release notes --------- Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> --- .../ACTIIndicatorFeed_test.py | 2 +- .../AccentureCTI_Feed/ReleaseNotes/1_1_38.md | 6 + Packs/AccentureCTI_Feed/pack_metadata.json | 2 +- .../HTTPFeedApiModule/HTTPFeedApiModule.py | 65 +++++++---- .../HTTPFeedApiModule/HTTPFeedApiModule.yml | 2 +- .../HTTPFeedApiModule_test.py | 65 +++++++++++ .../JSONFeedApiModule/JSONFeedApiModule.py | 24 +++- .../JSONFeedApiModule/JSONFeedApiModule.yml | 4 +- .../JSONFeedApiModule_test.py | 40 +++++++ .../CiscoWebExFeed/CiscoWebExFeed.py | 20 +++- .../CiscoWebExFeed/CiscoWebExFeed.yml | 32 +++++- .../CiscoWebExFeed/CiscoWebExFeed_test.py | 43 +++++++- .../Integrations/CiscoWebExFeed/README.md | 3 +- Packs/CiscoWebExFeed/ReleaseNotes/1_2_15.md | 9 ++ Packs/CiscoWebExFeed/pack_metadata.json | 2 +- .../FeedAWS/Integrations/FeedAWS/FeedAWS.yml | 34 +++++- Packs/FeedAWS/Integrations/FeedAWS/README.md | 1 + Packs/FeedAWS/ReleaseNotes/1_1_52.md | 9 ++ Packs/FeedAWS/pack_metadata.json | 2 +- .../Integrations/FeedAzure/FeedAzure.py | 41 ++++--- .../Integrations/FeedAzure/FeedAzure.yml | 48 ++++++-- .../Integrations/FeedAzure/FeedAzure_test.py | 104 +++++++++++++++++- .../Integrations/FeedAzure/README.md | 1 + Packs/FeedAzure/ReleaseNotes/1_0_28.md | 9 ++ Packs/FeedAzure/pack_metadata.json | 2 +- .../FeedBlocklist_de/FeedBlocklist_de.py | 4 +- .../FeedBlocklist_de/FeedBlocklist_de.yml | 2 +- Packs/FeedBlocklist_de/ReleaseNotes/1_1_29.md | 7 ++ Packs/FeedBlocklist_de/pack_metadata.json | 2 +- .../ReleaseNotes/1_1_27.md | 6 + .../FeedBruteForceBlocker/pack_metadata.json | 2 +- .../FeedCloudflare/FeedCloudflare.yml | 41 ++++++- .../Integrations/FeedCloudflare/README.md | 1 + Packs/FeedCloudflare/ReleaseNotes/1_1_27.md | 9 ++ Packs/FeedCloudflare/pack_metadata.json | 2 +- Packs/FeedDShield/ReleaseNotes/1_1_31.md | 6 + Packs/FeedDShield/pack_metadata.json | 2 +- .../Integrations/FeedFastly/FeedFastly.yml | 40 ++++++- .../Integrations/FeedFastly/README.md | 1 + Packs/FeedFastly/ReleaseNotes/1_1_29.md | 9 ++ Packs/FeedFastly/pack_metadata.json | 2 +- .../FeedGoogleIPRanges/FeedGoogleIPRanges.py | 12 +- .../FeedGoogleIPRanges/FeedGoogleIPRanges.yml | 33 +++++- .../Integrations/FeedGoogleIPRanges/README.md | 1 + Packs/FeedGCPWhitelist/ReleaseNotes/2_0_39.md | 9 ++ Packs/FeedGCPWhitelist/pack_metadata.json | 2 +- Packs/FeedJSON/ReleaseNotes/1_1_31.md | 6 + Packs/FeedJSON/pack_metadata.json | 2 +- .../FeedMalwareBazaar/ReleaseNotes/1_0_39.md | 6 + Packs/FeedMalwareBazaar/pack_metadata.json | 2 +- .../FeedOffice365/FeedOffice365.py | 45 +++++--- .../FeedOffice365/FeedOffice365.yml | 56 ++++++++-- .../FeedOffice365/FeedOffice365_test.py | 27 ++++- .../Integrations/FeedOffice365/README.md | 1 + Packs/FeedOffice365/ReleaseNotes/1_2_13.md | 9 ++ Packs/FeedOffice365/pack_metadata.json | 2 +- Packs/FeedPlainText/ReleaseNotes/1_1_27.md | 6 + Packs/FeedPlainText/pack_metadata.json | 2 +- .../FeedPublicDNS/FeedPublicDNS.py | 29 +++-- .../FeedPublicDNS/FeedPublicDNS.yml | 35 +++++- .../FeedPublicDNS/FeedPublicDNS_test.py | 51 +++++++++ .../Integrations/FeedPublicDNS/README.md | 1 + Packs/FeedPublicDNS/ReleaseNotes/1_0_16.md | 9 ++ Packs/FeedPublicDNS/pack_metadata.json | 2 +- Packs/FeedSpamhaus/ReleaseNotes/1_1_24.md | 6 + Packs/FeedSpamhaus/pack_metadata.json | 2 +- .../Integrations/FeedZoom/FeedZoom.py | 30 +++-- .../Integrations/FeedZoom/FeedZoom.yml | 36 +++++- .../Integrations/FeedZoom/FeedZoom_test.py | 80 +++++++++++++- .../FeedZoom/Integrations/FeedZoom/README.md | 1 + Packs/FeedZoom/ReleaseNotes/1_1_15.md | 10 ++ Packs/FeedZoom/pack_metadata.json | 2 +- 72 files changed, 1054 insertions(+), 167 deletions(-) create mode 100644 Packs/AccentureCTI_Feed/ReleaseNotes/1_1_38.md create mode 100644 Packs/CiscoWebExFeed/ReleaseNotes/1_2_15.md create mode 100644 Packs/FeedAWS/ReleaseNotes/1_1_52.md create mode 100644 Packs/FeedAzure/ReleaseNotes/1_0_28.md create mode 100644 Packs/FeedBlocklist_de/ReleaseNotes/1_1_29.md create mode 100644 Packs/FeedBruteForceBlocker/ReleaseNotes/1_1_27.md create mode 100644 Packs/FeedCloudflare/ReleaseNotes/1_1_27.md create mode 100644 Packs/FeedDShield/ReleaseNotes/1_1_31.md create mode 100644 Packs/FeedFastly/ReleaseNotes/1_1_29.md create mode 100644 Packs/FeedGCPWhitelist/ReleaseNotes/2_0_39.md create mode 100644 Packs/FeedJSON/ReleaseNotes/1_1_31.md create mode 100644 Packs/FeedMalwareBazaar/ReleaseNotes/1_0_39.md create mode 100644 Packs/FeedOffice365/ReleaseNotes/1_2_13.md create mode 100644 Packs/FeedPlainText/ReleaseNotes/1_1_27.md create mode 100644 Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS_test.py create mode 100644 Packs/FeedPublicDNS/ReleaseNotes/1_0_16.md create mode 100644 Packs/FeedSpamhaus/ReleaseNotes/1_1_24.md create mode 100644 Packs/FeedZoom/ReleaseNotes/1_1_15.md diff --git a/Packs/AccentureCTI_Feed/Integrations/ACTIIndicatorFeed/ACTIIndicatorFeed_test.py b/Packs/AccentureCTI_Feed/Integrations/ACTIIndicatorFeed/ACTIIndicatorFeed_test.py index 3626f05448b4..afbc946429d7 100644 --- a/Packs/AccentureCTI_Feed/Integrations/ACTIIndicatorFeed/ACTIIndicatorFeed_test.py +++ b/Packs/AccentureCTI_Feed/Integrations/ACTIIndicatorFeed/ACTIIndicatorFeed_test.py @@ -81,4 +81,4 @@ def test_build_iterator_change_extractor(): with pytest.raises(TypeError) as e: custom_build_iterator(client, PARAMS['feed_name_to_config']['Domain'], 0) if not e: - assert False + raise AssertionError diff --git a/Packs/AccentureCTI_Feed/ReleaseNotes/1_1_38.md b/Packs/AccentureCTI_Feed/ReleaseNotes/1_1_38.md new file mode 100644 index 000000000000..d9c80ab568a4 --- /dev/null +++ b/Packs/AccentureCTI_Feed/ReleaseNotes/1_1_38.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### ACTI Indicator Feed + +Enhanced **JSONFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. \ No newline at end of file diff --git a/Packs/AccentureCTI_Feed/pack_metadata.json b/Packs/AccentureCTI_Feed/pack_metadata.json index d9b73d3244a8..7361318e8061 100644 --- a/Packs/AccentureCTI_Feed/pack_metadata.json +++ b/Packs/AccentureCTI_Feed/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Accenture CTI Feed", "description": "Accenture Cyber Threat Intelligence Feed", "support": "partner", - "currentVersion": "1.1.37", + "currentVersion": "1.1.38", "author": "Accenture", "url": "https://www.accenture.com/us-en/services/security/cyber-defense", "email": "CTI.AcctManagement@accenture.com", diff --git a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py index faf1918dfc4b..0a6fc0ad0dee 100644 --- a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py +++ b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py @@ -379,10 +379,7 @@ def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], c field = {f: {}} if 'regex' in fattrs: field[f]['regex'] = re.compile(fattrs['regex']) - if 'transform' not in fattrs: - field[f]['transform'] = r'\g<0>' - else: - field[f]['transform'] = fattrs['transform'] + field[f]['transform'] = fattrs.get('transform', '\\g<0>') fields_to_extract.append(field) line = line.strip() @@ -420,7 +417,14 @@ def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], c return attributes, value -def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, create_relationships=False, **kwargs): +def fetch_indicators_command(client, + feed_tags, + tlp_color, + itype, + auto_detect, + create_relationships=False, + enrichment_excluded: bool = False, + **kwargs): iterators = client.build_iterator(**kwargs) indicators = [] @@ -440,22 +444,27 @@ def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, c indicator_type = determine_indicator_type( client.feed_url_to_config.get(url, {}).get('indicator_type'), itype, auto_detect, value) indicator_data = { - "value": value, - "type": indicator_type, - "rawJSON": attributes, + 'value': value, + 'type': indicator_type, + 'rawJSON': attributes, } - if create_relationships and client.feed_url_to_config.get(url, {}).get('relationship_name'): - if attributes.get('relationship_entity_b'): - relationships_lst = EntityRelationship( - name=client.feed_url_to_config.get(url, {}).get('relationship_name'), - entity_a=value, - entity_a_type=indicator_type, - entity_b=attributes.get('relationship_entity_b'), - entity_b_type=FeedIndicatorType.indicator_type_by_server_version( - client.feed_url_to_config.get(url, {}).get('relationship_entity_b_type')), - ) - relationships_of_indicator = [relationships_lst.to_indicator()] - indicator_data['relationships'] = relationships_of_indicator + if enrichment_excluded: + indicator_data['enrichmentExcluded'] = enrichment_excluded + + if (create_relationships + and client.feed_url_to_config.get(url, {}).get('relationship_name') + and attributes.get('relationship_entity_b') + ): + relationships_lst = EntityRelationship( + name=client.feed_url_to_config.get(url, {}).get('relationship_name'), + entity_a=value, + entity_a_type=indicator_type, + entity_b=attributes.get('relationship_entity_b'), + entity_b_type=FeedIndicatorType.indicator_type_by_server_version( + client.feed_url_to_config.get(url, {}).get('relationship_entity_b_type')), + ) + relationships_of_indicator = [relationships_lst.to_indicator()] + indicator_data['relationships'] = relationships_of_indicator if len(client.custom_fields_mapping.keys()) > 0 or TAGS in attributes: custom_fields = client.custom_fields_creator(attributes) @@ -483,14 +492,20 @@ def determine_indicator_type(indicator_type, default_indicator_type, auto_detect return indicator_type -def get_indicators_command(client: Client, args): +def get_indicators_command(client: Client, args, enrichment_excluded: bool = False): itype = args.get('indicator_type', client.indicator_type) limit = int(args.get('limit')) feed_tags = args.get('feedTags') tlp_color = args.get('tlp_color') auto_detect = demisto.params().get('auto_detect_type') create_relationships = demisto.params().get('create_relationships') - indicators_list, _ = fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, create_relationships)[:limit] + indicators_list, _ = fetch_indicators_command(client, + feed_tags, + tlp_color, + itype, + auto_detect, + create_relationships, + enrichment_excluded)[:limit] entry_result = camelize(indicators_list) hr = tableToMarkdown('Indicators', entry_result, headers=['Value', 'Type', 'Rawjson']) return hr, {}, indicators_list @@ -502,7 +517,7 @@ def test_module(client: Client, args): if not FeedIndicatorType.is_valid_type(indicator_type): indicator_types = [] for key, val in vars(FeedIndicatorType).items(): - if not key.startswith('__') and type(val) == str: + if not key.startswith('__') and isinstance(val, str): indicator_types.append(val) supported_values = ', '.join(indicator_types) raise ValueError(f'Indicator type of {indicator_type} is not supported. Supported values are:' @@ -518,6 +533,7 @@ def feed_main(feed_name, params=None, prefix=''): params['feed_name'] = feed_name feed_tags = argToList(demisto.params().get('feedTags')) tlp_color = demisto.params().get('tlp_color') + enrichment_excluded = demisto.params().get('enrichmentExcluded', False) client = Client(**params) command = demisto.command() if command != 'fetch-indicators': @@ -534,7 +550,8 @@ def feed_main(feed_name, params=None, prefix=''): indicators, no_update = fetch_indicators_command(client, feed_tags, tlp_color, params.get('indicator_type'), params.get('auto_detect_type'), - params.get('create_relationships')) + params.get('create_relationships'), + enrichment_excluded=enrichment_excluded) # check if the version is higher than 6.5.0 so we can use noUpdate parameter if is_demisto_version_ge('6.5.0'): diff --git a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.yml b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.yml index 9d9d71a0c6bb..b3a3380a9487 100644 --- a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.yml +++ b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.yml @@ -13,7 +13,7 @@ system: true scripttarget: 0 dependson: {} timeout: 0s -dockerimage: demisto/py3-tools:0.0.1.25751 +dockerimage: demisto/py3-tools:1.0.0.108682 fromversion: 5.0.0 tests: - No tests diff --git a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py index 7a55211503e7..6b2a24025ad3 100644 --- a/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py +++ b/Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py @@ -445,6 +445,71 @@ def test_get_indicators_without_relations(): assert indicators == expected_res +def test_fetch_indicators_exclude_enrichment(): + """ + Given: + - Exclude enrichment parameter is used + When: + - Calling the fetch_indicators_command + Then: + - The indicators should include the enrichmentExcluded field if exclude is True. + """ + + feed_url_to_config = { + 'https://www.spamhaus.org/drop/asndrop.txt': { + "indicator_type": 'IP', + "indicator": { + "regex": r"^.+,\"?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\"?", + "transform": "\\1" + }, + 'relationship_name': 'indicator-of', + 'relationship_entity_b_type': 'STIX Malware', + "fields": [{ + 'firstseenbysource': { + "regex": r"^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})", + "transform": "\\1" + }, + "port": { + "regex": r"^.+,.+,(\d{1,5}),", + "transform": "\\1" + }, + "updatedate": { + "regex": r"^.+,.+,.+,(\d{4}-\d{2}-\d{2})", + "transform": "\\1" + }, + "malwarefamily": { + "regex": r"^.+,.+,.+,.+,(.+)", + "transform": "\\1" + }, + "relationship_entity_b": { + "regex": r"^.+,.+,.+,.+,\"(.+)\"", + "transform": "\\1" + } + }], + } + } + expected_res = ([{'value': '127.0.0.1', 'type': 'IP', + 'rawJSON': {'malwarefamily': '"Test"', 'relationship_entity_b': 'Test', 'value': '127.0.0.1', + 'type': 'IP', 'tags': []}, + 'fields': {'tags': []}, + 'enrichmentExcluded': True}], True) + + asn_ranges = '"2021-01-17 07:44:49","127.0.0.1","3889","online","2021-04-22","Test"' + with requests_mock.Mocker() as m: + m.get('https://www.spamhaus.org/drop/asndrop.txt', content=asn_ranges.encode('utf-8')) + client = Client( + url="https://www.spamhaus.org/drop/asndrop.txt", + source_name='spamhaus', + ignore_regex='^;.*', + feed_url_to_config=feed_url_to_config, + indicator_type='ASN' + ) + indicators = fetch_indicators_command(client, feed_tags=[], tlp_color=[], itype='IP', auto_detect=False, + create_relationships=False, enrichment_excluded=True) + + assert indicators == expected_res + + def test_get_no_update_value(mocker): """ Given diff --git a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py index 2ac1f7940f98..632853952e04 100644 --- a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py +++ b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py @@ -240,7 +240,7 @@ def test_module(client: Client, limit) -> str: # pragma: no cover def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list, auto_detect: bool, create_relationships: bool = False, limit: int = 0, remove_ports: bool = False, - **kwargs) -> Tuple[List[dict], bool]: + enrichment_excluded: bool = False, **kwargs) -> Tuple[List[dict], bool]: """ Fetches the indicators from client. :param client: Client of a JSON Feed @@ -296,7 +296,9 @@ def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list indicators.extend( handle_indicator_function(client, item, feed_config, service_name, indicator_type, indicator_field, use_prefix_flat, feedTags, auto_detect, mapping_function, - create_relationships, create_relationships_function, remove_ports)) + create_relationships, create_relationships_function, remove_ports, + enrichment_excluded=enrichment_excluded, + )) if limit and len(indicators) >= limit: # We have a limitation only when get-indicators command is # called, and then we return for each service_name "limit" of indicators @@ -320,8 +322,9 @@ def indicator_mapping(mapping: Dict, indicator: Dict, attributes: Dict): def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name: str, indicator_type: str, indicator_field: str, use_prefix_flat: bool, feedTags: list, auto_detect: bool, mapping_function: Callable = indicator_mapping, - create_relationships: bool = False, relationships_func: Callable = None, - remove_ports: bool = False) -> List[dict]: + create_relationships: bool = False, relationships_func: Callable | None = None, + remove_ports: bool = False, + enrichment_excluded: bool = False) -> List[dict]: indicator_list = [] mapping = feed_config.get('mapping') take_value_from_flatten = False @@ -367,6 +370,9 @@ def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name indicator['rawJSON'] = item + if enrichment_excluded: + indicator['enrichmentExcluded'] = enrichment_excluded + indicator_list.append(indicator) return indicator_list @@ -436,6 +442,7 @@ def feed_main(params, feed_name, prefix): # pragma: no cover auto_detect = params.get('auto_detect_type') feedTags = argToList(params.get('feedTags')) limit = int(demisto.args().get('limit', 10)) + enrichment_excluded = params.get('enrichmentExcluded', False) command = demisto.command() if prefix and not prefix.endswith('-'): prefix += '-' @@ -448,8 +455,13 @@ def feed_main(params, feed_name, prefix): # pragma: no cover elif command == 'fetch-indicators': remove_ports = argToBoolean(params.get('remove_ports', False)) create_relationships = params.get('create_relationships') - indicators, no_update = fetch_indicators_command(client, indicator_type, feedTags, auto_detect, - create_relationships, remove_ports=remove_ports) + indicators, no_update = fetch_indicators_command(client, + indicator_type, + feedTags, + auto_detect, + create_relationships, + remove_ports=remove_ports, + enrichment_excluded=enrichment_excluded) # check if the version is higher than 6.5.0 so we can use noUpdate parameter if is_demisto_version_ge('6.5.0'): diff --git a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.yml b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.yml index 15e5772974f6..58b7348b10f6 100644 --- a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.yml +++ b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.yml @@ -1,4 +1,4 @@ -comment: Common code that will be appended into each JSON Feed integration when it's deployed +comment: Common code that will be appended into each JSON Feed integration when it's deployed. commonfields: id: JSONFeedApiModule version: -1 @@ -11,7 +11,7 @@ tags: - server timeout: 0s type: python -dockerimage: demisto/py3-tools:0.0.1.25751 +dockerimage: demisto/py3-tools:1.0.0.108682 dependson: {} fromversion: 5.5.0 tests: diff --git a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py index 974a2ee42c12..c2a5faba7ad4 100644 --- a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py +++ b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py @@ -165,6 +165,46 @@ def test_list_of_indicators_with_no_json_object(): assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'} +def test_fetch_indicators_with_exclude_enrichment(): + """ + Given: + - Exclude enrichment parameter is used + When: + - Calling the fetch_indicators_command + Then: + - The indicators should include the enrichmentExcluded field if exclude is True. + """ + + feed_name_to_config = { + 'Github': { + 'url': 'https://api.github.com/meta', + 'extractor': "hooks", + 'indicator': None, + 'remove_ports': "true" + } + } + + with requests_mock.Mocker() as m: + m.get('https://api.github.com/meta', json=json.loads(FLAT_LIST_OF_INDICATORS)) + + client = Client( + url='https://api.github.com/meta', + feed_name_to_config=feed_name_to_config, + insecure=True + ) + + indicators, _ = fetch_indicators_command(client=client, indicator_type=None, feedTags=['test'], + auto_detect=True, remove_ports=True, enrichment_excluded=True) + + assert len(indicators) == 3 + assert indicators[0].get('value') == '1.1.1.1' + assert indicators[0].get('type') == 'IP' + assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'} + + for ind in indicators: + assert ind['enrichmentExcluded'] + + def test_post_of_indicators_with_no_json_object(): feed_name_to_config = { 'Github': { diff --git a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.py b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.py index f5d1dca8be0f..5c6dfd5368fd 100644 --- a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.py +++ b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.py @@ -189,7 +189,10 @@ def get_indicators_command(client: Client, **args) -> CommandResults: ) -def fetch_indicators_command(client: Client, tags: tuple = None, tlp_color: str = None) -> list: +def fetch_indicators_command(client: Client, + tags: tuple = None, + tlp_color: str = None, + enrichment_excluded: bool = False) -> list: """Wrapper for fetching indicators from the feed to the Indicators tab. Args: @@ -204,11 +207,17 @@ def fetch_indicators_command(client: Client, tags: tuple = None, tlp_color: str results = [] indicator_mapping_fields = {'tags': tags, 'trafficlightprotocol': tlp_color} for indicator in clean_res[CIDR] + clean_res[DOMAIN]: - results.append({ + indicator_obj: dict[str, Any] = { 'value': indicator, 'type': check_indicator_type(indicator), - 'fields': indicator_mapping_fields - }) + 'fields': indicator_mapping_fields, + } + + if enrichment_excluded: + indicator_obj['enrichmentExcluded'] = enrichment_excluded + + results.append(indicator_obj) + return results @@ -219,6 +228,7 @@ def main(): proxy = params.get('proxy', False) tags = params.get('feedTags'), tlp_color = params.get('tlp_color') + enrichment_excluded = demisto.params().get('enrichmentExcluded', False) command = demisto.command() try: @@ -231,7 +241,7 @@ def main(): if command == 'test-module': return_results(test_module(client=client)) elif command == 'fetch-indicators': - res = fetch_indicators_command(client=client, tags=tags, tlp_color=tlp_color) + res = fetch_indicators_command(client=client, tags=tags, tlp_color=tlp_color, enrichment_excluded=enrichment_excluded) for iter_ in batch(res, batch_size=2000): demisto.createIndicators(iter_) elif command == 'webex-get-indicators': diff --git a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.yml b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.yml index 75b097b29d4b..9b349fb3240b 100644 --- a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.yml +++ b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed.yml @@ -8,6 +8,7 @@ configuration: name: feed type: 8 required: false + section: Collect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -19,6 +20,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -32,6 +34,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -42,6 +45,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - defaultvalue: indicatorType display: "" name: feedExpirationPolicy @@ -52,36 +56,59 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - defaultvalue: "20160" display: "" name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - defaultvalue: "30" display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: Supports CSV values. defaultvalue: Webex display: Tags name: feedTags type: 0 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. defaultvalue: "true" display: Bypass exclusion list name: feedBypassExclusionList type: 8 required: false + section: Collect + advanced: true +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: Use the Cisco Webex Feed integration to fetch indicators from Webex. display: Cisco Webex Feed name: Cisco WebEx Feed @@ -101,7 +128,7 @@ script: - Both description: Gets indicators from the feed. name: webex-get-indicators - dockerimage: demisto/btfl-soup:1.0.1.87353 + dockerimage: demisto/btfl-soup:1.0.1.107991 feed: true script: '' subtype: python3 @@ -109,3 +136,6 @@ script: fromversion: 6.0.0 tests: - No tests (auto formatted) +sectionOrder: +- Connect +- Collect diff --git a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed_test.py b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed_test.py index e46688efa28e..1d8ae7e067a2 100644 --- a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed_test.py +++ b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/CiscoWebExFeed_test.py @@ -2,13 +2,12 @@ import bs4 import pytest import CiscoWebExFeed -import io import json from CommonServerPython import * # noqa: F401 def util_load_json(path): - with io.open(path, mode='r', encoding='utf-8') as f: + with open(path, encoding='utf-8') as f: return json.loads(f.read()) @@ -39,13 +38,20 @@ def MockedClient(Client): FETCH_INDICATORS_INPUT_1 = {'CIDR': ['ipmock'], 'DOMAIN': ['domainmock']} -FETCH_INDICATORS_UOTPUT_1 = [{'value': 'ipmock', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), +FETCH_INDICATORS_OUTPUT_1 = [{'value': 'ipmock', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), 'trafficlightprotocol': 'very_yellow'}}, {'value': 'domainmock', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), 'trafficlightprotocol': 'very_yellow'}}] +FETCH_INDICATORS_NO_ENRICH_OUTPUT_1 = [{'value': 'ipmock', 'type': 'Domain', + 'fields': {'tags': ('very_good', 'very_bad'), + 'trafficlightprotocol': 'very_yellow'}, 'enrichmentExcluded': True}, + {'value': 'domainmock', 'type': 'Domain', + 'fields': {'tags': ('very_good', 'very_bad'), + 'trafficlightprotocol': 'very_yellow'}, 'enrichmentExcluded': True}] + FETCH_INDICATORS_INPUT_2 = {'CIDR': ['ipmock1', 'ipmock2'], 'DOMAIN': ['domainmock1', 'domainmock2']} -FETCH_INDICATORS_UOTPUT_2 = [{'value': 'ipmock1', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), +FETCH_INDICATORS_OUTPUT_2 = [{'value': 'ipmock1', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), 'trafficlightprotocol': 'very_yellow'}}, {'value': 'ipmock2', 'type': 'Domain', 'fields': {'tags': ('very_good', 'very_bad'), 'trafficlightprotocol': 'very_yellow'}}, @@ -178,8 +184,8 @@ def test_get_indicators_command__wrong_indicator_type(mocker): assert e.value.message == 'The indicator_type argument must be one of the following: Both, CIDR, DOMAIN' -@pytest.mark.parametrize('input, expected', [(FETCH_INDICATORS_INPUT_1, FETCH_INDICATORS_UOTPUT_1), - (FETCH_INDICATORS_INPUT_2, FETCH_INDICATORS_UOTPUT_2)]) +@pytest.mark.parametrize('input, expected', [(FETCH_INDICATORS_INPUT_1, FETCH_INDICATORS_OUTPUT_1), + (FETCH_INDICATORS_INPUT_2, FETCH_INDICATORS_OUTPUT_2)]) def test_fetch_indicators_command__different_sizes_of_inputs(mocker, input, expected): """ Given: @@ -198,6 +204,31 @@ def test_fetch_indicators_command__different_sizes_of_inputs(mocker, input, expe assert fetch_indicators_command(client=client, tags=("very_good", "very_bad"), tlp_color="very_yellow") == expected_result +def test_fetch_indicators_command__exclude_enrichment(mocker): + """ + Given: + - Exclude enrichment parameter is used + When: + - Calling the fetch_indicators_command + Then: + - The indicators should include the enrichmentExcluded field if exclude is True. + """ + from CiscoWebExFeed import fetch_indicators_command, Client + + input = FETCH_INDICATORS_INPUT_1 + expected_result = FETCH_INDICATORS_NO_ENRICH_OUTPUT_1 + + client = MockedClient(Client) + mocker.patch.object(Client, 'all_raw_data', return_value='gg') + mocker.patch.object(CiscoWebExFeed, 'parse_indicators_from_response', + return_value=input) + + assert fetch_indicators_command(client=client, + tags=("very_good", "very_bad"), + tlp_color="very_yellow", + enrichment_excluded=True) == expected_result + + def test_parse_indicators_from_response__fail_to_parse(mocker, requests_mock): """ Given: diff --git a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/README.md b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/README.md index 89c7806a0e3a..d6ea5149ac0b 100644 --- a/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/README.md +++ b/Packs/CiscoWebExFeed/Integrations/CiscoWebExFeed/README.md @@ -17,10 +17,12 @@ Use the Cisco Webex Feed integration to fetch indicators from WeBex. | Feed Fetch Interval | | False | | Tags | Supports CSV values. | False | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | Trust any certificate (not secure) | | False | | Use system proxy settings | | False | 4. Click **Test** to validate the URLs, token, and connection. + ## Commands You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. After you successfully execute a command, a DBot message appears in the War Room with the command details. @@ -56,4 +58,3 @@ There is no context output for this command. >| *.wbx2.com | DomainGlob | >| *.ciscospark.com | DomainGlob | >| *.webexcontent.com | DomainGlob | - diff --git a/Packs/CiscoWebExFeed/ReleaseNotes/1_2_15.md b/Packs/CiscoWebExFeed/ReleaseNotes/1_2_15.md new file mode 100644 index 000000000000..f4eb31105ce6 --- /dev/null +++ b/Packs/CiscoWebExFeed/ReleaseNotes/1_2_15.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Cisco Webex Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/btfl-soup:1.0.1.107991*. diff --git a/Packs/CiscoWebExFeed/pack_metadata.json b/Packs/CiscoWebExFeed/pack_metadata.json index 83b2c5bbcd4e..a29cb44f8b82 100644 --- a/Packs/CiscoWebExFeed/pack_metadata.json +++ b/Packs/CiscoWebExFeed/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Cisco WebEx Feed", "description": "Whitelist feed for Cisco Webex using a screen scrape of the website.", "support": "xsoar", - "currentVersion": "1.2.14", + "currentVersion": "1.2.15", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml b/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml index dd053d3df5e5..599722124f3a 100644 --- a/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml +++ b/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml @@ -33,6 +33,7 @@ configuration: type: 16 additionalinfo: The services to fetch indicators from. Default value is 'All'. If empty, all services will be included. defaultvalue: 'All' + section: Connect - additionalinfo: The AWS Regions to fetch indicators by. Default value is 'All'. If empty, all regions will be included. display: Regions name: regions @@ -74,11 +75,13 @@ configuration: type: 16 defaultvalue: 'All' required: false + section: Connect - display: Fetch indicators name: feed defaultvalue: 'true' type: 8 required: false + section: Collect - defaultvalue: Good display: Indicator Reputation name: feedReputation @@ -90,6 +93,7 @@ configuration: type: 18 additionalinfo: Indicators from this integration instance will be marked with this reputation required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -103,6 +107,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -113,6 +118,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - defaultvalue: indicatorType name: feedExpirationPolicy display: "" @@ -123,34 +129,57 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - name: feedExpirationInterval display: "" type: 1 required: false + section: Collect + advanced: true - defaultvalue: '5' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: Supports CSV values. display: Tags name: feedTags type: 0 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList defaultvalue: "true" type: 8 required: false + section: Collect + advanced: true +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + defaultvalue: 'false' + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: Use the AWS feed integration to fetch indicators from the feed. display: AWS Feed name: AWS Feed @@ -162,7 +191,7 @@ script: name: limit description: Fetches indicators from the feed. name: aws-get-indicators - dockerimage: demisto/py3-tools:1.0.0.99035 + dockerimage: demisto/py3-tools:1.0.0.108682 feed: true runonce: false script: '-' @@ -173,3 +202,6 @@ defaultclassifier: AWS Feed Classifier defaultmapperin: AWS Feed Mapper tests: - No tests (auto formatted) +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedAWS/Integrations/FeedAWS/README.md b/Packs/FeedAWS/Integrations/FeedAWS/README.md index 286c9a7d27f1..28c3cefcf1b8 100644 --- a/Packs/FeedAWS/Integrations/FeedAWS/README.md +++ b/Packs/FeedAWS/Integrations/FeedAWS/README.md @@ -41,6 +41,7 @@ Use the AWS feed integration to fetch indicators from the feed. * __Skip Exclusion List__: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. + * __Enrichment Excluded__: Select this option to exclude the fetched indicators from the enrichment process. * __Indicator reputation__: Indicators from this integration instance will be marked with this reputation. * __Trust any certificate (not secure)__ diff --git a/Packs/FeedAWS/ReleaseNotes/1_1_52.md b/Packs/FeedAWS/ReleaseNotes/1_1_52.md new file mode 100644 index 000000000000..d6490b78b289 --- /dev/null +++ b/Packs/FeedAWS/ReleaseNotes/1_1_52.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### AWS Feed + +<~XSOAR_SAAS> +Support for instance-wide indicator enrichment exclusion + +- Updated the Docker image to: *demisto/py3-tools:1.0.0.108682*. diff --git a/Packs/FeedAWS/pack_metadata.json b/Packs/FeedAWS/pack_metadata.json index d37162b55294..a1bd7484c59e 100644 --- a/Packs/FeedAWS/pack_metadata.json +++ b/Packs/FeedAWS/pack_metadata.json @@ -2,7 +2,7 @@ "name": "AWS Feed", "description": "Indicators feed from AWS", "support": "xsoar", - "currentVersion": "1.1.51", + "currentVersion": "1.1.52", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.py b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.py index 9024f7f3c42d..92d45e42f0bd 100644 --- a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.py +++ b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.py @@ -1,6 +1,5 @@ import re import urllib3 -from typing import Dict, List, Tuple, Optional from CommonServerPython import * @@ -36,7 +35,7 @@ def __init__(self, regions_list: list, services_list: list, polling_timeout: int self._polling_timeout = polling_timeout @staticmethod - def build_ip_indicator(azure_ip_address, **indicator_metadata) -> Dict: + def build_ip_indicator(azure_ip_address, **indicator_metadata) -> dict: """Creates an IP data dict. Args: @@ -96,7 +95,7 @@ def get_azure_download_link(self): return download_link - def get_download_file_content_values(self, download_link: str) -> Dict: + def get_download_file_content_values(self, download_link: str) -> dict: """Create a request to receive file content from link. Args: @@ -116,7 +115,7 @@ def get_download_file_content_values(self, download_link: str) -> Dict: return file_download_response.get('values') @staticmethod - def extract_metadata_of_indicators_group(indicators_group_data: Dict) -> Dict: + def extract_metadata_of_indicators_group(indicators_group_data: dict) -> dict: """Extracts metadata of an indicators group. Args: @@ -125,7 +124,7 @@ def extract_metadata_of_indicators_group(indicators_group_data: Dict) -> Dict: Returns: Dict. Indicators group metadata. """ - indicator_metadata = dict() + indicator_metadata = {} indicator_metadata['id'] = indicators_group_data.get('id') indicator_metadata['name'] = indicators_group_data.get('name') @@ -147,7 +146,7 @@ def extract_metadata_of_indicators_group(indicators_group_data: Dict) -> Dict: return indicator_metadata @staticmethod - def filter_and_aggregate_values(address_list: List) -> List: + def filter_and_aggregate_values(address_list: list) -> list: """For each indicator value from the given list we aggregate the all the different keys found. Args: @@ -165,9 +164,9 @@ def filter_and_aggregate_values(address_list: List) -> List: else: indicator_objects[current_value] = item_to_search - return [value for value in indicator_objects.values()] + return list(indicator_objects.values()) - def extract_indicators_from_values_dict(self, values_from_file: Dict) -> List: + def extract_indicators_from_values_dict(self, values_from_file: dict) -> list: """Builds a list of all IP indicators in the input dict. Args: @@ -208,7 +207,7 @@ def extract_indicators_from_values_dict(self, values_from_file: Dict) -> List: ) return self.filter_and_aggregate_values(results) - def build_iterator(self) -> List: + def build_iterator(self) -> list: """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. @@ -233,7 +232,7 @@ def build_iterator(self) -> List: raise ValueError(f'Could not parse returned data to Json. \n\nError massage: {err}') -def test_module(client: Client) -> Tuple[str, Dict, Dict]: +def test_module(client: Client) -> tuple[str, dict, dict]: """Test the ability to fetch Azure file. Args: client: Client object. @@ -259,7 +258,10 @@ def test_module(client: Client) -> Tuple[str, Dict, Dict]: return 'ok', {}, {} -def get_indicators_command(client: Client, feedTags: list, tlp_color: Optional[str]) -> Tuple[str, Dict, Dict]: +def get_indicators_command(client: Client, + feedTags: list, + tlp_color: str | None, + enrichment_excluded: bool = False) -> tuple[str, dict, dict]: """Retrieves indicators from the feed to the war-room. Args: @@ -273,7 +275,7 @@ def get_indicators_command(client: Client, feedTags: list, tlp_color: Optional[s Dict. The raw data of the indicators. """ limit = int(demisto.args().get('limit')) if 'limit' in demisto.args() else 10 - indicators, raw_response = fetch_indicators_command(client, feedTags, tlp_color, limit) + indicators, raw_response = fetch_indicators_command(client, feedTags, tlp_color, limit, enrichment_excluded) human_readable = tableToMarkdown('Indicators from Azure Feed:', indicators, headers=['value', 'type'], removeNull=True) @@ -281,8 +283,11 @@ def get_indicators_command(client: Client, feedTags: list, tlp_color: Optional[s return human_readable, {}, {'raw_response': raw_response} -def fetch_indicators_command(client: Client, feedTags: list, tlp_color: Optional[str], limit: int = -1) \ - -> Tuple[List[Dict], List]: +def fetch_indicators_command(client: Client, + feedTags: list, + tlp_color: str | None, + limit: int = -1, + enrichment_excluded: bool = False) -> tuple[list[dict], list]: """Fetches indicators from the feed to the indicators tab. Args: client (Client): Client object configured according to instance arguments. @@ -311,12 +316,15 @@ def fetch_indicators_command(client: Client, feedTags: list, tlp_color: Optional 'service': indicator.get('azure_system_service'), 'tags': feedTags, }, - 'rawJSON': indicator + 'rawJSON': indicator, } if tlp_color: indicator_obj['fields']['trafficlightprotocol'] = tlp_color + if enrichment_excluded: + indicator_obj['enrichmentExcluded'] = enrichment_excluded + indicators.append(indicator_obj) raw_response.append(indicator) @@ -337,6 +345,7 @@ def main(): feedTags = argToList(demisto.params().get('feedTags')) tlp_color = demisto.params().get('tlp_color') + enrichment_excluded = demisto.params().get('enrichmentExcluded', False) polling_arg = demisto.params().get('polling_timeout', '') polling_timeout = int(polling_arg) if polling_arg.isdigit() else 20 @@ -355,7 +364,7 @@ def main(): feedTags['tags'] = feedTags return_outputs(*get_indicators_command(client, feedTags, tlp_color)) elif command == 'fetch-indicators': - indicators, _ = fetch_indicators_command(client, feedTags, tlp_color) + indicators, _ = fetch_indicators_command(client, feedTags, tlp_color, enrichment_excluded=enrichment_excluded) for single_batch in batch(indicators, batch_size=2000): demisto.createIndicators(single_batch) diff --git a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml index 5a470dfe52f1..34e1303b947c 100644 --- a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml +++ b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml @@ -8,6 +8,7 @@ configuration: name: feed type: 8 required: false + section: Collect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -19,6 +20,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -32,6 +34,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -42,6 +45,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - defaultvalue: indicatorType display: '' name: feedExpirationPolicy @@ -52,22 +56,39 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - defaultvalue: '20160' display: '' name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - defaultvalue: '5' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList type: 8 defaultvalue: "true" required: false + section: Collect + advanced: true +- defaultvalue: 'false' + display: Enrichment Excluded + name: enrichmentExcluded + required: false + type: 8 + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + hidden: + - xsoar_on_prem + section: Collect - defaultvalue: All display: Regions name: regions @@ -120,9 +141,12 @@ configuration: - westus2 required: true type: 16 -- defaultvalue: All - display: Services + section: Connect +- display: Services name: services + type: 16 + required: true + defaultvalue: All options: - All - ActionGroup @@ -193,26 +217,33 @@ configuration: - StorageSyncService - WindowsAdminCenter - WindowsVirtualDesktop - required: true - type: 16 -- additionalinfo: Supports CSV values. - display: Tags + section: Connect +- display: Tags name: feedTags type: 0 required: false + additionalinfo: Supports CSV values. + section: Collect + advanced: true - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true - defaultvalue: '20' display: Request Timeout name: polling_timeout type: 0 required: false + section: Collect + advanced: true description: Azure.CloudIPs Feed Integration. display: Azure Feed name: AzureFeed @@ -224,7 +255,7 @@ script: name: limit description: Gets indicators from the feed. name: azure-get-indicators - dockerimage: demisto/python3:3.10.13.86272 + dockerimage: demisto/python3:3.11.9.107902 feed: true runonce: false script: '-' @@ -233,3 +264,6 @@ script: tests: - AzureFeed - Test fromversion: 5.5.0 +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure_test.py b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure_test.py index 5f4c184aa5ab..24658016109d 100644 --- a/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure_test.py +++ b/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure_test.py @@ -1,5 +1,5 @@ import pytest -from FeedAzure import Client +from FeedAzure import Client, fetch_indicators_command, AZUREJSON_URL @pytest.mark.parametrize('regions_list, services_list', [(['All'], ['All'])]) @@ -255,3 +255,105 @@ def test_filter_duplicate_addresses(list_to_filter, expected_result): """ client = Client([], []) assert expected_result == client.filter_and_aggregate_values(list_to_filter) + + +@pytest.mark.parametrize('enrichment_excluded', [True, False]) +def test_fetch_indicators_command(requests_mock, enrichment_excluded): + """ + Given: + Parameters (regions_list, services_list, enrichment_excluded) for fetching indicators + When: + Calling fetch_indicators_command + Then: + The indicators will be returned as expected, with enrichmentExcluded if requested + """ + url = "https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20240819.json" + downloadData = ''' + downloadData={ + "base_0":{ + "url":"https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20240819.json", + "id":"56519", + "oldid":"3a1b5c65-0f86-41d9-b2fe-24708260c0f1" + } + } + ''' + mock_json = { + "changeNumber": 320, + "cloud": "Public", + "values": [ + { + "name": "AzureAdvancedThreatProtection", + "id": "AzureAdvancedThreatProtection", + "properties": { + "changeNumber": 24, + "region": "", + "regionId": 0, + "platform": "Azure", + "systemService": "AzureAdvancedThreatProtection", + "addressPrefixes": [ + "192.168.0.1/29", + "10.0.0.1/29", + ], + "networkFeatures": [ + "API", + "NSG", + "UDR", + "FW" + ] + } + } + ] + } + expected = [ + { + 'value': '192.168.0.1/29', + 'type': 'CIDR', + 'fields': { + 'region': '', + 'service': 'AzureAdvancedThreatProtection', + 'tags': ['test'], + 'trafficlightprotocol': 'test_color' + }, + 'rawJSON': { + 'value': '192.168.0.1/29', + 'type': 'CIDR', + 'azure_name': 'AzureAdvancedThreatProtection', + 'azure_id': 'AzureAdvancedThreatProtection', + 'azure_region': '', + 'azure_platform': 'Azure', + 'azure_system_service': 'AzureAdvancedThreatProtection' + }, + }, + { + 'value': '10.0.0.1/29', + 'type': 'CIDR', + 'fields': { + 'region': '', + 'service': 'AzureAdvancedThreatProtection', + 'tags': ['test'], + 'trafficlightprotocol': 'test_color' + }, + 'rawJSON': { + 'value': '10.0.0.1/29', + 'type': 'CIDR', + 'azure_name': 'AzureAdvancedThreatProtection', + 'azure_id': 'AzureAdvancedThreatProtection', + 'azure_region': '', + 'azure_platform': 'Azure', + 'azure_system_service': 'AzureAdvancedThreatProtection' + }, + } + ] + if enrichment_excluded: + for ind in expected: + ind['enrichmentExcluded'] = True + regions_list = ['All'] + services_list = ['All'] + requests_mock.get(AZUREJSON_URL, text=f'{downloadData=}') + requests_mock.get(url, json=mock_json) + client = Client(regions_list, services_list) + indicators, _ = fetch_indicators_command(client, + feedTags=['test'], + tlp_color='test_color', + enrichment_excluded=enrichment_excluded) + assert indicators == expected diff --git a/Packs/FeedAzure/Integrations/FeedAzure/README.md b/Packs/FeedAzure/Integrations/FeedAzure/README.md index 29f112c96d7d..7ea18941c0b8 100644 --- a/Packs/FeedAzure/Integrations/FeedAzure/README.md +++ b/Packs/FeedAzure/Integrations/FeedAzure/README.md @@ -15,6 +15,7 @@ Azure.CloudIPs Feed Integration. | feedExpirationInterval | | False | | feedFetchInterval | Feed Fetch Interval | False | | feedBypassExclusionList | Bypass exclusion list | False | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | regions | Regions | True | | services | Services | True | | feedTags | Tags | False | diff --git a/Packs/FeedAzure/ReleaseNotes/1_0_28.md b/Packs/FeedAzure/ReleaseNotes/1_0_28.md new file mode 100644 index 000000000000..a22307f323d4 --- /dev/null +++ b/Packs/FeedAzure/ReleaseNotes/1_0_28.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Azure Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. diff --git a/Packs/FeedAzure/pack_metadata.json b/Packs/FeedAzure/pack_metadata.json index c2afb5b13217..d11f50474330 100644 --- a/Packs/FeedAzure/pack_metadata.json +++ b/Packs/FeedAzure/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Azure Feed", "description": "Indicators feed from Azure", "support": "xsoar", - "currentVersion": "1.0.27", + "currentVersion": "1.0.28", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.py b/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.py index 72408671e84e..44d8360e663e 100644 --- a/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.py +++ b/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.py @@ -6,7 +6,7 @@ def main(): services = ['all', 'ssh', 'mail', 'apache', 'imap', 'ftp', 'sip', 'bots', 'strongips', 'bruteforcelogin'] - feed_types = dict() + feed_types = {} for service in services: feed_types[F'https://lists.blocklist.de/lists/{service}.txt'] = { @@ -18,7 +18,7 @@ def main(): # Automatically infer the indicator type params['auto_detect_type'] = True - chosen_services = list() + chosen_services = [] for service in argToList(demisto.params().get('services', [])): chosen_services.append(F'https://lists.blocklist.de/lists/{service}.txt') diff --git a/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.yml b/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.yml index c4da82a95d73..7c1799eb5244 100644 --- a/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.yml +++ b/Packs/FeedBlocklist_de/Integrations/FeedBlocklist_de/FeedBlocklist_de.yml @@ -113,7 +113,7 @@ script: name: indicator_type description: Gets the feed indicators. name: blocklist_de-get-indicators - dockerimage: demisto/python3:3.10.13.74666 + dockerimage: demisto/python3:3.11.9.107902 feed: true runonce: false script: '-' diff --git a/Packs/FeedBlocklist_de/ReleaseNotes/1_1_29.md b/Packs/FeedBlocklist_de/ReleaseNotes/1_1_29.md new file mode 100644 index 000000000000..b7ced483096d --- /dev/null +++ b/Packs/FeedBlocklist_de/ReleaseNotes/1_1_29.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### Blocklist_de Feed + +- Enhanced **HTTPFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. diff --git a/Packs/FeedBlocklist_de/pack_metadata.json b/Packs/FeedBlocklist_de/pack_metadata.json index dd9719a2f4ce..1b7af161c355 100644 --- a/Packs/FeedBlocklist_de/pack_metadata.json +++ b/Packs/FeedBlocklist_de/pack_metadata.json @@ -2,7 +2,7 @@ "name": "BlockList DE Feed", "description": "Indicators feed from BlockList DE", "support": "xsoar", - "currentVersion": "1.1.28", + "currentVersion": "1.1.29", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedBruteForceBlocker/ReleaseNotes/1_1_27.md b/Packs/FeedBruteForceBlocker/ReleaseNotes/1_1_27.md new file mode 100644 index 000000000000..eade739ef658 --- /dev/null +++ b/Packs/FeedBruteForceBlocker/ReleaseNotes/1_1_27.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### BruteForceBlocker Feed + +Enhanced **HTTPFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedBruteForceBlocker/pack_metadata.json b/Packs/FeedBruteForceBlocker/pack_metadata.json index 53070d17bdb7..0c0a6bae59c8 100644 --- a/Packs/FeedBruteForceBlocker/pack_metadata.json +++ b/Packs/FeedBruteForceBlocker/pack_metadata.json @@ -2,7 +2,7 @@ "name": "BruteForce Feed", "description": "Indicators feed from BruteForceBlocker", "support": "xsoar", - "currentVersion": "1.1.26", + "currentVersion": "1.1.27", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedCloudflare/Integrations/FeedCloudflare/FeedCloudflare.yml b/Packs/FeedCloudflare/Integrations/FeedCloudflare/FeedCloudflare.yml index 6466ebcbc8ac..27509eba9cd3 100644 --- a/Packs/FeedCloudflare/Integrations/FeedCloudflare/FeedCloudflare.yml +++ b/Packs/FeedCloudflare/Integrations/FeedCloudflare/FeedCloudflare.yml @@ -10,11 +10,13 @@ configuration: - https://www.cloudflare.com/ips-v6 required: true type: 16 + section: Connect - defaultvalue: 'true' display: Fetch indicators name: feed type: 8 required: false + section: Collect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -26,6 +28,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -39,6 +42,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -49,6 +53,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - defaultvalue: indicatorType display: '' name: feedExpirationPolicy @@ -59,41 +64,66 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - defaultvalue: '20160' display: '' name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - defaultvalue: '5' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList type: 8 defaultvalue: "true" required: false + section: Collect + advanced: true +- additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + hidden: + - xsoar_on_prem + section: Collect - additionalinfo: Timeout of the polling request in seconds. - defaultvalue: '20' display: Request Timeout name: polling_timeout type: 0 required: false -- additionalinfo: Supports CSV values. - display: Tags + defaultvalue: '20' + section: Collect + advanced: true +- display: Tags name: feedTags type: 0 required: false + additionalinfo: Supports CSV values. + section: Collect + advanced: true - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: Use the Cloudflare feed integration to fetch indicators from the feed. display: Cloudflare Feed name: Cloudflare Feed @@ -105,7 +135,7 @@ script: name: limit description: Gets the feed indicators. name: cloudflare-get-indicators - dockerimage: demisto/python3:3.10.13.74666 + dockerimage: demisto/python3:3.11.9.107902 feed: true runonce: false script: '-' @@ -114,3 +144,6 @@ script: fromversion: 5.5.0 tests: - No tests (auto formatted) +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedCloudflare/Integrations/FeedCloudflare/README.md b/Packs/FeedCloudflare/Integrations/FeedCloudflare/README.md index 38422c417aa3..af95762dc7d4 100644 --- a/Packs/FeedCloudflare/Integrations/FeedCloudflare/README.md +++ b/Packs/FeedCloudflare/Integrations/FeedCloudflare/README.md @@ -15,6 +15,7 @@ Use the Cloudflare feed integration to fetch indicators from the feed. | Traffic Light Protocol Color | The Traffic Light Protocol \(TLP\) designation to apply to indicators fetched from the feed | False | | Feed Fetch Interval | | False | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | +| Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | Request Timeout | Timeout of the polling request in seconds. | False | | Tags | Supports CSV values. | False | | Trust any certificate (not secure) | | False | diff --git a/Packs/FeedCloudflare/ReleaseNotes/1_1_27.md b/Packs/FeedCloudflare/ReleaseNotes/1_1_27.md new file mode 100644 index 000000000000..dbefcbc42c48 --- /dev/null +++ b/Packs/FeedCloudflare/ReleaseNotes/1_1_27.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Cloudflare Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. diff --git a/Packs/FeedCloudflare/pack_metadata.json b/Packs/FeedCloudflare/pack_metadata.json index 49c367dbc4b1..92bb6d276fdf 100644 --- a/Packs/FeedCloudflare/pack_metadata.json +++ b/Packs/FeedCloudflare/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Cloudflare Feed", "description": "Indicators feed from Cloudflare", "support": "xsoar", - "currentVersion": "1.1.26", + "currentVersion": "1.1.27", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedDShield/ReleaseNotes/1_1_31.md b/Packs/FeedDShield/ReleaseNotes/1_1_31.md new file mode 100644 index 000000000000..ad80ae1f310d --- /dev/null +++ b/Packs/FeedDShield/ReleaseNotes/1_1_31.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### DShield Feed + +Enhanced **HTTPFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedDShield/pack_metadata.json b/Packs/FeedDShield/pack_metadata.json index c96dc47b7b00..fcf377211905 100644 --- a/Packs/FeedDShield/pack_metadata.json +++ b/Packs/FeedDShield/pack_metadata.json @@ -2,7 +2,7 @@ "name": "DShield Feed", "description": "Indicators feed from DShield", "support": "xsoar", - "currentVersion": "1.1.30", + "currentVersion": "1.1.31", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedFastly/Integrations/FeedFastly/FeedFastly.yml b/Packs/FeedFastly/Integrations/FeedFastly/FeedFastly.yml index d92c771a767c..19df2c28d11f 100644 --- a/Packs/FeedFastly/Integrations/FeedFastly/FeedFastly.yml +++ b/Packs/FeedFastly/Integrations/FeedFastly/FeedFastly.yml @@ -8,6 +8,7 @@ configuration: name: feed type: 8 required: false + section: Collect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -19,6 +20,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -32,6 +34,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -42,6 +45,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - defaultvalue: indicatorType display: '' name: feedExpirationPolicy @@ -52,35 +56,58 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - defaultvalue: '20160' display: '' name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - defaultvalue: '5' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList type: 8 defaultvalue: "true" required: false -- additionalinfo: Supports CSV values. - display: Tags + section: Collect + advanced: true +- additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect +- display: Tags name: feedTags type: 0 required: false + additionalinfo: Supports CSV values. + section: Collect + advanced: true - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: Use Fastly Feed to get assigned CIDRs and add them to your firewall's allowlist in order to enable using Fastly's services. display: Fastly Feed name: Fastly Feed @@ -88,14 +115,19 @@ script: commands: - arguments: - defaultValue: '50' - description: limits the number of context indicators to output + description: limits the number of context indicators to output. name: limit description: Fetches indicators from the feed. name: fastly-get-indicators - dockerimage: demisto/py3-tools:1.0.0.91504 + dockerimage: demisto/py3-tools:1.0.0.108682 feed: true runonce: false script: '-' subtype: python3 type: python fromversion: 5.5.0 +sectionOrder: +- Connect +- Collect +tests: +- No tests (auto formatted) diff --git a/Packs/FeedFastly/Integrations/FeedFastly/README.md b/Packs/FeedFastly/Integrations/FeedFastly/README.md index 3bda6e303c3f..4692698b1968 100644 --- a/Packs/FeedFastly/Integrations/FeedFastly/README.md +++ b/Packs/FeedFastly/Integrations/FeedFastly/README.md @@ -16,6 +16,7 @@ Use Fastly Feed to get assigned CIDRs and add them to your firewall's allow-list | | | False | | Feed Fetch Interval | | False | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | Tags | Supports CSV values. | False | | Trust any certificate (not secure) | | False | | Use system proxy settings | | False | diff --git a/Packs/FeedFastly/ReleaseNotes/1_1_29.md b/Packs/FeedFastly/ReleaseNotes/1_1_29.md new file mode 100644 index 000000000000..be976310b98a --- /dev/null +++ b/Packs/FeedFastly/ReleaseNotes/1_1_29.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Fastly Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/py3-tools:1.0.0.108682*. diff --git a/Packs/FeedFastly/pack_metadata.json b/Packs/FeedFastly/pack_metadata.json index 960e66b7164f..940770ac2c9c 100644 --- a/Packs/FeedFastly/pack_metadata.json +++ b/Packs/FeedFastly/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Fastly Feed", "description": "Indicators feed from Fastly", "support": "xsoar", - "currentVersion": "1.1.28", + "currentVersion": "1.1.29", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.py b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.py index 635c0f1fd69f..627ab672b83f 100644 --- a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.py +++ b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.py @@ -5,29 +5,29 @@ feed_config = { 'All GCP customer global and regional external IP ranges': - dict(CIDR={ + {'CIDR': { 'url': 'https://www.gstatic.com/ipranges/cloud.json', 'extractor': "prefixes[]", 'indicator': 'ipv4Prefix', 'indicator_type': FeedIndicatorType.CIDR, - }, IPv6CIDR={ + }, 'IPv6CIDR': { 'url': 'https://www.gstatic.com/ipranges/cloud.json', 'extractor': "prefixes[]", 'indicator': 'ipv6Prefix', 'indicator_type': FeedIndicatorType.IPv6CIDR, - }), + }}, 'All available Google IP ranges': - dict(CIDR={ + {'CIDR': { 'url': 'https://www.gstatic.com/ipranges/goog.json', 'extractor': "prefixes[]", 'indicator': 'ipv4Prefix', 'indicator_type': FeedIndicatorType.CIDR, - }, IPv6CIDR={ + }, 'IPv6CIDR': { 'url': 'https://www.gstatic.com/ipranges/goog.json', 'extractor': "prefixes[]", 'indicator': 'ipv6Prefix', 'indicator_type': FeedIndicatorType.IPv6CIDR, - }), + }}, } diff --git a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.yml b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.yml index bb35c8131357..1c181c9b2e37 100644 --- a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.yml +++ b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/FeedGoogleIPRanges.yml @@ -8,6 +8,7 @@ configuration: name: feed type: 8 required: false + section: Collect - defaultvalue: All GCP customer global and regional external IP ranges display: IP Address Ranges name: ip_ranges @@ -17,6 +18,7 @@ configuration: required: true type: 15 additionalinfo: IP address ranges group to be fetched. See integration help for more information. + section: Connect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: None display: Indicator Reputation @@ -28,6 +30,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -41,6 +44,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -51,6 +55,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - display: "" name: feedExpirationPolicy defaultvalue: suddenDeath @@ -61,34 +66,57 @@ configuration: - indicatorType - suddenDeath required: false + section: Collect + advanced: true - defaultvalue: '20160' display: "" name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - defaultvalue: '240' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - additionalinfo: Supports CSV values. display: Tags name: feedTags type: 0 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList type: 8 required: false + section: Collect + advanced: true +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: Use the Google IP Ranges integration to get GCP and Google global IP ranges. display: Google IP Ranges Feed name: Google IP Ranges Feed @@ -100,7 +128,7 @@ script: defaultValue: "10" description: Gets indicators from the feed. name: google-ip-ranges-get-indicators - dockerimage: demisto/py3-tools:1.0.0.86612 + dockerimage: demisto/py3-tools:1.0.0.108682 feed: true runonce: false script: '-' @@ -109,3 +137,6 @@ script: fromversion: 6.0.0 tests: - Fetch Indicators Test +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/README.md b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/README.md index 77400d37df99..4d62b274f3fe 100644 --- a/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/README.md +++ b/Packs/FeedGCPWhitelist/Integrations/FeedGoogleIPRanges/README.md @@ -18,6 +18,7 @@ Use the Google IP Ranges Feed integration to get GCP and Google global IP ranges | feedFetchInterval | Feed Fetch Interval | False | | feedTags | Tags | False | | feedBypassExclusionList | Bypass exclusion list | False | +| Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | insecure | Trust any certificate \(not secure\) | False | | proxy | Use system proxy settings | False | diff --git a/Packs/FeedGCPWhitelist/ReleaseNotes/2_0_39.md b/Packs/FeedGCPWhitelist/ReleaseNotes/2_0_39.md new file mode 100644 index 000000000000..ae50edd8a8e8 --- /dev/null +++ b/Packs/FeedGCPWhitelist/ReleaseNotes/2_0_39.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Google IP Ranges Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/py3-tools:1.0.0.108682*. diff --git a/Packs/FeedGCPWhitelist/pack_metadata.json b/Packs/FeedGCPWhitelist/pack_metadata.json index 70901ed74eee..cf35d4d48ef6 100644 --- a/Packs/FeedGCPWhitelist/pack_metadata.json +++ b/Packs/FeedGCPWhitelist/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Google IP Ranges Feed", "description": "Use the Google IP Ranges Feed integration to get GCP and Google global IP ranges.", "support": "xsoar", - "currentVersion": "2.0.38", + "currentVersion": "2.0.39", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedJSON/ReleaseNotes/1_1_31.md b/Packs/FeedJSON/ReleaseNotes/1_1_31.md new file mode 100644 index 000000000000..3c6b6d8e5211 --- /dev/null +++ b/Packs/FeedJSON/ReleaseNotes/1_1_31.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### JSON Feed + +Enhanced **JSONFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedJSON/pack_metadata.json b/Packs/FeedJSON/pack_metadata.json index 311b6cbac478..874bc5508388 100644 --- a/Packs/FeedJSON/pack_metadata.json +++ b/Packs/FeedJSON/pack_metadata.json @@ -2,7 +2,7 @@ "name": "JSON Feed", "description": "Indicators feed from a JSON file", "support": "xsoar", - "currentVersion": "1.1.30", + "currentVersion": "1.1.31", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedMalwareBazaar/ReleaseNotes/1_0_39.md b/Packs/FeedMalwareBazaar/ReleaseNotes/1_0_39.md new file mode 100644 index 000000000000..1a2088054ea7 --- /dev/null +++ b/Packs/FeedMalwareBazaar/ReleaseNotes/1_0_39.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### MalwareBazaar Feed + +Enhanced **JSONFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedMalwareBazaar/pack_metadata.json b/Packs/FeedMalwareBazaar/pack_metadata.json index 6ec26f54a9a8..74303d474044 100644 --- a/Packs/FeedMalwareBazaar/pack_metadata.json +++ b/Packs/FeedMalwareBazaar/pack_metadata.json @@ -2,7 +2,7 @@ "name": "MalwareBazaar Feed", "description": "MalwareBazaar is a project from abuse.ch with the goal of sharing malware samples with the infosec community, AV vendors and threat intelligence providers.", "support": "xsoar", - "currentVersion": "1.0.38", + "currentVersion": "1.0.39", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.py b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.py index 30407c928bdd..373a1159b3b5 100644 --- a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.py +++ b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.py @@ -1,4 +1,5 @@ -from typing import Dict, List, Tuple, Any, Callable, Optional +from typing import Any +from collections.abc import Callable import uuid import urllib3 @@ -34,7 +35,7 @@ def build_region_or_category_list(param_list: list, all_config_list: list, allow return param_list -def build_urls_dict(regions_list: list, services_list: list, unique_id) -> List[Dict[str, Any]]: +def build_urls_dict(regions_list: list, services_list: list, unique_id) -> list[dict[str, Any]]: """Builds a URL dictionary with the relevant data for each service Args: @@ -69,22 +70,22 @@ class Client: https://techcommunity.microsoft.com/t5/Office-365-Blog/Announcing-Office-365-endpoint-categories-and-Office-365-IP/ba-p/177638 """ - def __init__(self, urls_list: list, category_list: list, insecure: bool = False, tags: Optional[list] = None, - tlp_color: Optional[str] = None): + def __init__(self, urls_list: list, category_list: list, insecure: bool = False, tags: list | None = None, + tlp_color: str | None = None): """ Implements class for Office 365 feeds. :param urls_list: List of url, regions and service of each service. :param insecure: boolean, if *false* feed HTTPS server certificate is verified. Default: *false* :param tlp_color: Traffic Light Protocol color. """ - self._urls_list: List[dict] = urls_list + self._urls_list: list[dict] = urls_list self._verify: bool = insecure self.tags = [] if tags is None else tags self.tlp_color = tlp_color self._proxies = handle_proxy(proxy_param_name='proxy', checkbox_default_value=False) self.category_list = category_list - def build_iterator(self) -> List: + def build_iterator(self) -> list: """Retrieves all entries from the feed. Returns: @@ -158,7 +159,7 @@ def check_indicator_type(indicator): return FeedIndicatorType.Domain -def test_module(client: Client, *_) -> Tuple[str, Dict[Any, Any], Dict[Any, Any]]: +def test_module(client: Client, *_) -> tuple[str, dict[Any, Any], dict[Any, Any]]: """Builds the iterator to check that the feed is accessible. Args: client: Client object. @@ -170,7 +171,7 @@ def test_module(client: Client, *_) -> Tuple[str, Dict[Any, Any], Dict[Any, Any] return 'ok', {}, {} -def fetch_indicators(client: Client, indicator_type_lower: str, limit: int = -1) -> List[Dict]: +def fetch_indicators(client: Client, indicator_type_lower: str, limit: int = -1, enrichment_excluded: bool = False) -> list[dict]: """Retrieves indicators from the feed Args: @@ -183,7 +184,7 @@ def fetch_indicators(client: Client, indicator_type_lower: str, limit: int = -1) """ iterator = client.build_iterator() # filter indicator_type specific entries - if not indicator_type_lower == 'both': + if indicator_type_lower != 'both': iterator = [i for i in iterator if indicator_type_lower in i] indicators = [] if limit > 0: @@ -222,17 +223,24 @@ def fetch_indicators(client: Client, indicator_type_lower: str, limit: int = -1) if client.tlp_color: indicator_mapping_fields['trafficlightprotocol'] = client.tlp_color - indicators.append({ + indicator_obj = { 'value': value, 'type': type_, 'rawJSON': raw_data, - 'fields': indicator_mapping_fields - }) + 'fields': indicator_mapping_fields, + } + + if enrichment_excluded: + indicator_obj['enrichmentExcluded'] = enrichment_excluded + + indicators.append(indicator_obj) return indicators -def get_indicators_command(client: Client, args: Dict[str, str]) -> Tuple[str, Dict[Any, Any], Dict[Any, Any]]: +def get_indicators_command(client: Client, + args: dict[str, str], + enrichment_excluded: bool = False) -> tuple[str, dict[Any, Any], dict[Any, Any]]: """Wrapper for retrieving indicators from the feed to the war-room. Args: @@ -245,14 +253,14 @@ def get_indicators_command(client: Client, args: Dict[str, str]) -> Tuple[str, D indicator_type = str(args.get('indicator_type')) indicator_type_lower = indicator_type.lower() limit = int(demisto.args().get('limit')) if 'limit' in demisto.args() else 10 - indicators = fetch_indicators(client, indicator_type_lower, limit) + indicators = fetch_indicators(client, indicator_type_lower, limit, enrichment_excluded) human_readable = tableToMarkdown('Indicators from Office 365 Feed:', indicators, headers=['value', 'type'], removeNull=True) return human_readable, {}, {'raw_response': indicators} -def fetch_indicators_command(client: Client) -> List[Dict]: +def fetch_indicators_command(client: Client, enrichment_excluded: bool = False) -> list[dict]: """Wrapper for fetching indicators from the feed to the Indicators tab. Args: @@ -261,7 +269,7 @@ def fetch_indicators_command(client: Client) -> List[Dict]: Returns: Indicators. """ - indicators = fetch_indicators(client, 'both') + indicators = fetch_indicators(client, 'both', enrichment_excluded=enrichment_excluded) return indicators @@ -279,13 +287,14 @@ def main(): use_ssl = not params.get('insecure', False) tags = argToList(params.get('feedTags')) tlp_color = params.get('tlp_color') + enrichment_excluded = demisto.params().get('enrichmentExcluded', False) command = demisto.command() demisto.info(f'Command being called is {command}') try: client = Client(urls_list, category_list, use_ssl, tags, tlp_color) - commands: Dict[str, Callable[[Client, Dict[str, str]], Tuple[str, Dict[Any, Any], Dict[Any, Any]]]] = { + commands: dict[str, Callable[[Client, dict[str, str]], tuple[str, dict[Any, Any], dict[Any, Any]]]] = { 'test-module': test_module, 'office365-get-indicators': get_indicators_command } @@ -293,7 +302,7 @@ def main(): return_outputs(*commands[command](client, demisto.args())) elif command == 'fetch-indicators': - indicators = fetch_indicators_command(client) + indicators = fetch_indicators_command(client, enrichment_excluded) for iter_ in batch(indicators, batch_size=2000): demisto.createIndicators(iter_) diff --git a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.yml b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.yml index c437cd2766c5..015c467b7a93 100644 --- a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.yml +++ b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365.yml @@ -8,6 +8,7 @@ configuration: defaultvalue: 'true' type: 8 required: false + section: Collect - defaultvalue: All display: Category name: category @@ -18,6 +19,7 @@ configuration: - Default type: 16 required: false + section: Connect - defaultvalue: All display: Regions name: regions @@ -30,23 +32,26 @@ configuration: - Worldwide required: true type: 16 -- display: Allow Germany - additionalinfo: In some cases, the Germany endpoints can be unavailable for some users. By default, we exclude Germany to prevent the fetch indicators from failing. - name: allow_germany - defaultvalue: 'false' - type: 8 - required: false -- defaultvalue: All - display: Services + section: Connect +- display: Services name: services + defaultvalue: 'All' + type: 16 + required: true options: - Common - Exchange - Sharepoint - Skype - All - required: true - type: 16 + section: Connect +- defaultvalue: 'false' + display: Allow Germany + name: allow_germany + required: false + type: 8 + additionalinfo: In some cases, the Germany endpoints can be unavailable for some users. By default, we exclude Germany to prevent the fetch indicators from failing. + section: Connect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -58,6 +63,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data display: Source Reliability name: feedReliability @@ -71,6 +77,7 @@ configuration: required: true type: 15 defaultvalue: A - Completely reliable + section: Collect - name: tlp_color display: Traffic Light Protocol Color options: @@ -81,6 +88,7 @@ configuration: type: 15 additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed required: false + section: Collect - defaultvalue: suddenDeath display: '' name: feedExpirationPolicy @@ -91,35 +99,58 @@ configuration: - indicatorType - suddenDeath required: false + section: Collect + advanced: true - defaultvalue: '20160' display: '' name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - display: Feed Fetch Interval name: feedFetchInterval type: 19 defaultvalue: '30' required: false + section: Collect + advanced: true - additionalinfo: Supports CSV values. display: Tags name: feedTags type: 0 required: false + section: Collect + advanced: true - display: Bypass exclusion list name: feedBypassExclusionList type: 8 additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. defaultvalue: 'true' required: false + section: Collect + advanced: true +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: The Office 365 IP Address and URL web service is a read-only API provided by Microsoft to expose the URLs and IPs used by Office 365. The Office 365 Feed integration fetches indicators from the service, with which you can create a list (allow list, block list, EDL, etc.) for your SIEM or firewall service to ingest and apply to its policy rules. display: Office 365 Feed name: Office 365 Feed @@ -139,7 +170,7 @@ script: - Both description: Gets indicators from the feed. name: office365-get-indicators - dockerimage: demisto/python3:3.10.12.68714 + dockerimage: demisto/python3:3.11.9.107902 feed: true runonce: false script: '-' @@ -148,3 +179,6 @@ script: tests: - Office365_Feed_Test fromversion: 5.5.0 +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365_test.py b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365_test.py index 443551770029..f294b1fd764e 100644 --- a/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365_test.py +++ b/Packs/FeedOffice365/Integrations/FeedOffice365/FeedOffice365_test.py @@ -43,6 +43,28 @@ def test_fetch_indicators_command(category_list, expected_indicators): assert len(indicators) == expected_indicators +def test_fetch_indicators_command__exclude_enrichment(): + """ + Given: + - Exclude enrichment parameter is used + When: + - Calling the fetch_indicators_command + Then: + - The indicators should include the enrichmentExcluded field if exclude is True. + """ + with requests_mock.Mocker() as mock: + url_dict = { + "FeedURL": 'https://endpoints.office.com/endpoints/worldwide', + "Region": 'Worldwide', + "Service": 'Any' + } + mock.get(url_dict.get('FeedURL'), json=RESPONSE_DATA) + client = Client([url_dict], ALL_CATEGORY_LIST) + indicators = fetch_indicators_command(client, enrichment_excluded=True) + for ind in indicators: + assert ind['enrichmentExcluded'] + + @pytest.mark.parametrize('command, args, response, length', [ (get_indicators_command, {'limit': 2, 'indicator_type': 'IPs'}, RESPONSE_DATA, 4), (get_indicators_command, {'limit': 2, 'indicator_type': 'URLs'}, RESPONSE_DATA, 6), @@ -131,9 +153,8 @@ def test_build_iterator_success(self): category_list = ['category1'] client = Client(urls_list, category_list) result = client.build_iterator() - self.assertEqual(result, [ - {'ips': ['1.1.1.1'], 'category': 'category1', 'Region': 'Region1', 'Service': 'Service1', - 'FeedURL': 'http://example.com'}]) + assert result == [{'ips': ['1.1.1.1'], 'category': 'category1', + 'Region': 'Region1', 'Service': 'Service1', 'FeedURL': 'http://example.com'}] def test_build_iterator_connection_error(self): # Mock the requests library to raise a ConnectionError diff --git a/Packs/FeedOffice365/Integrations/FeedOffice365/README.md b/Packs/FeedOffice365/Integrations/FeedOffice365/README.md index 3a9c9eb9a7c3..b7bbef1a363f 100644 --- a/Packs/FeedOffice365/Integrations/FeedOffice365/README.md +++ b/Packs/FeedOffice365/Integrations/FeedOffice365/README.md @@ -21,6 +21,7 @@ The Office 365 IP Address and URL web service is a read-only API provided by Mic | feedExpirationInterval | | | | Feed Fetch Interval | How often to fetch indicators from this integration instance. You can specify the interval in days, hours, or minutes. | 30 minutes | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | N/A | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | Trust any certificate (not secure) | When selected, certificates are not checked. | N/A | | Use system proxy settings | Runs the integration instance using the proxy server (HTTP or HTTPS) that you defined in the server configuration. | False | 4. Click __Test__ to validate the URLs and connection. diff --git a/Packs/FeedOffice365/ReleaseNotes/1_2_13.md b/Packs/FeedOffice365/ReleaseNotes/1_2_13.md new file mode 100644 index 000000000000..a7a27ad16ed0 --- /dev/null +++ b/Packs/FeedOffice365/ReleaseNotes/1_2_13.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Office 365 Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. diff --git a/Packs/FeedOffice365/pack_metadata.json b/Packs/FeedOffice365/pack_metadata.json index a7e88cf1b3db..6ba9e95ebb93 100644 --- a/Packs/FeedOffice365/pack_metadata.json +++ b/Packs/FeedOffice365/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Office 365 Feed", "description": "The Office 365 IP Address and URL web service is a read-only API provided by Microsoft to expose the URLs and IPs used by Office 365. The Office 365 Feed integration fetches indicators from the service, with which you can create a list (whitelist, blacklist, EDL, etc.) for your SIEM or firewall service to ingest and apply to its policy rules.", "support": "xsoar", - "currentVersion": "1.2.12", + "currentVersion": "1.2.13", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedPlainText/ReleaseNotes/1_1_27.md b/Packs/FeedPlainText/ReleaseNotes/1_1_27.md new file mode 100644 index 000000000000..2bac49e1a497 --- /dev/null +++ b/Packs/FeedPlainText/ReleaseNotes/1_1_27.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Plain Text Feed + +Enhanced **HTTPFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedPlainText/pack_metadata.json b/Packs/FeedPlainText/pack_metadata.json index d16f6b3314ee..55a60166dc2a 100644 --- a/Packs/FeedPlainText/pack_metadata.json +++ b/Packs/FeedPlainText/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Plain Text Feed", "description": "Fetches indicators from a plain text feed.", "support": "xsoar", - "currentVersion": "1.1.26", + "currentVersion": "1.1.27", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.py b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.py index 4bcf7e39330b..a95a2fc27574 100644 --- a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.py +++ b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.py @@ -3,7 +3,7 @@ from CommonServerUserPython import * ''' IMPORTS ''' -from typing import Dict, List, Tuple, Any +from typing import Any from netaddr import IPAddress import urllib3 @@ -24,7 +24,7 @@ def __init__(self, feed_url: str, tags: Optional[list] = None, self.Tags = [] if tags is None else tags self.Tlp_color = tlp_color - def build_iterator(self) -> List: + def build_iterator(self) -> list: """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. @@ -54,7 +54,7 @@ def build_iterator(self) -> List: return indicators -def test_module(client: Client) -> Tuple[str, Dict[Any, Any], Dict[Any, Any]]: +def test_module(client: Client) -> tuple[str, dict[Any, Any], dict[Any, Any]]: """Builds the iterator to check that the feed is accessible. Args: client: Client object. @@ -65,7 +65,7 @@ def test_module(client: Client) -> Tuple[str, Dict[Any, Any], Dict[Any, Any]]: return 'ok', {}, {} -def fetch_indicators(client: Client, limit: int = -1) -> List[Dict]: +def fetch_indicators(client: Client, limit: int = -1, enrichment_excluded: bool = False) -> list[dict]: """Retrieves indicators from the feed Args: client: Client object with request @@ -90,9 +90,12 @@ def fetch_indicators(client: Client, limit: int = -1) -> List[Dict]: 'value': item, 'type': type_, 'rawJSON': {'value': item, 'type': type_}, - 'fields': {'tags': client.Tags} + 'fields': {'tags': client.Tags}, } + if enrichment_excluded: + indicator_obj['enrichmentExcluded'] = enrichment_excluded + if client.Tlp_color: indicator_obj['fields']['trafficlightprotocol'] = client.Tlp_color @@ -101,7 +104,7 @@ def fetch_indicators(client: Client, limit: int = -1) -> List[Dict]: return indicators -def get_indicators_command(client: Client) -> Tuple[str, Dict[Any, Any], Dict[Any, Any]]: +def get_indicators_command(client: Client) -> tuple[str, dict[Any, Any], dict[Any, Any]]: """Wrapper for retrieving indicators from the feed to the war-room. Args: client: Client object with request @@ -110,36 +113,38 @@ def get_indicators_command(client: Client) -> Tuple[str, Dict[Any, Any], Dict[An """ limit = int(demisto.args().get('limit')) if 'limit' in demisto.args() else 10 - indicators = fetch_indicators(client, limit) + enrichment_excluded = demisto.params().get('enrichmentExcluded', False) + indicators = fetch_indicators(client, limit, enrichment_excluded=enrichment_excluded) human_readable = tableToMarkdown(f'{INTEGRATION_NAME}:', indicators, headers=['value', 'type'], removeNull=True) return human_readable, {'Indicator': indicators}, {'raw_response': indicators} -def fetch_indicators_command(client: Client) -> List[Dict]: +def fetch_indicators_command(client: Client, enrichment_excluded: bool = False) -> list[dict]: """Wrapper for fetching indicators from the feed to the Indicators tab. Args: client: Client object with request Returns: Indicators. """ - indicators = fetch_indicators(client) + indicators = fetch_indicators(client, enrichment_excluded=enrichment_excluded) return indicators -def main(): +def main(): # pragma: no cover params = demisto.params() url = params.get('url', 'https://public-dns.info/nameservers-all.txt') tags = argToList(params.get('feedTags')) tlp_color = params.get('tlp_color') use_ssl = not params.get('insecure', False) + enrichment_excluded = params.get('enrichmentExcluded', False) command = demisto.command() demisto.info(f'Command being called is {command}') try: client = Client(url, tags, tlp_color, use_ssl) - commands: Dict = { + commands: dict = { 'test-module': test_module, 'public-dns-get-indicators': get_indicators_command } @@ -147,7 +152,7 @@ def main(): return_outputs(*commands[command](client)) elif command == 'fetch-indicators': - indicators = fetch_indicators_command(client) + indicators = fetch_indicators_command(client, enrichment_excluded=enrichment_excluded) for iter_ in batch(indicators, batch_size=2000): demisto.createIndicators(iter_) diff --git a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.yml b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.yml index 5bc4ef84706f..010f9be55e70 100644 --- a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.yml +++ b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS.yml @@ -8,11 +8,13 @@ configuration: name: url required: true type: 1 + section: Connect - defaultvalue: 'true' display: Fetch indicators name: feed type: 8 required: false + section: Collect - additionalinfo: Indicators from this integration instance will be marked with this reputation defaultvalue: Good display: Indicator Reputation @@ -24,6 +26,7 @@ configuration: - Bad type: 18 required: false + section: Collect - additionalinfo: Reliability of the source providing the intelligence data defaultvalue: A - Completely reliable display: Source Reliability @@ -37,6 +40,7 @@ configuration: - F - Reliability cannot be judged required: true type: 15 + section: Collect - defaultvalue: indicatorType name: feedExpirationPolicy display: "" @@ -47,27 +51,37 @@ configuration: - suddenDeath type: 17 required: false + section: Collect + advanced: true - defaultvalue: '240' display: Feed Fetch Interval name: feedFetchInterval type: 19 required: false + section: Collect + advanced: true - defaultvalue: '20160' display: "" name: feedExpirationInterval type: 1 required: false + section: Collect + advanced: true - additionalinfo: Supports CSV values. display: Tags name: feedTags type: 0 required: false + section: Collect + advanced: true - additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. display: Bypass exclusion list name: feedBypassExclusionList type: 8 required: false -- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed + section: Collect + advanced: true +- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed. display: Traffic Light Protocol Color name: tlp_color options: @@ -77,14 +91,28 @@ configuration: - WHITE type: 15 required: false + section: Collect +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true description: A feed of known benign IPs of public DNS servers. display: Public DNS Feed name: Public DNS Feed @@ -96,7 +124,7 @@ script: name: limit description: Gets indicators from the feed. name: public-dns-get-indicators - dockerimage: demisto/netutils:1.0.0.86390 + dockerimage: demisto/netutils:1.0.0.108034 feed: true runonce: false script: '-' @@ -105,3 +133,6 @@ script: tests: - Public_DNS_Feed_Test fromversion: 5.5.0 +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS_test.py b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS_test.py new file mode 100644 index 000000000000..e0a37420dc89 --- /dev/null +++ b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/FeedPublicDNS_test.py @@ -0,0 +1,51 @@ +import pytest + +from FeedPublicDNS import Client, fetch_indicators_command + + +@pytest.mark.parametrize('enrichment_excluded', [True, False]) +def test_fetch_indicators_command(requests_mock, enrichment_excluded): + """ + Given: + When: + Then: + """ + expected = [ + { + 'value': '192.168.0.1', + 'type': 'IP', + 'rawJSON': { + 'value': '192.168.0.1', + 'type': 'IP' + }, + 'fields': { + 'tags': ['test'], + 'trafficlightprotocol': 'test color' + } + }, + { + 'value': '10.0.0.1', + 'type': 'IP', + 'rawJSON': { + 'value': '10.0.0.1', + 'type': 'IP' + }, + 'fields': { + 'tags': ['test'], + 'trafficlightprotocol': 'test color' + } + } + ] + if enrichment_excluded: + for ind in expected: + ind['enrichmentExcluded'] = True + + url = 'https://public-dns.info/nameservers-all.txt' + mock_response = '192.168.0.1\n10.0.0.1' + requests_mock.get(url, text=mock_response) + + client = Client(url, tags=['test'], tlp_color='test color') + + indicators = fetch_indicators_command(client, enrichment_excluded=enrichment_excluded) + + assert indicators == expected diff --git a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/README.md b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/README.md index 461d57ffd435..96f069e12920 100644 --- a/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/README.md +++ b/Packs/FeedPublicDNS/Integrations/FeedPublicDNS/README.md @@ -17,6 +17,7 @@ A feed of known benign IPs of public DNS servers. | feedTags | Tags | False | | feedBypassExclusionList | Bypass exclusion list | False | | tlp_color | Traffic Light Protocol Color | False | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | insecure | Trust any certificate \(not secure\) | False | | proxy | Use system proxy settings | False | diff --git a/Packs/FeedPublicDNS/ReleaseNotes/1_0_16.md b/Packs/FeedPublicDNS/ReleaseNotes/1_0_16.md new file mode 100644 index 000000000000..d23487e276dc --- /dev/null +++ b/Packs/FeedPublicDNS/ReleaseNotes/1_0_16.md @@ -0,0 +1,9 @@ + +#### Integrations + +##### Public DNS Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + +- Updated the Docker image to: *demisto/netutils:1.0.0.108034*. diff --git a/Packs/FeedPublicDNS/pack_metadata.json b/Packs/FeedPublicDNS/pack_metadata.json index aedb00b8f166..2bb82de466a7 100644 --- a/Packs/FeedPublicDNS/pack_metadata.json +++ b/Packs/FeedPublicDNS/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Public DNS Feed", "description": "The Public DNS Feed fetches known IPs associated with public DNS servers from https://public-dns.info/", "support": "xsoar", - "currentVersion": "1.0.15", + "currentVersion": "1.0.16", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedSpamhaus/ReleaseNotes/1_1_24.md b/Packs/FeedSpamhaus/ReleaseNotes/1_1_24.md new file mode 100644 index 000000000000..0afa273dd518 --- /dev/null +++ b/Packs/FeedSpamhaus/ReleaseNotes/1_1_24.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Spamhaus Feed + +Enhanced **HTTPFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration. diff --git a/Packs/FeedSpamhaus/pack_metadata.json b/Packs/FeedSpamhaus/pack_metadata.json index 9492e2ab92f1..b67aad5733a3 100644 --- a/Packs/FeedSpamhaus/pack_metadata.json +++ b/Packs/FeedSpamhaus/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Spamhaus Feed", "description": "The Spamhaus DROP (Don't Route Or Peer) lists are advisory \"drop all traffic\" lists, consisting of netblocks that are \"hijacked\" or leased by professional spam or cyber-crime operations (used for dissemination of malware, trojan downloaders, botnet controllers). The DROP lists are a tiny subset of the SBL, designed for use by firewalls and routing equipment to filter out the malicious traffic from these netblocks.", "support": "xsoar", - "currentVersion": "1.1.23", + "currentVersion": "1.1.24", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.py b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.py index 552dd9a3d7f4..9afdf6b7bf56 100644 --- a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.py +++ b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.py @@ -1,6 +1,6 @@ import demistomock as demisto from CommonServerPython import * -from typing import Dict, List, Callable, Optional, Union +from collections.abc import Callable import urllib3 @@ -49,7 +49,7 @@ def get_indicators(self) -> Set: indicators.add(ip) return indicators - def build_iterator(self) -> List: + def build_iterator(self) -> list: """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. @@ -98,8 +98,8 @@ def test_module(client: Client, *_) -> str: return "ok" -def fetch_indicators(client: Client, feed_tags: List = [], tlp_color: Optional[str] = None, - limit: int = -1) -> List[Dict]: +def fetch_indicators(client: Client, feed_tags: list = [], tlp_color: str | None = None, + limit: int = -1, enrichment_excluded: bool = False) -> list[dict]: """Retrieves indicators from the feed Args: client (Client): Client object with request @@ -122,24 +122,30 @@ def fetch_indicators(client: Client, feed_tags: List = [], tlp_color: Optional[s } for key, val in item.items(): raw_data.update({key: val}) + indicator_obj = { "value": value, "type": type_, "service": "Zoom Feed", "rawJSON": raw_data, - 'fields': {} + 'fields': {}, } + if feed_tags: indicator_obj["fields"]['tags'] = feed_tags + if tlp_color: indicator_obj["fields"]['trafficlightprotocol'] = tlp_color + if enrichment_excluded: + indicator_obj['enrichmentExcluded'] = enrichment_excluded + indicators.append(indicator_obj) return indicators def get_indicators_command( - client: Client, params: Dict[str, str], args: Dict[str, str] + client: Client, params: dict, args: dict[str, str] ) -> CommandResults: """Wrapper for retrieving indicators from the feed to the war-room. Args: @@ -152,7 +158,8 @@ def get_indicators_command( feed_tags = argToList(params.get("feedTags", "")) tlp_color = params.get('tlp_color') limit = arg_to_number(args.get('limit')) or 10 - indicators = fetch_indicators(client, feed_tags, tlp_color, limit) + enrichment_excluded = params.get('enrichmentExcluded', False) + indicators = fetch_indicators(client, feed_tags, tlp_color, limit, enrichment_excluded) if indicators: human_readable = tableToMarkdown( @@ -165,7 +172,7 @@ def get_indicators_command( raw_response=indicators) -def fetch_indicators_command(client: Client, params: Dict[str, str]) -> List[Dict]: +def fetch_indicators_command(client: Client, params: dict) -> list[dict]: """Wrapper for fetching indicators from the feed to the Indicators tab. Args: client: Client object with request @@ -175,7 +182,8 @@ def fetch_indicators_command(client: Client, params: Dict[str, str]) -> List[Dic """ feed_tags = argToList(params.get("feedTags", "")) tlp_color = params.get('tlp_color') - indicators = fetch_indicators(client, feed_tags, tlp_color) + enrichment_excluded = params.get('enrichmentExcluded', False) + indicators = fetch_indicators(client, feed_tags, tlp_color, enrichment_excluded=enrichment_excluded) return indicators @@ -193,8 +201,8 @@ def main(): try: client = Client(base_url=ZOOM_DOCS_IP_RANGES_URL, verify=insecure, proxy=proxy) - commands: Dict[ - str, Callable[[Client, Dict[str, str], Dict[str, str]], Union[str, CommandResults]] + commands: dict[ + str, Callable[[Client, dict[str, str], dict[str, str]], str | CommandResults] ] = {"test-module": test_module, "zoom-get-indicators": get_indicators_command} if command in commands: return_results(commands[command](client, demisto.params(), demisto.args())) diff --git a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.yml b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.yml index 83d84847b6bf..a593ba386aa0 100644 --- a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.yml +++ b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom.yml @@ -12,6 +12,7 @@ configuration: defaultvalue: "true" type: 8 required: false + section: Collect - additionalinfo: Zoom clients for certificate validation defaultvalue: 'crl3.digicert.com,crl4.digicert.com,ocsp.digicert.com,certificates.godaddy.com,crl.godaddy.com,ocsp.godaddy.com,certificates.starfieldtech.com,crl.starfieldtech.com,ocsp.starfieldtech.com' display: Firewall rules for certificate validation @@ -28,6 +29,8 @@ configuration: - ocsp.starfieldtech.com type: 16 required: false + section: Connect + advanced: true - additionalinfo: All Zoom Clients. User's web browser defaultvalue: '*.zoom.us,*.cloudfront.net' display: Firewall rules for Zoom website @@ -37,6 +40,8 @@ configuration: - '*.cloudfront.net' type: 16 required: false + section: Connect + advanced: true - display: Indicator Reputation name: feedReputation defaultvalue: Good @@ -48,6 +53,7 @@ configuration: - Bad additionalinfo: Indicators from this integration instance will be marked with this reputation. required: false + section: Collect - display: Source Reliability name: feedReliability defaultvalue: A - Completely reliable @@ -61,6 +67,7 @@ configuration: - E - Unreliable - F - Reliability cannot be judged additionalinfo: Reliability of the source providing the intelligence data + section: Collect - additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed display: Traffic Light Protocol Color name: tlp_color @@ -71,6 +78,7 @@ configuration: - WHITE type: 15 required: false + section: Collect - display: "" name: feedExpirationPolicy defaultvalue: suddenDeath @@ -81,37 +89,60 @@ configuration: - indicatorType - suddenDeath required: false + section: Collect + advanced: true - display: "" name: feedExpirationInterval defaultvalue: "20160" type: 1 required: false + section: Collect + advanced: true - display: Feed Fetch Interval name: feedFetchInterval defaultvalue: '240' type: 19 additionalinfo: Setting a more frequent fetch interval may cause errors from the vendor. required: false + section: Collect + advanced: true - display: Tags name: feedTags defaultvalue: "" type: 0 additionalinfo: Supports CSV values. required: false + section: Collect + advanced: true - display: Bypass exclusion list name: feedBypassExclusionList defaultvalue: "true" type: 8 additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. required: false + section: Collect + advanced: true +- display: Enrichment Excluded + name: enrichmentExcluded + type: 8 + required: false + additionalinfo: Select this option to exclude the fetched indicators from the enrichment process. + defaultvalue: 'false' + hidden: + - xsoar_on_prem + section: Collect - display: Trust any certificate (not secure) name: insecure type: 8 required: false + section: Connect + advanced: true - display: Use system proxy settings name: proxy type: 8 required: false + section: Connect + advanced: true script: script: '' type: python @@ -122,8 +153,11 @@ script: description: The maximum number of results to return. The default value is 10. defaultValue: "10" description: Gets indicators from the feed. - dockerimage: demisto/btfl-soup:1.0.1.86352 + dockerimage: demisto/btfl-soup:1.0.1.107991 feed: true subtype: python3 tests: - FeedZoom_Test +sectionOrder: +- Connect +- Collect diff --git a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom_test.py b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom_test.py index d09c69fc484b..77f14c2d5233 100644 --- a/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom_test.py +++ b/Packs/FeedZoom/Integrations/FeedZoom/FeedZoom_test.py @@ -1,4 +1,7 @@ -from FeedZoom import Client +import re +import pytest + +from FeedZoom import Client, fetch_indicators_command import demistomock as demisto @@ -15,7 +18,7 @@ def test_build_iterator(mocker): Then: Build iterator of indicators from the API. """ - with open('test_data/zoom_endpoint.txt', 'r') as file: + with open('test_data/zoom_endpoint.txt') as file: response = file.read() mocker.patch.object(Client, '_http_request', return_value=response) mocker.patch.object(demisto, 'params', @@ -38,3 +41,76 @@ def test_build_iterator(mocker): assert expected_cidr in cidr_indicators assert expected_ipv4 in ip_indicators assert expected_glob in domain_glob_indicators + + +@pytest.mark.parametrize('enrichment_excluded', [True, False]) +def test_fetch_indicators_command(mocker, requests_mock, enrichment_excluded): + """ + Given: + Parameters (zoom_clients_certificate_validation, zoom_clients_user_browser, enrichment_excluded) for fetching indicators + When: + Calling fetch_indicators_command + Then: + The indicators will be returned as expected, with enrichmentExcluded if requested + """ + expected = [ + { + 'value': '3.7.35.0/25', + 'type': 'CIDR', + 'service': 'Zoom Feed', + 'rawJSON': {'value': '3.7.35.0/25', 'type': 'CIDR', 'FeedURL': 'https://assets.zoom.us/docs/ipranges'}, + 'fields': {}, + }, + { + 'value': '1.2.3.4', + 'type': 'IP', + 'service': 'Zoom Feed', + 'rawJSON': {'value': '1.2.3.4', 'type': 'IP', 'FeedURL': 'https://assets.zoom.us/docs/ipranges'}, + 'fields': {}, + }, + { + 'value': '*.zoom.us', + 'type': 'DomainGlob', + 'service': 'Zoom Feed', + 'rawJSON': {'value': '*.zoom.us', 'type': 'DomainGlob', 'FeedURL': 'https://assets.zoom.us/docs/ipranges'}, + 'fields': {}, + }, + { + 'value': 'crl4.digicert.com', + 'type': 'Domain', + 'service': 'Zoom Feed', + 'rawJSON': {'value': 'crl4.digicert.com', 'type': 'Domain', 'FeedURL': 'https://assets.zoom.us/docs/ipranges'}, + 'fields': {}, + }, + { + 'value': 'crl3.digicert.com', + 'type': 'Domain', + 'service': 'Zoom Feed', + 'rawJSON': {'value': 'crl3.digicert.com', 'type': 'Domain', 'FeedURL': 'https://assets.zoom.us/docs/ipranges'}, + 'fields': {}, + } + ] + + if enrichment_excluded: + for ind in expected: + ind['enrichmentExcluded'] = True + + with open('test_data/zoom_endpoint.txt') as file: + response = file.read() + requests_mock.register_uri('GET', re.compile(rf'{URL}.*'), text=response) + + mocker.patch.object(demisto, 'params', + return_value={'zoom_clients_certificate_validation': 'crl3.digicert.com,crl4.digicert.com', + 'zoom_clients_user_browser': '*.zoom.us', + 'enrichmentExcluded': enrichment_excluded}) + + client = Client( + base_url=URL, + verify=False, + proxy=False, + ) + + indicators = fetch_indicators_command(client, demisto.params()) + + for ind in expected: + assert ind in indicators diff --git a/Packs/FeedZoom/Integrations/FeedZoom/README.md b/Packs/FeedZoom/Integrations/FeedZoom/README.md index 5e9751a76baf..90796f75d156 100644 --- a/Packs/FeedZoom/Integrations/FeedZoom/README.md +++ b/Packs/FeedZoom/Integrations/FeedZoom/README.md @@ -27,6 +27,7 @@ For information about Zoom network settings, see the [Zoom documentation](https: | Feed Fetch Interval | Setting a more frequent fetch interval may cause errors from the vendor. | False | | Tags | Supports CSV values. | False | | Bypass exclusion list | When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system. | False | + | Enrichment Excluded | Select this option to exclude the fetched indicators from the enrichment process. | False | | Trust any certificate (not secure) | | False | | Use system proxy settings | | False | diff --git a/Packs/FeedZoom/ReleaseNotes/1_1_15.md b/Packs/FeedZoom/ReleaseNotes/1_1_15.md new file mode 100644 index 000000000000..67dcab6789b9 --- /dev/null +++ b/Packs/FeedZoom/ReleaseNotes/1_1_15.md @@ -0,0 +1,10 @@ + +#### Integrations + +##### Zoom Feed + +<~XSOAR_SAAS> +- Support for instance-wide indicator enrichment exclusion. + + +- Updated the Docker image to: *demisto/btfl-soup:1.0.1.107991*. diff --git a/Packs/FeedZoom/pack_metadata.json b/Packs/FeedZoom/pack_metadata.json index ff6abcc95394..2f7395960863 100644 --- a/Packs/FeedZoom/pack_metadata.json +++ b/Packs/FeedZoom/pack_metadata.json @@ -5,7 +5,7 @@ "videos": [ "https://www.youtube.com/embed/s9lRtJltTGI" ], - "currentVersion": "1.1.14", + "currentVersion": "1.1.15", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 7f1624bb7f736318edae938f08ba4f0d12616dcb Mon Sep 17 00:00:00 2001 From: Yehuda Rosenberg <90599084+RosenbergYehuda@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:10:31 +0300 Subject: [PATCH 4/5] init (#36096) --- Packs/AccentureCTI/.pack-ignore | 8 ++++- Packs/Armis/.pack-ignore | 12 ++++--- Packs/Claroty/.pack-ignore | 6 +++- Packs/Confluera/.pack-ignore | 6 ++++ Packs/Cryptosim/.pack-ignore | 6 ++++ Packs/Cyberpion/.pack-ignore | 6 ++++ Packs/CybleEvents/.pack-ignore | 9 ++++- Packs/Cymulate/.pack-ignore | 3 ++ Packs/CyrenInboxSecurity/.pack-ignore | 3 ++ Packs/Darktrace/.pack-ignore | 6 ++++ Packs/DevSecOps/.pack-ignore | 6 ++++ Packs/ExodusIntelligence/.pack-ignore | 3 ++ Packs/GigamonThreatINSIGHT/.pack-ignore | 3 ++ Packs/GoogleCloudSCC/.pack-ignore | 3 ++ .../.pack-ignore | 36 ++++++++++++++++++- Packs/Gurucul/.pack-ignore | 3 ++ Packs/HealthCheck/.pack-ignore | 13 +++++++ Packs/Inventa/.pack-ignore | 5 +++ Packs/IronDefense/.pack-ignore | 5 ++- Packs/Lumu/.pack-ignore | 6 +++- Packs/OTSecurity/.pack-ignore | 8 ++++- Packs/PingCastle/.pack-ignore | 3 ++ Packs/Qintel/.pack-ignore | 6 +++- Packs/RSANetWitness_v11_1/.pack-ignore | 10 +++++- Packs/RecordedFuture/.pack-ignore | 6 +++- Packs/Respond/.pack-ignore | 3 ++ Packs/SOCRadar/.pack-ignore | 3 ++ Packs/SSLCertificates/.pack-ignore | 6 +++- Packs/SafeBreach/.pack-ignore | 19 ++++++++++ Packs/SafeNet_Trusted_Access/.pack-ignore | 5 +++ Packs/SalesforceV2/.pack-ignore | 21 +++++++++++ Packs/SecurityScorecard/.pack-ignore | 4 +++ .../.pack-ignore | 3 ++ Packs/SumoLogic_Cloud_SIEM/.pack-ignore | 3 ++ Packs/Trello/.pack-ignore | 6 +++- .../.pack-ignore | 6 +++- Packs/XDRBestPracticeAssessment/.pack-ignore | 3 ++ Packs/XsoarWebserver/.pack-ignore | 7 ++++ Packs/knowbe4Phisher/.pack-ignore | 6 +++- 39 files changed, 258 insertions(+), 18 deletions(-) diff --git a/Packs/AccentureCTI/.pack-ignore b/Packs/AccentureCTI/.pack-ignore index 0fde09867904..50d3f1c172d9 100644 --- a/Packs/AccentureCTI/.pack-ignore +++ b/Packs/AccentureCTI/.pack-ignore @@ -7,4 +7,10 @@ uuid CustomCVE acti getThreatIntelReport -get-fundamentals-by-uuid \ No newline at end of file +get-fundamentals-by-uuid + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-ACTI_Intelligence_Report.json] +ignore=GR103 +[file:layoutscontainer-ACTI_Intelligence_Alert.json] +ignore=GR103 diff --git a/Packs/Armis/.pack-ignore b/Packs/Armis/.pack-ignore index 7144a232caaa..6be32c2ecf9d 100644 --- a/Packs/Armis/.pack-ignore +++ b/Packs/Armis/.pack-ignore @@ -1,17 +1,21 @@ +# GR103 is temporary, see CIAC-11656 [file:incidentfields_Armis_Alert_Desctiption.json] -ignore=IF100 +ignore=IF100,GR103 [file:incidentfields_Armis_Alert_ID.json] -ignore=IF100 +ignore=IF100,GR103 [file:incidentfields_Armis_Alert_Severity.json] -ignore=IF100 +ignore=IF100,GR103 [file:incidentfields_Armis_Alert_Status.json] ignore=IF100,BA113 [file:incidentfields_Armis_Alert_Type.json] -ignore=IF100 +ignore=IF100,GR103 + +[file:classifier-Armis_mapper-incoming.json] +ignore=GR103 [file:Armis_image.png] ignore=IM111 diff --git a/Packs/Claroty/.pack-ignore b/Packs/Claroty/.pack-ignore index 98fab619d653..6521aafa339a 100644 --- a/Packs/Claroty/.pack-ignore +++ b/Packs/Claroty/.pack-ignore @@ -4,8 +4,12 @@ ignore=IN126 [file:README.md] ignore=RM104 +# GR103 is temporary, see CIAC-11656 [file:classifier-mapper-incoming-Claroty.json] -ignore=BA101 +ignore=BA101,GR103 + +[file:incidentfield-Claroty_Site_ID.json] +ignore=GR103 [file:Claroty_image.png] ignore=IM111 diff --git a/Packs/Confluera/.pack-ignore b/Packs/Confluera/.pack-ignore index ba768d4ad053..8ed689e42275 100644 --- a/Packs/Confluera/.pack-ignore +++ b/Packs/Confluera/.pack-ignore @@ -25,3 +25,9 @@ ignore=BA124 [file:ConflueraDetectionsData.yml] ignore=BA124 +# GR103 is temporary, see CIAC-11656 +[file:report-Iqhub_Report.json] +ignore=GR103 + +[file:dashboard-Confluera.json] +ignore=GR103 diff --git a/Packs/Cryptosim/.pack-ignore b/Packs/Cryptosim/.pack-ignore index e69de29bb2d1..201b41ecd6e9 100644 --- a/Packs/Cryptosim/.pack-ignore +++ b/Packs/Cryptosim/.pack-ignore @@ -0,0 +1,6 @@ +# GR103 is temporary, see CIAC-11656 +[file:classifier-CRYPTTECH_Generic.json] +ignore=GR103 + +[file:layoutscontainer-Correlation_Alerts.json] +ignore=GR103 diff --git a/Packs/Cyberpion/.pack-ignore b/Packs/Cyberpion/.pack-ignore index e69de29bb2d1..074c2113dc79 100644 --- a/Packs/Cyberpion/.pack-ignore +++ b/Packs/Cyberpion/.pack-ignore @@ -0,0 +1,6 @@ +# GR103 is temporary, see CIAC-11656 +[file:classifier-Cyberpion_-_Mapper.json] +ignore=GR103 + +[file:layoutscontainer-Cyberpion_-_Action_Item.json] +ignore=GR103 diff --git a/Packs/CybleEvents/.pack-ignore b/Packs/CybleEvents/.pack-ignore index 636abb483934..4272c6f0977a 100644 --- a/Packs/CybleEvents/.pack-ignore +++ b/Packs/CybleEvents/.pack-ignore @@ -5,4 +5,11 @@ ignore=IN126 ignore=BA101 [known_words] -cyble \ No newline at end of file +cyble + +# GR103 is temporary, see CIAC-11656 +[file:incidenttype-Cyble_Intel_Alert.json] +ignore=GR103 + +[file:layoutscontainer-Cyble_Intel_Alert.json] +ignore=GR103 diff --git a/Packs/Cymulate/.pack-ignore b/Packs/Cymulate/.pack-ignore index 38ffece6b8e4..d5379b259cf3 100644 --- a/Packs/Cymulate/.pack-ignore +++ b/Packs/Cymulate/.pack-ignore @@ -34,3 +34,6 @@ ignore=BA101 [file:Cymulate_Immediate_Threats_Playbook.yml] ignore=PB121 +# GR103 is temporary, see CIAC-11656 +[file:incidenttype-cymulate.json] +ignore=GR103 diff --git a/Packs/CyrenInboxSecurity/.pack-ignore b/Packs/CyrenInboxSecurity/.pack-ignore index 649dfed9dceb..903430eff7fb 100644 --- a/Packs/CyrenInboxSecurity/.pack-ignore +++ b/Packs/CyrenInboxSecurity/.pack-ignore @@ -10,3 +10,6 @@ ignore=BA124 [file:CyrenShowThreatIndicators.yml] ignore=BA124 +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Cyren_Inbox_Security_layout.json] +ignore=GR103 diff --git a/Packs/Darktrace/.pack-ignore b/Packs/Darktrace/.pack-ignore index a3547fd3db6b..2e876768cfe0 100644 --- a/Packs/Darktrace/.pack-ignore +++ b/Packs/Darktrace/.pack-ignore @@ -13,3 +13,9 @@ ignore=BA101 [file:playbook-HandleDarktraceModelBreach.yml] ignore=PB121 +# GR103 is temporary, see CIAC-11656 +[file:classifier-Darktrace_Model_Breach.json] +ignore=GR103 + +[file:layoutscontainer-Darktrace_MBs_Layout.json] +ignore=GR103 diff --git a/Packs/DevSecOps/.pack-ignore b/Packs/DevSecOps/.pack-ignore index 56767510dca8..5b49021fe0cd 100644 --- a/Packs/DevSecOps/.pack-ignore +++ b/Packs/DevSecOps/.pack-ignore @@ -30,3 +30,9 @@ ignore=RM112 [file:1_1_8.md] ignore=RN116 + +# GR103 is temporary, see CIAC-11656 +[file:DevSecOps_-_Fetch_PR_-_Triage.yml] +ignore=GR103 +[file:DevSecOps_-_LGTM_-_Analysis_-_SubPB.yml] +ignore=GR103 diff --git a/Packs/ExodusIntelligence/.pack-ignore b/Packs/ExodusIntelligence/.pack-ignore index abac82a32240..44eabe3f59ce 100644 --- a/Packs/ExodusIntelligence/.pack-ignore +++ b/Packs/ExodusIntelligence/.pack-ignore @@ -1,3 +1,6 @@ [file:ExodusVulnerabilityEnrichment.yml] ignore=RM110 +# GR103 is temporary, see CIAC-11656 +[file:ExodusIntelligence.json] +ignore=GR103 diff --git a/Packs/GigamonThreatINSIGHT/.pack-ignore b/Packs/GigamonThreatINSIGHT/.pack-ignore index e69de29bb2d1..0bf42c2b4c05 100644 --- a/Packs/GigamonThreatINSIGHT/.pack-ignore +++ b/Packs/GigamonThreatINSIGHT/.pack-ignore @@ -0,0 +1,3 @@ +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Gigamon_ThreatINSIGHT_Detection.json] +ignore=GR103 diff --git a/Packs/GoogleCloudSCC/.pack-ignore b/Packs/GoogleCloudSCC/.pack-ignore index 80d2455f7c10..a932bdc4eb00 100644 --- a/Packs/GoogleCloudSCC/.pack-ignore +++ b/Packs/GoogleCloudSCC/.pack-ignore @@ -13,3 +13,6 @@ ignore=IF100 [file:incidentfield-GoogleCloudSCC_Finding_OrganizationID.json] ignore=IF115 +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Google_Cloud_SCC_Finding.json] +ignore=GR103 diff --git a/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore b/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore index 22616273f986..074d7564c461 100644 --- a/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore +++ b/Packs/GroupIB_ThreatIntelligenceAttribution/.pack-ignore @@ -17,4 +17,38 @@ ignore=IM111 ignore=BA101 [file:1_4_1.md] -ignore=RN116 \ No newline at end of file +ignore=RN116 + +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-GIB_Screenshot.json] +ignore=GR103 +[file:incidentfield-GIB_Related_Indicators_Data.json] +ignore=GR103 +[file:incidentfield-GIB_Date_Expired.json] +ignore=GR103 +[file:incidentfield-GIB_Phishing_Status.json] +ignore=GR103 +[file:incidentfield-GIB_HTML.json] +ignore=GR103 +[file:incidentfield-GIB_Date_Created.json] +ignore=GR103 +[file:incidentfield-GIB_Address.json] +ignore=GR103 +[file:incidentfield-GIB_Phishing_Domain.json] +ignore=GR103 +[file:incidentfield-GIB_Phishing_Type.json] +ignore=GR103 +[file:incidentfield-GIB_Title.json] +ignore=GR103 +[file:classifier-Group-IB_Threat_Intelligence_mapper.json] +ignore=GR103 +[file:incidentfield-GIB_Name_Servers.json] +ignore=GR103 +[file:incidentfield-GIB_Email.json] +ignore=GR103 +[file:incidentfield-GIB_Person.json] +ignore=GR103 +[file:incidentfield-GIB_ID.json] +ignore=GR103 +[file:incidentfield-GIB_Favicon.json] +ignore=GR103 diff --git a/Packs/Gurucul/.pack-ignore b/Packs/Gurucul/.pack-ignore index 5e707e044f07..9e40380241f1 100644 --- a/Packs/Gurucul/.pack-ignore +++ b/Packs/Gurucul/.pack-ignore @@ -7,3 +7,6 @@ ignore=RM102 [file:GRAAnomaliesDisplay.yml] ignore=BA124 +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-GRACaseLayout.json] +ignore=GR103 diff --git a/Packs/HealthCheck/.pack-ignore b/Packs/HealthCheck/.pack-ignore index 4aefa6cef02c..7206e387a952 100644 --- a/Packs/HealthCheck/.pack-ignore +++ b/Packs/HealthCheck/.pack-ignore @@ -13,3 +13,16 @@ ignore=BA124 [file:HealthCheckOutdatedPacks.yml] ignore=BA124 +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-XSOAR_Dev-Prod.json] +ignore=GR103 +[file:incidentfield-Health_Check_Installed_Packs.json] +ignore=GR103 +[file:incidentfield-XSOAR_Architecture.json] +ignore=GR103 +[file:incidentfield-XSOAR_DR.json] +ignore=GR103 +[file:incidentfield-Health_Check_Total_Packs_Installed.json] +ignore=GR103 +[file:HealthCheckGetLargestInputsAndOutputsInIncidents.yml] +ignore=GR103 diff --git a/Packs/Inventa/.pack-ignore b/Packs/Inventa/.pack-ignore index e69de29bb2d1..bd8ed08b5c9c 100644 --- a/Packs/Inventa/.pack-ignore +++ b/Packs/Inventa/.pack-ignore @@ -0,0 +1,5 @@ +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-Inventa_Vehicle_Number.json] +ignore=GR103 +[file:layoutscontainer-Inventa_DSAR_Layout.json] +ignore=GR103 diff --git a/Packs/IronDefense/.pack-ignore b/Packs/IronDefense/.pack-ignore index 9ec1f56fc8e1..b8f85e464142 100644 --- a/Packs/IronDefense/.pack-ignore +++ b/Packs/IronDefense/.pack-ignore @@ -29,4 +29,7 @@ ignore=IF100 ignore=IM111 [file:classifier-IronDefense.json] -ignore=BA101 \ No newline at end of file +ignore=BA101 +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-IronDefense_Event_Notification.json] +ignore=GR103 diff --git a/Packs/Lumu/.pack-ignore b/Packs/Lumu/.pack-ignore index 6c6041d42b19..6ac823440928 100644 --- a/Packs/Lumu/.pack-ignore +++ b/Packs/Lumu/.pack-ignore @@ -52,5 +52,9 @@ ignore=IF113 [file:incidentfield-Lumu-lumu_event_type.json] ignore=IF113 +# GR103 is temporary, see CIAC-11656 [file:classifier-mapper-incoming-Lumu.json] -ignore=MP106 +ignore=MP106,GR103 + +[file:layoutscontainer-layouts-Lumu.json] +ignore=GR103 diff --git a/Packs/OTSecurity/.pack-ignore b/Packs/OTSecurity/.pack-ignore index 4ba5c5006cda..e4cf4081ddca 100644 --- a/Packs/OTSecurity/.pack-ignore +++ b/Packs/OTSecurity/.pack-ignore @@ -2,4 +2,10 @@ ignore=BA101 [file:1_0_3.md] -ignore=RN116 \ No newline at end of file +ignore=RN116 + +# GR103 is temporary, see CIAC-11656 +[file:classifier-mapper-incoming-OTSecurity_API_Incoming_Mapper_v1.json] +ignore=GR103 +[file:OTSecurity_-_Rogue_Device_Investigation.yml] +ignore=GR103 diff --git a/Packs/PingCastle/.pack-ignore b/Packs/PingCastle/.pack-ignore index fe9ad1bd7779..cae0621ab2d6 100644 --- a/Packs/PingCastle/.pack-ignore +++ b/Packs/PingCastle/.pack-ignore @@ -4,3 +4,6 @@ ignore=PB115 [file:PingCastle_image.png] ignore=IM111 +# GR103 is temporary, see CIAC-11656 +[file:incidenttype-PingCastle.json] +ignore=GR103 diff --git a/Packs/Qintel/.pack-ignore b/Packs/Qintel/.pack-ignore index 05c39b0f9744..c51b1e04e827 100644 --- a/Packs/Qintel/.pack-ignore +++ b/Packs/Qintel/.pack-ignore @@ -11,4 +11,8 @@ ignore=IM111 ignore=RM113 [file:QintelPMI.yml] -ignore=IN154 \ No newline at end of file +ignore=IN154 + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Qintel_-_QWatch_Alert.json] +ignore=GR103 diff --git a/Packs/RSANetWitness_v11_1/.pack-ignore b/Packs/RSANetWitness_v11_1/.pack-ignore index c950a3274671..af6f418c67a8 100644 --- a/Packs/RSANetWitness_v11_1/.pack-ignore +++ b/Packs/RSANetWitness_v11_1/.pack-ignore @@ -15,4 +15,12 @@ ignore=IM111 RSANetWitness [file:incidentfield-RSA_Metas_Events.json] -ignore=IF115 \ No newline at end of file +ignore=IF115 + +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-RSA_Event_Count.json] +ignore=GR103 +[file:incidentfield-RSA_Alert_Count.json] +ignore=GR103 +[file:incidentfield-RSA_Rule_Id.json] +ignore=GR103 diff --git a/Packs/RecordedFuture/.pack-ignore b/Packs/RecordedFuture/.pack-ignore index b24204efa183..6419595a6ba7 100644 --- a/Packs/RecordedFuture/.pack-ignore +++ b/Packs/RecordedFuture/.pack-ignore @@ -5,4 +5,8 @@ ignore=PB121 ignore=PB121 [file:playbook-Recorded_Future_Playbook_Alert.yml] -ignore=BA110 \ No newline at end of file +ignore=BA110 + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Recorded_Future_Vulnerability_Layout.json] +ignore=GR103 diff --git a/Packs/Respond/.pack-ignore b/Packs/Respond/.pack-ignore index a3d96055a8dd..538d9c5c1a11 100644 --- a/Packs/Respond/.pack-ignore +++ b/Packs/Respond/.pack-ignore @@ -19,3 +19,6 @@ ignore=IM111 [file:classifier-Mandiant_Automated_Defense.json] ignore=BA101 +# GR103 is temporary, see CIAC-11656 +[file:incidenttype-Mandiant_Automated_Defense_Incident.json] +ignore=GR103 diff --git a/Packs/SOCRadar/.pack-ignore b/Packs/SOCRadar/.pack-ignore index 774e3086ffb5..3b289ae97682 100644 --- a/Packs/SOCRadar/.pack-ignore +++ b/Packs/SOCRadar/.pack-ignore @@ -7,3 +7,6 @@ ignore=IM111 [file:README.md] ignore=RM108 +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-SOCRadar_Incident.json] +ignore=GR103 diff --git a/Packs/SSLCertificates/.pack-ignore b/Packs/SSLCertificates/.pack-ignore index 52a08804c45f..2b57160f36b5 100644 --- a/Packs/SSLCertificates/.pack-ignore +++ b/Packs/SSLCertificates/.pack-ignore @@ -1,2 +1,6 @@ [file:README.md] -ignore=RM104 \ No newline at end of file +ignore=RM104 + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-SSL_Certificates.json] +ignore=GR103 diff --git a/Packs/SafeBreach/.pack-ignore b/Packs/SafeBreach/.pack-ignore index 31be26c509de..117728592ba4 100644 --- a/Packs/SafeBreach/.pack-ignore +++ b/Packs/SafeBreach/.pack-ignore @@ -4,3 +4,22 @@ ignore=BA108,BA109 [file:SafeBreach_v2_image.png] ignore=IM111 +# GR103 is temporary, see CIAC-11656 +[file:reputation-SafeBreach_IP.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_URL.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Command.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Domain.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Hash.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Protocol.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Registry.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Process.json] +ignore=GR103 +[file:layoutscontainer-SafeBreach_Port.json] +ignore=GR103 diff --git a/Packs/SafeNet_Trusted_Access/.pack-ignore b/Packs/SafeNet_Trusted_Access/.pack-ignore index eebac8aa85be..524653a5c853 100644 --- a/Packs/SafeNet_Trusted_Access/.pack-ignore +++ b/Packs/SafeNet_Trusted_Access/.pack-ignore @@ -13,3 +13,8 @@ ignore=PR101 [file:SafeNetTrustedAccessModelingRules.yml] ignore=MR108 +# GR103 is temporary, see CIAC-11656 +[file:classifier-SafeNet_Trusted_Access_–_Alert_Classifier.json] +ignore=GR103 +[file:classifier-SafeNet_Trusted_Access_–_Push_Reject_Alert_Classifier.json] +ignore=GR103 diff --git a/Packs/SalesforceV2/.pack-ignore b/Packs/SalesforceV2/.pack-ignore index e69de29bb2d1..0c0ba279bc43 100644 --- a/Packs/SalesforceV2/.pack-ignore +++ b/Packs/SalesforceV2/.pack-ignore @@ -0,0 +1,21 @@ +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-SalesforceV2_Priority.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Last_Modified_Date.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Owner.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Case_Number.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Origin.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Status.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Escalated.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Subject.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Milestone_Status.json] +ignore=GR103 +[file:incidentfield-SalesforceV2_Closed_Date.json] +ignore=GR103 diff --git a/Packs/SecurityScorecard/.pack-ignore b/Packs/SecurityScorecard/.pack-ignore index 70c679866483..b4433ba65a91 100644 --- a/Packs/SecurityScorecard/.pack-ignore +++ b/Packs/SecurityScorecard/.pack-ignore @@ -1,2 +1,6 @@ [known_words] SecurityScorecard + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-SecurityScorecard_Alert_Layout.json] +ignore=GR103 diff --git a/Packs/SocialEngineeringDomainAnalysis/.pack-ignore b/Packs/SocialEngineeringDomainAnalysis/.pack-ignore index e69de29bb2d1..a97bd03b211e 100644 --- a/Packs/SocialEngineeringDomainAnalysis/.pack-ignore +++ b/Packs/SocialEngineeringDomainAnalysis/.pack-ignore @@ -0,0 +1,3 @@ +# GR103 is temporary, see CIAC-11656 +[file:incidentfield-Social_Engineering_Domain_Summary.json] +ignore=GR103 diff --git a/Packs/SumoLogic_Cloud_SIEM/.pack-ignore b/Packs/SumoLogic_Cloud_SIEM/.pack-ignore index 45aedf8dbbf8..910559ed6e73 100644 --- a/Packs/SumoLogic_Cloud_SIEM/.pack-ignore +++ b/Packs/SumoLogic_Cloud_SIEM/.pack-ignore @@ -4,3 +4,6 @@ ignore=IM111 [known_words] sumourl +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Sumo_Logic_Insight.json] +ignore=GR103 diff --git a/Packs/Trello/.pack-ignore b/Packs/Trello/.pack-ignore index 52a08804c45f..1f1cc019ffb9 100644 --- a/Packs/Trello/.pack-ignore +++ b/Packs/Trello/.pack-ignore @@ -1,2 +1,6 @@ [file:README.md] -ignore=RM104 \ No newline at end of file +ignore=RM104 + +# GR103 is temporary, see CIAC-11656 +[file:playbook-Trello_-_Generic.yml] +ignore=GR103 diff --git a/Packs/UncoverUnknownMalwareUsingSSDeep/.pack-ignore b/Packs/UncoverUnknownMalwareUsingSSDeep/.pack-ignore index b0e016afac63..3d768a436daf 100644 --- a/Packs/UncoverUnknownMalwareUsingSSDeep/.pack-ignore +++ b/Packs/UncoverUnknownMalwareUsingSSDeep/.pack-ignore @@ -1,2 +1,6 @@ [file:incidentfield-Similar_File_Hashes_Based_on_SSDeep.json] -ignore=IF113 \ No newline at end of file +ignore=IF113 + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Uncover_Unknown_Malware_Using_SSDeep.json] +ignore=GR103 diff --git a/Packs/XDRBestPracticeAssessment/.pack-ignore b/Packs/XDRBestPracticeAssessment/.pack-ignore index e69de29bb2d1..1fbd94f4ed3a 100644 --- a/Packs/XDRBestPracticeAssessment/.pack-ignore +++ b/Packs/XDRBestPracticeAssessment/.pack-ignore @@ -0,0 +1,3 @@ +# GR103 is temporary, see CIAC-11656 +[file:incidenttype-XDR_Best_Practice_Assessment.json] +ignore=GR103 diff --git a/Packs/XsoarWebserver/.pack-ignore b/Packs/XsoarWebserver/.pack-ignore index e69de29bb2d1..9c2a4c9665b6 100644 --- a/Packs/XsoarWebserver/.pack-ignore +++ b/Packs/XsoarWebserver/.pack-ignore @@ -0,0 +1,7 @@ +# GR103 is temporary, see CIAC-11656 +[file:playbook-xsoarwebserver-email-data-collection.yml] +ignore=GR103 +[file:playbook-xsoarwebserver-data-collection-response-tracking.yml] +ignore=GR103 +[file:playbook-xsoarwebserver-email-acknowledgement.yml] +ignore=GR103 diff --git a/Packs/knowbe4Phisher/.pack-ignore b/Packs/knowbe4Phisher/.pack-ignore index 9742b35dc887..0ad51a978775 100644 --- a/Packs/knowbe4Phisher/.pack-ignore +++ b/Packs/knowbe4Phisher/.pack-ignore @@ -1,2 +1,6 @@ [known_words] -PhishER \ No newline at end of file +PhishER + +# GR103 is temporary, see CIAC-11656 +[file:layoutscontainer-Phisher.json] +ignore=GR103 From 2d428a960d10210e7bb9cbe057eee99f1201fb0f Mon Sep 17 00:00:00 2001 From: Yaakov Praisler <59408745+yaakovpraisler@users.noreply.github.com> Date: Sun, 1 Sep 2024 16:11:55 +0300 Subject: [PATCH 5/5] AzureStorageContainer] Fix XSUP-40794 (#36101) * add account key argument * fix readme and ignore validation IN124 * add test * fix test * Update Packs/AzureStorageContainer/ReleaseNotes/1_0_22.md Co-authored-by: Dan Tavori <38749041+dantavori@users.noreply.github.com> --------- Co-authored-by: Dan Tavori <38749041+dantavori@users.noreply.github.com> --- Packs/AzureStorageContainer/.pack-ignore | 2 ++ .../AzureStorageContainer.py | 6 +++++- .../AzureStorageContainer.yml | 3 +++ .../AzureStorageContainer_test.py | 18 ++++++++++++++++++ .../AzureStorageContainer/README.md | 6 ++++-- .../ReleaseNotes/1_0_22.md | 6 ++++++ Packs/AzureStorageContainer/pack_metadata.json | 2 +- 7 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Packs/AzureStorageContainer/ReleaseNotes/1_0_22.md diff --git a/Packs/AzureStorageContainer/.pack-ignore b/Packs/AzureStorageContainer/.pack-ignore index e69de29bb2d1..c8adfe7fbcd7 100644 --- a/Packs/AzureStorageContainer/.pack-ignore +++ b/Packs/AzureStorageContainer/.pack-ignore @@ -0,0 +1,2 @@ +[file:AzureStorageContainer.yml] +ignore=IN124 \ No newline at end of file diff --git a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.py b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.py index a0af4fba3751..e1bf73747740 100644 --- a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.py +++ b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.py @@ -956,7 +956,11 @@ def generate_sas_token_command(client: Client, args: dict) -> CommandResults: # if check_valid_permission(valid_permissions, signed_permissions): # type: ignore # Set start time signed_start = str((datetime.utcnow() - timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ")) - account_key = demisto.params().get("key") + account_key = demisto.params().get("key") or args.get("account_key") + + if not account_key: + raise DemistoException("An account key must be given to generate the SAS token.") + time_taken = int(args.get('expiry_time')) # type: ignore signed_expiry = str((datetime.utcnow() + timedelta(hours=time_taken)).strftime("%Y-%m-%dT%H:%M:%SZ")) url_suffix = f"{container_name}" diff --git a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.yml b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.yml index 12ff1c010f7a..adc77bb51980 100644 --- a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.yml +++ b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer.yml @@ -31,6 +31,7 @@ configuration: name: key type: 0 required: false + hidden: true description: Create and Manage Azure Storage Container services. display: Azure Storage Container name: Azure Storage Container @@ -313,6 +314,8 @@ script: - description: specifies a public IP address or a range of public IP addresses from which to accept requests. name: signed_ip type: unknown + - description: The account key to create the SAS token with. + name: account_key description: create SAS token for container. name: azure-storage-container-sas-create dockerimage: demisto/python3:3.10.14.100715 diff --git a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer_test.py b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer_test.py index 4bc67ab43748..448e8c2dbec4 100644 --- a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer_test.py +++ b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/AzureStorageContainer_test.py @@ -438,6 +438,24 @@ def test_generate_sas_signature(): 'test', ) == 'sp=test&st=test&se=test&sip=test&spr=https&sv=test&sr=test&sig=pyUQ25%2BIijJ2TstI5Q6Sre3jJWI0b4qwvRg2LtD9uhc%3D' # noqa +def test_generate_sas_signature_no_key(mocker): + """ + Given: + - User hasn't provided an account key to create the SAS token. + When: + - azure-storage-container-sas-create called. + Then: + - Ensure command raises an exception. + """ + from AzureStorageContainer import generate_sas_token_command, Client + mocker.patch.object(demisto, "params", return_value={}) + client = Client(server_url=BASE_URL, verify=False, proxy=False, + account_sas_token=SAS_TOKEN, + storage_account_name=ACCOUNT_NAME, api_version=API_VERSION) + with pytest.raises(DemistoException): + generate_sas_token_command(client, {"signed_permissions": "c"}) + + def test_check_valid_permission(): from AzureStorageContainer import check_valid_permission assert check_valid_permission('cr', 'c') diff --git a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/README.md b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/README.md index 2f04f23d8c71..78cb56c72997 100644 --- a/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/README.md +++ b/Packs/AzureStorageContainer/Integrations/AzureStorageContainer/README.md @@ -590,7 +590,8 @@ There is no context output for this command. >Blob xsoar.txt properties successfully updated. -### azure-storage-container-blob-property-get +### azure-storage-container-sas-create + *** Retrieve Blob properties. @@ -607,6 +608,7 @@ Retrieve Blob properties. | signed_resources | specifies which resources are accessible via the shared access signature. Options available c(container), b(blob), bv(blob version),bs(blob snapshot),d(directory) | Required | | signed_permissions | The permissions that are associated with the shared access signature. The user is restricted to operations that are allowed by the permissions. Possible permission: r = Read, a=access, c=create, w=write. Also must follow the this order "racwdxltmeop"Example: r,c,a,w,rac, racw. | Required | | signed_ip | specifies a public IP address or a range of public IP addresses from which to accept requests. | Required | +| account_key | The account key to create the SAS token with. | | #### Command Example -```!azure-storage-container-sas-create expiry_time="1" signed_resources="test signed_permissions="test signed_ip="127.0.0.1"``` +```!azure-storage-container-sas-create account_key="TestAccountKey" expiry_time="1" signed_resources="test signed_permissions="test signed_ip="127.0.0.1"``` diff --git a/Packs/AzureStorageContainer/ReleaseNotes/1_0_22.md b/Packs/AzureStorageContainer/ReleaseNotes/1_0_22.md new file mode 100644 index 000000000000..e8ca1b6865ba --- /dev/null +++ b/Packs/AzureStorageContainer/ReleaseNotes/1_0_22.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Azure Storage Container + +Added the *account_key* argument to the ***azure-storage-container-sas-create*** command. diff --git a/Packs/AzureStorageContainer/pack_metadata.json b/Packs/AzureStorageContainer/pack_metadata.json index ca2c2bd5cba8..b0e315d8dd32 100644 --- a/Packs/AzureStorageContainer/pack_metadata.json +++ b/Packs/AzureStorageContainer/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Azure Storage Container", "description": "Create and Manage Azure Storage Container services.", "support": "xsoar", - "currentVersion": "1.0.21", + "currentVersion": "1.0.22", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",