Merge branch 'Add/SekoiaXDR' of github.com:SEKOIA-IO/Cortex-XSOAR-int…

…egration into Add/SekoiaXDR

Config/corepacks_override.json

@@ -1,6 +1,6 @@
 {
     "server_version": "8.7.0",
-    "file_version": "1",
+    "file_version": "2",
     "xsoar_saas": {
         "updated_corepacks_content": {
             "corePacks": [
@@ -69,7 +69,7 @@
                 "FeedMitreAttackv2/1.1.38/FeedMitreAttackv2.zip",
                 "ThreatIntelligenceManagement/1.1.10/ThreatIntelligenceManagement.zip",
                 "FiltersAndTransformers/1.2.72/FiltersAndTransformers.zip",
-                "CoreAlertFields/1.0.34/CoreAlertFields.zip",
+                "CoreAlertFields/1.0.37/CoreAlertFields.zip",
                 "FeedUnit42v2/1.0.54/FeedUnit42v2.zip",
                 "AutoFocus/2.2.1/AutoFocus.zip",
                 "EDL/3.3.1/EDL.zip",
@@ -86,7 +86,7 @@
                 "Unit42Intel/1.0.23/Unit42Intel.zip",
                 "CommonTypes/3.5.4/CommonTypes.zip",
                 "CortexAttackSurfaceManagement/1.7.39/CortexAttackSurfaceManagement.zip",
-                "Base/1.34.12/Base.zip",
+                "Base/1.34.35/Base.zip",
                 "DemistoRESTAPI/1.3.55/DemistoRESTAPI.zip",
                 "rasterize/2.0.13/rasterize.zip"
             ],

Packs/AccentureCTI/.pack-ignore

@@ -7,4 +7,10 @@ uuid
 CustomCVE
 acti
 getThreatIntelReport
-get-fundamentals-by-uuid
+get-fundamentals-by-uuid
+
+# GR103 is temporary, see CIAC-11656
+[file:layoutscontainer-ACTI_Intelligence_Report.json]
+ignore=GR103
+[file:layoutscontainer-ACTI_Intelligence_Alert.json]
+ignore=GR103

Packs/AccentureCTI_Feed/Integrations/ACTIIndicatorFeed/ACTIIndicatorFeed_test.py

@@ -81,4 +81,4 @@ def test_build_iterator_change_extractor():
         with pytest.raises(TypeError) as e:
             custom_build_iterator(client, PARAMS['feed_name_to_config']['Domain'], 0)
         if not e:
-            assert False
+            raise AssertionError

Packs/AccentureCTI_Feed/ReleaseNotes/1_1_38.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### ACTI Indicator Feed
+
+Enhanced **JSONFeedApiModule** to support setting indicator enrichment exclusion in select feeds. The change has no impact on this integration.

Packs/AccentureCTI_Feed/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Accenture CTI Feed",
     "description": "Accenture Cyber Threat Intelligence Feed",
     "support": "partner",
-    "currentVersion": "1.1.37",
+    "currentVersion": "1.1.38",
     "author": "Accenture",
     "url": "https://www.accenture.com/us-en/services/security/cyber-defense",
     "email": "[email protected]",

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py

@@ -379,10 +379,7 @@ def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], c
                 field = {f: {}}
                 if 'regex' in fattrs:
                     field[f]['regex'] = re.compile(fattrs['regex'])
-                if 'transform' not in fattrs:
-                    field[f]['transform'] = r'\g<0>'
-                else:
-                    field[f]['transform'] = fattrs['transform']
+                field[f]['transform'] = fattrs.get('transform', '\\g<0>')
                 fields_to_extract.append(field)
 
     line = line.strip()
@@ -420,7 +417,14 @@ def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], c
     return attributes, value
 
 
-def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, create_relationships=False, **kwargs):
+def fetch_indicators_command(client,
+                             feed_tags,
+                             tlp_color,
+                             itype,
+                             auto_detect,
+                             create_relationships=False,
+                             enrichment_excluded: bool = False,
+                             **kwargs):
     iterators = client.build_iterator(**kwargs)
     indicators = []
 
@@ -440,22 +444,27 @@ def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, c
                     indicator_type = determine_indicator_type(
                         client.feed_url_to_config.get(url, {}).get('indicator_type'), itype, auto_detect, value)
                     indicator_data = {
-                        "value": value,
-                        "type": indicator_type,
-                        "rawJSON": attributes,
+                        'value': value,
+                        'type': indicator_type,
+                        'rawJSON': attributes,
                     }
-                    if create_relationships and client.feed_url_to_config.get(url, {}).get('relationship_name'):
-                        if attributes.get('relationship_entity_b'):
-                            relationships_lst = EntityRelationship(
-                                name=client.feed_url_to_config.get(url, {}).get('relationship_name'),
-                                entity_a=value,
-                                entity_a_type=indicator_type,
-                                entity_b=attributes.get('relationship_entity_b'),
-                                entity_b_type=FeedIndicatorType.indicator_type_by_server_version(
-                                    client.feed_url_to_config.get(url, {}).get('relationship_entity_b_type')),
-                            )
-                            relationships_of_indicator = [relationships_lst.to_indicator()]
-                            indicator_data['relationships'] = relationships_of_indicator
+                    if enrichment_excluded:
+                        indicator_data['enrichmentExcluded'] = enrichment_excluded
+
+                    if (create_relationships
+                            and client.feed_url_to_config.get(url, {}).get('relationship_name')
+                            and attributes.get('relationship_entity_b')
+                            ):
+                        relationships_lst = EntityRelationship(
+                            name=client.feed_url_to_config.get(url, {}).get('relationship_name'),
+                            entity_a=value,
+                            entity_a_type=indicator_type,
+                            entity_b=attributes.get('relationship_entity_b'),
+                            entity_b_type=FeedIndicatorType.indicator_type_by_server_version(
+                                client.feed_url_to_config.get(url, {}).get('relationship_entity_b_type')),
+                        )
+                        relationships_of_indicator = [relationships_lst.to_indicator()]
+                        indicator_data['relationships'] = relationships_of_indicator
 
                     if len(client.custom_fields_mapping.keys()) > 0 or TAGS in attributes:
                         custom_fields = client.custom_fields_creator(attributes)
@@ -483,14 +492,20 @@ def determine_indicator_type(indicator_type, default_indicator_type, auto_detect
     return indicator_type
 
 
-def get_indicators_command(client: Client, args):
+def get_indicators_command(client: Client, args, enrichment_excluded: bool = False):
     itype = args.get('indicator_type', client.indicator_type)
     limit = int(args.get('limit'))
     feed_tags = args.get('feedTags')
     tlp_color = args.get('tlp_color')
     auto_detect = demisto.params().get('auto_detect_type')
     create_relationships = demisto.params().get('create_relationships')
-    indicators_list, _ = fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, create_relationships)[:limit]
+    indicators_list, _ = fetch_indicators_command(client,
+                                                  feed_tags,
+                                                  tlp_color,
+                                                  itype,
+                                                  auto_detect,
+                                                  create_relationships,
+                                                  enrichment_excluded)[:limit]
     entry_result = camelize(indicators_list)
     hr = tableToMarkdown('Indicators', entry_result, headers=['Value', 'Type', 'Rawjson'])
     return hr, {}, indicators_list
@@ -502,7 +517,7 @@ def test_module(client: Client, args):
         if not FeedIndicatorType.is_valid_type(indicator_type):
             indicator_types = []
             for key, val in vars(FeedIndicatorType).items():
-                if not key.startswith('__') and type(val) == str:
+                if not key.startswith('__') and isinstance(val, str):
                     indicator_types.append(val)
             supported_values = ', '.join(indicator_types)
             raise ValueError(f'Indicator type of {indicator_type} is not supported. Supported values are:'
@@ -518,6 +533,7 @@ def feed_main(feed_name, params=None, prefix=''):
         params['feed_name'] = feed_name
     feed_tags = argToList(demisto.params().get('feedTags'))
     tlp_color = demisto.params().get('tlp_color')
+    enrichment_excluded = demisto.params().get('enrichmentExcluded', False)
     client = Client(**params)
     command = demisto.command()
     if command != 'fetch-indicators':
@@ -534,7 +550,8 @@ def feed_main(feed_name, params=None, prefix=''):
             indicators, no_update = fetch_indicators_command(client, feed_tags, tlp_color,
                                                              params.get('indicator_type'),
                                                              params.get('auto_detect_type'),
-                                                             params.get('create_relationships'))
+                                                             params.get('create_relationships'),
+                                                             enrichment_excluded=enrichment_excluded)
 
             # check if the version is higher than 6.5.0 so we can use noUpdate parameter
             if is_demisto_version_ge('6.5.0'):

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.yml

@@ -13,7 +13,7 @@ system: true
 scripttarget: 0
 dependson: {}
 timeout: 0s
-dockerimage: demisto/py3-tools:0.0.1.25751
+dockerimage: demisto/py3-tools:1.0.0.108682
 fromversion: 5.0.0
 tests:
 - No tests

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py

@@ -445,6 +445,71 @@ def test_get_indicators_without_relations():
         assert indicators == expected_res
 
 
+def test_fetch_indicators_exclude_enrichment():
+    """
+    Given:
+        - Exclude enrichment parameter is used
+    When:
+        - Calling the fetch_indicators_command
+    Then:
+        - The indicators should include the enrichmentExcluded field if exclude is True.
+    """
+
+    feed_url_to_config = {
+        'https://www.spamhaus.org/drop/asndrop.txt': {
+            "indicator_type": 'IP',
+            "indicator": {
+                "regex": r"^.+,\"?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\"?",
+                "transform": "\\1"
+            },
+            'relationship_name': 'indicator-of',
+            'relationship_entity_b_type': 'STIX Malware',
+            "fields": [{
+                'firstseenbysource': {
+                    "regex": r"^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})",
+                    "transform": "\\1"
+                },
+                "port": {
+                    "regex": r"^.+,.+,(\d{1,5}),",
+                    "transform": "\\1"
+                },
+                "updatedate": {
+                    "regex": r"^.+,.+,.+,(\d{4}-\d{2}-\d{2})",
+                    "transform": "\\1"
+                },
+                "malwarefamily": {
+                    "regex": r"^.+,.+,.+,.+,(.+)",
+                    "transform": "\\1"
+                },
+                "relationship_entity_b": {
+                    "regex": r"^.+,.+,.+,.+,\"(.+)\"",
+                    "transform": "\\1"
+                }
+            }],
+        }
+    }
+    expected_res = ([{'value': '127.0.0.1', 'type': 'IP',
+                     'rawJSON': {'malwarefamily': '"Test"', 'relationship_entity_b': 'Test', 'value': '127.0.0.1',
+                                 'type': 'IP', 'tags': []},
+                      'fields': {'tags': []},
+                      'enrichmentExcluded': True}], True)
+
+    asn_ranges = '"2021-01-17 07:44:49","127.0.0.1","3889","online","2021-04-22","Test"'
+    with requests_mock.Mocker() as m:
+        m.get('https://www.spamhaus.org/drop/asndrop.txt', content=asn_ranges.encode('utf-8'))
+        client = Client(
+            url="https://www.spamhaus.org/drop/asndrop.txt",
+            source_name='spamhaus',
+            ignore_regex='^;.*',
+            feed_url_to_config=feed_url_to_config,
+            indicator_type='ASN'
+        )
+        indicators = fetch_indicators_command(client, feed_tags=[], tlp_color=[], itype='IP', auto_detect=False,
+                                              create_relationships=False, enrichment_excluded=True)
+
+        assert indicators == expected_res
+
+
 def test_get_no_update_value(mocker):
     """
     Given

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py

@@ -240,7 +240,7 @@ def test_module(client: Client, limit) -> str:  # pragma: no cover
 
 def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list, auto_detect: bool,
                              create_relationships: bool = False, limit: int = 0, remove_ports: bool = False,
-                             **kwargs) -> Tuple[List[dict], bool]:
+                             enrichment_excluded: bool = False, **kwargs) -> Tuple[List[dict], bool]:
     """
     Fetches the indicators from client.
     :param client: Client of a JSON Feed
@@ -296,7 +296,9 @@ def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list
             indicators.extend(
                 handle_indicator_function(client, item, feed_config, service_name, indicator_type, indicator_field,
                                           use_prefix_flat, feedTags, auto_detect, mapping_function,
-                                          create_relationships, create_relationships_function, remove_ports))
+                                          create_relationships, create_relationships_function, remove_ports,
+                                          enrichment_excluded=enrichment_excluded,
+                                          ))
 
             if limit and len(indicators) >= limit:  # We have a limitation only when get-indicators command is
                 # called, and then we return for each service_name "limit" of indicators
@@ -320,8 +322,9 @@ def indicator_mapping(mapping: Dict, indicator: Dict, attributes: Dict):
 def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name: str,
                      indicator_type: str, indicator_field: str, use_prefix_flat: bool,
                      feedTags: list, auto_detect: bool, mapping_function: Callable = indicator_mapping,
-                     create_relationships: bool = False, relationships_func: Callable = None,
-                     remove_ports: bool = False) -> List[dict]:
+                     create_relationships: bool = False, relationships_func: Callable | None = None,
+                     remove_ports: bool = False,
+                     enrichment_excluded: bool = False) -> List[dict]:
     indicator_list = []
     mapping = feed_config.get('mapping')
     take_value_from_flatten = False
@@ -367,6 +370,9 @@ def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name
 
     indicator['rawJSON'] = item
 
+    if enrichment_excluded:
+        indicator['enrichmentExcluded'] = enrichment_excluded
+
     indicator_list.append(indicator)
 
     return indicator_list
@@ -436,6 +442,7 @@ def feed_main(params, feed_name, prefix):  # pragma: no cover
     auto_detect = params.get('auto_detect_type')
     feedTags = argToList(params.get('feedTags'))
     limit = int(demisto.args().get('limit', 10))
+    enrichment_excluded = params.get('enrichmentExcluded', False)
     command = demisto.command()
     if prefix and not prefix.endswith('-'):
         prefix += '-'
@@ -448,8 +455,13 @@ def feed_main(params, feed_name, prefix):  # pragma: no cover
         elif command == 'fetch-indicators':
             remove_ports = argToBoolean(params.get('remove_ports', False))
             create_relationships = params.get('create_relationships')
-            indicators, no_update = fetch_indicators_command(client, indicator_type, feedTags, auto_detect,
-                                                             create_relationships, remove_ports=remove_ports)
+            indicators, no_update = fetch_indicators_command(client,
+                                                             indicator_type,
+                                                             feedTags,
+                                                             auto_detect,
+                                                             create_relationships,
+                                                             remove_ports=remove_ports,
+                                                             enrichment_excluded=enrichment_excluded)
 
             # check if the version is higher than 6.5.0 so we can use noUpdate parameter
             if is_demisto_version_ge('6.5.0'):

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.yml

@@ -1,4 +1,4 @@
-comment: Common code that will be appended into each JSON Feed integration when it's deployed
+comment: Common code that will be appended into each JSON Feed integration when it's deployed.
 commonfields:
   id: JSONFeedApiModule
   version: -1
@@ -11,7 +11,7 @@ tags:
 - server
 timeout: 0s
 type: python
-dockerimage: demisto/py3-tools:0.0.1.25751
+dockerimage: demisto/py3-tools:1.0.0.108682
 dependson: {}
 fromversion: 5.5.0
 tests:

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py

@@ -165,6 +165,46 @@ def test_list_of_indicators_with_no_json_object():
         assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'}
 
 
+def test_fetch_indicators_with_exclude_enrichment():
+    """
+    Given:
+        - Exclude enrichment parameter is used
+    When:
+        - Calling the fetch_indicators_command
+    Then:
+        - The indicators should include the enrichmentExcluded field if exclude is True.
+    """
+
+    feed_name_to_config = {
+        'Github': {
+            'url': 'https://api.github.com/meta',
+            'extractor': "hooks",
+            'indicator': None,
+            'remove_ports': "true"
+        }
+    }
+
+    with requests_mock.Mocker() as m:
+        m.get('https://api.github.com/meta', json=json.loads(FLAT_LIST_OF_INDICATORS))
+
+        client = Client(
+            url='https://api.github.com/meta',
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        indicators, _ = fetch_indicators_command(client=client, indicator_type=None, feedTags=['test'],
+                                                 auto_detect=True, remove_ports=True, enrichment_excluded=True)
+
+        assert len(indicators) == 3
+        assert indicators[0].get('value') == '1.1.1.1'
+        assert indicators[0].get('type') == 'IP'
+        assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'}
+
+        for ind in indicators:
+            assert ind['enrichmentExcluded']
+
+
 def test_post_of_indicators_with_no_json_object():
     feed_name_to_config = {
         'Github': {