Merge branch 'contrib/metron-labs_devo_V2-dev' into devo_V2-dev

.github/workflows/create-internal-pr-from-external.yml

@@ -19,11 +19,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v8
+        uses: Gr1N/setup-poetry@v9
       - name: Print Context
         run: |
           echo "$GITHUB_CONTEXT"

.github/workflows/handle-new-external-pr.yml

@@ -17,11 +17,11 @@ jobs:
           fetch-depth: 2
 
       - name: Setup Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Setup Poetry
-        uses: Gr1N/setup-poetry@v8
+        uses: Gr1N/setup-poetry@v9
       - name: Print Context
         run: |
           echo "$GITHUB_CONTEXT"

Packs/AWS-SecurityLake/Integrations/AWSSecurityLake/AWSSecurityLake.yml

@@ -79,7 +79,7 @@ configuration:
   type: 8
   advanced: true
 description: "Amazon Security Lake is a fully managed security data lake service."
-display: AWS-SecurityLake
+display: Amazon Security Lake
 name: AWS Security Lake
 script:
   commands:

Packs/AWS-SecurityLake/Integrations/AWSSecurityLake/README.md

@@ -1,7 +1,7 @@
 Amazon Security Lake is a fully managed security data lake service.
 This integration was integrated and tested with version 1.34.20 of AWS Security Lake SDK (boto3).
 
-## Configure AWS-SecurityLake on Cortex XSOAR
+## Configure Amazon Security Lake on Cortex XSOAR
 
 1. Navigate to **Settings** > **Integrations** > **Servers & Services**.
 2. Search for AWS-SecurityLake.

Packs/AWS-SecurityLake/README.md

@@ -1,4 +1,4 @@
-### AWS Security Lake
+### Amazon Security Lake
 
 Amazon Security Lake is a fully managed security data lake service. You can use Security Lake to automatically centralize security data from AWS environments, SaaS providers, on premises, cloud sources, and third-party sources into a purpose-built data lake that's stored in your AWS account.
 Security Lake helps you analyze security data, so you can get a more complete understanding of your security posture across the entire organization. With Security Lake, you can also improve the protection of your workloads, applications, and data.

Packs/AWS-SecurityLake/ReleaseNotes/1_0_6.md

@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### Amazon Security Lake
+
+- Renamed the integration to ***Amazon Security Lake***.
+

Packs/AWS-SecurityLake/pack_metadata.json

@@ -1,8 +1,8 @@
 {
-    "name": "AWS - Security Lake",
+    "name": "Amazon - Security Lake",
     "description": "Amazon Security Lake is a fully managed security data lake service.",
     "support": "xsoar",
-    "currentVersion": "1.0.5",
+    "currentVersion": "1.0.6",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/ApiModules/Scripts/TAXII2ApiModule/TAXII2ApiModule.py

@@ -1,4 +1,3 @@
-import logging
 
 import demistomock as demisto  # noqa: F401
 from CommonServerPython import *  # noqa: F401
@@ -9,6 +8,8 @@
 from requests.sessions import merge_setting, CaseInsensitiveDict
 import re
 import copy
+import logging
+import traceback
 import types
 import urllib3
 from taxii2client import v20, v21
@@ -20,7 +21,7 @@
 urllib3.disable_warnings()
 
 
-class SuppressWarningFilter(logging.Filter):    # pragma: no cover
+class XsoarSuppressWarningFilter(logging.Filter):    # pragma: no cover
     def filter(self, record):
         # Suppress all logger records, but send the important ones to demisto logger
         if record.levelno == logging.WARNING:
@@ -30,8 +31,14 @@ def filter(self, record):
         return False
 
 
+# Make sure we have only one XsoarSuppressWarningFilter
 v21_logger = logging.getLogger("taxii2client.v21")
-v21_logger.addFilter(SuppressWarningFilter())
+demisto.debug(f'Logging Filters before cleaning: {v21_logger.filters=}')
+for current_filter in list(v21_logger.filters):    # pragma: no cover
+    if 'XsoarSuppressWarningFilter' in type(current_filter).__name__:
+        v21_logger.removeFilter(current_filter)
+v21_logger.addFilter(XsoarSuppressWarningFilter())
+demisto.debug(f'Logging Filters: {v21_logger.filters=}')
 
 # CONSTANTS
 TAXII_VER_2_0 = "2.0"
@@ -382,7 +389,7 @@ def init_collection_to_fetch(self, collection_to_fetch=None):
                     break
             if not collection_found:
                 raise DemistoException(
-                    "Could not find the provided Collection name in the available collections. "
+                    f"Could not find the provided Collection name {collection_to_fetch} in the available collections. "
                     "Please make sure you entered the name correctly."
                 )
 
@@ -1181,7 +1188,7 @@ def build_iterator(self, limit: int = -1, **kwargs) -> list[dict[str, str]]:
             envelopes = self.poll_collection(page_size, **kwargs)  # got data from server
             indicators = self.load_stix_objects_from_envelope(envelopes, limit)
         except InvalidJSONError as e:
-            demisto.debug(f'Excepted InvalidJSONError, continuing with empty result.\nError: {e}')
+            demisto.debug(f'Excepted InvalidJSONError, continuing with empty result.\nError: {e}, {traceback.format_exc()}')
             # raised when the response is empty, because {} is parsed into '筽'
             indicators = []
 
@@ -1219,20 +1226,35 @@ def load_stix_objects_from_envelope(self, envelopes: types.GeneratorType, limit:
         if relationships_lst:
             indicators.extend(self.parse_relationships(relationships_lst))
         demisto.debug(
-            f"TAXII 2 Feed has extracted {len(indicators)} indicators"
+            f"TAXII 2 Feed has extracted {len(list(indicators))} indicators"
         )
 
         return indicators
 
+    def increase_count(self, counter: Dict[str, int], id: str):
+        if id in counter:
+            counter[id] = counter[id] + 1
+        else:
+            counter[id] = 1
+
     def parse_generator_type_envelope(self, envelopes: types.GeneratorType, parse_objects_func, limit: int = -1):
         indicators = []
         relationships_lst = []
+        # Used mainly for logging
+        parsed_objects_counter: Dict[str, int] = {}
         try:
             for envelope in envelopes:
-                stix_objects = envelope.get("objects")
-                if not stix_objects:
-                    # no fetched objects
-                    break
+                self.increase_count(parsed_objects_counter, 'envelope')
+                try:
+                    stix_objects = envelope.get("objects")
+                    if not stix_objects:
+                        # no fetched objects
+                        self.increase_count(parsed_objects_counter, 'not-parsed-envelope-not-stix')
+                        break
+                except Exception as e:
+                    demisto.info(f"Exception trying to get envelope objects: {e}, {traceback.format_exc()}")
+                    self.increase_count(parsed_objects_counter, 'exception-envelope-get-objects')
+                    continue
 
                 # we should build the id_to_object dict before iteration as some object reference each other
                 self.id_to_object.update(
@@ -1243,33 +1265,51 @@ def parse_generator_type_envelope(self, envelopes: types.GeneratorType, parse_ob
                 )
                 # now we have a list of objects, go over each obj, save id with obj, parse the obj
                 for obj in stix_objects:
-                    obj_type = obj.get('type')
+                    try:
+                        obj_type = obj.get('type')
+                    except Exception as e:
+                        demisto.info(f"Exception trying to get stix_object-type: {e}, {traceback.format_exc()}")
+                        self.increase_count(parsed_objects_counter, 'exception-stix-object-type')
+                        continue
 
                     # we currently don't support extension object
                     if obj_type == 'extension-definition':
                         demisto.debug(f'There is no parsing function for object type "extension-definition", for object {obj}.')
+                        self.increase_count(parsed_objects_counter, 'not-parsed-extension-definition')
                         continue
                     elif obj_type == 'relationship':
                         relationships_lst.append(obj)
+                        self.increase_count(parsed_objects_counter, 'not-parsed-relationship')
                         continue
 
                     if not parse_objects_func.get(obj_type):
                         demisto.debug(f'There is no parsing function for object type {obj_type}, for object {obj}.')
-
+                        self.increase_count(parsed_objects_counter, f'not-parsed-{obj_type}')
                         continue
-                    if result := parse_objects_func[obj_type](obj):
-                        indicators.extend(result)
-                        self.update_last_modified_indicator_date(obj.get("modified"))
+                    try:
+                        if result := parse_objects_func[obj_type](obj):
+                            indicators.extend(result)
+                            self.update_last_modified_indicator_date(obj.get("modified"))
+                    except Exception as e:
+                        demisto.info(f"Exception parsing stix_object-type {obj_type}: {e}, {traceback.format_exc()}")
+                        self.increase_count(parsed_objects_counter, f'exception-parsing-{obj_type}')
+                        continue
+                    self.increase_count(parsed_objects_counter, f'parsed-{obj_type}')
 
                     if reached_limit(limit, len(indicators)):
-                        demisto.debug("Reached limit of indicators to fetch")
+                        demisto.debug(f"Reached the limit ({limit}) of indicators to fetch. Indicators len: {len(indicators)}."
+                                      f' Got {len(indicators)} indicators and {len(list(relationships_lst))} relationships.'
+                                      f' Objects counters: {parsed_objects_counter}')
+
                         return indicators, relationships_lst
         except Exception as e:
+            demisto.info(f"Exception trying to parse envelope: {e}, {traceback.format_exc()}")
             if len(indicators) == 0:
                 demisto.debug("No Indicator were parsed")
                 raise e
             demisto.debug(f"Failed while parsing envelopes, succeeded to retrieve {len(indicators)} indicators.")
-        demisto.debug("Finished parsing all objects")
+        demisto.debug(f'Finished parsing all objects. Got {len(list(indicators))} indicators '
+                      f'and {len(list(relationships_lst))} relationships. Objects counters: {parsed_objects_counter}')
         return indicators, relationships_lst
 
     def poll_collection(
@@ -1286,7 +1326,9 @@ def poll_collection(
                 self.objects_to_fetch.append('relationship')
             kwargs['type'] = self.objects_to_fetch
         if isinstance(self.collection_to_fetch, v20.Collection):
+            demisto.debug(f'Collection is a v20 type collction, {self.collection_to_fetch}')
             return v20.as_pages(get_objects, per_request=page_size, **kwargs)
+        demisto.debug(f'Collection is a v21 type collction, {self.collection_to_fetch}')
         return v21.as_pages(get_objects, per_request=page_size, **kwargs)
 
     def get_page_size(self, max_limit: int, cur_limit: int) -> int:
@@ -1314,6 +1356,8 @@ def extract_indicators_from_stix_objects(
         extracted_objs = [
             item for item in stix_objs if item.get("type") in required_objects
         ]  # retrieve only required type
+        demisto.debug(f'Extracted {len(list(extracted_objs))} out of {len(list(stix_objs))} Stix objects with the types: '
+                      f'{required_objects}')
 
         return extracted_objs
 

Packs/ApiModules/Scripts/TAXII2ApiModule/TAXII2ApiModule_test.py

@@ -1446,3 +1446,25 @@ def test_reached_limit(limit, element_count, return_value):
     """
     from TAXII2ApiModule import reached_limit
     assert reached_limit(limit, element_count) == return_value
+
+
+def test_increase_count():
+    """
+    Given:
+        - A counters dict.
+    When:
+        - Increasing various counters.
+    Then:
+        - Assert that the counters reflect the expected values.
+    """
+    mock_client = Taxii2FeedClient(url='', collection_to_fetch='', proxies=[], verify=False, objects_to_fetch=[])
+    objects_counter: Dict[str, int] = {}
+
+    mock_client.increase_count(objects_counter, 'counter_a')
+    assert objects_counter == {'counter_a': 1}
+
+    mock_client.increase_count(objects_counter, 'counter_a')
+    assert objects_counter == {'counter_a': 2}
+
+    mock_client.increase_count(objects_counter, 'counter_b')
+    assert objects_counter == {'counter_a': 2, 'counter_b': 1}