Merge pull request #31 from splunk-soar-connectors/next

Merging next to main for release 2.9.0

README.md

@@ -2,11 +2,11 @@
 # Parser
 
 Publisher: Splunk  
-Connector Version: 2\.8\.0  
+Connector Version: 2\.9\.0  
 Product Vendor: Splunk  
 Product Name: Parser  
 Product Version Supported (regex): "\.\*"  
-Minimum Product Version: 5\.3\.3  
+Minimum Product Version: 5\.3\.4  
 
 This app extracts IOCs from various files such as PDFs, emails, or raw text
 
@@ -31,6 +31,16 @@ This app will ignore the HTTP_PROXY and HTTPS_PROXY environment variables.
 This app uses the defusedxml module, which is licensed under the Python Software Foundation License
 (PSFL), Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam, The Netherlands.
 
+<div>
+
+## URL Extraction
+
+The app extracts defanged URL's that start with hxxp/hxxps. The defanged URL with \[.\] is not
+considered valid. Therefore it does not get ingested. Hence, the app will not extract URLs defanged
+with \[.\]
+
+</div>
+
 
 ### Supported Actions  
 [extract ioc](#action-extract-ioc) - Create IOC artifacts from a file in the vault or raw text  

parser.json

@@ -9,13 +9,13 @@
     "product_name": "Parser",
     "product_version_regex": ".*",
     "publisher": "Splunk",
-    "app_version": "2.8.0",
+    "app_version": "2.9.0",
     "fips_compliant": true,
     "license": "Copyright (c) 2017-2022 Splunk Inc.",
     "utctime_updated": "2022-03-15T18:28:49.000000Z",
     "package_name": "phantom_parser",
     "main_module": "parser_connector.py",
-    "min_phantom_version": "5.3.3",
+    "min_phantom_version": "5.3.4",
     "python_version": "3",
     "latest_tested_versions": [
         "Module pdfminer.six-20211012, September 20 2022"
@@ -25,7 +25,7 @@
         "wheel": [
             {
                 "module": "Django",
-                "input_file": "wheels/py3/Django-3.2.15-py3-none-any.whl"
+                "input_file": "wheels/py3/Django-3.2.16-py3-none-any.whl"
             },
             {
                 "module": "asgiref",
@@ -499,7 +499,7 @@
         "wheel": [
             {
                 "module": "Django",
-                "input_file": "wheels/py3/Django-3.2.15-py3-none-any.whl"
+                "input_file": "wheels/py3/Django-3.2.16-py3-none-any.whl"
             },
             {
                 "module": "asgiref",
@@ -571,4 +571,4 @@
             }
         ]
     }
-}
+}

parser_email.py

@@ -715,7 +715,7 @@ def _handle_attachment(part, file_name, file_path, parsed_mail):
             "Error occurred while adding file to Vault. Error Details: {}".format(error_msg))
         return
 
-    file_hash = hashlib.sha1(part_payload).hexdigest()
+    file_hash = hashlib.sha1(part_payload).hexdigest()  # nosemgrep
     files.append({'file_name': file_name, 'file_path': file_path,
                  'file_hash': file_hash, 'meta_info': attach_meta_info})
 
@@ -851,7 +851,7 @@ def _add_body_in_email_headers(parsed_mail, file_path, charset, content_type, fi
         _debug_print('Saving image {} to files'.format(file_name))
 
         try:
-            file_hash = hashlib.sha1(body_content.encode()).hexdigest()
+            file_hash = hashlib.sha1(body_content.encode()).hexdigest()  # nosemgrep
             files = parsed_mail[PROC_EMAIL_JSON_FILES]
             files.append({'file_name': file_name, 'file_path': file_path, 'file_hash': file_hash})
         except Exception as e:
@@ -977,7 +977,7 @@ def _handle_mail_object(mail, email_id, rfc822_email, tmp_dir, start_time_epoch)
     container_data = dict(parsed_mail)
 
     # delete the header info, we don't make it a part of the container json
-    del(container_data[PROC_EMAIL_JSON_EMAIL_HEADERS])
+    del (container_data[PROC_EMAIL_JSON_EMAIL_HEADERS])
     container.update(_container_common)
     _container['source_data_identifier'] = email_id
     _container['name'] = container_name
@@ -1406,7 +1406,7 @@ def _create_dict_hash(input_dict):
     # to not impact existing customers
     dict_hash = UnicodeDammit(input_dict_str).unicode_markup.encode()
     if not fips_enabled:
-        dict_hash = hashlib.md5(dict_hash)
+        dict_hash = hashlib.md5(dict_hash)  # nosemgrep
     else:
         dict_hash = hashlib.sha256(dict_hash)
     return dict_hash.hexdigest()

readme.html

@@ -14,7 +14,16 @@
 -->
 <p>
 	This app will ignore the HTTP_PROXY and HTTPS_PROXY environment variables.
-<br>
-	<h2>Defusedxml</h2>
-	This app uses the defusedxml module, which is licensed under the Python Software Foundation License (PSFL), Copyright 1991-1995 by Stichting Mathematisch Centrum, Amsterdam, The Netherlands.
+	<br>
+<h2>Defusedxml</h2>
+This app uses the defusedxml module, which is licensed under the Python Software Foundation License (PSFL), Copyright
+1991-1995 by Stichting Mathematisch Centrum, Amsterdam, The Netherlands.
 </p>
+
+<div>
+	<h2>URL Extraction</h2>
+	<p>
+		The app extracts defanged URL's that start with hxxp/hxxps. The defanged URL with [.] is not considered valid.
+		Therefore it does not get ingested. Hence, the app will not extract URLs defanged with [.]
+	</p>
+</div>

release_notes/2.9.0.md

@@ -0,0 +1 @@
+* Updated documentation for Defang URL [PAPP-27852]

requirements.txt

@@ -1,6 +1,6 @@
 beautifulsoup4==4.9.1
 defusedxml==0.7.1
-Django==3.2.15
+Django==3.2.16
 git+https://github.com/phantomcyber/pdfminer.six.git@20211012-fips
 python-magic==0.4.18
 requests==2.25.0