updated test cases & bug fixes

Packs/Devo/Integrations/Devo_v2/Devo_v2.py

@@ -520,48 +520,6 @@ def fetch_incidents():
     return incidents
 
 
-def filter_results_by_fields(results, filtered_columns_string):
-    """
-    Filter a list of dictionaries by including only specified fields.
-
-    Parameters:
-        - results (list): A list of dictionaries representing rows of data.
-        - filtered_columns_string (str): A comma-separated string containing field names to include.
-
-    Returns:
-        list: A new list of dictionaries with only the specified fields.
-
-    Raises:
-        ValueError: If the filtered_columns_string contains invalid column names.
-    """
-    if filtered_columns_string == "":
-        raise ValueError("filtered_columns cannot be empty.")
-
-    if not filtered_columns_string:
-        return results
-
-    if not results:
-        return results
-
-    filtered_columns_list = argToList(filtered_columns_string)
-
-    # Check if all fields from filtered_columns_list are present in the first dictionary in results
-    first_dict = results[0]
-    missing_columns = set(filtered_columns_list) - set(first_dict)
-    if missing_columns:
-        raise ValueError(f"Fields {list(missing_columns)} not found in query result")
-
-    filtered_results = []
-
-    for result in results:
-        filtered_result = {
-            column: result.get(column) for column in filtered_columns_list
-        }
-        filtered_results.append(filtered_result)
-
-    return filtered_results
-
-
 def run_query_command(offset, items):
     to_query = demisto.args()["query"]
     timestamp_from = demisto.args()["from"]
@@ -571,7 +529,6 @@ def run_query_command(offset, items):
     linq_base = demisto.args().get("linqLinkBase", None)
     time_range = get_time_range(timestamp_from, timestamp_to)
     to_query = f"{to_query} offset {offset} limit {items}"
-    filtered_columns = demisto.args().get("filtered_columns", None)
     results = list(
         ds.Reader(
             oauth_token=READER_OAUTH_TOKEN,
@@ -597,8 +554,6 @@ def run_query_command(offset, items):
         )
     }
 
-    results = filter_results_by_fields(results, filtered_columns)
-
     entry = {
         "Type": entryTypes["note"],
         "Contents": results,
@@ -643,7 +598,6 @@ def get_alerts_command(offset, items):
     linq_base = demisto.args().get("linqLinkBase", None)
     user_alert_table = demisto.args().get("table_name", None)
     user_prefix = demisto.args().get("prefix", "")
-    filtered_columns = demisto.args().get("filtered_columns", None)
     user_alert_table = user_alert_table if user_alert_table else DEFAULT_ALERT_TABLE
     if user_prefix:
         user_prefix = f"{user_prefix}_"
@@ -708,8 +662,6 @@ def get_alerts_command(offset, items):
         for ed in res[extra_data]:
             res[extra_data][ed] = urllib.parse.unquote_plus(res[extra_data][ed])
 
-    results = filter_results_by_fields(results, filtered_columns)
-
     entry = {
         "Type": entryTypes["note"],
         "Contents": results,
@@ -749,13 +701,15 @@ def get_alerts_command(offset, items):
 
 
 def multi_table_query_command(offset, items):
+    # Check if items is negative
+    if items < 0:
+        raise ValueError("The 'limit' parameter cannot be negative.")
     tables_to_query = check_type(demisto.args()["tables"], list)
     search_token = demisto.args()["searchToken"]
     timestamp_from = demisto.args()["from"]
     timestamp_to = demisto.args().get("to", None)
     write_context = demisto.args()["writeToContext"].lower()
     query_timeout = int(demisto.args().get("queryTimeout", TIMEOUT))
-    filtered_columns = demisto.args().get("filtered_columns", None)
     global COUNT_MULTI_TABLE
     time_range = get_time_range(timestamp_from, timestamp_to)
 
@@ -799,8 +753,6 @@ def multi_table_query_command(offset, items):
 
     concurrent.futures.wait(futures)
 
-    all_results = filter_results_by_fields(all_results, filtered_columns)
-
     entry = {
         "Type": entryTypes["note"],
         "Contents": all_results,
@@ -963,6 +915,7 @@ def write_to_lookup_table_command():
         con.flush_buffer()
         con.socket.shutdown(0)
 
+        # Return three lines as output
         return f"Lookup Table Name: {lookup_table_name}.\nTotal Records Sent: {total_events}.\nTotal Bytes Sent: {total_bytes}."
 
     except Exception as e:

Packs/Devo/Integrations/Devo_v2/Devo_v2.yml

@@ -194,6 +194,8 @@ script:
     description: Queries multiple tables for a given token and returns relevant results.
   - name: devo-write-to-table
     arguments:
+    - name: tag
+      description: The tag to assign to the records.
     - name: tableName
       required: true
       description: The name of the table to write to.
@@ -221,7 +223,6 @@ script:
     - name: headers
       required: true
       description: Headers for lookup table control.
-      isArray: true
     - name: records
       required: true
       description: Records to write to the specified table.

Packs/Devo/Integrations/Devo_v2/Devo_v2_test.py

@@ -4,7 +4,6 @@
 from unittest.mock import MagicMock, patch
 from datetime import datetime
 import pytest
-import re
 from freezegun import freeze_time
 
 from Devo_v2 import (
@@ -102,60 +101,34 @@
     "from": time.time() - 60,
     "to": time.time(),
     "writeToContext": "true",
-    "filtered_columns": "alertId,extraData,context"
-}
-MOCK_QUERY_ARGS_INVALIDE_COLUMN_NAME = {
-    "query": "from whatever",
-    "from": time.time() - 60,
-    "to": time.time(),
-    "writeToContext": "true",
-    "filtered_columns": "eventdate,abcd"
-}
-MOCK_ALERT_ARGS_REPEATED_FIELDS = {
-    "filters": MOCK_FETCH_INCIDENTS_FILTER,
-    "from": time.time() - 60,
-    "to": time.time(),
-    "writeToContext": "true",
-    "filtered_columns": "alertId,extraData,context,alertId,extraData,context",
 }
 MOCK_ALERT_ARGS = {
     "filters": MOCK_FETCH_INCIDENTS_FILTER,
     "from": time.time() - 60,
     "to": time.time(),
     "writeToContext": "true",
-    "filtered_columns": "alertId,extraData,context"
-}
-MOCK_ALERT_ARGS_EMPTY_filtered_columns_PRAM = {
-    "filters": MOCK_FETCH_INCIDENTS_FILTER,
-    "from": time.time() - 60,
-    "to": time.time(),
-    "writeToContext": "true",
-    "filtered_columns": ""
 }
 MOCK_MULTI_ARGS = {
     "tables": ["app", "charlie", "test"],
     "searchToken": "searching",
     "from": time.time() - 60,
     "to": time.time(),
     "writeToContext": "true",
-    "filtered_columns": "alertId,extraData,context"
 }
 MOCK_WRITER_ARGS = {
     "tableName": "whatever.table",
     "records": [{"foo": "hello"}, {"foo": "world"}, {"foo": "demisto"}],
 }
 MOCK_WRITE_TO_TABLE_RECORDS = {
     "tableName": "whatever.table",
-    "records": ['{"foo": "hello"}', '{"foo": "world"}', '{"foo": "demisto"}'],
+    "records": '[{"hello": "world"}, {"abc": "xyz"}, {"data": "test"}]',
 }
 MOCK_LOOKUP_WRITER_ARGS = {
     "lookupTableName": "hello.world.lookup",
-    "headers": ["foo", "bar", "baz"],
-    "records": [
-        {"key": "fookey", "values": ["fookey", "bar0", "baz0"]},
-        {"key": "keyfoo", "values": ["keyfoo", "bar1", "baz1"]},
-        {"key": "keykey", "values": ["keykey", "bar5", "baz6"]},
-    ],
+    "headers": '{"headers": ["foo", "bar", "baz"], "key_index": 0, "action": "FULL"}',
+    "records": ('[{"fields": ["foo1", "bar1", "baz1"], "delete": false}, '
+                '{"fields": ["foo2", "bar2", "baz2"]}, '
+                '{"fields": ["foo3", "bar3", "baz3"]}]')
 }
 MOCK_KEYS = {"foo": "bar", "baz": "bug"}
 OFFSET = 0
@@ -235,6 +208,9 @@
 
 
 class MOCK_LOOKUP:
+    def send_headers(*args, **kw):
+        pass
+
     def send_control(*args, **kw):
         pass
 
@@ -346,61 +322,7 @@ def test_get_alerts(mock_query_results, mock_args_results):
     mock_args_results.return_value = MOCK_ALERT_ARGS
     results = get_alerts_command(OFFSET, ITEMS_PER_PAGE)
     assert len(results) == 2
-    assert results[0]["Contents"][0]["context"] == "CPU_Usage_Alert"
-
-
-@patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
-@patch("Devo_v2.READER_OAUTH_TOKEN", MOCK_READER_OAUTH_TOKEN, create=True)
-@patch("Devo_v2.demisto.args")
-@patch("Devo_v2.ds.Reader.query")
-def test_get_alerts_check_result_columns(mock_query_results, mock_args_results):
-    mock_query_results.return_value = copy.deepcopy(MOCK_QUERY_RESULTS)
-    mock_args_results.return_value = MOCK_ALERT_ARGS
-    results = get_alerts_command(OFFSET, ITEMS_PER_PAGE)
-    assert len(results) == 2
-    assert results[0]["Contents"][0]["context"] == "CPU_Usage_Alert"
-    # Check if all expected columns are present in the dictionary
-    # Convert filtered_columns from a list to a comma-separated string
-    expected_columns = ','.join(field.strip() for field in MOCK_ALERT_ARGS['filtered_columns'].split(','))
-    result = results[0]["Contents"][0]
-    assert all(column in result for column in expected_columns.split(',')), (
-        f"Not all columns present in the dictionary. Missing columns: "
-        f"{', '.join(column for column in expected_columns.split(',') if column not in result)}"
-    )
-
-
-@patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
-@patch("Devo_v2.READER_OAUTH_TOKEN", MOCK_READER_OAUTH_TOKEN, create=True)
-@patch("Devo_v2.demisto.args")
-@patch("Devo_v2.ds.Reader.query")
-def test_get_alerts_with_repeated_fields(mock_query_results, mock_args_results):
-    mock_query_results.return_value = copy.deepcopy(MOCK_QUERY_RESULTS)
-    mock_args_results.return_value = MOCK_ALERT_ARGS_REPEATED_FIELDS
-
-    results = get_alerts_command(OFFSET, ITEMS_PER_PAGE)
-
-    assert len(results) == 2
-    assert results[0]["Contents"][0]["context"] == "CPU_Usage_Alert"
-
-    # Check if all expected columns are present in the dictionary
-    expected_columns = ','.join(field.strip() for field in MOCK_ALERT_ARGS_REPEATED_FIELDS['filtered_columns'].split(','))
-    result = results[0]["Contents"][0]
-
-    # Assert that each field appears only once in the result
-    assert all(result[column] == result.get(column) for column in expected_columns.split(',')), (
-        f"Repeated fields not handled properly. Result: {result}"
-    )
-
-
-@patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
-@patch("Devo_v2.READER_OAUTH_TOKEN", MOCK_READER_OAUTH_TOKEN, create=True)
-@patch("Devo_v2.demisto.args")
-@patch("Devo_v2.ds.Reader.query")
-def test_get_alerts_with_empty_filtered_columns_param(mock_query_results, mock_args_results):
-    mock_query_results.return_value = copy.deepcopy(MOCK_QUERY_RESULTS)
-    mock_args_results.return_value = MOCK_ALERT_ARGS_EMPTY_filtered_columns_PRAM
-    with pytest.raises(ValueError, match="filtered_columns cannot be empty."):
-        get_alerts_command(OFFSET, ITEMS_PER_PAGE)
+    assert results[0]["Contents"][0]["engine"] == "CPU_Usage_Alert"
 
 
 @patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
@@ -413,18 +335,7 @@ def test_run_query(mock_query_results, mock_args_results):
     results = run_query_command(OFFSET, ITEMS_PER_PAGE)
     assert (results[1]["HumanReadable"]).find("Devo Direct Link") != -1
     assert len(results) == 2
-    assert results[0]["Contents"][0]["context"] == "CPU_Usage_Alert"
-
-
-@patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
-@patch("Devo_v2.READER_OAUTH_TOKEN", MOCK_READER_OAUTH_TOKEN, create=True)
-@patch("Devo_v2.demisto.args")
-@patch("Devo_v2.ds.Reader.query")
-def test_run_query_with_invalid_column_name(mock_query_results, mock_args_results):
-    mock_query_results.return_value = copy.deepcopy(MOCK_QUERY_RESULTS)
-    mock_args_results.return_value = MOCK_QUERY_ARGS_INVALIDE_COLUMN_NAME
-    with pytest.raises(ValueError, match=re.escape("Fields ['abcd'] not found in query result")):
-        run_query_command(OFFSET, ITEMS_PER_PAGE)
+    assert results[0]["Contents"][0]["engine"] == "CPU_Usage_Alert"
 
 
 @patch("Devo_v2.READER_ENDPOINT", MOCK_READER_ENDPOINT, create=True)
@@ -461,7 +372,7 @@ def test_write_devo(mock_load_results, mock_write_args):
     mock_load_results.return_value.load.return_value = MOCK_LINQ_RETURN
     mock_write_args.return_value = MOCK_WRITE_TO_TABLE_RECORDS
     results = write_to_table_command()
-    assert len(results[0]["EntryContext"]["Devo.RecordsWritten"]) == 3
+    assert len(results) == 2  # We expect two entries in the results list
     assert results[0]["EntryContext"]["Devo.LinqQuery"] == "from whatever.table"
 
 
@@ -477,7 +388,10 @@ def test_write_lookup_devo(
     mock_lookup_writer_sender.return_value = MOCK_SENDER()
     mock_lookup_writer_lookup.return_value = MOCK_LOOKUP()
     results = write_to_lookup_table_command()
-    assert len(results[0]["EntryContext"]["Devo.RecordsWritten"]) == 3
+    assert isinstance(results, str)  # We expect a string result
+    assert "Lookup Table Name: hello.world.lookup." in results
+    assert "Total Records Sent: 3." in results
+    assert "Total Bytes Sent: 125." in results
 
 
 @patch("Devo_v2.demisto_ISO", return_value="2022-03-15T15:01:23.456Z")

Packs/Devo/ReleaseNotes/1_3_1.md

@@ -0,0 +1,9 @@
+#### Integrations
+
+##### Devo v2
+
+- **Release Note Update:**
+  - Improved functionality for writing records to Devo tables and lookup tables has been added.
+  - The Docker image has been updated to version *demisto/devo:1.0.0.86778*.
+
+These updates enhance the capabilities of the Devo integration by introducing improved methods for writing records to both Devo tables and lookup tables. The new functions provide enhanced flexibility and efficiency in managing and sending data to Devo's platform. Additionally, users can benefit from the latest version of the Docker image, ensuring compatibility and stability with the integration.

Packs/Devo/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Devo",
     "description": "Use the Devo integration to query Devo for alerts, lookup tables, and to write to lookup tables.",
     "support": "partner",
-    "currentVersion": "1.3.0",
+    "currentVersion": "1.3.1",
     "author": "Devo",
     "url": "https://www.devo.com",
     "email": "[email protected]",