feat: update eap json report spec (#3964)

- use a new EAP-specific datasource instead of adding a generic entry
- foreach_collect is used 

Signed-off-by: Chenlizhong <[email protected]>

insights/specs/datasources/__init__.py

@@ -8,6 +8,10 @@
 parser/combiner/component required by a custom datasource must be included in the YAML
 template to ensure it is loaded.
 """
+
+import os
+import time
+
 DEFAULT_SHELL_TIMEOUT = 10
 """ int: Default timeout in seconds for ctx.shell_out() commands, must be provided as an arg """
 
@@ -46,3 +50,26 @@ def get_running_commands(ps, ctx, commands):
             continue
         ret.add(which[0]) if which else None
     return sorted(ret)
+
+
+def get_recent_files(target_path, last_modify_hours):
+    """
+    Get all recent updated files or created
+
+    Arguments:
+        target_path (string): target path to search
+        last_modify_hours (int): Specify the recent hours
+
+    Returns:
+        list: List of files that updated or creates just in last_modify_hours hours.
+    """
+    result_files = []
+    if os.path.exists(target_path):
+        for parent_path, _, report_files in os.walk(target_path):
+            current_time = time.time()
+            for one_file in report_files:
+                t_full_file = os.path.join(parent_path, one_file)
+                file_time = os.path.getmtime(t_full_file)
+                if (current_time - file_time) // 3600 < last_modify_hours:
+                    result_files.append(t_full_file)
+    return result_files

insights/specs/datasources/eap_reports.py

@@ -0,0 +1,27 @@
+"""
+Custom datasource to get all updated/created eap reports in last 24 hours
+"""
+
+import os
+
+from insights.core.context import HostContext
+from insights.core.plugins import datasource
+from insights.core.exceptions import SkipComponent
+from insights.specs.datasources import get_recent_files
+
+
+@datasource(HostContext)
+def eap_report_files(broker):
+    """
+    Get all updated/created eap reports in last 24 hours.
+
+    Returns:
+        function: return the list of EAP reports file in last 24 hours.
+    """
+    ctx = broker[HostContext]
+    root = ctx.root
+    EAP_REPORTS_JSON_PATH = os.path.join(root, "var/tmp/insights-runtimes/uploads/")
+    final_files = get_recent_files(EAP_REPORTS_JSON_PATH, 24)
+    if final_files:
+        return [f[len(root):] for f in final_files]
+    raise SkipComponent

insights/specs/default.py

@@ -24,7 +24,7 @@
 from insights.specs import Specs
 from insights.specs.datasources import (
     aws, awx_manage, client_metadata, cloud_init, corosync as corosync_ds,
-    dir_list, ethernet, httpd, ipcs, intersystems, kernel, kernel_module_list,
+    dir_list, eap_reports, ethernet, httpd, ipcs, intersystems, kernel, kernel_module_list,
     leapp, ls, lpstat, luks_devices, machine_ids, malware_detection, md5chk,
     mount as mount_ds, package_provides, ps as ps_datasource, rpm_pkgs, sap,
     satellite_missed_queues, ssl_certificate, sys_fs_cgroup_memory,
@@ -189,7 +189,7 @@ class DefaultSpecs(Specs):
     dse_ldif = glob_file("/etc/dirsrv/*/dse.ldif")
     du_dirs = foreach_execute(dir_list.du_dir_list, "/bin/du -s -k %s")  # empty filter
     duplicate_machine_id = machine_ids.dup_machine_id_info
-    eap_json_reports = glob_file(r"/var/tmp/insights-runtimes/uploads/*.json")
+    eap_json_reports = foreach_collect(eap_reports.eap_report_files, "%s")
     engine_log = simple_file("/var/log/ovirt-engine/engine.log")
     etc_journald_conf = simple_file(r"etc/systemd/journald.conf")
     etc_journald_conf_d = glob_file(r"etc/systemd/journald.conf.d/*.conf")

insights/specs/sos_archive.py

@@ -77,6 +77,7 @@ class SosSpecs(Specs):
     docker_list_images = simple_file("sos_commands/docker/docker_images")
     docker_storage = simple_file("/etc/sysconfig/docker-storage")
     dumpe2fs_h = glob_file("sos_commands/filesys/dumpe2fs_-h_*")
+    eap_json_reports = glob_file(r"/var/tmp/insights-runtimes/uploads/*.json")
     ethtool = glob_file("sos_commands/networking/ethtool_*", ignore="ethtool_-.*")
     ethtool_S = glob_file("sos_commands/networking/ethtool_-S_*")
     ethtool_T = glob_file("sos_commands/networking/ethtool_-T_*")

insights/tests/datasources/test_eap_reports.py

@@ -0,0 +1,68 @@
+import os
+import pytest
+import shutil
+import tempfile
+from datetime import datetime, timedelta
+
+from insights.core import dr
+from insights.core.spec_factory import foreach_collect
+from insights.core.context import HostContext
+from insights.specs.datasources.eap_reports import eap_report_files
+
+
+@pytest.fixture
+def sample_directory(scope="module"):
+    tmpdir = tempfile.mkdtemp()
+    os.makedirs(tmpdir + "/var/tmp/insights-runtimes/uploads/")
+    for f in ["file_a.json", "file_b.json"]:
+        fd = open(tmpdir + "/var/tmp/insights-runtimes/uploads/" + f, "w")
+        fd.write('{"name":"example_json", "count_number":30}')
+        fd.close()
+    yield tmpdir
+    shutil.rmtree(tmpdir)
+
+
+def run_eap_files_test(sample_directory, spec):
+    ctx = HostContext()
+    ctx.root = sample_directory
+    broker = dr.Broker()
+    broker[HostContext] = ctx
+    broker = dr.run([spec], broker)
+    return broker
+
+
+def test_eap_match_results(sample_directory):
+    spec = eap_report_files
+    broker = run_eap_files_test(sample_directory, spec)
+    file_names = [p.split('/')[-1] for p in broker[spec]]
+    assert 'file_a.json' in file_names
+    assert 'file_b.json' in file_names
+
+
+def test_foreach_eap_match_results(sample_directory):
+    spec = foreach_collect(eap_report_files, "%s")
+    broker = run_eap_files_test(sample_directory, spec)
+    file_names = [p.file_name for p in broker[spec]]
+    assert 'file_a.json' in file_names
+    assert 'file_b.json' in file_names
+
+
+def test_eap_one_match_paths(sample_directory):
+    spec = eap_report_files
+    now = datetime.today()
+    previous_day = now - timedelta(days=1)
+    os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_a.json")
+    broker = run_eap_files_test(sample_directory, spec)
+    file_names = [p for p in broker[spec]]
+    assert len(broker[spec]) == 1
+    assert 'file_b.json' in file_names[-1]
+
+
+def test_eap_zero_match_paths(sample_directory):
+    spec = eap_report_files
+    now = datetime.today()
+    previous_day = now - timedelta(days=1)
+    os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_a.json")
+    os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_b.json")
+    broker = run_eap_files_test(sample_directory, spec)
+    assert spec not in broker

insights/util/autology/datasources.py

@@ -18,7 +18,7 @@
 SIMPLE_FILE_TYPE = 'simple_file'
 """ str: Literal constant for a simple_file Spec object """
 GLOB_FILE_TYPE = 'glob_file'
-""" str: Literal constant for a simple_file Spec object """
+""" str: Literal constant for a glob_file Spec object """
 FOREACH_EXECUTE_TYPE = 'foreach_execute'
 """ str: Literal constant for a foreach_execute Spec object """
 CONTAINER_EXECUTE_TYPE = 'container_execute'