diff --git a/insights/specs/datasources/__init__.py b/insights/specs/datasources/__init__.py index 4113b50bf2..273caf42a2 100644 --- a/insights/specs/datasources/__init__.py +++ b/insights/specs/datasources/__init__.py @@ -8,6 +8,10 @@ parser/combiner/component required by a custom datasource must be included in the YAML template to ensure it is loaded. """ + +import os +import time + DEFAULT_SHELL_TIMEOUT = 10 """ int: Default timeout in seconds for ctx.shell_out() commands, must be provided as an arg """ @@ -46,3 +50,26 @@ def get_running_commands(ps, ctx, commands): continue ret.add(which[0]) if which else None return sorted(ret) + + +def get_recent_files(target_path, last_modify_hours): + """ + Get all recent updated files or created + + Arguments: + target_path (string): target path to search + last_modify_hours (int): Specify the recent hours + + Returns: + list: List of files that updated or creates just in last_modify_hours hours. + """ + result_files = [] + if os.path.exists(target_path): + for parent_path, _, report_files in os.walk(target_path): + current_time = time.time() + for one_file in report_files: + t_full_file = os.path.join(parent_path, one_file) + file_time = os.path.getmtime(t_full_file) + if (current_time - file_time) // 3600 < last_modify_hours: + result_files.append(t_full_file) + return result_files diff --git a/insights/specs/datasources/eap_reports.py b/insights/specs/datasources/eap_reports.py new file mode 100644 index 0000000000..d792b62933 --- /dev/null +++ b/insights/specs/datasources/eap_reports.py @@ -0,0 +1,27 @@ +""" +Custom datasource to get all updated/created eap reports in last 24 hours +""" + +import os + +from insights.core.context import HostContext +from insights.core.plugins import datasource +from insights.core.exceptions import SkipComponent +from insights.specs.datasources import get_recent_files + + +@datasource(HostContext) +def eap_report_files(broker): + """ + Get all updated/created eap reports in last 24 hours. + + Returns: + function: return the list of EAP reports file in last 24 hours. + """ + ctx = broker[HostContext] + root = ctx.root + EAP_REPORTS_JSON_PATH = os.path.join(root, "var/tmp/insights-runtimes/uploads/") + final_files = get_recent_files(EAP_REPORTS_JSON_PATH, 24) + if final_files: + return [f[len(root):] for f in final_files] + raise SkipComponent diff --git a/insights/specs/default.py b/insights/specs/default.py index 6cd67afb41..63730b87fd 100644 --- a/insights/specs/default.py +++ b/insights/specs/default.py @@ -24,7 +24,7 @@ from insights.specs import Specs from insights.specs.datasources import ( aws, awx_manage, client_metadata, cloud_init, corosync as corosync_ds, - dir_list, ethernet, httpd, ipcs, intersystems, kernel, kernel_module_list, + dir_list, eap_reports, ethernet, httpd, ipcs, intersystems, kernel, kernel_module_list, leapp, ls, lpstat, luks_devices, machine_ids, malware_detection, md5chk, mount as mount_ds, package_provides, ps as ps_datasource, rpm_pkgs, sap, satellite_missed_queues, ssl_certificate, sys_fs_cgroup_memory, @@ -189,7 +189,7 @@ class DefaultSpecs(Specs): dse_ldif = glob_file("/etc/dirsrv/*/dse.ldif") du_dirs = foreach_execute(dir_list.du_dir_list, "/bin/du -s -k %s") # empty filter duplicate_machine_id = machine_ids.dup_machine_id_info - eap_json_reports = glob_file(r"/var/tmp/insights-runtimes/uploads/*.json") + eap_json_reports = foreach_collect(eap_reports.eap_report_files, "%s") engine_log = simple_file("/var/log/ovirt-engine/engine.log") etc_journald_conf = simple_file(r"etc/systemd/journald.conf") etc_journald_conf_d = glob_file(r"etc/systemd/journald.conf.d/*.conf") diff --git a/insights/specs/sos_archive.py b/insights/specs/sos_archive.py index b159445bcb..17b13bf87e 100644 --- a/insights/specs/sos_archive.py +++ b/insights/specs/sos_archive.py @@ -77,6 +77,7 @@ class SosSpecs(Specs): docker_list_images = simple_file("sos_commands/docker/docker_images") docker_storage = simple_file("/etc/sysconfig/docker-storage") dumpe2fs_h = glob_file("sos_commands/filesys/dumpe2fs_-h_*") + eap_json_reports = glob_file(r"/var/tmp/insights-runtimes/uploads/*.json") ethtool = glob_file("sos_commands/networking/ethtool_*", ignore="ethtool_-.*") ethtool_S = glob_file("sos_commands/networking/ethtool_-S_*") ethtool_T = glob_file("sos_commands/networking/ethtool_-T_*") diff --git a/insights/tests/datasources/test_eap_reports.py b/insights/tests/datasources/test_eap_reports.py new file mode 100644 index 0000000000..a729934827 --- /dev/null +++ b/insights/tests/datasources/test_eap_reports.py @@ -0,0 +1,68 @@ +import os +import pytest +import shutil +import tempfile +from datetime import datetime, timedelta + +from insights.core import dr +from insights.core.spec_factory import foreach_collect +from insights.core.context import HostContext +from insights.specs.datasources.eap_reports import eap_report_files + + +@pytest.fixture +def sample_directory(scope="module"): + tmpdir = tempfile.mkdtemp() + os.makedirs(tmpdir + "/var/tmp/insights-runtimes/uploads/") + for f in ["file_a.json", "file_b.json"]: + fd = open(tmpdir + "/var/tmp/insights-runtimes/uploads/" + f, "w") + fd.write('{"name":"example_json", "count_number":30}') + fd.close() + yield tmpdir + shutil.rmtree(tmpdir) + + +def run_eap_files_test(sample_directory, spec): + ctx = HostContext() + ctx.root = sample_directory + broker = dr.Broker() + broker[HostContext] = ctx + broker = dr.run([spec], broker) + return broker + + +def test_eap_match_results(sample_directory): + spec = eap_report_files + broker = run_eap_files_test(sample_directory, spec) + file_names = [p.split('/')[-1] for p in broker[spec]] + assert 'file_a.json' in file_names + assert 'file_b.json' in file_names + + +def test_foreach_eap_match_results(sample_directory): + spec = foreach_collect(eap_report_files, "%s") + broker = run_eap_files_test(sample_directory, spec) + file_names = [p.file_name for p in broker[spec]] + assert 'file_a.json' in file_names + assert 'file_b.json' in file_names + + +def test_eap_one_match_paths(sample_directory): + spec = eap_report_files + now = datetime.today() + previous_day = now - timedelta(days=1) + os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_a.json") + broker = run_eap_files_test(sample_directory, spec) + file_names = [p for p in broker[spec]] + assert len(broker[spec]) == 1 + assert 'file_b.json' in file_names[-1] + + +def test_eap_zero_match_paths(sample_directory): + spec = eap_report_files + now = datetime.today() + previous_day = now - timedelta(days=1) + os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_a.json") + os.system('touch -d "{0}" '.format(previous_day.isoformat()) + sample_directory + "/var/tmp/insights-runtimes/uploads/file_b.json") + broker = run_eap_files_test(sample_directory, spec) + assert spec not in broker diff --git a/insights/util/autology/datasources.py b/insights/util/autology/datasources.py index 5019297c81..3e8e12a763 100644 --- a/insights/util/autology/datasources.py +++ b/insights/util/autology/datasources.py @@ -18,7 +18,7 @@ SIMPLE_FILE_TYPE = 'simple_file' """ str: Literal constant for a simple_file Spec object """ GLOB_FILE_TYPE = 'glob_file' -""" str: Literal constant for a simple_file Spec object """ +""" str: Literal constant for a glob_file Spec object """ FOREACH_EXECUTE_TYPE = 'foreach_execute' """ str: Literal constant for a foreach_execute Spec object """ CONTAINER_EXECUTE_TYPE = 'container_execute'