Skip to content

Commit

Permalink
refactor: remove legacy collection (#4009)
Browse files Browse the repository at this point in the history 
- and remove all dependent codes and relevant tests
- update the script that generates the filters.yaml
- see RHINENG-6982

Signed-off-by: Xiangce Liu <[email protected]>
  • Loading branch information
@xiangce
xiangce authored Sep 30, 2024
1 parent 6892a58 commit 7ff8c5e
Show file tree
Hide file tree
Showing 31 changed files with 230 additions and 1,988 deletions.
12 changes: 0 additions & 12 deletions insights/client/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,17 +373,6 @@ def delete_tmpdir(self):
logger.debug("Deleting temp directory %s." % (self.tmpdir))
shutil.rmtree(self.tmpdir, True)

@_net
def update_rules(self):
"""
returns (dict): new client rules
"""
if self.config.offline or not self.config.auto_update:
logger.debug("Bypassing rule update due to config "
"running in offline mode or auto updating turned off.")
else:
return client.update_rules(self.config, self.connection)

@_net
def collect(self):
# return collection results
Expand Down Expand Up @@ -650,7 +639,6 @@ def list_specs(self):
logger.info("For a full list of insights-core datasources, please refer to https://insights-core.readthedocs.io/en/latest/specs_catalog.html")
logger.info("The items in General Datasources can be selected for omission by adding them to the 'components' section of file-redaction.yaml")
logger.info("When specifying these items in file-redaction.yaml, they must be prefixed with 'insights.specs.default.DefaultSpecs.', i.e. 'insights.specs.default.DefaultSpecs.httpd_V'")
logger.info("This information applies only to Insights Core collection. To use Core collection, set core_collect=True in %s", self.config.conf)

@_net
def checkin(self):
Expand Down
39 changes: 0 additions & 39 deletions insights/client/archive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import atexit
import logging
import os
import re
import shlex
import shutil
import subprocess
Expand All @@ -18,7 +17,6 @@
from signal import SIGTERM, signal

from insights.client.constants import InsightsConstants as constants
from insights.client.insights_spec import InsightsFile, InsightsCommand
from insights.client.utilities import determine_hostname, _expand_paths, write_data_to_file

logger = logging.getLogger(__name__)
Expand All @@ -34,7 +32,6 @@ class InsightsArchive(object):
tmp_dir - a temporary directory in /var/tmp
archive_dir - location to collect archive data inside tmp_dir
archive_name - filename of the archive and archive_dir
cmd_dir - insights_commands directory inside archive_dir
compressor - tar compression flag to use
tar_file - path of the final archive file
"""
Expand Down Expand Up @@ -67,7 +64,6 @@ def __init__(self, config):
# classic collection and compliance needs these
# core collection will set "archive_dir" on its own
self.archive_dir = None
self.cmd_dir = None

self.compressor = config.compressor
self.archive_stored = None
Expand All @@ -91,18 +87,6 @@ def create_archive_dir(self):
self.archive_dir = archive_dir
return self.archive_dir

def create_command_dir(self):
"""
Create the "insights_commands" dir
"""
self.create_archive_dir()
cmd_dir = os.path.join(self.archive_dir, "insights_commands")
logger.debug('Creating command directory %s...', cmd_dir)
if not os.path.exists(cmd_dir):
os.makedirs(cmd_dir, 0o700)
self.cmd_dir = cmd_dir
return self.cmd_dir

def get_full_archive_path(self, path):
"""
Returns the full archive path
Expand Down Expand Up @@ -203,29 +187,6 @@ def delete_archive_dir(self):
logger.debug("Deleting: " + self.archive_dir)
shutil.rmtree(self.archive_dir, True)

def add_to_archive(self, spec, cleaner=None, spec_info=None):
'''
Add files and commands to archive
Use InsightsSpec.get_output() to get data
'''
ab_regex = [
"^timeout: failed to run command .+: No such file or directory$",
"^Missing Dependencies:"
]
if isinstance(spec, InsightsCommand):
archive_path = os.path.join(self.cmd_dir, spec.archive_path.lstrip('/'))
if isinstance(spec, InsightsFile):
archive_path = self.get_full_archive_path(spec.archive_path.lstrip('/'))
output = spec.get_output()
if output and not any(re.search(rg, output) for rg in ab_regex):
write_data_to_file(output, archive_path)
if cleaner:
no_obfuscate, no_redact = spec_info if spec_info else ([], False)
# Redact and Obfuscate for data collection
cleaner.clean_file(archive_path,
no_obfuscate=no_obfuscate,
no_redact=no_redact)

def add_metadata_to_archive(self, metadata, meta_path):
'''
Add metadata to archive
Expand Down
21 changes: 2 additions & 19 deletions insights/client/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,8 @@
determine_hostname,
get_version_info)
from .collection_rules import InsightsUploadConf
from .data_collector import DataCollector
from .core_collector import CoreCollector
from .connection import InsightsConnection
from .archive import InsightsArchive
from .support import registration_check
from .constants import InsightsConstants as constants

Expand Down Expand Up @@ -308,15 +306,6 @@ def get_machine_id():
return generate_machine_id()


def update_rules(config, pconn):
if not pconn:
raise ValueError('ERROR: Cannot update rules in --offline mode. '
'Disable auto_update in config file.')

pc = InsightsUploadConf(config, conn=pconn)
return pc.get_conf_update()


def get_branch_info(config):
"""
Get branch info for a system
Expand All @@ -335,15 +324,9 @@ def collect(config):
All the heavy lifting done here
"""
pc = InsightsUploadConf(config)
archive = InsightsArchive(config)

if config.core_collect:
dc = CoreCollector(config, archive)
else:
dc = DataCollector(config, archive, spec_conf=pc.get_conf_file())
dc = CoreCollector(config)

logger.info('Starting to collect Insights data for %s',
determine_hostname(config.display_name))
logger.info('Starting to collect Insights data for %s' % determine_hostname(config.display_name))

dc.run_collection(pc.get_rm_conf(),
get_branch_info(config),
Expand Down
191 changes: 7 additions & 184 deletions insights/client/collection_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,17 @@
Rules for data collection
"""
from __future__ import absolute_import
import hashlib

import json
import logging
import six
import os
import requests
import yaml
import six
import stat
import yaml

from six.moves import configparser as ConfigParser

from tempfile import NamedTemporaryFile
from . import crypto
from .constants import InsightsConstants as constants
from insights.client.constants import InsightsConstants as constants

APP_NAME = constants.app_name
logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -95,26 +93,18 @@ def verify_permissions(f):

class InsightsUploadConf(object):
"""
Insights spec configuration from uploader.json
Insights spec configuration
"""

def __init__(self, config, conn=None):
"""
Load config from parent
Load configuration from parent
"""
self.config = config
self.fallback_file = constants.collection_fallback_file
self.remove_file = config.remove_file
self.redaction_file = config.redaction_file
self.content_redaction_file = config.content_redaction_file
self.tags_file = config.tags_file
self.collection_rules_file = constants.collection_rules_file
self.collection_rules_url = self.config.collection_rules_url
self.gpg = self.config.gpg

# initialize an attribute to store the content of uploader.json
# once it is loaded and verified
self.uploader_json = None

# set rm_conf as a class attribute so we can observe it
# in create_report
Expand All @@ -125,173 +115,6 @@ def __init__(self, config, conn=None):
# since new format is favored.
self.using_new_format = True

if conn:
if self.collection_rules_url is None:
if config.legacy_upload:
self.collection_rules_url = conn.base_url + '/v1/static/uploader.v2.json'
else:
self.collection_rules_url = conn.base_url.split('/platform')[0] + '/v1/static/uploader.v2.json'
self.conn = conn

def validate_gpg_sig(self, path, sig=None):
"""
Validate the collection rules
"""
logger.debug("Verifying GPG signature of Insights configuration")

if sig is None:
sig = path + ".asc"

result = crypto.verify_gpg_signed_file(
file=path, signature=sig,
key=constants.pub_gpg_path,
)
if not result.ok:
logger.error("ERROR: Unable to validate GPG signature: %s", path)
return False
else:
logger.debug("GPG signature verified")
return True

def try_disk(self, path, gpg=True):
"""
Try to load json off disk
"""
if not os.path.isfile(path):
return

if not gpg or self.validate_gpg_sig(path):
stream = open(path, 'r')
json_stream = stream.read()
if len(json_stream):
try:
json_config = json.loads(json_stream)
return json_config
except ValueError:
logger.error("ERROR: Invalid JSON in %s", path)
return False
else:
logger.warn("WARNING: %s was an empty file", path)
return

def get_collection_rules(self, raw=False):
"""
Download the collection rules
"""
logger.debug("Attemping to download collection rules from %s",
self.collection_rules_url)

try:
req = self.conn.get(
self.collection_rules_url, headers=({'accept': 'text/plain'}))

if req.status_code == 200:
logger.debug("Successfully downloaded collection rules")

json_response = NamedTemporaryFile()
json_response.write(req.text.encode('utf-8'))
json_response.file.flush()
else:
logger.error("ERROR: Could not download dynamic configuration")
logger.error("Debug Info: \nConf status: %s", req.status_code)
logger.error("Debug Info: \nConf message: %s", req.text)
return None
except requests.ConnectionError as e:
logger.error(
"ERROR: Could not download dynamic configuration: %s", e)
return None

if self.gpg:
self.get_collection_rules_gpg(json_response)

self.write_collection_data(self.collection_rules_file, req.text)

if raw:
return req.text
else:
return json.loads(req.text)

def fetch_gpg(self):
logger.debug("Attemping to download collection "
"rules GPG signature from %s",
self.collection_rules_url + ".asc")

headers = ({'accept': 'text/plain'})
config_sig = self.conn.get(self.collection_rules_url + '.asc',
headers=headers)
if config_sig.status_code == 200:
logger.debug("Successfully downloaded GPG signature")
return config_sig.text
else:
logger.error("ERROR: Download of GPG Signature failed!")
logger.error("Sig status: %s", config_sig.status_code)
return False

def get_collection_rules_gpg(self, collection_rules):
"""
Download the collection rules gpg signature
"""
sig_text = self.fetch_gpg()
sig_response = NamedTemporaryFile(suffix=".asc")
sig_response.write(sig_text.encode('utf-8'))
sig_response.file.flush()
self.validate_gpg_sig(collection_rules.name, sig_response.name)
self.write_collection_data(self.collection_rules_file + ".asc", sig_text)

def write_collection_data(self, path, data):
"""
Write collections rules to disk
"""
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
fd = os.open(path, flags, 0o600)
with os.fdopen(fd, 'w') as dyn_conf_file:
dyn_conf_file.write(data)

def get_conf_file(self):
"""
Get config from local config file, first try cache, then fallback.
"""
if self.uploader_json:
return self.uploader_json

for conf_file in [self.collection_rules_file, self.fallback_file]:
logger.debug("trying to read conf from: " + conf_file)
conf = self.try_disk(conf_file, self.gpg)

if not conf:
continue

version = conf.get('version', None)
if version is None:
raise ValueError("ERROR: Could not find version in json")

conf['file'] = conf_file
logger.debug("Success reading config")
logger.debug(json.dumps(conf))
self.uploader_json = conf
return conf

raise RuntimeError("ERROR: Unable to download conf or read it from disk!")

def get_conf_update(self):
"""
Get updated config from URL.
"""
dyn_conf = self.get_collection_rules()

if not dyn_conf:
return self.get_conf_file()

version = dyn_conf.get('version', None)
if version is None:
raise ValueError("ERROR: Could not find version in json")

dyn_conf['file'] = self.collection_rules_file
logger.debug("Success reading config")
config_hash = hashlib.sha1(json.dumps(dyn_conf).encode('utf-8')).hexdigest()
logger.debug('sha1 of config: %s', config_hash)
return dyn_conf

def get_rm_conf_old(self):
"""
Get excluded files config from remove_file.
Expand Down
Loading

0 comments on commit 7ff8c5e

Please sign in to comment.