From ae1354c11acb287e642bf74b089e8ad6ab6fdf3c Mon Sep 17 00:00:00 2001 From: vh Date: Wed, 5 Feb 2020 09:47:56 +0200 Subject: [PATCH 1/2] TDM api integration tool --- .../SOCPrimeTDM/requirements.txt | 2 + .../SOCPrimeTDM/tdm_api_integration_tool.py | 342 ++++++++++++++++++ sigma_hunting_app/bin/setup.py | 9 +- sigma_hunting_app/bin/update.sh | 28 +- sigma_hunting_app/default/settings.conf | 1 + sigma_hunting_app/default/setup.xml | 11 +- 6 files changed, 388 insertions(+), 5 deletions(-) create mode 100644 sigma_hunting_app/SOCPrimeTDM/requirements.txt create mode 100755 sigma_hunting_app/SOCPrimeTDM/tdm_api_integration_tool.py diff --git a/sigma_hunting_app/SOCPrimeTDM/requirements.txt b/sigma_hunting_app/SOCPrimeTDM/requirements.txt new file mode 100644 index 0000000..b436c27 --- /dev/null +++ b/sigma_hunting_app/SOCPrimeTDM/requirements.txt @@ -0,0 +1,2 @@ +requests==2.22.0 +pyyaml==5.3 diff --git a/sigma_hunting_app/SOCPrimeTDM/tdm_api_integration_tool.py b/sigma_hunting_app/SOCPrimeTDM/tdm_api_integration_tool.py new file mode 100755 index 0000000..bce694f --- /dev/null +++ b/sigma_hunting_app/SOCPrimeTDM/tdm_api_integration_tool.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +__author__ = 'Andriy Yatsynyak' +__version__ = '2.0' +__company__ = 'SOC Prime' + + +import argparse +import os +import locale +import datetime +import json +import logging +import logging.handlers +from copy import copy +import requests +import yaml + + +locale.setlocale(locale.LC_ALL, '') + + +BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +USE_DATETIME = '' + +LAST_DATETIME = datetime.datetime.utcnow().replace(microsecond=0) +FIRST_DATETIME = LAST_DATETIME - datetime.timedelta(days=30) + + +BASE_URL = 'https://api.tdm.socprime.com/v1/' +PREFIX_SEARCH = 'search-sigmas' +PREFIX_MAPPING = 'custom-field-mapping' + + +RES_FRM_FILE = 'txt' # default +FRM_FILES = ['yaml'] +FRM_FILES.append(RES_FRM_FILE) + +RES_DIR = os.path.join(BASE_DIR, 'output') + + +API_KEY = '' +MAPPING = False +CACHE_FILE_DATETIME = 'last_datetime.json' + + +FRM_DATETIME = '%Y-%m-%dT%H:%M:%S' +KEY_DATE_END = 'date_end' +KEY_DATE_START = 'date_start' + + +class Logger: + def __init__(self, logger_name): + logging.captureWarnings(True) + self.logger = logging.getLogger(logger_name) + self.logger.setLevel(logging.INFO) + + self.logPath = BASE_DIR + + if not os.path.exists(self.logPath): + self.logPath = os.path.dirname(os.path.abspath(__file__)) + + LOG_FILENAME = os.path.normpath('{}/{}.log'.format(self.logPath, logger_name)) + + fh = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=5242880, backupCount=10) + fh.setLevel(logging.INFO) + fh.setFormatter(logging.Formatter('[%(asctime)s][%(name)s][%(levelname)s] %(message)s')) + self.logger.addHandler(fh) + + def debug(self, msg): + self.log(logging.DEBUG, msg) + + def info(self, msg): + self.log(logging.INFO, msg) + + def warning(self, msg): + self.log(logging.WARNING, msg) + + def error(self, msg): + self.log(logging.ERROR,msg) + + def critical(self, msg): + self.log(logging.CRITICAL, msg) + + def exception(self, msg): + self.logger.exception(msg) + + def log(self, level, msg): + msg = str(msg).replace('%', '') + self.logger.log(level, str(msg) +' %s', '') + + +def query_api(logger, **kwargs): + headers = { + 'client_secret_id': API_KEY + } + headers.update(**kwargs) + + response = requests.get(f'{BASE_URL}{PREFIX_SEARCH}/', headers=headers) + if not response.ok: + logger.info(f'response data: {response.status_code} {response.content} filter: {kwargs}') + return False, response.content + return True, response.json() + + +def get_mapping_api(logger): + headers = { + 'client_secret_id': API_KEY + } + + response = requests.get(f'{BASE_URL}{PREFIX_MAPPING}/', headers=headers) + if not response.ok: + logger.info(f'response data: {response.status_code} {response.content}') + return False, [] + return True, response.json() + + +def convert_name(s: str) -> str: + return s.lower().strip().replace(' ', '_').replace('/', '_').replace('(', '').replace(')', '').replace('.', '').replace('-', '_') + + +def save_info_in_file(siem_list: list): + for siem in siem_list: + siem_type = siem['siem_type'] + name = siem['case']['name'] + text = siem['sigma']['text'] + name_file = f'{convert_name(name)}_{convert_name(siem_type)}' + + path = os.path.join(RES_DIR, f'{name_file}.{RES_FRM_FILE}') + if os.path.exists(path): + if not os.path.isfile(path): + print(f'error: this dir {path}') + continue + elif not os.access(path, os.W_OK): + print(f"error: this file {path} and script " + f"{os.path.basename(__file__)} don't have rules of write") + continue + + if RES_FRM_FILE == 'yaml': + with open(path, 'w') as f: + yaml.dump(text, f, default_flow_style=False) + else: + with open(path, 'w') as f: + f.write(text) + + +def convert_str_into_datetime(s: str) -> datetime: + return datetime.datetime.strptime(s, FRM_DATETIME) + + +def change_last_datetime(date_json): + datetime_end = convert_str_into_datetime(date_json[KEY_DATE_END]) + date_json[KEY_DATE_END] = (datetime_end + datetime.timedelta(days=1)).strftime(FRM_DATETIME) + date_json[KEY_DATE_START] = datetime_end.replace(second=0).strftime(FRM_DATETIME) + return date_json + + +def save_last_datetime(date_json): + with open(CACHE_FILE_DATETIME, 'w') as json_file: + json.dump(date_json, json_file) + + +def is_date(logger, string: str) -> bool: + try: + datetime.datetime.strptime(string, FRM_DATETIME) + except ValueError: + logger.error(f'incorrect data format {string}, should be {FRM_DATETIME}') + return False + return True + + +def validate_json_frm(logger, data_json) -> bool: + if not all(k in data_json for k in (KEY_DATE_END, KEY_DATE_START)): + return False + if not all(is_date(logger, string) for string in data_json.values()): + return False + return True + + +def pre_validate_global_variable(logger): + if not FRM_FILES: + variable_msg = { + 'cache_file_datetime': CACHE_FILE_DATETIME, + 'frm_files': FRM_FILES + } + + msg = """Error some variable empty or aren't correct: + CACHE_FILE_DATETIME - {cache_file_datetime}, + FRM_FILES - {frm_files} + """ + + logger.error(msg.format(**variable_msg)) + exit(msg.format(**variable_msg)) + + +def post_validate_global_variable(logger): + if not (BASE_URL and API_KEY and CACHE_FILE_DATETIME and FRM_DATETIME and RES_DIR): + variable_msg = { + 'base_url': BASE_URL, + 'api_key': 'XXXXXXXXXXXXX', + 'cache_file_datetime': CACHE_FILE_DATETIME, + 'frm_datetime': FRM_DATETIME, + 'res_dir': RES_DIR, + 'res_frm_file': RES_FRM_FILE + } + + msg = """Error some variable empty or aren't correct: + URL - {base_url} + API_KEY - {api_key} + CACHE_FILE_DATETIME - {cache_file_datetime} + FRM_DATETIME_FILTER - {frm_datetime} + RES_DIR - {res_dir} + RES_FRM_FILE - {res_frm_file} + """ + + logger.error(msg.format(**variable_msg)) + variable_msg['api_key'] = API_KEY + exit(msg.format(**variable_msg)) + + msg_err = 'error:' + if not os.path.isdir(RES_DIR): + try: + os.mkdir(RES_DIR) + except OSError as e: + logger.error(f'{msg_err} to try create dir for path: {RES_DIR} error: {e}') + exit(f'{msg_err} to try create dir for path: {RES_DIR}') + elif not os.access(RES_DIR, os.W_OK): + logger.error(f'{msg_err} this dir not have writeable rule: {RES_DIR}') + exit(f'{msg_err} this dir not have writeable rule: {RES_DIR}') + + if USE_DATETIME and not is_date(logger, USE_DATETIME): + logger.error(f'{msg_err} not correct variable {USE_DATETIME} for this format {FRM_DATETIME}') + exit(f'{msg_err} not correct variable {USE_DATETIME} for this format {FRM_DATETIME}') + + +def run_query_apis(logger): + mapping_list = list() + logger = logger + logger.info(f'current last time: {LAST_DATETIME}') + + if MAPPING: + status_mapping, mapping_list = get_mapping_api(logger) + logger.info(f'information mapping list {mapping_list}') + status_mapping or exit('error: to try get sigma mapping') + + while True: + if not os.path.isfile(CACHE_FILE_DATETIME) or os.path.isfile(CACHE_FILE_DATETIME) \ + and not os.stat(CACHE_FILE_DATETIME).st_size: + + date_filter = dict.fromkeys((KEY_DATE_END, KEY_DATE_START), + USE_DATETIME or FIRST_DATETIME.strftime(FRM_DATETIME)) + date_filter = change_last_datetime(date_filter) + else: + with open(CACHE_FILE_DATETIME) as json_file: + date_filter = json.load(json_file) + + if not validate_json_frm(logger, date_filter): + logger.error(f'not validate format file {CACHE_FILE_DATETIME} json-frm: {date_filter}') + raise Exception(f'not validate file {CACHE_FILE_DATETIME}, need remove this file {CACHE_FILE_DATETIME}') + + logger.info(f'show date filter: {date_filter}') + if MAPPING and mapping_list: + kwargs = copy(date_filter) + for mapping_name in mapping_list: + kwargs['mapping_name'] = mapping_name + status, data_json = query_api(logger, **kwargs) + status and save_info_in_file(data_json) + else: + status, data_json = query_api(logger, **date_filter) + status and save_info_in_file(data_json) + + datetime_obj = convert_str_into_datetime(date_filter[KEY_DATE_END]) + if datetime_obj >= LAST_DATETIME: + date_filter[KEY_DATE_END] = LAST_DATETIME.strftime(FRM_DATETIME) + date_filter[KEY_DATE_START] = (LAST_DATETIME.replace(second=0) - + datetime.timedelta(minutes=5)).strftime(FRM_DATETIME) + save_last_datetime(date_filter) + logger.info(f'finish script: {datetime_obj} >= {LAST_DATETIME}') + return + date_filter = change_last_datetime(date_filter) + save_last_datetime(date_filter) + + +def valid_str_date(s: str) -> str or None: + now_date = LAST_DATETIME.replace(second=0) + try: + input_date = datetime.datetime.strptime(s, '%Y-%m-%d') + if now_date <= input_date: + raise AttributeError + return input_date.strftime(FRM_DATETIME) + except ValueError: + msg = "Not a valid date: '{}'".format(s) + raise argparse.ArgumentTypeError(msg) + except AttributeError: + msg = "Not a valid date, this future date: '{}'".format(s) + raise argparse.ArgumentTypeError(msg) + + +if __name__ == '__main__': + logger = Logger('run_script') + pre_validate_global_variable(logger) + + parser = argparse.ArgumentParser( + description=f'List commands for "{os.path.basename(__file__)}" script.') + parser.add_argument('-d', '--path-dir', + type=str, + help='full path to directory') + parser.add_argument('-k', '--api-key', + type=str, + help='secret api key') + parser.add_argument('-f', '--format-file', + default='txt', + choices=list(FRM_FILES), + help='save format file:') + parser.add_argument('-s', + '--startdate', + help='the start date - format: YYYY-MM-DD', + required=False, + type=valid_str_date) + parser.add_argument('-m', + '--mapping-field', + action='store_true', + default=False, + help='get sigma mapping field rules') + + args = parser.parse_args() + RES_DIR = args.path_dir if args.path_dir else RES_DIR + RES_FRM_FILE = args.format_file if args.format_file else RES_FRM_FILE + API_KEY = args.api_key if args.api_key else API_KEY + MAPPING = args.mapping_field if args.mapping_field else MAPPING + + if args.startdate: + USE_DATETIME = args.startdate + if os.path.exists(CACHE_FILE_DATETIME): + try: + os.remove(CACHE_FILE_DATETIME) + except: + logger.error(f"can't remove file {CACHE_FILE_DATETIME}") + exit(f"error: can't remove file {CACHE_FILE_DATETIME}") + + post_validate_global_variable(logger) + run_query_apis(logger) + diff --git a/sigma_hunting_app/bin/setup.py b/sigma_hunting_app/bin/setup.py index 9d89462..c7cf8aa 100755 --- a/sigma_hunting_app/bin/setup.py +++ b/sigma_hunting_app/bin/setup.py @@ -7,7 +7,7 @@ class ConfigApp(admin.MConfigHandler): ''' def setup(self): if self.requestedAction == admin.ACTION_EDIT: - for arg in ['repository', 'folder']: + for arg in ['repository', 'folder', 'tdm_api_key']: self.supportedArgs.addOptArg(arg) ''' @@ -36,6 +36,8 @@ def handleList(self, confInfo): val = '' if key in ['folder'] and val in [None, '']: val = '' + if key in ['tdm_api_key'] and val in [None, '']: + val = '' confInfo[stanza].append(key, val) ''' @@ -50,7 +52,10 @@ def handleEdit(self, confInfo): self.callerArgs.data['repository'][0] = '' if self.callerArgs.data['folder'][0] in [None, '']: - self.callerArgs.data['folder'][0] = '' + self.callerArgs.data['folder'][0] = '' + + if self.callerArgs.data['tdm_api_key'][0] in [None, '']: + self.callerArgs.data['tdm_api_key'][0] = '' ''' diff --git a/sigma_hunting_app/bin/update.sh b/sigma_hunting_app/bin/update.sh index 36c2787..19fb4de 100755 --- a/sigma_hunting_app/bin/update.sh +++ b/sigma_hunting_app/bin/update.sh @@ -2,6 +2,16 @@ unset PYTHONPATH unset LD_LIBRARY_PATH +##### +SHELL=/bin/bash +USER=root +MAIL=/var/spool/mail/root +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin +PWD=/root +HOME=/root +LOGNAME=root +##### + if [ -z $SPLUNK_HOME ]; then splunk_path='/opt/splunk' else @@ -16,6 +26,7 @@ FILE=$splunk_path/etc/apps/sigma_hunting_app/local/settings.conf repository=$(cat $splunk_path/etc/apps/sigma_hunting_app/default/settings.conf | grep repository | cut -d ' ' -f3) folder=$(cat $splunk_path/etc/apps/sigma_hunting_app/default/settings.conf | grep folder | cut -d ' ' -f3) +tdm_api_key=$(cat $splunk_path/etc/apps/sigma_hunting_app/default/settings.conf | grep folder | cut -d ' ' -f3) if [ -f $FILE ]; then check_repository=$(cat $splunk_path/etc/apps/sigma_hunting_app/local/settings.conf | grep repository | cut -d ' ' -f3) @@ -27,13 +38,28 @@ if [ -f $FILE ]; then if ! [ -z $check_folder ]; then folder=$check_folder fi + + check_tdm_api_key=$(cat $splunk_path/etc/apps/sigma_hunting_app/local/settings.conf | grep tdm_api_key | cut -d ' ' -f3) + if ! [ -z $check_tdm_api_key ]; then + tdm_api_key=$check_tdm_api_key + fi + fi cd $splunk_path/etc/apps/sigma_hunting_app/Sigma2SplunkAlert rm -rf rules/* cd rules git clone $repository > /dev/null 2>&1 -cd .. + +if [ -n "${tdm_api_key}" ];then + { + mkdir -p $splunk_path/etc/apps/sigma_hunting_app/Sigma2SplunkAlert/rules/$folder/SOCPrimeTDMrules + cd $splunk_path/etc/apps/sigma_hunting_app/SOCPrimeTDM/ + ./tdm_api_integration_tool.py -d $splunk_path/etc/apps/sigma_hunting_app/Sigma2SplunkAlert/rules/$folder/SOCPrimeTDMrules -k ${tdm_api_key} -f yaml -s $(date +%Y-%m-%d -d "1 year ago") + } +fi + +cd $splunk_path/etc/apps/sigma_hunting_app/Sigma2SplunkAlert diff --git a/sigma_hunting_app/default/settings.conf b/sigma_hunting_app/default/settings.conf index ddc5021..a3600be 100644 --- a/sigma_hunting_app/default/settings.conf +++ b/sigma_hunting_app/default/settings.conf @@ -1,3 +1,4 @@ [settings] repository = https://github.com/P4T12ICK/Sigma-Rule-Repository.git folder = +tdm_api_key = diff --git a/sigma_hunting_app/default/setup.xml b/sigma_hunting_app/default/setup.xml index 1401e0c..d9fb022 100644 --- a/sigma_hunting_app/default/setup.xml +++ b/sigma_hunting_app/default/setup.xml @@ -15,6 +15,13 @@ text - - + + + Enter your TDM API key here. Leave empty if you do not want to use SOC Prime TDM sigma rule feed. +To get a valid TDM API key go to tdm.socprime.com. + + + text + + From e2feba4e7ac261a6137dbefbbf666f45d86d329e Mon Sep 17 00:00:00 2001 From: vh Date: Thu, 6 Feb 2020 09:25:05 +0200 Subject: [PATCH 2/2] update setupt title --- sigma_hunting_app/bin/setup.py | 1 - sigma_hunting_app/default/setup.xml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sigma_hunting_app/bin/setup.py b/sigma_hunting_app/bin/setup.py index c7cf8aa..c81f799 100755 --- a/sigma_hunting_app/bin/setup.py +++ b/sigma_hunting_app/bin/setup.py @@ -56,7 +56,6 @@ def handleEdit(self, confInfo): if self.callerArgs.data['tdm_api_key'][0] in [None, '']: self.callerArgs.data['tdm_api_key'][0] = '' - ''' Since we are using a conf file to store parameters, diff --git a/sigma_hunting_app/default/setup.xml b/sigma_hunting_app/default/setup.xml index d9fb022..7085024 100644 --- a/sigma_hunting_app/default/setup.xml +++ b/sigma_hunting_app/default/setup.xml @@ -16,7 +16,7 @@ text - + Enter your TDM API key here. Leave empty if you do not want to use SOC Prime TDM sigma rule feed. To get a valid TDM API key go to tdm.socprime.com.