From 4be12481c1909dc7e8bc5061deb25c1249196fa2 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Mon, 23 Dec 2024 17:05:25 -0500 Subject: [PATCH 01/10] fix(engine_secret): adjust deserialize config tool, use configure external checks from utils, add flags to scan specific path --- .../src/domain/model/gateway/tool_gateway.py | 5 +- .../src/domain/usecases/secret_scan.py | 44 +++++++------ .../src/domain/usecases/set_input_core.py | 15 ++--- .../trufflehog/trufflehog_run.py | 66 ++++++------------- .../entry_points/entry_point_tool.py | 2 +- 5 files changed, 54 insertions(+), 78 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py index 2803cde2..d57564dd 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py @@ -11,8 +11,9 @@ def run_tool_secret_scan(self, agent_os: str, agent_work_folder: str, repository_name: str, - config_tool: DeserializeConfigTool, + config_tool, secret_tool, secret_external_checks, - agent_tem_dir:str) -> str: + agent_tem_dir:str, + tool) -> str: "run tool secret scan" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index f05712e6..ee1d755c 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -1,8 +1,5 @@ import re -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) + from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) @@ -29,22 +26,25 @@ def __init__( self.tool_deserialize = tool_deserialize self.git_gateway = git_gateway - def process(self, skip_tool, config_tool, secret_tool, dict_args): + def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): + tool = str(tool).lower() finding_list = [] file_path_findings = "" secret_external_checks=dict_args["token_external_checks"] + files_pullrequest = [dict_args["folder_path"]] if skip_tool == False: - self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool.tool_version) - files_pullrequest = self.git_gateway.get_files_pull_request( - self.devops_platform_gateway.get_variable("path_directory"), - self.devops_platform_gateway.get_variable("target_branch"), - config_tool.target_branches, - self.devops_platform_gateway.get_variable("source_branch"), - self.devops_platform_gateway.get_variable("access_token"), - self.devops_platform_gateway.get_variable("organization"), - self.devops_platform_gateway.get_variable("project_name"), - self.devops_platform_gateway.get_variable("repository"), - self.devops_platform_gateway.get_variable("repository_provider")) + self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"]) + if files_pullrequest is None: + files_pullrequest = self.git_gateway.get_files_pull_request( + self.devops_platform_gateway.get_variable("path_directory"), + self.devops_platform_gateway.get_variable("target_branch"), + config_tool["TARGET_BRANCHES"], + self.devops_platform_gateway.get_variable("source_branch"), + self.devops_platform_gateway.get_variable("access_token"), + self.devops_platform_gateway.get_variable("organization"), + self.devops_platform_gateway.get_variable("project_name"), + self.devops_platform_gateway.get_variable("repository"), + self.devops_platform_gateway.get_variable("repository_provider")) findings, file_path_findings = self.tool_gateway.run_tool_secret_scan( files_pullrequest, self.devops_platform_gateway.get_variable("os"), @@ -53,7 +53,8 @@ def process(self, skip_tool, config_tool, secret_tool, dict_args): config_tool, secret_tool, secret_external_checks, - self.devops_platform_gateway.get_variable("temp_directory")) + self.devops_platform_gateway.get_variable("temp_directory"), + tool) finding_list = self.tool_deserialize.get_list_vulnerability( findings, self.devops_platform_gateway.get_variable("os"), @@ -69,12 +70,13 @@ def complete_config_tool(self, dict_args, tool): init_config_tool = self.devops_platform_gateway.get_remote_config( dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json", dict_args["remote_config_branch"] ) - config_tool = DeserializeConfigTool(json_data=init_config_tool, tool=tool) - config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") + init_config_tool['SCOPE_PIPELINE'] = self.devops_platform_gateway.get_variable("pipeline_name") + # config_tool = DeserializeConfigTool(json_data=init_config_tool, tool=tool) + # config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") - skip_tool = bool(re.match(config_tool.ignore_search_pattern, config_tool.scope_pipeline, re.IGNORECASE)) + skip_tool = bool(re.match(init_config_tool["IGNORE_SEARCH_PATTERN"], init_config_tool["SCOPE_PIPELINE"], re.IGNORECASE)) - return config_tool, skip_tool + return init_config_tool, skip_tool def skip_from_exclusion(self, exclusions, skip_tool_isp): """ diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py index 23bde240..a1f034c3 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py @@ -2,12 +2,9 @@ from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_utilities.utils.utils import Utils - +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold class SetInputCore: def __init__( @@ -15,7 +12,7 @@ def __init__( tool_remote: DevopsPlatformGateway, dict_args, tool, - config_tool: DeserializeConfigTool, + config_tool, ): self.tool_remote = tool_remote self.dict_args = dict_args @@ -80,12 +77,12 @@ def set_input_core(self, finding_list): ), threshold_defined=Utils.update_threshold( self, - self.config_tool.level_compliance, + Threshold(self.config_tool['THRESHOLD']), exclusions_config, - self.config_tool.scope_pipeline, + self.config_tool["SCOPE_PIPELINE"], ), path_file_results=finding_list, - custom_message_break_build=self.config_tool.message_info_engine_secret, - scope_pipeline=self.config_tool.scope_pipeline, + custom_message_break_build=self.config_tool["MESSAGE_INFO_ENGINE_SECRET"], + scope_pipeline=self.config_tool["SCOPE_PIPELINE"], stage_pipeline=self.tool_remote.get_variable("stage").capitalize(), ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py index 5559af01..1fa9e8a2 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py @@ -7,11 +7,10 @@ from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) -from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( - GithubApi, -) + from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @@ -44,7 +43,7 @@ def install_tool(self, agent_os, agent_temp_dir, tool_version) -> any: def run_install(self, tool_version): command = f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin v{tool_version}" - res = subprocess.run(command, capture_output=True, shell=True) + subprocess.run(command, capture_output=True, shell=True) def run_install_win(self, agent_temp_dir, tool_version): command_complete = f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://github.com/trufflesecurity/trufflehog/releases/download/v{tool_version}/trufflehog_{tool_version}_windows_amd64.tar.gz' -OutFile {agent_temp_dir}/trufflehog.tar.gz -UseBasicParsing; tar -xzf {agent_temp_dir}/trufflehog.tar.gz -C {agent_temp_dir}; Remove-Item {agent_temp_dir}/trufflehog.tar.gz; $env:Path += '; + {agent_temp_dir}'; & {agent_temp_dir}/trufflehog.exe --version" @@ -62,35 +61,20 @@ def run_tool_secret_scan( config_tool, secret_tool, secret_external_checks, - agent_temp_dir + agent_temp_dir, + tool ): trufflehog_command = "trufflehog" if "Windows" in agent_os: trufflehog_command = f"{agent_temp_dir}/trufflehog.exe" with open(f"{agent_work_folder}/excludedPath.txt", "w") as file: - file.write("\n".join(config_tool.exclude_path)) + file.write("\n".join(config_tool[tool]["EXCLUDE_PATH"])) exclude_path = f"{agent_work_folder}/excludedPath.txt" include_paths = self.config_include_path(files_commits, agent_work_folder, agent_os) - enable_custom_rules = config_tool.enable_custom_rules.lower() - secret = None - github_api = GithubApi() - - if secret_tool is not None: - secret_tmp = secret_tool - secret = github_api.get_installation_access_token( - secret_tmp["github_token"], - config_tool.app_id_github, - config_tool.installation_id_github - ) - elif secret_external_checks is not None: - secret = secret_external_checks.split("github:")[1] if "github" in secret_external_checks else None + enable_custom_rules = config_tool[tool]["ENABLE_CUSTOM_RULES"].lower() + Utils().configurate_external_checks(tool, config_tool, secret_tool, secret_external_checks, agent_work_folder) - if enable_custom_rules == "true" and secret is not None: - self.configurate_external_checks(config_tool, secret) - else: #In case that remote config from tool is enable but in the args dont send any type of secrets. So dont modified command - enable_custom_rules = "false" - - with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool.number_threads) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool[tool]["NUMBER_THREADS"]) as executor: results = executor.map( self.run_trufflehog, [trufflehog_command] * len(include_paths), @@ -99,8 +83,9 @@ def run_tool_secret_scan( include_paths, [repository_name] * len(include_paths), [enable_custom_rules] * len(include_paths), + [agent_os] * len(include_paths) ) - findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder, config_tool) + findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder, config_tool, tool) return findings, file_findings def config_include_path(self, files, agent_work_folder, agent_os): @@ -130,12 +115,15 @@ def run_trufflehog( exclude_path, include_path, repository_name, - enable_custom_rules + enable_custom_rules, + agent_os ): command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --no-update --json" - if str(enable_custom_rules).lower() == "true": - command = command.replace("--no-verification --no-update --json", "--config /tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json") + if str(enable_custom_rules).lower() == "true" and "Windows" in agent_os: + command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json") + if str(enable_custom_rules).lower() == "true" and "Linux" in agent_os: + command = command.replace("--no-verification --no-update --json", f"--config /tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json") result = subprocess.run(command, capture_output=True, shell=True, text=True, encoding='utf-8') return result.stdout.strip() @@ -150,7 +138,7 @@ def decode_output(self, results): result.append(json_obj) return result - def create_file(self, findings, agent_work_folder, config_tool): + def create_file(self, findings, agent_work_folder, config_tool, tool): file_findings = os.path.join(agent_work_folder, "secret_scan_result.json") with open(file_findings, "w") as file: for find in findings: @@ -159,20 +147,8 @@ def create_file(self, findings, agent_work_folder, config_tool): where_text = original_where.replace(agent_work_folder, "") find["SourceMetadata"]["Data"]["Filesystem"]["file"] = where_text find["Id"] = "MISCONFIGURATION_SCANNING" if "exposure" in find["Raw"] else "SECRET_SCANNING" - find["References"] = config_tool.extradata_rules[find["Id"]]["References"] if "SECRET_SCANNING" not in find["Id"] else "N.A" - find["Mitigation"] = config_tool.extradata_rules[find["Id"]]["Mitigation"] if "SECRET_SCANNING" not in find["Id"] else "N.A" + find["References"] = config_tool[tool]["RULES"][find["Id"]]["References"] if "SECRET_SCANNING" not in find["Id"] else "N.A" + find["Mitigation"] = config_tool[tool]["RULES"][find["Id"]]["Mitigation"] if "SECRET_SCANNING" not in find["Id"] else "N.A" json_str = json.dumps(find) file.write(json_str + '\n') - return findings, file_findings - - def configurate_external_checks(self, config_tool, secret): - try: - github_api = GithubApi() - github_api.download_latest_release_assets( - config_tool.external_dir_owner, - config_tool.external_dir_repo, - secret, - "/tmp", - ) - except Exception as ex: - logger.error(f"An error ocurred download external checks {ex}") \ No newline at end of file + return findings, file_findings \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py index 62908f49..25c1b590 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py @@ -11,6 +11,6 @@ def engine_secret_scan(devops_platform_gateway, tool_gateway, dict_args, tool, t secret_scan = SecretScan(tool_gateway, devops_platform_gateway, tool_deserealizator, git_gateway) config_tool, skip_tool_isp = secret_scan.complete_config_tool(dict_args, tool) skip_tool = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) - finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args) + finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args, tool) input_core = SetInputCore(devops_platform_gateway, dict_args, tool, config_tool) return finding_list, input_core.set_input_core(file_path_findings) \ No newline at end of file From 8bac22153c51d109d7199b71f70b5d37e6e0fee4 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Tue, 24 Dec 2024 09:47:35 -0500 Subject: [PATCH 02/10] fix(engine_secret): adjust folder path flag --- .../src/domain/usecases/secret_scan.py | 2 +- .../engine_utilities/utils/utils.py | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index ee1d755c..3500b615 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -31,7 +31,7 @@ def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): finding_list = [] file_path_findings = "" secret_external_checks=dict_args["token_external_checks"] - files_pullrequest = [dict_args["folder_path"]] + files_pullrequest = None if dict_args["folder_path"] is None else [dict_args["folder_path"]] if skip_tool == False: self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"]) if files_pullrequest is None: diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py index 8471d1c5..166b89a5 100755 --- a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py @@ -30,7 +30,7 @@ def unzip_file(self, zip_file_path, extract_path): with zipfile.ZipFile(zip_file_path, "r") as zip_ref: zip_ref.extractall(extract_path) - def configurate_external_checks(self, tool, config_tool, secret_tool, secret_external_checks): + def configurate_external_checks(self, tool, config_tool, secret_tool, secret_external_checks, agent_work_folder="/tmp"): try: agent_env = None secret = None @@ -99,12 +99,20 @@ def configurate_external_checks(self, tool, config_tool, secret_tool, secret_ext config_tool[tool]["APP_ID_GITHUB"], config_tool[tool]["INSTALLATION_ID_GITHUB"] ) if secret.get("github_apps") else secret.get("github_token") - github_api.download_latest_release_assets( - config_tool[tool]["EXTERNAL_DIR_OWNER"], - config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], - github_token, - "/tmp", - ) + if platform.system() in "Windows": + github_api.download_latest_release_assets( + config_tool[tool]["EXTERNAL_DIR_OWNER"], + config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], + github_token, + agent_work_folder + ) + else: + github_api.download_latest_release_assets( + config_tool[tool]["EXTERNAL_DIR_OWNER"], + config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], + github_token, + "/tmp" + ) except Exception as ex: logger.error(f"An error occurred configuring external checks: {ex}") From 2fbdb121172c097c6a5e46001f7324c0d9e425fe Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 26 Dec 2024 09:23:13 -0500 Subject: [PATCH 03/10] fix(secrets): add unit test --- .../engine_secret/src/domain/usecases/secret_scan.py | 2 -- .../test/domain/usecases/test_secret_scan.py | 10 ++++------ .../driven_adapters/trufflehog/test_trufflehog_run.py | 7 +++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index 3500b615..78b5de25 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -71,8 +71,6 @@ def complete_config_tool(self, dict_args, tool): dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json", dict_args["remote_config_branch"] ) init_config_tool['SCOPE_PIPELINE'] = self.devops_platform_gateway.get_variable("pipeline_name") - # config_tool = DeserializeConfigTool(json_data=init_config_tool, tool=tool) - # config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") skip_tool = bool(re.match(init_config_tool["IGNORE_SEARCH_PATTERN"], init_config_tool["SCOPE_PIPELINE"], re.IGNORECASE)) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py index a64f4d02..64e3fc58 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py @@ -64,7 +64,7 @@ def test_process( mock_dict_args = { "remote_config_repo": "example_repo", "remote_config_branch": "", - "folder_path": ".", + "folder_path": None, "environment": "test", "platform": "local", "token_external_checks": "fake_github_token", @@ -83,7 +83,6 @@ def test_process( "vulnerability_data" ] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = ( @@ -91,7 +90,7 @@ def test_process( ) finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, ["vulnerability_data"]) @@ -137,13 +136,12 @@ def test_process_empty( mock_deserialize_gateway_instance.get_list_vulnerability.return_value = [] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = "", "" finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, []) @@ -277,7 +275,7 @@ def test_complete_config_tool( {"remote_config_repo": "repository", "remote_config_branch": ""}, "TRUFFLEHOG" ) - self.assertEqual(config_tool_instance.scope_pipeline, "example_pipeline") + self.assertEqual(config_tool_instance["SCOPE_PIPELINE"], "example_pipeline") if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py index 61b0fc73..378e2373 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py @@ -132,12 +132,11 @@ def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_e } } } - config_tool = DeserializeConfigTool(json_data=json_config_tool, tool="trufflehog") secret_tool = None trufflehog_run = TrufflehogRun() - result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, config_tool, secret_tool, secret_external_checks, agent_temp_dir) + result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, json_config_tool, secret_tool, secret_external_checks, agent_temp_dir, "trufflehog") expected_result = [ {"SourceMetadata": {"Data": {"Filesystem": {"file": "/usr/bin/local/file1.txt", "line": 1}}}, "SourceID": 1, @@ -168,7 +167,7 @@ def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): enable_custom_rules = "false" trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) @@ -179,7 +178,7 @@ def test_run_trufflehog_enable_rules_true(self, mock_subprocess_run): enable_custom_rules = "true" trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) From 3c9fc155c52db51114ba3fb013f83cb961bb82ce Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Mon, 30 Dec 2024 08:53:20 -0500 Subject: [PATCH 04/10] fix(engine_secret): add variables to documentation and fix validations --- .../engine_sast/engine_secret/ConfigTool.json | 10 +++++++++- .../src/applications/runner_engine_core.py | 2 +- .../src/domain/usecases/secret_scan.py | 8 ++++---- .../driven_adapters/trufflehog/trufflehog_run.py | 10 +++++----- .../engine_utilities/utils/utils.py | 14 +++----------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json index 5b3bf28d..3c4c260a 100755 --- a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json +++ b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json @@ -21,6 +21,14 @@ "EXTERNAL_DIR_OWNER": "ExternalOrg", "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", "APP_ID_GITHUB":"", - "INSTALLATION_ID_GITHUB":"" + "INSTALLATION_ID_GITHUB":"", + "USE_EXTERNAL_CHECKS_GIT": false, + "USE_EXTERNAL_CHECKS_DIR": true, + "RULES": { + "MISCONFIGURATION_SCANNING": { + "References": "https://reference.url/", + "Mitigation": "Make sure you only enable the Spring Boot Actuator endpoints that you really need and restrict access to these endpoints." + } + } } } \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py index cd3dd2c0..d39ff755 100755 --- a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py +++ b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py @@ -101,7 +101,7 @@ def get_inputs_from_cli(args): "--folder_path", type=str, required=False, - help="Folder Path to scan, only apply engine_iac, engine_code and engine_dependencies tools", + help="Folder Path to scan, only apply engine_iac, engine_code, engine_secret and engine_dependencies tools", ) parser.add_argument( "-p", diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index 78b5de25..644cb407 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -31,11 +31,11 @@ def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): finding_list = [] file_path_findings = "" secret_external_checks=dict_args["token_external_checks"] - files_pullrequest = None if dict_args["folder_path"] is None else [dict_args["folder_path"]] + files_to_scan = None if dict_args["folder_path"] is None else [dict_args["folder_path"]] if skip_tool == False: self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"]) - if files_pullrequest is None: - files_pullrequest = self.git_gateway.get_files_pull_request( + if files_to_scan is None: + files_to_scan = self.git_gateway.get_files_pull_request( self.devops_platform_gateway.get_variable("path_directory"), self.devops_platform_gateway.get_variable("target_branch"), config_tool["TARGET_BRANCHES"], @@ -46,7 +46,7 @@ def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): self.devops_platform_gateway.get_variable("repository"), self.devops_platform_gateway.get_variable("repository_provider")) findings, file_path_findings = self.tool_gateway.run_tool_secret_scan( - files_pullrequest, + files_to_scan, self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("path_directory"), self.devops_platform_gateway.get_variable("repository"), diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py index 1fa9e8a2..6635156b 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py @@ -120,11 +120,11 @@ def run_trufflehog( ): command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --no-update --json" - if str(enable_custom_rules).lower() == "true" and "Windows" in agent_os: - command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json") - if str(enable_custom_rules).lower() == "true" and "Linux" in agent_os: - command = command.replace("--no-verification --no-update --json", f"--config /tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json") - + if str(enable_custom_rules).lower() == "true": + command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json" if "Windows" in agent_os else + "/tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json" if "Linux" in agent_os else + "--no-verification --no-update --json") + result = subprocess.run(command, capture_output=True, shell=True, text=True, encoding='utf-8') return result.stdout.strip() diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py index 166b89a5..a42d7602 100755 --- a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py @@ -99,21 +99,13 @@ def configurate_external_checks(self, tool, config_tool, secret_tool, secret_ext config_tool[tool]["APP_ID_GITHUB"], config_tool[tool]["INSTALLATION_ID_GITHUB"] ) if secret.get("github_apps") else secret.get("github_token") - if platform.system() in "Windows": - github_api.download_latest_release_assets( + github_api.download_latest_release_assets( config_tool[tool]["EXTERNAL_DIR_OWNER"], config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], github_token, - agent_work_folder + agent_work_folder if platform.system() in "Windows" else "/tmp" ) - else: - github_api.download_latest_release_assets( - config_tool[tool]["EXTERNAL_DIR_OWNER"], - config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], - github_token, - "/tmp" - ) - + except Exception as ex: logger.error(f"An error occurred configuring external checks: {ex}") return agent_env From 929c315ddffe5c1e3e722919965883c13c84fe7c Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Mon, 30 Dec 2024 08:56:14 -0500 Subject: [PATCH 05/10] fix(engine_secret): add variables to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5092d39b..e206c59f 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ pip3 install devsecops-engine-tools ### Scan running - flags (CLI) ```bash -devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code and engine_dependencies"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] +devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code and engine_dependencies, engine_secret"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] ``` ### Structure Remote Config From ff618e742c223df3289c04ff7407ff1169ed6b76 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 2 Jan 2025 14:20:59 -0500 Subject: [PATCH 06/10] fix(engine_secret): change enable_custom_rules variable to boolean --- README.md | 2 +- .../engine_sast/engine_secret/ConfigTool.json | 2 +- .../driven_adapters/trufflehog/trufflehog_run.py | 4 ++-- .../driven_adapters/trufflehog/test_trufflehog_run.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e206c59f..ba315b81 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ pip3 install devsecops-engine-tools ### Scan running - flags (CLI) ```bash -devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code and engine_dependencies, engine_secret"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] +devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code, engine_dependencies and engine_secret"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] ``` ### Structure Remote Config diff --git a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json index 3c4c260a..b90897ce 100755 --- a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json +++ b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json @@ -17,7 +17,7 @@ "VERSION": "1.2.3", "EXCLUDE_PATH": [".git"], "NUMBER_THREADS": 4, - "ENABLE_CUSTOM_RULES" : "True", + "ENABLE_CUSTOM_RULES" : true, "EXTERNAL_DIR_OWNER": "ExternalOrg", "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", "APP_ID_GITHUB":"", diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py index 6635156b..7339f6c4 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py @@ -71,7 +71,7 @@ def run_tool_secret_scan( file.write("\n".join(config_tool[tool]["EXCLUDE_PATH"])) exclude_path = f"{agent_work_folder}/excludedPath.txt" include_paths = self.config_include_path(files_commits, agent_work_folder, agent_os) - enable_custom_rules = config_tool[tool]["ENABLE_CUSTOM_RULES"].lower() + enable_custom_rules = config_tool[tool]["ENABLE_CUSTOM_RULES"] Utils().configurate_external_checks(tool, config_tool, secret_tool, secret_external_checks, agent_work_folder) with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool[tool]["NUMBER_THREADS"]) as executor: @@ -120,7 +120,7 @@ def run_trufflehog( ): command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --no-update --json" - if str(enable_custom_rules).lower() == "true": + if enable_custom_rules: command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json" if "Windows" in agent_os else "/tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json" if "Linux" in agent_os else "--no-verification --no-update --json") diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py index 378e2373..d60ba05f 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py @@ -164,7 +164,7 @@ def test_config_include_path(self, mock_open): @patch('subprocess.run') def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "false" + enable_custom_rules = False trufflehog_run = TrufflehogRun() result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") @@ -175,7 +175,7 @@ def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): @patch('subprocess.run') def test_run_trufflehog_enable_rules_true(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "true" + enable_custom_rules = True trufflehog_run = TrufflehogRun() result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") From e0f1e715df5f49a9ad9c38654070196ecc442c0e Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 2 Jan 2025 14:32:21 -0500 Subject: [PATCH 07/10] fix(engine_secret): delete import and file DeserializeConfigTool --- .../src/domain/model/DeserializeConfigTool.py | 18 ------------------ .../src/domain/model/gateway/tool_gateway.py | 1 - 2 files changed, 19 deletions(-) delete mode 100755 tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py deleted file mode 100755 index 100cd35d..00000000 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py +++ /dev/null @@ -1,18 +0,0 @@ -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold - -class DeserializeConfigTool: - def __init__(self, json_data, tool): - self.ignore_search_pattern = json_data["IGNORE_SEARCH_PATTERN"] - self.message_info_engine_secret = json_data["MESSAGE_INFO_ENGINE_SECRET"] - self.level_compliance = Threshold(json_data['THRESHOLD']) - self.scope_pipeline = '' - self.exclude_path = json_data[tool]["EXCLUDE_PATH"] - self.number_threads = json_data[tool]["NUMBER_THREADS"] - self.target_branches = json_data["TARGET_BRANCHES"] - self.enable_custom_rules = json_data[tool]["ENABLE_CUSTOM_RULES"] - self.external_dir_owner = json_data[tool]["EXTERNAL_DIR_OWNER"] - self.external_dir_repo = json_data[tool]["EXTERNAL_DIR_REPOSITORY"] - self.app_id_github = json_data[tool]["APP_ID_GITHUB"] - self.installation_id_github = json_data[tool]["INSTALLATION_ID_GITHUB"] - self.tool_version = json_data[tool]["VERSION"] - self.extradata_rules = json_data[tool]["RULES"] diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py index d57564dd..37302051 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py @@ -1,5 +1,4 @@ from abc import ABCMeta, abstractmethod -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool class ToolGateway(metaclass=ABCMeta): @abstractmethod From 9640e813f52cbdd691c5c155ce8780db28601ce5 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 2 Jan 2025 14:56:27 -0500 Subject: [PATCH 08/10] fix(engine_secret): delete import DeserializeConfigTool in test --- .../engine_secret/test/domain/usecases/test_secret_scan.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py index 64e3fc58..39e5eb61 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py @@ -1,12 +1,8 @@ import unittest from unittest.mock import patch -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.secret_scan import ( SecretScan, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) class TestSecretScan(unittest.TestCase): def setUp(self) -> None: From ce5370197ae0217ef446c6d2dc5f32f146ec6f48 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 2 Jan 2025 15:03:33 -0500 Subject: [PATCH 09/10] fix(engine_secret): delete import DeserializeConfigTool in test --- .../engine_secret/test/domain/usecases/test_set_input_core.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py index daa4031a..b9701e31 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py @@ -8,9 +8,6 @@ ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.set_input_core import SetInputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool - ) @pytest.fixture From cbc043d943a3900b3b6e2beabe851f5aed7e8865 Mon Sep 17 00:00:00 2001 From: Carlos Javier Lopez Ortega Date: Thu, 2 Jan 2025 15:27:44 -0500 Subject: [PATCH 10/10] fix(engine_secret): delete import DeserializeConfigTool in test --- .../engine_secret/test/domain/usecases/test_set_input_core.py | 3 --- .../driven_adapters/trufflehog/test_trufflehog_run.py | 2 -- .../test/infrastructure/entry_points/test_entry_point_tool.py | 4 +--- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py index b9701e31..da7602fb 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py @@ -1,8 +1,5 @@ import pytest -import json from unittest.mock import MagicMock, Mock -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py index d60ba05f..91172634 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py @@ -1,7 +1,5 @@ -import json import unittest from unittest.mock import patch, MagicMock -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run import TrufflehogRun import os diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py index 56a2ff98..dcc05e3f 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py @@ -1,6 +1,5 @@ import unittest from unittest.mock import Mock, patch -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.entry_points.entry_point_tool import engine_secret_scan class TestEngineSecretScan(unittest.TestCase): @@ -62,13 +61,12 @@ def test_engine_secret_scan(self, MockSetInputCore, MockSecretScan): } } } - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_platform_gateway.get_remote_config.side_effect = [json_exclusion ,json_config, json_exclusion] secret_tool = "secret" skip_tool_isp = False mock_secret_scan_instance = MockSecretScan.return_value - mock_secret_scan_instance.complete_config_tool.return_value = obj_config_tool, skip_tool_isp + mock_secret_scan_instance.complete_config_tool.return_value = json_config, skip_tool_isp mock_devops_platform_gateway.get_variable.side_effect = ["pipeline_name_carlos","pipeline_name_carlos", "pipeline_name", "build"] mock_secret_scan_instance.process.return_value = ([], "")