diff --git a/README.md b/README.md index 5092d39b..ba315b81 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ pip3 install devsecops-engine-tools ### Scan running - flags (CLI) ```bash -devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code and engine_dependencies"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] +devsecops-engine-tools --platform_devops ["local","azure","github"] --remote_config_repo ["remote_config_repo"] --remote_config_branch ["remote_config_branch"] --tool ["engine_iac", "engine_dast", "engine_secret", "engine_dependencies", "engine_container", "engine_risk", "engine_code"] --folder_path ["Folder path scan engine_iac, engine_code, engine_dependencies and engine_secret"] --platform ["k8s","cloudformation","docker", "openapi", "terraform"] --use_secrets_manager ["false", "true"] --use_vulnerability_management ["false", "true"] --send_metrics ["false", "true"] --token_cmdb ["token_cmdb"] --token_vulnerability_management ["token_vulnerability_management"] --token_engine_container ["token_engine_container"] --token_engine_dependencies ["token_engine_dependencies"] --token_external_checks ["token_external_checks"] --xray_mode ["scan", "audit"] --image_to_scan ["image_to_scan"] ``` ### Structure Remote Config diff --git a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json index 5b3bf28d..b90897ce 100755 --- a/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json +++ b/example_remote_config_local/engine_sast/engine_secret/ConfigTool.json @@ -17,10 +17,18 @@ "VERSION": "1.2.3", "EXCLUDE_PATH": [".git"], "NUMBER_THREADS": 4, - "ENABLE_CUSTOM_RULES" : "True", + "ENABLE_CUSTOM_RULES" : true, "EXTERNAL_DIR_OWNER": "ExternalOrg", "EXTERNAL_DIR_REPOSITORY": "DevSecOps_Checks", "APP_ID_GITHUB":"", - "INSTALLATION_ID_GITHUB":"" + "INSTALLATION_ID_GITHUB":"", + "USE_EXTERNAL_CHECKS_GIT": false, + "USE_EXTERNAL_CHECKS_DIR": true, + "RULES": { + "MISCONFIGURATION_SCANNING": { + "References": "https://reference.url/", + "Mitigation": "Make sure you only enable the Spring Boot Actuator endpoints that you really need and restrict access to these endpoints." + } + } } } \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py index cd3dd2c0..d39ff755 100755 --- a/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py +++ b/tools/devsecops_engine_tools/engine_core/src/applications/runner_engine_core.py @@ -101,7 +101,7 @@ def get_inputs_from_cli(args): "--folder_path", type=str, required=False, - help="Folder Path to scan, only apply engine_iac, engine_code and engine_dependencies tools", + help="Folder Path to scan, only apply engine_iac, engine_code, engine_secret and engine_dependencies tools", ) parser.add_argument( "-p", diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py deleted file mode 100755 index 100cd35d..00000000 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/DeserializeConfigTool.py +++ /dev/null @@ -1,18 +0,0 @@ -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold - -class DeserializeConfigTool: - def __init__(self, json_data, tool): - self.ignore_search_pattern = json_data["IGNORE_SEARCH_PATTERN"] - self.message_info_engine_secret = json_data["MESSAGE_INFO_ENGINE_SECRET"] - self.level_compliance = Threshold(json_data['THRESHOLD']) - self.scope_pipeline = '' - self.exclude_path = json_data[tool]["EXCLUDE_PATH"] - self.number_threads = json_data[tool]["NUMBER_THREADS"] - self.target_branches = json_data["TARGET_BRANCHES"] - self.enable_custom_rules = json_data[tool]["ENABLE_CUSTOM_RULES"] - self.external_dir_owner = json_data[tool]["EXTERNAL_DIR_OWNER"] - self.external_dir_repo = json_data[tool]["EXTERNAL_DIR_REPOSITORY"] - self.app_id_github = json_data[tool]["APP_ID_GITHUB"] - self.installation_id_github = json_data[tool]["INSTALLATION_ID_GITHUB"] - self.tool_version = json_data[tool]["VERSION"] - self.extradata_rules = json_data[tool]["RULES"] diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py index 2803cde2..37302051 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/model/gateway/tool_gateway.py @@ -1,5 +1,4 @@ from abc import ABCMeta, abstractmethod -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool class ToolGateway(metaclass=ABCMeta): @abstractmethod @@ -11,8 +10,9 @@ def run_tool_secret_scan(self, agent_os: str, agent_work_folder: str, repository_name: str, - config_tool: DeserializeConfigTool, + config_tool, secret_tool, secret_external_checks, - agent_tem_dir:str) -> str: + agent_tem_dir:str, + tool) -> str: "run tool secret scan" \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py index f05712e6..644cb407 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/secret_scan.py @@ -1,8 +1,5 @@ import re -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) + from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) @@ -29,31 +26,35 @@ def __init__( self.tool_deserialize = tool_deserialize self.git_gateway = git_gateway - def process(self, skip_tool, config_tool, secret_tool, dict_args): + def process(self, skip_tool, config_tool, secret_tool, dict_args, tool): + tool = str(tool).lower() finding_list = [] file_path_findings = "" secret_external_checks=dict_args["token_external_checks"] + files_to_scan = None if dict_args["folder_path"] is None else [dict_args["folder_path"]] if skip_tool == False: - self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool.tool_version) - files_pullrequest = self.git_gateway.get_files_pull_request( - self.devops_platform_gateway.get_variable("path_directory"), - self.devops_platform_gateway.get_variable("target_branch"), - config_tool.target_branches, - self.devops_platform_gateway.get_variable("source_branch"), - self.devops_platform_gateway.get_variable("access_token"), - self.devops_platform_gateway.get_variable("organization"), - self.devops_platform_gateway.get_variable("project_name"), - self.devops_platform_gateway.get_variable("repository"), - self.devops_platform_gateway.get_variable("repository_provider")) + self.tool_gateway.install_tool(self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("temp_directory"), config_tool[tool]["VERSION"]) + if files_to_scan is None: + files_to_scan = self.git_gateway.get_files_pull_request( + self.devops_platform_gateway.get_variable("path_directory"), + self.devops_platform_gateway.get_variable("target_branch"), + config_tool["TARGET_BRANCHES"], + self.devops_platform_gateway.get_variable("source_branch"), + self.devops_platform_gateway.get_variable("access_token"), + self.devops_platform_gateway.get_variable("organization"), + self.devops_platform_gateway.get_variable("project_name"), + self.devops_platform_gateway.get_variable("repository"), + self.devops_platform_gateway.get_variable("repository_provider")) findings, file_path_findings = self.tool_gateway.run_tool_secret_scan( - files_pullrequest, + files_to_scan, self.devops_platform_gateway.get_variable("os"), self.devops_platform_gateway.get_variable("path_directory"), self.devops_platform_gateway.get_variable("repository"), config_tool, secret_tool, secret_external_checks, - self.devops_platform_gateway.get_variable("temp_directory")) + self.devops_platform_gateway.get_variable("temp_directory"), + tool) finding_list = self.tool_deserialize.get_list_vulnerability( findings, self.devops_platform_gateway.get_variable("os"), @@ -69,12 +70,11 @@ def complete_config_tool(self, dict_args, tool): init_config_tool = self.devops_platform_gateway.get_remote_config( dict_args["remote_config_repo"], "engine_sast/engine_secret/ConfigTool.json", dict_args["remote_config_branch"] ) - config_tool = DeserializeConfigTool(json_data=init_config_tool, tool=tool) - config_tool.scope_pipeline = self.devops_platform_gateway.get_variable("pipeline_name") + init_config_tool['SCOPE_PIPELINE'] = self.devops_platform_gateway.get_variable("pipeline_name") - skip_tool = bool(re.match(config_tool.ignore_search_pattern, config_tool.scope_pipeline, re.IGNORECASE)) + skip_tool = bool(re.match(init_config_tool["IGNORE_SEARCH_PATTERN"], init_config_tool["SCOPE_PIPELINE"], re.IGNORECASE)) - return config_tool, skip_tool + return init_config_tool, skip_tool def skip_from_exclusion(self, exclusions, skip_tool_isp): """ diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py index 23bde240..a1f034c3 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/domain/usecases/set_input_core.py @@ -2,12 +2,9 @@ from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_utilities.utils.utils import Utils - +from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold class SetInputCore: def __init__( @@ -15,7 +12,7 @@ def __init__( tool_remote: DevopsPlatformGateway, dict_args, tool, - config_tool: DeserializeConfigTool, + config_tool, ): self.tool_remote = tool_remote self.dict_args = dict_args @@ -80,12 +77,12 @@ def set_input_core(self, finding_list): ), threshold_defined=Utils.update_threshold( self, - self.config_tool.level_compliance, + Threshold(self.config_tool['THRESHOLD']), exclusions_config, - self.config_tool.scope_pipeline, + self.config_tool["SCOPE_PIPELINE"], ), path_file_results=finding_list, - custom_message_break_build=self.config_tool.message_info_engine_secret, - scope_pipeline=self.config_tool.scope_pipeline, + custom_message_break_build=self.config_tool["MESSAGE_INFO_ENGINE_SECRET"], + scope_pipeline=self.config_tool["SCOPE_PIPELINE"], stage_pipeline=self.tool_remote.get_variable("stage").capitalize(), ) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py index 5559af01..7339f6c4 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/driven_adapters/trufflehog/trufflehog_run.py @@ -7,11 +7,10 @@ from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.gateway.tool_gateway import ( ToolGateway, ) -from devsecops_engine_tools.engine_utilities.github.infrastructure.github_api import ( - GithubApi, -) + from devsecops_engine_tools.engine_utilities.utils.logger_info import MyLogger from devsecops_engine_tools.engine_utilities import settings +from devsecops_engine_tools.engine_utilities.utils.utils import Utils logger = MyLogger.__call__(**settings.SETTING_LOGGER).get_logger() @@ -44,7 +43,7 @@ def install_tool(self, agent_os, agent_temp_dir, tool_version) -> any: def run_install(self, tool_version): command = f"curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sh -s -- -b /usr/local/bin v{tool_version}" - res = subprocess.run(command, capture_output=True, shell=True) + subprocess.run(command, capture_output=True, shell=True) def run_install_win(self, agent_temp_dir, tool_version): command_complete = f"powershell -Command [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; [Net.ServicePointManager]::SecurityProtocol; New-Item -Path {agent_temp_dir} -ItemType Directory -Force; Invoke-WebRequest -Uri 'https://github.com/trufflesecurity/trufflehog/releases/download/v{tool_version}/trufflehog_{tool_version}_windows_amd64.tar.gz' -OutFile {agent_temp_dir}/trufflehog.tar.gz -UseBasicParsing; tar -xzf {agent_temp_dir}/trufflehog.tar.gz -C {agent_temp_dir}; Remove-Item {agent_temp_dir}/trufflehog.tar.gz; $env:Path += '; + {agent_temp_dir}'; & {agent_temp_dir}/trufflehog.exe --version" @@ -62,35 +61,20 @@ def run_tool_secret_scan( config_tool, secret_tool, secret_external_checks, - agent_temp_dir + agent_temp_dir, + tool ): trufflehog_command = "trufflehog" if "Windows" in agent_os: trufflehog_command = f"{agent_temp_dir}/trufflehog.exe" with open(f"{agent_work_folder}/excludedPath.txt", "w") as file: - file.write("\n".join(config_tool.exclude_path)) + file.write("\n".join(config_tool[tool]["EXCLUDE_PATH"])) exclude_path = f"{agent_work_folder}/excludedPath.txt" include_paths = self.config_include_path(files_commits, agent_work_folder, agent_os) - enable_custom_rules = config_tool.enable_custom_rules.lower() - secret = None - github_api = GithubApi() - - if secret_tool is not None: - secret_tmp = secret_tool - secret = github_api.get_installation_access_token( - secret_tmp["github_token"], - config_tool.app_id_github, - config_tool.installation_id_github - ) - elif secret_external_checks is not None: - secret = secret_external_checks.split("github:")[1] if "github" in secret_external_checks else None + enable_custom_rules = config_tool[tool]["ENABLE_CUSTOM_RULES"] + Utils().configurate_external_checks(tool, config_tool, secret_tool, secret_external_checks, agent_work_folder) - if enable_custom_rules == "true" and secret is not None: - self.configurate_external_checks(config_tool, secret) - else: #In case that remote config from tool is enable but in the args dont send any type of secrets. So dont modified command - enable_custom_rules = "false" - - with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool.number_threads) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=config_tool[tool]["NUMBER_THREADS"]) as executor: results = executor.map( self.run_trufflehog, [trufflehog_command] * len(include_paths), @@ -99,8 +83,9 @@ def run_tool_secret_scan( include_paths, [repository_name] * len(include_paths), [enable_custom_rules] * len(include_paths), + [agent_os] * len(include_paths) ) - findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder, config_tool) + findings, file_findings = self.create_file(self.decode_output(results), agent_work_folder, config_tool, tool) return findings, file_findings def config_include_path(self, files, agent_work_folder, agent_os): @@ -130,13 +115,16 @@ def run_trufflehog( exclude_path, include_path, repository_name, - enable_custom_rules + enable_custom_rules, + agent_os ): command = f"{trufflehog_command} filesystem {agent_work_folder + '/' + repository_name} --include-paths {include_path} --exclude-paths {exclude_path} --no-verification --no-update --json" - if str(enable_custom_rules).lower() == "true": - command = command.replace("--no-verification --no-update --json", "--config /tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json") - + if enable_custom_rules: + command = command.replace("--no-verification --no-update --json", f"--config {agent_work_folder}//rules//trufflehog//custom-rules.yaml --no-verification --no-update --json" if "Windows" in agent_os else + "/tmp/rules/trufflehog/custom-rules.yaml --no-verification --no-update --json" if "Linux" in agent_os else + "--no-verification --no-update --json") + result = subprocess.run(command, capture_output=True, shell=True, text=True, encoding='utf-8') return result.stdout.strip() @@ -150,7 +138,7 @@ def decode_output(self, results): result.append(json_obj) return result - def create_file(self, findings, agent_work_folder, config_tool): + def create_file(self, findings, agent_work_folder, config_tool, tool): file_findings = os.path.join(agent_work_folder, "secret_scan_result.json") with open(file_findings, "w") as file: for find in findings: @@ -159,20 +147,8 @@ def create_file(self, findings, agent_work_folder, config_tool): where_text = original_where.replace(agent_work_folder, "") find["SourceMetadata"]["Data"]["Filesystem"]["file"] = where_text find["Id"] = "MISCONFIGURATION_SCANNING" if "exposure" in find["Raw"] else "SECRET_SCANNING" - find["References"] = config_tool.extradata_rules[find["Id"]]["References"] if "SECRET_SCANNING" not in find["Id"] else "N.A" - find["Mitigation"] = config_tool.extradata_rules[find["Id"]]["Mitigation"] if "SECRET_SCANNING" not in find["Id"] else "N.A" + find["References"] = config_tool[tool]["RULES"][find["Id"]]["References"] if "SECRET_SCANNING" not in find["Id"] else "N.A" + find["Mitigation"] = config_tool[tool]["RULES"][find["Id"]]["Mitigation"] if "SECRET_SCANNING" not in find["Id"] else "N.A" json_str = json.dumps(find) file.write(json_str + '\n') - return findings, file_findings - - def configurate_external_checks(self, config_tool, secret): - try: - github_api = GithubApi() - github_api.download_latest_release_assets( - config_tool.external_dir_owner, - config_tool.external_dir_repo, - secret, - "/tmp", - ) - except Exception as ex: - logger.error(f"An error ocurred download external checks {ex}") \ No newline at end of file + return findings, file_findings \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py index 62908f49..25c1b590 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/src/infrastructure/entry_points/entry_point_tool.py @@ -11,6 +11,6 @@ def engine_secret_scan(devops_platform_gateway, tool_gateway, dict_args, tool, t secret_scan = SecretScan(tool_gateway, devops_platform_gateway, tool_deserealizator, git_gateway) config_tool, skip_tool_isp = secret_scan.complete_config_tool(dict_args, tool) skip_tool = secret_scan.skip_from_exclusion(exclusions, skip_tool_isp) - finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args) + finding_list, file_path_findings = secret_scan.process(skip_tool, config_tool, secret_tool, dict_args, tool) input_core = SetInputCore(devops_platform_gateway, dict_args, tool, config_tool) return finding_list, input_core.set_input_core(file_path_findings) \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py index a64f4d02..39e5eb61 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_secret_scan.py @@ -1,12 +1,8 @@ import unittest from unittest.mock import patch -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.secret_scan import ( SecretScan, ) -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool, -) class TestSecretScan(unittest.TestCase): def setUp(self) -> None: @@ -64,7 +60,7 @@ def test_process( mock_dict_args = { "remote_config_repo": "example_repo", "remote_config_branch": "", - "folder_path": ".", + "folder_path": None, "environment": "test", "platform": "local", "token_external_checks": "fake_github_token", @@ -83,7 +79,6 @@ def test_process( "vulnerability_data" ] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = ( @@ -91,7 +86,7 @@ def test_process( ) finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, ["vulnerability_data"]) @@ -137,13 +132,12 @@ def test_process_empty( mock_deserialize_gateway_instance.get_list_vulnerability.return_value = [] - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_gateway_instance.get_remote_config.return_value = json_config mock_devops_gateway_instance.get_variable.return_value = "example_pipeline" mock_tool_gateway_instance.run_tool_secret_scan.return_value = "", "" finding_list, file_path_findings = secret_scan.process( - False, obj_config_tool, secret_tool, mock_dict_args + False, json_config, secret_tool, mock_dict_args, "trufflehog" ) self.assertEqual(finding_list, []) @@ -277,7 +271,7 @@ def test_complete_config_tool( {"remote_config_repo": "repository", "remote_config_branch": ""}, "TRUFFLEHOG" ) - self.assertEqual(config_tool_instance.scope_pipeline, "example_pipeline") + self.assertEqual(config_tool_instance["SCOPE_PIPELINE"], "example_pipeline") if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py index daa4031a..da7602fb 100644 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/domain/usecases/test_set_input_core.py @@ -1,16 +1,10 @@ import pytest -import json from unittest.mock import MagicMock, Mock -from devsecops_engine_tools.engine_core.src.domain.model.input_core import InputCore -from devsecops_engine_tools.engine_core.src.domain.model.threshold import Threshold from devsecops_engine_tools.engine_core.src.domain.model.gateway.devops_platform_gateway import ( DevopsPlatformGateway, ) from devsecops_engine_tools.engine_core.src.domain.model.exclusions import Exclusions from devsecops_engine_tools.engine_sast.engine_secret.src.domain.usecases.set_input_core import SetInputCore -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import ( - DeserializeConfigTool - ) @pytest.fixture diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py index 61b0fc73..91172634 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/driven_adapters/trufflehog/test_trufflehog_run.py @@ -1,7 +1,5 @@ -import json import unittest from unittest.mock import patch, MagicMock -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.driven_adapters.trufflehog.trufflehog_run import TrufflehogRun import os @@ -132,12 +130,11 @@ def test_run_tool_secret_scan(self, mock_config_include_path, mock_thread_pool_e } } } - config_tool = DeserializeConfigTool(json_data=json_config_tool, tool="trufflehog") secret_tool = None trufflehog_run = TrufflehogRun() - result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, config_tool, secret_tool, secret_external_checks, agent_temp_dir) + result, file_findings = trufflehog_run.run_tool_secret_scan(files_commits, agent_os, agent_work_folder, repository_name, json_config_tool, secret_tool, secret_external_checks, agent_temp_dir, "trufflehog") expected_result = [ {"SourceMetadata": {"Data": {"Filesystem": {"file": "/usr/bin/local/file1.txt", "line": 1}}}, "SourceID": 1, @@ -165,10 +162,10 @@ def test_config_include_path(self, mock_open): @patch('subprocess.run') def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "false" + enable_custom_rules = False trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) @@ -176,10 +173,10 @@ def test_run_trufflehog_enable_rules_false(self, mock_subprocess_run): @patch('subprocess.run') def test_run_trufflehog_enable_rules_true(self, mock_subprocess_run): mock_subprocess_run.return_value.stdout.strip.return_value = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' - enable_custom_rules = "true" + enable_custom_rules = True trufflehog_run = TrufflehogRun() - result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules) + result = trufflehog_run.run_trufflehog('trufflehog', '/usr/local', '/usr/temp/excludedPath.txt', '/usr/temp/includePath0.txt', 'NU00000_Repo_Test', enable_custom_rules, "trufflehog") expected_result = '{"SourceMetadata":{"Data":{"Filesystem":{"file":"/usr/bin/local/file1.txt","line":1}}},"SourceID":1,"SourceType":15,"SourceName":"trufflehog - filesystem","DetectorType":17,"DetectorName":"URI","DecoderName":"BASE64","Verified":false,"Raw":"https://admin:admin@the-internet.herokuapp.com","RawV2":"https://admin:admin@the-internet.herokuapp.com/basic_auth","Redacted":"https://admin:********@the-internet.herokuapp.com","ExtraData":null,"StructuredData":null}' self.assertEqual(result, expected_result) diff --git a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py index 56a2ff98..dcc05e3f 100755 --- a/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py +++ b/tools/devsecops_engine_tools/engine_sast/engine_secret/test/infrastructure/entry_points/test_entry_point_tool.py @@ -1,6 +1,5 @@ import unittest from unittest.mock import Mock, patch -from devsecops_engine_tools.engine_sast.engine_secret.src.domain.model.DeserializeConfigTool import DeserializeConfigTool from devsecops_engine_tools.engine_sast.engine_secret.src.infrastructure.entry_points.entry_point_tool import engine_secret_scan class TestEngineSecretScan(unittest.TestCase): @@ -62,13 +61,12 @@ def test_engine_secret_scan(self, MockSetInputCore, MockSecretScan): } } } - obj_config_tool = DeserializeConfigTool(json_config, 'trufflehog') mock_devops_platform_gateway.get_remote_config.side_effect = [json_exclusion ,json_config, json_exclusion] secret_tool = "secret" skip_tool_isp = False mock_secret_scan_instance = MockSecretScan.return_value - mock_secret_scan_instance.complete_config_tool.return_value = obj_config_tool, skip_tool_isp + mock_secret_scan_instance.complete_config_tool.return_value = json_config, skip_tool_isp mock_devops_platform_gateway.get_variable.side_effect = ["pipeline_name_carlos","pipeline_name_carlos", "pipeline_name", "build"] mock_secret_scan_instance.process.return_value = ([], "") diff --git a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py index 8471d1c5..a42d7602 100755 --- a/tools/devsecops_engine_tools/engine_utilities/utils/utils.py +++ b/tools/devsecops_engine_tools/engine_utilities/utils/utils.py @@ -30,7 +30,7 @@ def unzip_file(self, zip_file_path, extract_path): with zipfile.ZipFile(zip_file_path, "r") as zip_ref: zip_ref.extractall(extract_path) - def configurate_external_checks(self, tool, config_tool, secret_tool, secret_external_checks): + def configurate_external_checks(self, tool, config_tool, secret_tool, secret_external_checks, agent_work_folder="/tmp"): try: agent_env = None secret = None @@ -100,12 +100,12 @@ def configurate_external_checks(self, tool, config_tool, secret_tool, secret_ext config_tool[tool]["INSTALLATION_ID_GITHUB"] ) if secret.get("github_apps") else secret.get("github_token") github_api.download_latest_release_assets( - config_tool[tool]["EXTERNAL_DIR_OWNER"], - config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], - github_token, - "/tmp", - ) - + config_tool[tool]["EXTERNAL_DIR_OWNER"], + config_tool[tool]["EXTERNAL_DIR_REPOSITORY"], + github_token, + agent_work_folder if platform.system() in "Windows" else "/tmp" + ) + except Exception as ex: logger.error(f"An error occurred configuring external checks: {ex}") return agent_env