From 20e9faaf912ca5808bedd50e3c564ccfd41c2520 Mon Sep 17 00:00:00 2001 From: Barak Fatal <35402131+bo156@users.noreply.github.com> Date: Sun, 20 Oct 2024 17:46:58 +0300 Subject: [PATCH 01/14] feat(terraform_plan): Support after_unknown evaluation of complex attributes (#6784) * Added function to handle complex case of after_unknwon in terraform plan * Added test --- checkov/terraform/plan_parser.py | 25 ++++++++++ .../resources/plan_after_unknown/tfplan.json | 49 +++++++++++++++++++ tests/terraform/parser/test_plan_parser.py | 12 +++++ 3 files changed, 86 insertions(+) create mode 100644 tests/terraform/parser/resources/plan_after_unknown/tfplan.json diff --git a/checkov/terraform/plan_parser.py b/checkov/terraform/plan_parser.py index 041b442cb5f..d37065cc6d4 100644 --- a/checkov/terraform/plan_parser.py +++ b/checkov/terraform/plan_parser.py @@ -234,6 +234,31 @@ def _eval_after_unknown(changes: dict[str, Any], resource_conf: dict[str, Any]) # In these cases, policies checking the existence of a value will succeed, # but policies checking for concrete values will fail resource_conf[k] = _clean_simple_type_list([TRUE_AFTER_UNKNOWN]) + elif isinstance(v, list) and len(v) == 1 and isinstance(v[0], dict): + _handle_complex_after_unknown(k, resource_conf, v) + + +def _handle_complex_after_unknown(k: str, resource_conf: dict[str, Any], v: Any) -> None: + """ + Handles a case of an inner key generated with "after_unknown" value. + Example: + ` + after_unknown: { + "logging_config": [ + { + "bucket": true + } + ], + } + ` + """ + inner_keys = list(v[0].keys()) + for inner_key in inner_keys: + if inner_key in (START_LINE, END_LINE): + # skip inner checkov keys + continue + if inner_key not in resource_conf[k]: + resource_conf[k][0][inner_key] = _clean_simple_type_list([TRUE_AFTER_UNKNOWN]) def _find_child_modules( diff --git a/tests/terraform/parser/resources/plan_after_unknown/tfplan.json b/tests/terraform/parser/resources/plan_after_unknown/tfplan.json new file mode 100644 index 00000000000..92851fa94c9 --- /dev/null +++ b/tests/terraform/parser/resources/plan_after_unknown/tfplan.json @@ -0,0 +1,49 @@ +{ + "format_version": "1.0", + "terraform_version": "1.1.3", + "planned_values": { + "root_module": { + "resources": [ + { + "address": "aws_cloudfront_distribution.cloudfront", + "mode": "managed", + "type": "aws_cloudfront_distribution", + "name": "cloudfront", + "provider_name": "registry.terraform.io/hashicorp/aws", + "schema_version": 1, + "values": { + "logging_config": [ + { + "include_cookies": false, + "prefix": "cloudfront" + } + ] + } + } + ] + } + }, + "resource_changes": [ + { + "address": "aws_cloudfront_distribution.cloudfront", + "mode": "managed", + "type": "aws_cloudfront_distribution", + "name": "cloudfront", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": { + "actions": [ + "create" + ], + "before": null, + "after": null, + "after_unknown": { + "logging_config": [ + { + "bucket": true + } + ] + } + } + } + ] +} diff --git a/tests/terraform/parser/test_plan_parser.py b/tests/terraform/parser/test_plan_parser.py index ad7e343c2c0..be193d981dc 100644 --- a/tests/terraform/parser/test_plan_parser.py +++ b/tests/terraform/parser/test_plan_parser.py @@ -1,9 +1,11 @@ import os import unittest from pathlib import Path +from unittest import mock from pytest_mock import MockerFixture +from checkov.common.util.consts import TRUE_AFTER_UNKNOWN from checkov.terraform.plan_parser import parse_tf_plan from checkov.common.parsers.node import StrNode @@ -80,6 +82,16 @@ def test_module_with_connected_resources(self): resource_attributes = next(iter(resource_definition.values())) self.assertTrue(resource_attributes['references_']) + @mock.patch.dict(os.environ, {"EVAL_TF_PLAN_AFTER_UNKNOWN": "True"}) + def test_after_unknown_handling(self): + current_dir = os.path.dirname(os.path.realpath(__file__)) + valid_plan_path = current_dir + "/resources/plan_after_unknown/tfplan.json" + tf_definition, _ = parse_tf_plan(valid_plan_path, {}) + file_resource_definition = tf_definition['resource'][0] + resource_definition = next(iter(file_resource_definition.values())) + resource_attributes = next(iter(resource_definition.values())) + self.assertEqual(resource_attributes['logging_config'][0]["bucket"], [TRUE_AFTER_UNKNOWN]) + def test_large_file(mocker: MockerFixture): # given test_file = Path(__file__).parent / "resources/plan_encodings/tfplan_mac_utf8.json" From c7146bfe1fa7e4e7c963d0658e0c6778c4981ed7 Mon Sep 17 00:00:00 2001 From: Barak Fatal <35402131+bo156@users.noreply.github.com> Date: Sun, 20 Oct 2024 17:46:58 +0300 Subject: [PATCH 02/14] feat(terraform_plan): Support after_unknown evaluation of complex attributes (#6784) * Added function to handle complex case of after_unknwon in terraform plan * Added test --- checkov/version.py | 2 +- kubernetes/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/version.py b/checkov/version.py index 7fb74b97b81..a31b2973bbd 100644 --- a/checkov/version.py +++ b/checkov/version.py @@ -1 +1 @@ -version = '3.2.267' +version = '3.2.268' diff --git a/kubernetes/requirements.txt b/kubernetes/requirements.txt index b4e956f9d58..3b7f587ca9f 100644 --- a/kubernetes/requirements.txt +++ b/kubernetes/requirements.txt @@ -1 +1 @@ -checkov==3.2.267 +checkov==3.2.268 From 398496e0970b117015c2385c3f349780e2f8f782 Mon Sep 17 00:00:00 2001 From: gruebel Date: Sun, 20 Oct 2024 23:01:30 +0000 Subject: [PATCH 03/14] chore: update release notes --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12851ae7328..27a87e5a34f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # CHANGELOG -## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.267...HEAD) +## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.268...HEAD) + +## [3.2.268](https://github.com/bridgecrewio/checkov/compare/3.2.267...3.2.268) - 2024-10-20 + +### Feature + +- **terraform_plan:** Support after_unknown evaluation of complex attributes - [#6784](https://github.com/bridgecrewio/checkov/pull/6784) ## [3.2.267](https://github.com/bridgecrewio/checkov/compare/3.2.266...3.2.267) - 2024-10-16 From a873b2019588bf5917758d65eb12331a2e585768 Mon Sep 17 00:00:00 2001 From: Taylor <28880387+tsmithv11@users.noreply.github.com> Date: Sun, 20 Oct 2024 22:39:32 -0700 Subject: [PATCH 04/14] fix(terraform): Fix crash when version isn't a float (#6783) * Catch non-floats * Add tests * fix test --- .../gcp/GKEPodSecurityPolicyEnabled.py | 5 +- .../checks/resource/gcp/GKEUseCosImage.py | 5 +- .../main.tf | 7 +++ .../gcp/example_GKEUseCosImage/main.tf | 58 +++++++++++++++++++ 4 files changed, 73 insertions(+), 2 deletions(-) diff --git a/checkov/terraform/checks/resource/gcp/GKEPodSecurityPolicyEnabled.py b/checkov/terraform/checks/resource/gcp/GKEPodSecurityPolicyEnabled.py index 314759268ff..266043f7f99 100644 --- a/checkov/terraform/checks/resource/gcp/GKEPodSecurityPolicyEnabled.py +++ b/checkov/terraform/checks/resource/gcp/GKEPodSecurityPolicyEnabled.py @@ -26,7 +26,10 @@ def scan_resource_conf(self, conf: dict[str, list[Any]]) -> CheckResult: splitter = raw.split(".") if len(splitter) >= 2: str_version = splitter[0] + "." + splitter[1] - version = float(str_version) + try: + version = float(str_version) + except (ValueError, IndexError): + return CheckResult.UNKNOWN if version < 1.25: if conf.get('pod_security_policy_config') and isinstance(conf.get('pod_security_policy_config'), list): policy = conf.get('pod_security_policy_config')[0] diff --git a/checkov/terraform/checks/resource/gcp/GKEUseCosImage.py b/checkov/terraform/checks/resource/gcp/GKEUseCosImage.py index 5995d9771ed..6ef354c255a 100644 --- a/checkov/terraform/checks/resource/gcp/GKEUseCosImage.py +++ b/checkov/terraform/checks/resource/gcp/GKEUseCosImage.py @@ -21,7 +21,10 @@ def scan_resource_conf(self, conf): splitter = raw.split(".") if len(splitter) >= 2: - version = float(splitter[0] + "." + splitter[1]) + try: + version = float(splitter[0] + "." + splitter[1]) + except (ValueError, IndexError): + return CheckResult.UNKNOWN if version >= 1.24: return CheckResult.UNKNOWN diff --git a/tests/terraform/checks/resource/gcp/example_GKEPodSecurityPolicyEnabled/main.tf b/tests/terraform/checks/resource/gcp/example_GKEPodSecurityPolicyEnabled/main.tf index 2cce34d39fa..0920dac8e04 100644 --- a/tests/terraform/checks/resource/gcp/example_GKEPodSecurityPolicyEnabled/main.tf +++ b/tests/terraform/checks/resource/gcp/example_GKEPodSecurityPolicyEnabled/main.tf @@ -24,5 +24,12 @@ resource "google_container_cluster" "fail2" { } } +resource "google_container_cluster" "unknown3" { + min_master_version = "not_a_float" + pod_security_policy_config { + enabled = false + } +} + diff --git a/tests/terraform/checks/resource/gcp/example_GKEUseCosImage/main.tf b/tests/terraform/checks/resource/gcp/example_GKEUseCosImage/main.tf index 236c141fde1..f9f92fb9dd9 100644 --- a/tests/terraform/checks/resource/gcp/example_GKEUseCosImage/main.tf +++ b/tests/terraform/checks/resource/gcp/example_GKEUseCosImage/main.tf @@ -172,3 +172,61 @@ resource "google_container_node_pool" "unknown" { version = "1.25.10-gke.36" zone = "us-west1" } + + +resource "google_container_node_pool" "unknown2" { + autoscaling { + max_node_count = "4" + min_node_count = "1" + } + + cluster = google_container_cluster.tfer.name + initial_node_count = "2" + location = "us-west1" + + management { + auto_repair = "true" + auto_upgrade = "true" + } + + max_pods_per_node = "110" + name = "async-pool-2" + + node_config { + disk_size_gb = "400" + disk_type = "pd-ssd" + + labels = { + async = "true" + } + + local_ssd_count = "0" + machine_type = "custom-32-65536" + + metadata = { + async = "true" + disable-legacy-endpoints = "true" + } + + oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] + preemptible = "false" + service_account = "default" + + shielded_instance_config { + enable_integrity_monitoring = "true" + enable_secure_boot = "true" + } + } + + node_count = "1" + node_locations = ["us-west1-b", "us-west1-a"] + project = "test-project" + + upgrade_settings { + max_surge = "1" + max_unavailable = "0" + } + + version = lookup(each.value,"auto_upgrade",True) ? "" : lookup(each.value,"version",latest) + zone = "us-west1" +} From 1da7e33ed85420564683a6ddddcbee7c69cc2b93 Mon Sep 17 00:00:00 2001 From: Taylor <28880387+tsmithv11@users.noreply.github.com> Date: Sun, 20 Oct 2024 22:39:32 -0700 Subject: [PATCH 05/14] fix(terraform): Fix crash when version isn't a float (#6783) * Catch non-floats * Add tests * fix test --- checkov/version.py | 2 +- kubernetes/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/version.py b/checkov/version.py index a31b2973bbd..805e511295c 100644 --- a/checkov/version.py +++ b/checkov/version.py @@ -1 +1 @@ -version = '3.2.268' +version = '3.2.269' diff --git a/kubernetes/requirements.txt b/kubernetes/requirements.txt index 3b7f587ca9f..a59cb02f2ff 100644 --- a/kubernetes/requirements.txt +++ b/kubernetes/requirements.txt @@ -1 +1 @@ -checkov==3.2.268 +checkov==3.2.269 From e2fa39dd51b0535929f35ff9aee5b9f8d231e2ef Mon Sep 17 00:00:00 2001 From: LirShindalman <49649760+lirshindalman@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:36:47 +0300 Subject: [PATCH 06/14] feat(arm): add variable and parameters edges and rendering (#6787) * vars and parameters edges * vars and parameters edges * add tests * add tests * add tests * lint * lint * lint * lint * lint * lint * . * . * . --- .../graph_components/block_types.py | 3 +- checkov/arm/graph_builder/local_graph.py | 69 ++++++++++++++++++- .../variable_rendering/__init__.py | 0 .../variable_rendering/renderer.py | 64 +++++++++++++++++ checkov/bicep/graph_builder/local_graph.py | 2 +- .../variable_rendering/renderer.py | 2 +- checkov/bicep/utils.py | 25 +------ checkov/common/graph/graph_builder/utils.py | 23 +++++++ tests/arm/graph_builder/test_local_graph.py | 5 +- tests/arm/rendering/__init__.py | 0 tests/arm/rendering/test_rendering.json | 48 +++++++++++++ tests/arm/rendering/test_rendering.py | 24 +++++++ tests/arm/test_graph_manager.py | 4 +- 13 files changed, 235 insertions(+), 34 deletions(-) create mode 100644 checkov/arm/graph_builder/variable_rendering/__init__.py create mode 100644 checkov/arm/graph_builder/variable_rendering/renderer.py create mode 100644 tests/arm/rendering/__init__.py create mode 100644 tests/arm/rendering/test_rendering.json create mode 100644 tests/arm/rendering/test_rendering.py diff --git a/checkov/arm/graph_builder/graph_components/block_types.py b/checkov/arm/graph_builder/graph_components/block_types.py index bf8022f4e54..73d46082d8c 100644 --- a/checkov/arm/graph_builder/graph_components/block_types.py +++ b/checkov/arm/graph_builder/graph_components/block_types.py @@ -8,4 +8,5 @@ @dataclass class BlockType(CommonBlockType): - PARAMETER: Literal["parameter"] = "parameter" + PARAMETER: Literal["parameters"] = "parameters" + VARIABLE: Literal["variables"] = "variables" diff --git a/checkov/arm/graph_builder/local_graph.py b/checkov/arm/graph_builder/local_graph.py index 35c37fede82..93ce8fab980 100644 --- a/checkov/arm/graph_builder/local_graph.py +++ b/checkov/arm/graph_builder/local_graph.py @@ -1,12 +1,14 @@ from __future__ import annotations import logging +import re from typing import Any, TYPE_CHECKING from checkov.arm.graph_builder.graph_components.block_types import BlockType from checkov.arm.graph_builder.graph_components.blocks import ArmBlock +from checkov.arm.graph_builder.variable_rendering.renderer import ArmVariableRenderer from checkov.arm.utils import ArmElements -from checkov.common.graph.graph_builder import CustomAttributes +from checkov.common.graph.graph_builder import CustomAttributes, Edge from checkov.common.graph.graph_builder.local_graph import LocalGraph from checkov.common.util.consts import START_LINE, END_LINE from checkov.common.util.data_structures_utils import pickle_deepcopy @@ -21,6 +23,7 @@ def __init__(self, definitions: dict[str, dict[str, Any]]) -> None: self.vertices: list[ArmBlock] = [] self.definitions = definitions self.vertices_by_path_and_id: dict[tuple[str, str], int] = {} + self.vertices_by_name: dict[str, int] = {} def build_graph(self, render_variables: bool = False) -> None: self._create_vertices() @@ -28,16 +31,49 @@ def build_graph(self, render_variables: bool = False) -> None: self._create_edges() logging.debug(f"[ArmLocalGraph] created {len(self.edges)} edges") + if render_variables: + renderer = ArmVariableRenderer(self) + renderer.render_variables_from_local_graph() def _create_vertices(self) -> None: for file_path, definition in self.definitions.items(): self._create_parameter_vertices(file_path=file_path, parameters=definition.get(ArmElements.PARAMETERS)) self._create_resource_vertices(file_path=file_path, resources=definition.get(ArmElements.RESOURCES)) + self._create_variables_vertices(file_path=file_path, variables=definition.get(ArmElements.VARIABLES)) for i, vertex in enumerate(self.vertices): self.vertices_by_block_type[vertex.block_type].append(i) self.vertices_block_name_map[vertex.block_type][vertex.name].append(i) self.vertices_by_path_and_id[(vertex.path, vertex.id)] = i + self.vertices_by_name[vertex.name] = i + + self.in_edges[i] = [] + self.out_edges[i] = [] + + def _create_variables_vertices(self, file_path: str, variables: dict[str, dict[str, Any]] | None) -> None: + if not variables: + return + + for name, conf in variables.items(): + if name in [START_LINE, END_LINE]: + continue + if not isinstance(conf, dict): + full_conf = {"value": pickle_deepcopy(conf)} + else: + full_conf = conf + config = pickle_deepcopy(full_conf) + attributes = pickle_deepcopy(full_conf) + + self.vertices.append( + ArmBlock( + name=name, + config=config, + path=file_path, + block_type=BlockType.VARIABLE, + attributes=attributes, + id=f"{BlockType.VARIABLE}.{name}", + ) + ) def _create_parameter_vertices(self, file_path: str, parameters: dict[str, dict[str, Any]] | None) -> None: if not parameters: @@ -90,8 +126,35 @@ def _create_resource_vertices(self, file_path: str, resources: list[dict[str, An ) def _create_edges(self) -> None: - # no edges yet - pass + self._create_vars_and_parameters_edges() + # todo add explicit references edges + + def _create_edge(self, element_name: str, origin_vertex_index: int, label: str) -> None: + vertex_name = element_name + if "." in vertex_name: + # special case for bicep and arm elements, when properties are accessed + vertex_name = vertex_name.split(".")[0] + + dest_vertex_index = self.vertices_by_name.get(vertex_name) + if dest_vertex_index or dest_vertex_index == 0: + if origin_vertex_index == dest_vertex_index: + return + edge = Edge(origin_vertex_index, dest_vertex_index, label) + self.edges.append(edge) + self.out_edges[origin_vertex_index].append(edge) + self.in_edges[dest_vertex_index].append(edge) + + def _create_vars_and_parameters_edges(self) -> None: + pattern = r"(variables|parameters)\('(\w+)'\)" + for origin_vertex_index, vertex in enumerate(self.vertices): + for attr_key, attr_value in vertex.attributes.items(): + if not isinstance(attr_value, str): + continue + if ArmElements.VARIABLES in attr_value or ArmElements.PARAMETERS in attr_value: + matches = re.findall(pattern, attr_value) + for match in matches: + var_name = match[1] + self._create_edge(var_name, origin_vertex_index, attr_key) def update_vertices_configs(self) -> None: # not used diff --git a/checkov/arm/graph_builder/variable_rendering/__init__.py b/checkov/arm/graph_builder/variable_rendering/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/checkov/arm/graph_builder/variable_rendering/renderer.py b/checkov/arm/graph_builder/variable_rendering/renderer.py new file mode 100644 index 00000000000..14f0c72c30c --- /dev/null +++ b/checkov/arm/graph_builder/variable_rendering/renderer.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from checkov.arm.graph_builder.graph_components.block_types import BlockType +from checkov.common.graph.graph_builder import Edge +from checkov.common.graph.graph_builder.utils import adjust_value +from checkov.common.graph.graph_builder.variable_rendering.renderer import VariableRenderer +from checkov.common.util.data_structures_utils import pickle_deepcopy + +if TYPE_CHECKING: + from checkov.arm.graph_builder.local_graph import ArmLocalGraph + + +class ArmVariableRenderer(VariableRenderer["ArmLocalGraph"]): + def __init__(self, local_graph: ArmLocalGraph) -> None: + super().__init__(local_graph) + + def _render_variables_from_vertices(self) -> None: + # need to add rendering to function like format, reference etc + pass + + def evaluate_vertex_attribute_from_edge(self, edge_list: list[Edge]) -> None: + origin_vertex_attributes = self.local_graph.vertices[edge_list[0].origin].attributes + val_to_eval = pickle_deepcopy(origin_vertex_attributes.get(edge_list[0].label, "")) + attr_path = None + for edge in edge_list: + attr_path, attr_value = self.extract_dest_attribute_path_and_value(dest_index=edge.dest, + origin_value=val_to_eval) + '''if the arg start with '[parameters'/ '[variables' its mean we need to eval the all attribute + like here - "addressPrefix": "[parameters('subnetAddressPrefix')]" ''' + if len(edge_list) == 1 and val_to_eval.startswith(("[parameters", "[variables")): + val_to_eval = attr_value + continue + ''' + if the value i need to eval is part of the full attribute like "[format('{0}/{1}', parameters('vnetName'), variables('subnetName'))]" + or "[resourceId('Microsoft.Network/networkProfiles', variables('networkProfileName'))]". + vertices[edge.dest].id = variables.networkProfileName -> variables('networkProfileName') + ''' + val_to_replace = self.local_graph.vertices[edge.dest].id.replace(".", "('") + "')" + val_to_eval = val_to_eval.replace(val_to_replace, attr_value) + + self.local_graph.update_vertex_attribute( + vertex_index=edge_list[0].origin, + attribute_key=edge_list[0].label, + attribute_value=val_to_eval, + change_origin_id=edge_list[0].dest, + attribute_at_dest=attr_path, + ) + + def extract_dest_attribute_path_and_value(self, dest_index: int, origin_value: Any) -> tuple[str, Any] | tuple[None, None]: + vertex = self.local_graph.vertices[dest_index] + if vertex.block_type == BlockType.PARAMETER: + new_value = vertex.attributes.get("defaultValue") + if new_value: + new_value = adjust_value(element_name=origin_value, value=new_value) + return "defaultValue", new_value + elif vertex.block_type == BlockType.VARIABLE: + new_value = adjust_value(element_name=origin_value, value=vertex.attributes["value"]) + return "value", new_value + return None, None + + def evaluate_non_rendered_values(self) -> None: + pass diff --git a/checkov/bicep/graph_builder/local_graph.py b/checkov/bicep/graph_builder/local_graph.py index 0bf98d1cb13..ea5efde60a0 100644 --- a/checkov/bicep/graph_builder/local_graph.py +++ b/checkov/bicep/graph_builder/local_graph.py @@ -11,10 +11,10 @@ from checkov.bicep.graph_builder.graph_components.block_types import BlockType from checkov.bicep.graph_builder.graph_components.blocks import BicepBlock from checkov.bicep.graph_builder.variable_rendering.renderer import BicepVariableRenderer -from checkov.bicep.utils import adjust_value from checkov.common.graph.graph_builder.graph_components.edge import Edge from checkov.common.graph.graph_builder.local_graph import LocalGraph from checkov.common.graph.graph_builder.utils import filter_sub_keys +from checkov.common.graph.graph_builder.utils import adjust_value from checkov.common.util.data_structures_utils import pickle_deepcopy from checkov.common.util.type_forcers import force_int diff --git a/checkov/bicep/graph_builder/variable_rendering/renderer.py b/checkov/bicep/graph_builder/variable_rendering/renderer.py index 589513d827d..d8ec3d22924 100644 --- a/checkov/bicep/graph_builder/variable_rendering/renderer.py +++ b/checkov/bicep/graph_builder/variable_rendering/renderer.py @@ -5,8 +5,8 @@ from pycep.transformer import BicepElement from checkov.bicep.graph_builder.graph_components.block_types import BlockType -from checkov.bicep.utils import adjust_value from checkov.common.graph.graph_builder import Edge +from checkov.common.graph.graph_builder.utils import adjust_value from checkov.common.graph.graph_builder.variable_rendering.renderer import VariableRenderer from checkov.common.util.data_structures_utils import pickle_deepcopy diff --git a/checkov/bicep/utils.py b/checkov/bicep/utils.py index fa2016c1a0f..4eee4a70b35 100644 --- a/checkov/bicep/utils.py +++ b/checkov/bicep/utils.py @@ -5,7 +5,7 @@ import re from collections.abc import Collection from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING from checkov.common.runners.base_runner import filter_ignored_paths from checkov.runner_filter import RunnerFilter @@ -88,26 +88,3 @@ def create_definitions( logging.warning(f"[bicep] found errors while parsing definitions: {parsing_errors}") return definitions, definitions_raw - - -def adjust_value(element_name: str, value: Any) -> Any: - """Adjusts the value, if the 'element_name' references a nested key - - Ex: - element_name = publicKey.keyData - value = {"keyData": "key-data", "path": "path"} - - returns new_value = "key-data" - """ - - if "." in element_name and isinstance(value, dict): - key_parts = element_name.split(".") - new_value = value.get(key_parts[1]) - - if new_value is None: - # couldn't find key in in value object - return None - - return adjust_value(".".join(key_parts[1:]), new_value) - - return value diff --git a/checkov/common/graph/graph_builder/utils.py b/checkov/common/graph/graph_builder/utils.py index 40ebf864ef6..319f24229ea 100644 --- a/checkov/common/graph/graph_builder/utils.py +++ b/checkov/common/graph/graph_builder/utils.py @@ -44,3 +44,26 @@ def filter_sub_keys(key_list: list[str]) -> list[str]: def is_include_dup_dynamic(key: str, list_keys: list[str]) -> bool: return f"dynamic.{key.split('.')[0]}" not in list_keys + + +def adjust_value(element_name: str, value: Any) -> Any: + """Adjusts the value, if the 'element_name' references a nested key + + Ex: + element_name = publicKey.keyData + value = {"keyData": "key-data", "path": "path"} + + returns new_value = "key-data" + """ + + if "." in element_name and isinstance(value, dict): + key_parts = element_name.split(".") + new_value = value.get(key_parts[1]) + + if new_value is None: + # couldn't find key in in value object + return None + + return adjust_value(".".join(key_parts[1:]), new_value) + + return value diff --git a/tests/arm/graph_builder/test_local_graph.py b/tests/arm/graph_builder/test_local_graph.py index bae80932eeb..17ef9342efe 100644 --- a/tests/arm/graph_builder/test_local_graph.py +++ b/tests/arm/graph_builder/test_local_graph.py @@ -18,8 +18,9 @@ def test_build_graph(): local_graph.build_graph(render_variables=False) # then - assert len(local_graph.vertices) == 15 - assert len(local_graph.edges) == 0 + assert len(local_graph.vertices) == 18 + assert len(local_graph.edges) == 20 assert len(local_graph.vertices_by_block_type[BlockType.PARAMETER]) == 11 assert len(local_graph.vertices_by_block_type[BlockType.RESOURCE]) == 4 + assert len(local_graph.vertices_by_block_type[BlockType.VARIABLE]) == 3 diff --git a/tests/arm/rendering/__init__.py b/tests/arm/rendering/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/arm/rendering/test_rendering.json b/tests/arm/rendering/test_rendering.json new file mode 100644 index 00000000000..cb7228a2aef --- /dev/null +++ b/tests/arm/rendering/test_rendering.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.5.6.12127", + "templateHash": "9027093124117826122" + } + }, + "parameters": { + "vnetName": { + "type": "string", + "defaultValue": "aci-vnet", + "metadata": { + "description": "VNet name" + } + }, + "vnetAddressPrefix": { + "type": "string", + "defaultValue": "10.0.0.0/16", + "metadata": { + "description": "Address prefix" + } + } + }, + "variables": { + "networkProfileName": "aci-networkProfile", + "location": "eth0" + + }, + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-11-01", + "name": "[format('{0}/{1}', parameters('vnetName'), variables('networkProfileName'))]", + "location": "[variables('location')]", + "id": "[resourceId('Microsoft.Network/networkProfiles', variables('networkProfileName'))]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[parameters('vnetAddressPrefix')]" + ] + } + } + } + ] +} \ No newline at end of file diff --git a/tests/arm/rendering/test_rendering.py b/tests/arm/rendering/test_rendering.py new file mode 100644 index 00000000000..1c6d5e725b1 --- /dev/null +++ b/tests/arm/rendering/test_rendering.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from checkov.arm.graph_builder.local_graph import ArmLocalGraph +from checkov.arm.utils import get_files_definitions + +EXAMPLES_DIR = Path(__file__).parent + +def test_rander_vars(): + # given + test_file = EXAMPLES_DIR / "test_rendering.json" + definitions, _, _ = get_files_definitions([str(test_file)]) + local_graph = ArmLocalGraph(definitions=definitions) + # when + local_graph.build_graph(render_variables=True) + + # then + assert len(local_graph.vertices) == 5 + assert len(local_graph.edges) == 5 + assert local_graph.vertices[2].attributes['name'] == "[format('{0}/{1}', aci-vnet, aci-networkProfile)]" + assert local_graph.vertices[2].attributes['id'] == "[resourceId('Microsoft.Network/networkProfiles', aci-networkProfile)]" + assert local_graph.vertices[2].attributes['location'] == "eth0" + assert local_graph.vertices[2].attributes['properties.addressSpace.addressPrefixes.0'] == "10.0.0.0/16" + assert local_graph.vertices[2].attributes['properties']['addressSpace']['addressPrefixes'][0] == "10.0.0.0/16" + diff --git a/tests/arm/test_graph_manager.py b/tests/arm/test_graph_manager.py index 21df6ebe102..4471bcf1839 100644 --- a/tests/arm/test_graph_manager.py +++ b/tests/arm/test_graph_manager.py @@ -32,8 +32,8 @@ def test_build_graph_from_definitions(graph_connector): local_graph = graph_manager.build_graph_from_definitions(definitions=definitions) # then - assert len(local_graph.vertices) == 15 - assert len(local_graph.edges) == 0 + assert len(local_graph.vertices) == 18 + assert len(local_graph.edges) == 20 # resource name will change, when variable rendering is supported container_idx = local_graph.vertices_by_path_and_id[(test_file, "Microsoft.ContainerInstance/containerGroups.[parameters('containerGroupName')]")] From 38009294c52e9b7aecbc00c6d139f58052dbd1c7 Mon Sep 17 00:00:00 2001 From: gruebel Date: Wed, 23 Oct 2024 23:01:43 +0000 Subject: [PATCH 07/14] chore: update release notes --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27a87e5a34f..4d18653a7f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # CHANGELOG -## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.268...HEAD) +## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.269...HEAD) + +## [3.2.269](https://github.com/bridgecrewio/checkov/compare/3.2.268...3.2.269) - 2024-10-23 + +### Bug Fix + +- **terraform:** Fix crash when version isn't a float - [#6783](https://github.com/bridgecrewio/checkov/pull/6783) ## [3.2.268](https://github.com/bridgecrewio/checkov/compare/3.2.267...3.2.268) - 2024-10-20 From 7713e7309c4d65028a8e6862f6264f645773846d Mon Sep 17 00:00:00 2001 From: omriyoffe-panw Date: Mon, 28 Oct 2024 14:55:54 +0200 Subject: [PATCH 08/14] feat(arm): arm custom policy support (#6769) * arm custom policy support --- .../arm/graph_builder/context_definitions.py | 10 +++ .../arm/graph_builder/graph_to_definitions.py | 13 +++ checkov/arm/graph_builder/local_graph.py | 81 +++++++++++++------ checkov/arm/utils.py | 31 ++++++- .../features/custom_policies_integration.py | 2 + checkov/common/checks_infra/registry.py | 2 +- .../ExplicitDepsResources/interface.json | 41 ++++++++++ .../ExplicitDepsResources/storage.json | 56 +++++++++++++ .../ExplicitDepsResources/subnet.json | 36 +++++++++ .../ImplicitDepsResources/interface.json | 37 +++++++++ .../ImplicitDepsResources/storage.json | 61 ++++++++++++++ .../ImplicitDepsResources/subnet.json | 34 ++++++++ tests/arm/graph_builder/test_local_graph.py | 34 +++++++- tests/arm/rendering/test_rendering.py | 3 +- tests/arm/test_utils.py | 16 +++- .../test_custom_policies_integration.py | 20 +++++ 16 files changed, 446 insertions(+), 31 deletions(-) create mode 100644 checkov/arm/graph_builder/context_definitions.py create mode 100644 checkov/arm/graph_builder/graph_to_definitions.py create mode 100644 tests/arm/examples/ExplicitDepsResources/interface.json create mode 100644 tests/arm/examples/ExplicitDepsResources/storage.json create mode 100644 tests/arm/examples/ExplicitDepsResources/subnet.json create mode 100644 tests/arm/examples/ImplicitDepsResources/interface.json create mode 100644 tests/arm/examples/ImplicitDepsResources/storage.json create mode 100644 tests/arm/examples/ImplicitDepsResources/subnet.json diff --git a/checkov/arm/graph_builder/context_definitions.py b/checkov/arm/graph_builder/context_definitions.py new file mode 100644 index 00000000000..9083b3a169c --- /dev/null +++ b/checkov/arm/graph_builder/context_definitions.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from pathlib import Path +from typing import List, Tuple, Dict, Any + + +def build_definitions_context(definitions: Dict[Path, Any], definitions_raw: Dict[Path, List[Tuple[int, str]]] + ) -> Dict[str, Dict[str, Any]]: + # TODO + return {} diff --git a/checkov/arm/graph_builder/graph_to_definitions.py b/checkov/arm/graph_builder/graph_to_definitions.py new file mode 100644 index 00000000000..ab371216165 --- /dev/null +++ b/checkov/arm/graph_builder/graph_to_definitions.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from checkov.arm.graph_builder.graph_components.blocks import ArmBlock + + +def convert_graph_vertices_to_definitions( + vertices: list[ArmBlock], root_folder: str | Path | None +) -> tuple[dict[Path, Any], dict[str, dict[str, Any]]]: + # TODO + return ({}, {}) diff --git a/checkov/arm/graph_builder/local_graph.py b/checkov/arm/graph_builder/local_graph.py index 93ce8fab980..8fa63f15a06 100644 --- a/checkov/arm/graph_builder/local_graph.py +++ b/checkov/arm/graph_builder/local_graph.py @@ -6,8 +6,9 @@ from checkov.arm.graph_builder.graph_components.block_types import BlockType from checkov.arm.graph_builder.graph_components.blocks import ArmBlock +from checkov.arm.utils import ArmElements, extract_resource_name_from_resource_id_func, \ + extract_resource_name_from_reference_func from checkov.arm.graph_builder.variable_rendering.renderer import ArmVariableRenderer -from checkov.arm.utils import ArmElements from checkov.common.graph.graph_builder import CustomAttributes, Edge from checkov.common.graph.graph_builder.local_graph import LocalGraph from checkov.common.util.consts import START_LINE, END_LINE @@ -16,6 +17,10 @@ if TYPE_CHECKING: from checkov.common.graph.graph_builder.local_graph import _Block +DEPENDS_ON_FIELD = 'dependsOn' +RESOURCE_ID_FUNC = 'resourceId(' +REFERENCE_FUNC = 'reference(' + class ArmLocalGraph(LocalGraph[ArmBlock]): def __init__(self, definitions: dict[str, dict[str, Any]]) -> None: @@ -27,14 +32,21 @@ def __init__(self, definitions: dict[str, dict[str, Any]]) -> None: def build_graph(self, render_variables: bool = False) -> None: self._create_vertices() - logging.debug(f"[ArmLocalGraph] created {len(self.vertices)} vertices") + logging.warning(f"[ArmLocalGraph] created {len(self.vertices)} vertices") - self._create_edges() - logging.debug(f"[ArmLocalGraph] created {len(self.edges)} edges") + ''' + In order to resolve the resources names for the dependencies we need to render the variables first + Examples: https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/resource-dependency + ''' + + self._create_vars_and_parameters_edges() if render_variables: renderer = ArmVariableRenderer(self) renderer.render_variables_from_local_graph() + self._create_edges() + logging.warning(f"[ArmLocalGraph] created {len(self.edges)} edges") + def _create_vertices(self) -> None: for file_path, definition in self.definitions.items(): self._create_parameter_vertices(file_path=file_path, parameters=definition.get(ArmElements.PARAMETERS)) @@ -83,7 +95,7 @@ def _create_parameter_vertices(self, file_path: str, parameters: dict[str, dict[ if name in (START_LINE, END_LINE): continue if not isinstance(config, dict): - logging.debug(f"[ArmLocalGraph] parameter {name} has wrong type {type(config)}") + logging.warning(f"[ArmLocalGraph] parameter {name} has wrong type {type(config)}") continue attributes = pickle_deepcopy(config) @@ -121,28 +133,30 @@ def _create_resource_vertices(self, file_path: str, resources: list[dict[str, An path=file_path, block_type=BlockType.RESOURCE, attributes=attributes, - id=f"{resource_type}.{resource_name}", + id=f"{resource_type}.{resource_name}" ) ) def _create_edges(self) -> None: - self._create_vars_and_parameters_edges() - # todo add explicit references edges - - def _create_edge(self, element_name: str, origin_vertex_index: int, label: str) -> None: - vertex_name = element_name - if "." in vertex_name: - # special case for bicep and arm elements, when properties are accessed - vertex_name = vertex_name.split(".")[0] - - dest_vertex_index = self.vertices_by_name.get(vertex_name) - if dest_vertex_index or dest_vertex_index == 0: - if origin_vertex_index == dest_vertex_index: - return - edge = Edge(origin_vertex_index, dest_vertex_index, label) - self.edges.append(edge) - self.out_edges[origin_vertex_index].append(edge) - self.in_edges[dest_vertex_index].append(edge) + for origin_vertex_index, vertex in enumerate(self.vertices): + if DEPENDS_ON_FIELD in vertex.attributes: + self._create_explicit_edge(origin_vertex_index, vertex.name, vertex.attributes['dependsOn']) + self._create_implicit_edges(origin_vertex_index, vertex.name, vertex.attributes) + + def _create_explicit_edge(self, origin_vertex_index: int, resource_name: str, deps: list[str]) -> None: + for dep in deps: + if RESOURCE_ID_FUNC in dep: + processed_dep = extract_resource_name_from_resource_id_func(dep) + else: + processed_dep = dep.split('/')[-1] + # Check if the processed dependency exists in the map + if processed_dep in self.vertices_by_name: + self._create_edge(processed_dep, origin_vertex_index, f'{resource_name}->{processed_dep}') + else: + # Dependency not found + logging.warning(f"[ArmLocalGraph] resource dependency {processed_dep} defined in {dep} for resource" + f" {resource_name} not found") + continue def _create_vars_and_parameters_edges(self) -> None: pattern = r"(variables|parameters)\('(\w+)'\)" @@ -156,13 +170,32 @@ def _create_vars_and_parameters_edges(self) -> None: var_name = match[1] self._create_edge(var_name, origin_vertex_index, attr_key) + def _create_edge(self, element_name: str, origin_vertex_index: int, label: str) -> None: + dest_vertex_index = self.vertices_by_name.get(element_name) + if origin_vertex_index == dest_vertex_index or dest_vertex_index is None: + return + edge = Edge(origin_vertex_index, dest_vertex_index, label) + self.edges.append(edge) + self.out_edges[origin_vertex_index].append(edge) + self.in_edges[dest_vertex_index].append(edge) + + def _create_implicit_edges(self, origin_vertex_index: int, resource_name: str, resource: dict[str, Any]) -> None: + for value in resource.values(): + if isinstance(value, str): + if REFERENCE_FUNC in value: + self._create_implicit_edge(origin_vertex_index, resource_name, value) + + def _create_implicit_edge(self, origin_vertex_index: int, resource_name: str, reference_string: str) -> None: + dep_name = extract_resource_name_from_reference_func(reference_string) + self._create_edge(dep_name, origin_vertex_index, f'{resource_name}->{dep_name}') + def update_vertices_configs(self) -> None: # not used pass @staticmethod def update_vertex_config( - vertex: _Block, changed_attributes: list[str] | dict[str, Any], has_dynamic_blocks: bool = False + vertex: _Block, changed_attributes: list[str] | dict[str, Any], has_dynamic_blocks: bool = False ) -> None: # not used pass diff --git a/checkov/arm/utils.py b/checkov/arm/utils.py index b837c3039b8..bf4da4f272c 100644 --- a/checkov/arm/utils.py +++ b/checkov/arm/utils.py @@ -40,7 +40,7 @@ def get_scannable_file_paths(root_folder: str | None = None, excluded_paths: lis def get_files_definitions( - files: Iterable[str], filepath_fn: Callable[[str], str] | None = None + files: Iterable[str], filepath_fn: Callable[[str], str] | None = None ) -> tuple[dict[str, dict[str, Any]], dict[str, list[tuple[int, str]]], list[str]]: """Parses ARM files into its definitions and raw data""" @@ -60,3 +60,32 @@ def get_files_definitions( parsing_errors.append(os.path.normpath(file)) return definitions, definitions_raw, parsing_errors + + +def extract_resource_name_from_resource_id_func(resource_id: str) -> str: + ''' + Examples: + resourceId('Microsoft.Network/virtualNetworks/', virtualNetworkName) -> virtualNetworkName + ''' + return clean_string(resource_id.split(',')[-1].split(')')[0]) + + +def extract_resource_name_from_reference_func(reference: str) -> str: + ''' + Examples: + reference('storageAccountName') -> storageAccountName + reference('myStorage').primaryEndpoints -> myStorage + reference('myStorage', '2022-09-01', 'Full').location -> myStorage + reference(resourceId('storageResourceGroup', 'Microsoft.Storage/storageAccounts', 'storageAccountName')), '2022-09-01') -> storageAccountName + reference(resourceId('Microsoft.Network/publicIPAddresses', 'ipAddressName')) -> ipAddressName + ''' + resource_name = ')'.join(reference.split('reference(', 1)[1].split(')')[:-1]) + if 'resourceId' in resource_name: + return clean_string( + ''.join(resource_name.split('resourceId(', 1)[1].split(')')[0]).split(',')[-1]) + else: + return clean_string(resource_name.split(',')[0].split('/')[-1]) + + +def clean_string(input: str) -> str: + return input.replace("'", '').replace(" ", "") diff --git a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py index 691fcb1c43d..cf31130cd47 100644 --- a/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/custom_policies_integration.py @@ -86,6 +86,8 @@ def pre_scan(self) -> None: get_graph_checks_registry("kubernetes").checks.append(check) elif f.lower() == "bicep": get_graph_checks_registry("bicep").checks.append(check) + elif f.lower() == "arm": + get_graph_checks_registry("arm").checks.append(check) else: for registry in get_all_graph_checks_registries(): registry.checks.append(check) diff --git a/checkov/common/checks_infra/registry.py b/checkov/common/checks_infra/registry.py index 993970849ac..074d9bdf724 100644 --- a/checkov/common/checks_infra/registry.py +++ b/checkov/common/checks_infra/registry.py @@ -23,7 +23,7 @@ GraphSupportedIACFrameworks = [GraphSource.TERRAFORM, GraphSource.CLOUDFORMATION, GraphSource.KUBERNETES, GraphSource.TERRAFORM_PLAN, GraphSource.KUSTOMIZE, GraphSource.BICEP, - GraphSource.GITHUB_ACTION, GraphSource.HELM, GraphSource.ANSIBLE] + GraphSource.GITHUB_ACTION, GraphSource.HELM, GraphSource.ANSIBLE, GraphSource.ARM] class Registry(BaseRegistry): diff --git a/tests/arm/examples/ExplicitDepsResources/interface.json b/tests/arm/examples/ExplicitDepsResources/interface.json new file mode 100644 index 00000000000..dbbc431be86 --- /dev/null +++ b/tests/arm/examples/ExplicitDepsResources/interface.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2020-06-01", + "name": "PublicIP1", + "location": "westus", + "properties": { + "publicIPAllocationMethod": "Dynamic" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2020-06-01", + "name": "NIC1", + "location": "westus", + "dependsOn": [ + "Microsoft.Network/virtualNetworks/VNet1/subnets/Subnet1", + "Microsoft.Network/publicIPAddresses/PublicIP1" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'VNet1', 'Subnet1')]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'PublicIP1')]" + } + } + } + ] + } + } + ] + } \ No newline at end of file diff --git a/tests/arm/examples/ExplicitDepsResources/storage.json b/tests/arm/examples/ExplicitDepsResources/storage.json new file mode 100644 index 00000000000..3992fd26ab7 --- /dev/null +++ b/tests/arm/examples/ExplicitDepsResources/storage.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/storageAccounts", + "apiVersion": "2019-06-01", + "name": "storaccount1", + "location": "westus", + "sku": { + "name": "Standard_LRS" + }, + "kind": "StorageV2", + "properties": {} + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2019-12-01", + "name": "VM1", + "location": "westus", + "dependsOn": [ + "Microsoft.Network/virtualNetworks/VNet1/subnets/Subnet1", + "Microsoft.Network/storageAccounts/storaccount1" + ], + "properties": { + "hardwareProfile": { + "vmSize": "Standard_D2s_v3" + }, + "storageProfile": { + "imageReference": { + "publisher": "MicrosoftWindowsServer", + "offer": "WindowsServer", + "sku": "2019-Datacenter", + "version": "latest" + }, + "osDisk": { + "createOption": "FromImage" + }, + "dataDisks": [] + }, + "osProfile": { + "computerName": "vm1", + "adminUsername": "adminuser", + "adminPassword": "Password123!" + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'nic1')]" + } + ] + } + } + } + ] + } \ No newline at end of file diff --git a/tests/arm/examples/ExplicitDepsResources/subnet.json b/tests/arm/examples/ExplicitDepsResources/subnet.json new file mode 100644 index 00000000000..a25d2d3ddad --- /dev/null +++ b/tests/arm/examples/ExplicitDepsResources/subnet.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-06-01", + "name": "VNet1", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": ["10.0.0.0/16"] + }, + "subnets": [ + { + "name": "Subnet1", + "properties": { + "addressPrefix": "10.0.1.0/24" + } + } + ] + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2020-06-01", + "name": "Subnet1", + "dependsOn": [ + "Microsoft.Network/virtualNetworks/VNet1" + ], + "properties": { + "addressPrefix": "10.0.1.0/24" + } + } + ] + } \ No newline at end of file diff --git a/tests/arm/examples/ImplicitDepsResources/interface.json b/tests/arm/examples/ImplicitDepsResources/interface.json new file mode 100644 index 00000000000..1e93aa1202e --- /dev/null +++ b/tests/arm/examples/ImplicitDepsResources/interface.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2020-06-01", + "name": "PublicIP1", + "location": "westus", + "properties": { + "publicIPAllocationMethod": "Dynamic" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2020-06-01", + "name": "NIC1", + "location": "westus", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[reference(resourceId('Microsoft.Network/virtualNetworks', 'myVNet')).subnets[0].id]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[reference(resourceId('Microsoft.Network/publicIPAddresses', 'PublicIP1')).id]" + } + } + } + ] + } + } + ] + } \ No newline at end of file diff --git a/tests/arm/examples/ImplicitDepsResources/storage.json b/tests/arm/examples/ImplicitDepsResources/storage.json new file mode 100644 index 00000000000..0cf4c744849 --- /dev/null +++ b/tests/arm/examples/ImplicitDepsResources/storage.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2019-09-01", + "name": "myKeyVault", + "location": "westus", + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "tenantId", + "accessPolicies": [] + }, + "resources": [ + { + "type": "secrets", + "name": "MySecret", + "apiVersion": "2019-09-01", + "properties": { + "value": "MySecretValue123" + } + } + ] + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2019-07-01", + "name": "myVM", + "location": "westus", + "properties": { + "hardwareProfile": { + "vmSize": "Standard_DS1_v2" + }, + "osProfile": { + "computerName": "myVM", + "adminUsername": "adminuser", + "customData": "[reference('Microsoft.KeyVault/vaults/myKeyVault', '2019-09-01').secrets['MySecret'].value]" + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', 'myNIC')]" + } + ] + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "Premium_LRS" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/tests/arm/examples/ImplicitDepsResources/subnet.json b/tests/arm/examples/ImplicitDepsResources/subnet.json new file mode 100644 index 00000000000..56d22b863f5 --- /dev/null +++ b/tests/arm/examples/ImplicitDepsResources/subnet.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [ + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2020-06-01", + "name": "myNSG", + "location": "westus" + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2020-06-01", + "name": "myVNet", + "location": "westus", + "properties": { + "addressSpace": { + "addressPrefixes": ["10.0.0.0/16"] + }, + "subnets": [ + { + "name": "mySubnet", + "properties": { + "addressPrefix": "10.0.1.0/24", + "networkSecurityGroup": { + "id": "[reference('myNSG', '2020-06-01').id]" + } + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/tests/arm/graph_builder/test_local_graph.py b/tests/arm/graph_builder/test_local_graph.py index 17ef9342efe..5d0fc02b768 100644 --- a/tests/arm/graph_builder/test_local_graph.py +++ b/tests/arm/graph_builder/test_local_graph.py @@ -5,15 +5,43 @@ from checkov.arm.utils import get_files_definitions EXAMPLES_DIR = Path(__file__).parent.parent / "examples" +EXPLICIT_DEPS_DIR = EXAMPLES_DIR / "ExplicitDepsResources" +IMPLICIT_DEPS_DIR = EXAMPLES_DIR / "ImplicitDepsResources" -def test_build_graph(): +def test_graph_explicit_deps(): + test_files = [str(EXPLICIT_DEPS_DIR / "subnet.json"), + str(EXPLICIT_DEPS_DIR / "storage.json"), + str(EXPLICIT_DEPS_DIR / "interface.json")] + definitions, _, _ = get_files_definitions(test_files) + test_graph = ArmLocalGraph(definitions) + test_graph.build_graph() + + assert len(test_graph.vertices) == 6 + assert len(test_graph.edges) == 5 + + assert len(test_graph.vertices_by_block_type[BlockType.RESOURCE]) == 6 + + +def test_graph_implicit_deps(): + test_files = [str(IMPLICIT_DEPS_DIR / "subnet.json"), + str(IMPLICIT_DEPS_DIR / "storage.json"), + str(IMPLICIT_DEPS_DIR / "interface.json")] + definitions, _, _ = get_files_definitions(test_files) + test_graph = ArmLocalGraph(definitions) + test_graph.build_graph() + + assert len(test_graph.vertices) == 6 + assert len(test_graph.edges) == 4 + + assert len(test_graph.vertices_by_block_type[BlockType.RESOURCE]) == 6 + + +def test_graph_params_vars(): # given test_file = EXAMPLES_DIR / "container_instance.json" definitions, _, _ = get_files_definitions([str(test_file)]) - local_graph = ArmLocalGraph(definitions=definitions) - # when local_graph.build_graph(render_variables=False) diff --git a/tests/arm/rendering/test_rendering.py b/tests/arm/rendering/test_rendering.py index 1c6d5e725b1..b29d101ac15 100644 --- a/tests/arm/rendering/test_rendering.py +++ b/tests/arm/rendering/test_rendering.py @@ -5,7 +5,8 @@ EXAMPLES_DIR = Path(__file__).parent -def test_rander_vars(): + +def test_render_vars(): # given test_file = EXAMPLES_DIR / "test_rendering.json" definitions, _, _ = get_files_definitions([str(test_file)]) diff --git a/tests/arm/test_utils.py b/tests/arm/test_utils.py index b71e44561a2..63363fcd06c 100644 --- a/tests/arm/test_utils.py +++ b/tests/arm/test_utils.py @@ -1,6 +1,6 @@ from pathlib import Path -from checkov.arm.utils import get_files_definitions +from checkov.arm.utils import get_files_definitions, extract_resource_name_from_reference_func def test_get_files_definitions_with_parsing_error(): @@ -15,3 +15,17 @@ def test_get_files_definitions_with_parsing_error(): assert definitions_raw == {} assert len(parsing_errors) == 1 assert parsing_errors[0].endswith("parser/examples/json/with_comments.json") + + +def test_extract_resource_name_from_reference_func(): + test_cases = ["reference('storageAccountName')", + "reference('myStorage').primaryEndpoints", + "reference('myStorage', '2022-09-01', 'Full').location", + "reference(resourceId('storageResourceGroup', 'Microsoft.Storage/storageAccounts', " + "'storageAccountName')), '2022-09-01')", + "reference(resourceId('Microsoft.Network/publicIPAddresses', 'ipAddressName'))"] + + expected = ["storageAccountName", "myStorage", "myStorage", "storageAccountName", "ipAddressName"] + + for i, test_case in enumerate(test_cases): + assert extract_resource_name_from_reference_func(test_case) == expected[i] diff --git a/tests/common/integration_features/test_custom_policies_integration.py b/tests/common/integration_features/test_custom_policies_integration.py index 46300323c8b..8e37328e37c 100644 --- a/tests/common/integration_features/test_custom_policies_integration.py +++ b/tests/common/integration_features/test_custom_policies_integration.py @@ -219,6 +219,7 @@ def test_pre_scan_with_cloned_checks(self): tf_registry = get_graph_checks_registry("terraform").checks k8s_registry = get_graph_checks_registry("kubernetes").checks bicep_registry = get_graph_checks_registry("bicep").checks + arm_registry = get_graph_checks_registry("arm").checks self.assertEqual(1, len(custom_policies_integration.bc_cloned_checks)) self.assertEqual('kpande_AZR_1648821862291', tf_registry[0].id, cfn_registry[0].id) self.assertEqual('kpande_AZR_1648821862291', tf_registry[0].bc_id, cfn_registry[0].bc_id) @@ -226,6 +227,8 @@ def test_pre_scan_with_cloned_checks(self): self.assertEqual('kpande_kubernetes_1650378013211', k8s_registry[0].bc_id) self.assertEqual('kpande_bicep_1650378013212', bicep_registry[0].id) self.assertEqual('kpande_bicep_1650378013212', bicep_registry[0].bc_id) + self.assertEqual('kpande_arm_1650378013213', arm_registry[0].bc_id) + self.assertEqual('kpande_arm_1650378013213', arm_registry[0].bc_id) def test_pre_scan_with_multiple_frameworks_graph_check(self): instance = BcPlatformIntegration() @@ -569,6 +572,23 @@ def mock_custom_policies_response(): "frameworks": [ "bicep" ] + }, + { + "id": "kpande_arm_1650378013213", + "code": "{\"operator\":\"exists\",\"attribute\":\"spec.runAsUser.rule\",\"cond_type\":\"attribute\"," + "\"resource_types\":[\"PodSecurityPolicy\"]}", + "title": "arm policy", + "guideline": "meaningful guideline for arm policy", + "severity": "HIGH", + "pcSeverity": None, + "category": "arm", + "pcPolicyId": None, + "additionalPcPolicyIds": None, + "sourceIncidentId": None, + "benchmarks": {}, + "frameworks": [ + "arm" + ] } ] } From ac5e35a6a99af308f3f8a0ef2954a24eaf1a9ced Mon Sep 17 00:00:00 2001 From: omriyoffe-panw Date: Mon, 28 Oct 2024 14:55:54 +0200 Subject: [PATCH 09/14] feat(arm): arm custom policy support (#6769) * arm custom policy support --- checkov/version.py | 2 +- kubernetes/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/version.py b/checkov/version.py index 805e511295c..998af35f0b1 100644 --- a/checkov/version.py +++ b/checkov/version.py @@ -1 +1 @@ -version = '3.2.269' +version = '3.2.270' diff --git a/kubernetes/requirements.txt b/kubernetes/requirements.txt index a59cb02f2ff..a27496327e0 100644 --- a/kubernetes/requirements.txt +++ b/kubernetes/requirements.txt @@ -1 +1 @@ -checkov==3.2.269 +checkov==3.2.270 From 02bf205e5fd3aa45ccde6636b1364e803e8a8e84 Mon Sep 17 00:00:00 2001 From: gruebel Date: Mon, 28 Oct 2024 23:01:27 +0000 Subject: [PATCH 10/14] chore: update release notes --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d18653a7f6..7499759e5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # CHANGELOG -## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.269...HEAD) +## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.270...HEAD) + +## [3.2.270](https://github.com/bridgecrewio/checkov/compare/3.2.269...3.2.270) - 2024-10-28 + +### Feature + +- **arm:** add variable and parameters edges and rendering - [#6787](https://github.com/bridgecrewio/checkov/pull/6787) +- **arm:** arm custom policy support - [#6769](https://github.com/bridgecrewio/checkov/pull/6769) ## [3.2.269](https://github.com/bridgecrewio/checkov/compare/3.2.268...3.2.269) - 2024-10-23 From 531e06aec1734ac42236c5df191556e9b7dfc8ae Mon Sep 17 00:00:00 2001 From: Ofek Shimko <70918223+OfekShimko@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:22:38 +0200 Subject: [PATCH 11/14] feat(sca): add enableDotnetCpm env var to sca scan request (#6786) * add enableDotnetCpm env var to sca * use EnvVarsConfig * bool not string * remove import os --------- Co-authored-by: oshimko --- checkov/common/util/env_vars_config.py | 1 + checkov/sca_package_2/scanner.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/checkov/common/util/env_vars_config.py b/checkov/common/util/env_vars_config.py index 15c12dbe832..35471ba0879 100644 --- a/checkov/common/util/env_vars_config.py +++ b/checkov/common/util/env_vars_config.py @@ -77,6 +77,7 @@ def __init__(self) -> None: self.ENABLE_DEFINITION_KEY = os.getenv("ENABLE_DEFINITION_KEY", False) self.CKV_SUPPORT_ALL_RESOURCE_TYPE = os.getenv('CKV_SUPPORT_ALL_RESOURCE_TYPE', False) self.HCL_PARSE_TIMEOUT_SEC = force_int(os.getenv("HCL_PARSE_TIMEOUT_SEC", 10)) + self.ENABLE_DOTNET_CPM = os.getenv('ENABLE_DOTNET_CPM', False) env_vars_config = EnvVarsConfig() diff --git a/checkov/sca_package_2/scanner.py b/checkov/sca_package_2/scanner.py index 5696d17544f..d453ecb81a0 100644 --- a/checkov/sca_package_2/scanner.py +++ b/checkov/sca_package_2/scanner.py @@ -8,6 +8,7 @@ from requests import JSONDecodeError from checkov.common.bridgecrew.platform_integration import bc_integration +from checkov.common.util.env_vars_config import env_vars_config from checkov.common.util.http_utils import request_wrapper from checkov.common.util.tqdm_utils import ProgressBar @@ -43,7 +44,8 @@ def run_scan(self) -> bool: "path": bc_integration.repo_path, "repoId": bc_integration.repo_id, "id": bc_integration.timestamp, - "repositoryId": "" + "repositoryId": "", + "enableDotnetCpm": env_vars_config.ENABLE_DOTNET_CPM, } response = request_wrapper( From 5be4f5781e21cf8a35b14195d604f31d50a682a9 Mon Sep 17 00:00:00 2001 From: Ofek Shimko <70918223+OfekShimko@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:22:38 +0200 Subject: [PATCH 12/14] feat(sca): add enableDotnetCpm env var to sca scan request (#6786) * add enableDotnetCpm env var to sca * use EnvVarsConfig * bool not string * remove import os --------- Co-authored-by: oshimko --- checkov/version.py | 2 +- kubernetes/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/version.py b/checkov/version.py index 998af35f0b1..74b0fee6c65 100644 --- a/checkov/version.py +++ b/checkov/version.py @@ -1 +1 @@ -version = '3.2.270' +version = '3.2.271' diff --git a/kubernetes/requirements.txt b/kubernetes/requirements.txt index a27496327e0..bddfbca7c04 100644 --- a/kubernetes/requirements.txt +++ b/kubernetes/requirements.txt @@ -1 +1 @@ -checkov==3.2.270 +checkov==3.2.271 From 5e0e82740f057450efcc2820b0181eb5cee0ce79 Mon Sep 17 00:00:00 2001 From: gruebel Date: Tue, 29 Oct 2024 23:02:00 +0000 Subject: [PATCH 13/14] chore: update release notes --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7499759e5e1..1be417d1a58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # CHANGELOG -## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.270...HEAD) +## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.271...HEAD) + +## [3.2.271](https://github.com/bridgecrewio/checkov/compare/3.2.270...3.2.271) - 2024-10-29 + +### Feature + +- **sca:** add enableDotnetCpm env var to sca scan request - [#6786](https://github.com/bridgecrewio/checkov/pull/6786) ## [3.2.270](https://github.com/bridgecrewio/checkov/compare/3.2.269...3.2.270) - 2024-10-28 From a05f1c255f7d85331a09584e8fd35868165140f0 Mon Sep 17 00:00:00 2001 From: LirShindalman <49649760+lirshindalman@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:42:00 +0200 Subject: [PATCH 14/14] fix(general): pipenv==2024.0.3 (#6803) pipenv==2024.0.3 --- .github/workflows/pr-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 118a6f32c04..3d1fe89913d 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -88,7 +88,12 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install pipenv run: | - python -m pip install --no-cache-dir --upgrade pipenv + if [ '${{ matrix.python }}' == '3.12' ]; then + # needed for numpy + python -m pip install --no-cache-dir --upgrade pipenv==2024.0.3 + else + python -m pip install --no-cache-dir --upgrade pipenv + fi - name: Install dependencies run: | # remove venv, if exists