Merge branch 'main' into main

.github/workflows/pr-test.yml

@@ -88,7 +88,12 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Install pipenv
         run: |
-          python -m pip install --no-cache-dir --upgrade pipenv
+          if [ '${{ matrix.python }}' == '3.12' ]; then
+            # needed for numpy
+            python -m pip install --no-cache-dir --upgrade pipenv==2024.0.3
+          else
+            python -m pip install --no-cache-dir --upgrade pipenv
+          fi
       - name: Install dependencies
         run: |
           # remove venv, if exists

CHANGELOG.md

@@ -1,6 +1,31 @@
 # CHANGELOG
 
-## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.267...HEAD)
+## [Unreleased](https://github.com/bridgecrewio/checkov/compare/3.2.271...HEAD)
+
+## [3.2.271](https://github.com/bridgecrewio/checkov/compare/3.2.270...3.2.271) - 2024-10-29
+
+### Feature
+
+- **sca:** add enableDotnetCpm env var to sca scan request  - [#6786](https://github.com/bridgecrewio/checkov/pull/6786)
+
+## [3.2.270](https://github.com/bridgecrewio/checkov/compare/3.2.269...3.2.270) - 2024-10-28
+
+### Feature
+
+- **arm:** add variable and parameters edges and rendering - [#6787](https://github.com/bridgecrewio/checkov/pull/6787)
+- **arm:** arm custom policy support - [#6769](https://github.com/bridgecrewio/checkov/pull/6769)
+
+## [3.2.269](https://github.com/bridgecrewio/checkov/compare/3.2.268...3.2.269) - 2024-10-23
+
+### Bug Fix
+
+- **terraform:** Fix crash when version isn't a float - [#6783](https://github.com/bridgecrewio/checkov/pull/6783)
+
+## [3.2.268](https://github.com/bridgecrewio/checkov/compare/3.2.267...3.2.268) - 2024-10-20
+
+### Feature
+
+- **terraform_plan:** Support after_unknown evaluation of complex attributes - [#6784](https://github.com/bridgecrewio/checkov/pull/6784)
 
 ## [3.2.267](https://github.com/bridgecrewio/checkov/compare/3.2.266...3.2.267) - 2024-10-16
 

checkov/arm/graph_builder/context_definitions.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Tuple, Dict, Any
+
+
+def build_definitions_context(definitions: Dict[Path, Any], definitions_raw: Dict[Path, List[Tuple[int, str]]]
+                              ) -> Dict[str, Dict[str, Any]]:
+    # TODO
+    return {}

checkov/arm/graph_builder/graph_components/block_types.py

@@ -8,4 +8,5 @@
 
 @dataclass
 class BlockType(CommonBlockType):
-    PARAMETER: Literal["parameter"] = "parameter"
+    PARAMETER: Literal["parameters"] = "parameters"
+    VARIABLE: Literal["variables"] = "variables"

checkov/arm/graph_builder/graph_to_definitions.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from checkov.arm.graph_builder.graph_components.blocks import ArmBlock
+
+
+def convert_graph_vertices_to_definitions(
+    vertices: list[ArmBlock], root_folder: str | Path | None
+) -> tuple[dict[Path, Any], dict[str, dict[str, Any]]]:
+    # TODO
+    return ({}, {})

checkov/arm/graph_builder/local_graph.py

@@ -1,43 +1,91 @@
 from __future__ import annotations
 
 import logging
+import re
 from typing import Any, TYPE_CHECKING
 
 from checkov.arm.graph_builder.graph_components.block_types import BlockType
 from checkov.arm.graph_builder.graph_components.blocks import ArmBlock
-from checkov.arm.utils import ArmElements
-from checkov.common.graph.graph_builder import CustomAttributes
+from checkov.arm.utils import ArmElements, extract_resource_name_from_resource_id_func, \
+    extract_resource_name_from_reference_func
+from checkov.arm.graph_builder.variable_rendering.renderer import ArmVariableRenderer
+from checkov.common.graph.graph_builder import CustomAttributes, Edge
 from checkov.common.graph.graph_builder.local_graph import LocalGraph
 from checkov.common.util.consts import START_LINE, END_LINE
 from checkov.common.util.data_structures_utils import pickle_deepcopy
 
 if TYPE_CHECKING:
     from checkov.common.graph.graph_builder.local_graph import _Block
 
+DEPENDS_ON_FIELD = 'dependsOn'
+RESOURCE_ID_FUNC = 'resourceId('
+REFERENCE_FUNC = 'reference('
+
 
 class ArmLocalGraph(LocalGraph[ArmBlock]):
     def __init__(self, definitions: dict[str, dict[str, Any]]) -> None:
         super().__init__()
         self.vertices: list[ArmBlock] = []
         self.definitions = definitions
         self.vertices_by_path_and_id: dict[tuple[str, str], int] = {}
+        self.vertices_by_name: dict[str, int] = {}
 
     def build_graph(self, render_variables: bool = False) -> None:
         self._create_vertices()
-        logging.debug(f"[ArmLocalGraph] created {len(self.vertices)} vertices")
+        logging.warning(f"[ArmLocalGraph] created {len(self.vertices)} vertices")
+
+        '''
+            In order to resolve the resources names for the dependencies we need to render the variables first
+            Examples: https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/resource-dependency
+        '''
+
+        self._create_vars_and_parameters_edges()
+        if render_variables:
+            renderer = ArmVariableRenderer(self)
+            renderer.render_variables_from_local_graph()
 
         self._create_edges()
-        logging.debug(f"[ArmLocalGraph] created {len(self.edges)} edges")
+        logging.warning(f"[ArmLocalGraph] created {len(self.edges)} edges")
 
     def _create_vertices(self) -> None:
         for file_path, definition in self.definitions.items():
             self._create_parameter_vertices(file_path=file_path, parameters=definition.get(ArmElements.PARAMETERS))
             self._create_resource_vertices(file_path=file_path, resources=definition.get(ArmElements.RESOURCES))
+            self._create_variables_vertices(file_path=file_path, variables=definition.get(ArmElements.VARIABLES))
 
         for i, vertex in enumerate(self.vertices):
             self.vertices_by_block_type[vertex.block_type].append(i)
             self.vertices_block_name_map[vertex.block_type][vertex.name].append(i)
             self.vertices_by_path_and_id[(vertex.path, vertex.id)] = i
+            self.vertices_by_name[vertex.name] = i
+
+            self.in_edges[i] = []
+            self.out_edges[i] = []
+
+    def _create_variables_vertices(self, file_path: str, variables: dict[str, dict[str, Any]] | None) -> None:
+        if not variables:
+            return
+
+        for name, conf in variables.items():
+            if name in [START_LINE, END_LINE]:
+                continue
+            if not isinstance(conf, dict):
+                full_conf = {"value": pickle_deepcopy(conf)}
+            else:
+                full_conf = conf
+            config = pickle_deepcopy(full_conf)
+            attributes = pickle_deepcopy(full_conf)
+
+            self.vertices.append(
+                ArmBlock(
+                    name=name,
+                    config=config,
+                    path=file_path,
+                    block_type=BlockType.VARIABLE,
+                    attributes=attributes,
+                    id=f"{BlockType.VARIABLE}.{name}",
+                )
+            )
 
     def _create_parameter_vertices(self, file_path: str, parameters: dict[str, dict[str, Any]] | None) -> None:
         if not parameters:
@@ -47,7 +95,7 @@ def _create_parameter_vertices(self, file_path: str, parameters: dict[str, dict[
             if name in (START_LINE, END_LINE):
                 continue
             if not isinstance(config, dict):
-                logging.debug(f"[ArmLocalGraph] parameter {name} has wrong type {type(config)}")
+                logging.warning(f"[ArmLocalGraph] parameter {name} has wrong type {type(config)}")
                 continue
 
             attributes = pickle_deepcopy(config)
@@ -85,21 +133,69 @@ def _create_resource_vertices(self, file_path: str, resources: list[dict[str, An
                     path=file_path,
                     block_type=BlockType.RESOURCE,
                     attributes=attributes,
-                    id=f"{resource_type}.{resource_name}",
+                    id=f"{resource_type}.{resource_name}"
                 )
             )
 
     def _create_edges(self) -> None:
-        # no edges yet
-        pass
+        for origin_vertex_index, vertex in enumerate(self.vertices):
+            if DEPENDS_ON_FIELD in vertex.attributes:
+                self._create_explicit_edge(origin_vertex_index, vertex.name, vertex.attributes['dependsOn'])
+            self._create_implicit_edges(origin_vertex_index, vertex.name, vertex.attributes)
+
+    def _create_explicit_edge(self, origin_vertex_index: int, resource_name: str, deps: list[str]) -> None:
+        for dep in deps:
+            if RESOURCE_ID_FUNC in dep:
+                processed_dep = extract_resource_name_from_resource_id_func(dep)
+            else:
+                processed_dep = dep.split('/')[-1]
+            # Check if the processed dependency exists in the map
+            if processed_dep in self.vertices_by_name:
+                self._create_edge(processed_dep, origin_vertex_index, f'{resource_name}->{processed_dep}')
+            else:
+                # Dependency not found
+                logging.warning(f"[ArmLocalGraph] resource dependency {processed_dep} defined in {dep} for resource"
+                                f" {resource_name} not found")
+                continue
+
+    def _create_vars_and_parameters_edges(self) -> None:
+        pattern = r"(variables|parameters)\('(\w+)'\)"
+        for origin_vertex_index, vertex in enumerate(self.vertices):
+            for attr_key, attr_value in vertex.attributes.items():
+                if not isinstance(attr_value, str):
+                    continue
+                if ArmElements.VARIABLES in attr_value or ArmElements.PARAMETERS in attr_value:
+                    matches = re.findall(pattern, attr_value)
+                    for match in matches:
+                        var_name = match[1]
+                        self._create_edge(var_name, origin_vertex_index, attr_key)
+
+    def _create_edge(self, element_name: str, origin_vertex_index: int, label: str) -> None:
+        dest_vertex_index = self.vertices_by_name.get(element_name)
+        if origin_vertex_index == dest_vertex_index or dest_vertex_index is None:
+            return
+        edge = Edge(origin_vertex_index, dest_vertex_index, label)
+        self.edges.append(edge)
+        self.out_edges[origin_vertex_index].append(edge)
+        self.in_edges[dest_vertex_index].append(edge)
+
+    def _create_implicit_edges(self, origin_vertex_index: int, resource_name: str, resource: dict[str, Any]) -> None:
+        for value in resource.values():
+            if isinstance(value, str):
+                if REFERENCE_FUNC in value:
+                    self._create_implicit_edge(origin_vertex_index, resource_name, value)
+
+    def _create_implicit_edge(self, origin_vertex_index: int, resource_name: str, reference_string: str) -> None:
+        dep_name = extract_resource_name_from_reference_func(reference_string)
+        self._create_edge(dep_name, origin_vertex_index, f'{resource_name}->{dep_name}')
 
     def update_vertices_configs(self) -> None:
         # not used
         pass
 
     @staticmethod
     def update_vertex_config(
-        vertex: _Block, changed_attributes: list[str] | dict[str, Any], has_dynamic_blocks: bool = False
+            vertex: _Block, changed_attributes: list[str] | dict[str, Any], has_dynamic_blocks: bool = False
     ) -> None:
         # not used
         pass

checkov/arm/graph_builder/variable_rendering/renderer.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from checkov.arm.graph_builder.graph_components.block_types import BlockType
+from checkov.common.graph.graph_builder import Edge
+from checkov.common.graph.graph_builder.utils import adjust_value
+from checkov.common.graph.graph_builder.variable_rendering.renderer import VariableRenderer
+from checkov.common.util.data_structures_utils import pickle_deepcopy
+
+if TYPE_CHECKING:
+    from checkov.arm.graph_builder.local_graph import ArmLocalGraph
+
+
+class ArmVariableRenderer(VariableRenderer["ArmLocalGraph"]):
+    def __init__(self, local_graph: ArmLocalGraph) -> None:
+        super().__init__(local_graph)
+
+    def _render_variables_from_vertices(self) -> None:
+        # need to add rendering to function like format, reference etc
+        pass
+
+    def evaluate_vertex_attribute_from_edge(self, edge_list: list[Edge]) -> None:
+        origin_vertex_attributes = self.local_graph.vertices[edge_list[0].origin].attributes
+        val_to_eval = pickle_deepcopy(origin_vertex_attributes.get(edge_list[0].label, ""))
+        attr_path = None
+        for edge in edge_list:
+            attr_path, attr_value = self.extract_dest_attribute_path_and_value(dest_index=edge.dest,
+                                                                               origin_value=val_to_eval)
+            '''if the arg start with '[parameters'/ '[variables' its mean we need to eval the all attribute
+            like here - "addressPrefix": "[parameters('subnetAddressPrefix')]" '''
+            if len(edge_list) == 1 and val_to_eval.startswith(("[parameters", "[variables")):
+                val_to_eval = attr_value
+                continue
+            '''
+            if the value i need to eval is part of the full attribute like "[format('{0}/{1}', parameters('vnetName'), variables('subnetName'))]"
+            or "[resourceId('Microsoft.Network/networkProfiles', variables('networkProfileName'))]".
+            vertices[edge.dest].id = variables.networkProfileName -> variables('networkProfileName')
+            '''
+            val_to_replace = self.local_graph.vertices[edge.dest].id.replace(".", "('") + "')"
+            val_to_eval = val_to_eval.replace(val_to_replace, attr_value)
+
+        self.local_graph.update_vertex_attribute(
+            vertex_index=edge_list[0].origin,
+            attribute_key=edge_list[0].label,
+            attribute_value=val_to_eval,
+            change_origin_id=edge_list[0].dest,
+            attribute_at_dest=attr_path,
+        )
+
+    def extract_dest_attribute_path_and_value(self, dest_index: int, origin_value: Any) -> tuple[str, Any] | tuple[None, None]:
+        vertex = self.local_graph.vertices[dest_index]
+        if vertex.block_type == BlockType.PARAMETER:
+            new_value = vertex.attributes.get("defaultValue")
+            if new_value:
+                new_value = adjust_value(element_name=origin_value, value=new_value)
+                return "defaultValue", new_value
+        elif vertex.block_type == BlockType.VARIABLE:
+            new_value = adjust_value(element_name=origin_value, value=vertex.attributes["value"])
+            return "value", new_value
+        return None, None
+
+    def evaluate_non_rendered_values(self) -> None:
+        pass

checkov/arm/utils.py

@@ -40,7 +40,7 @@ def get_scannable_file_paths(root_folder: str | None = None, excluded_paths: lis
 
 
 def get_files_definitions(
-    files: Iterable[str], filepath_fn: Callable[[str], str] | None = None
+        files: Iterable[str], filepath_fn: Callable[[str], str] | None = None
 ) -> tuple[dict[str, dict[str, Any]], dict[str, list[tuple[int, str]]], list[str]]:
     """Parses ARM files into its definitions and raw data"""
 
@@ -60,3 +60,32 @@ def get_files_definitions(
             parsing_errors.append(os.path.normpath(file))
 
     return definitions, definitions_raw, parsing_errors
+
+
+def extract_resource_name_from_resource_id_func(resource_id: str) -> str:
+    '''
+        Examples:
+            resourceId('Microsoft.Network/virtualNetworks/', virtualNetworkName) -> virtualNetworkName
+    '''
+    return clean_string(resource_id.split(',')[-1].split(')')[0])
+
+
+def extract_resource_name_from_reference_func(reference: str) -> str:
+    '''
+        Examples:
+                reference('storageAccountName') -> storageAccountName
+                reference('myStorage').primaryEndpoints -> myStorage
+                reference('myStorage', '2022-09-01', 'Full').location -> myStorage
+                reference(resourceId('storageResourceGroup', 'Microsoft.Storage/storageAccounts', 'storageAccountName')), '2022-09-01') -> storageAccountName
+                reference(resourceId('Microsoft.Network/publicIPAddresses', 'ipAddressName')) -> ipAddressName
+    '''
+    resource_name = ')'.join(reference.split('reference(', 1)[1].split(')')[:-1])
+    if 'resourceId' in resource_name:
+        return clean_string(
+            ''.join(resource_name.split('resourceId(', 1)[1].split(')')[0]).split(',')[-1])
+    else:
+        return clean_string(resource_name.split(',')[0].split('/')[-1])
+
+
+def clean_string(input: str) -> str:
+    return input.replace("'", '').replace(" ", "")

checkov/bicep/graph_builder/local_graph.py

@@ -11,10 +11,10 @@
 from checkov.bicep.graph_builder.graph_components.block_types import BlockType
 from checkov.bicep.graph_builder.graph_components.blocks import BicepBlock
 from checkov.bicep.graph_builder.variable_rendering.renderer import BicepVariableRenderer
-from checkov.bicep.utils import adjust_value
 from checkov.common.graph.graph_builder.graph_components.edge import Edge
 from checkov.common.graph.graph_builder.local_graph import LocalGraph
 from checkov.common.graph.graph_builder.utils import filter_sub_keys
+from checkov.common.graph.graph_builder.utils import adjust_value
 from checkov.common.util.data_structures_utils import pickle_deepcopy
 from checkov.common.util.type_forcers import force_int
 

checkov/bicep/graph_builder/variable_rendering/renderer.py

@@ -5,8 +5,8 @@
 from pycep.transformer import BicepElement
 
 from checkov.bicep.graph_builder.graph_components.block_types import BlockType
-from checkov.bicep.utils import adjust_value
 from checkov.common.graph.graph_builder import Edge
+from checkov.common.graph.graph_builder.utils import adjust_value
 from checkov.common.graph.graph_builder.variable_rendering.renderer import VariableRenderer
 from checkov.common.util.data_structures_utils import pickle_deepcopy