From 4fc338f137fa9a9815dbbe6be6aeffeb62aa364a Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Fri, 27 Jan 2023 20:54:31 +0100 Subject: [PATCH] [plugins/gcp][fix] Fix the GCP collection issues (#1389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [plugins/gcp][fix] Fix the GCP collection issues * remove old comments * quotas fix * Refactor machine_type usage * fix the quotas factory * fix quotas kwarg --------- Co-authored-by: Lukas Lösche --- plugins/cleanup_expired/setup.py | 1 + plugins/gcp/resoto_plugin_gcp/collector.py | 36 ++++++++++++---------- plugins/gcp/resoto_plugin_gcp/resources.py | 34 ++++++-------------- 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/plugins/cleanup_expired/setup.py b/plugins/cleanup_expired/setup.py index cbca8d5525..2e2f8b6dc2 100644 --- a/plugins/cleanup_expired/setup.py +++ b/plugins/cleanup_expired/setup.py @@ -2,6 +2,7 @@ import pkg_resources from setuptools import setup, find_packages + def read(file_name: str) -> str: with open(os.path.join(os.path.dirname(__file__), file_name)) as of: return of.read() diff --git a/plugins/gcp/resoto_plugin_gcp/collector.py b/plugins/gcp/resoto_plugin_gcp/collector.py index 8ee1d9d617..b37087a46b 100644 --- a/plugins/gcp/resoto_plugin_gcp/collector.py +++ b/plugins/gcp/resoto_plugin_gcp/collector.py @@ -456,7 +456,7 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di "ctime": iso2datetime(result.get("creationTimestamp")), "link": result.get("selfLink"), "label_fingerprint": result.get("labelFingerprint"), - "_account": self.project, + "account": self.project, } if attr_map is not None: @@ -469,7 +469,7 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di kwargs[map_to] = data # By default we search for a resources region and/or zone - default_search_map = {"_region": ["link", "region"], "_zone": ["link", "zone"]} + default_search_map = {"region": ["link", "region"], "zone": ["link", "zone"]} search_results = {} if search_map is None: search_map = dict(default_search_map) @@ -505,12 +505,12 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di # region based on the zone information we found. # E.g. if we know a disk is in zone us-central1-a then we can find # the region us-central1 from that. - if "_zone" in kwargs and "_region" not in kwargs and isinstance(kwargs["_zone"], BaseResource): - region = kwargs["_zone"].region(self.graph) + if "zone" in kwargs and "region" not in kwargs and isinstance(kwargs["zone"], BaseResource): + region = kwargs["zone"].region(self.graph) if region: - kwargs["_region"] = region - if "_region" in search_map.keys() and "_region" not in search_results: - search_results["_region"] = region + kwargs["region"] = region + if "region" in search_map.keys() and "region" not in search_results: + search_results["region"] = region return kwargs, search_results @@ -620,7 +620,7 @@ def collect_something( log.debug(f"Parent resource for {r.rtdname} set to {pr.rtdname}") if not isinstance(pr, BaseResource): - pr = kwargs.get("_zone", kwargs.get("_region", self.graph.root)) + pr = kwargs.get("zone", kwargs.get("region", self.graph.root)) log.debug(f"Parent resource for {r.rtdname} automatically set to {pr.rtdname}") self.graph.add_resource(pr, r, edge_type=EdgeType.default) @@ -692,7 +692,7 @@ def post_process(resource: GCPRegion, graph: Graph): self.collect_something( resource_class=GCPRegion, - attr_map={"region_status": "status", "quotas": "quotas"}, + attr_map={"region_status": "status", "_quotas": "quotas"}, post_process=post_process, ) @@ -756,7 +756,8 @@ def post_process(resource: GCPInstance, graph: Graph): Once added to the graph resoto will find it for successive instances of the same machine type. """ - if resource.instance_type == "" and "custom" in resource._machine_type_link: + resource.init_machine_type() + if resource.instance_type in ("", None) and "custom" in resource._machine_type_link: if resource.instance_status == InstanceStatus.TERMINATED: resource._cleaned = True log.debug(f"Fetching custom instance type for {resource.rtdname}") @@ -781,6 +782,9 @@ def post_process(resource: GCPInstance, graph: Graph): graph.add_edge(machine_type, resource, edge_type=EdgeType.default) self.post_process_machine_type(machine_type, graph) resource._machine_type = machine_type + resource.init_machine_type() + resource._machine_type = None + resource._machine_type_link = None instance_status_map: Dict[str, InstanceStatus] = { "PROVISIONING": InstanceStatus.BUSY, @@ -810,14 +814,14 @@ def post_process(resource: GCPInstance, graph: Graph): "link", (lambda r: next(iter(r.get("networkInterfaces", [])), {}).get("subnetwork")), ], - "machine_type": ["link", "machineType"], + "_machine_type": ["link", "machineType"], }, attr_map={ "instance_status": lambda i: instance_status_map.get(i["status"], InstanceStatus.UNKNOWN), - "machine_type_link": "machineType", + "_machine_type_link": "machineType", }, predecessors={ - EdgeType.default: ["__network", "__subnetwork", "machine_type"], + EdgeType.default: ["__network", "__subnetwork", "_machine_type"], EdgeType.delete: ["__network", "__subnetwork"], }, ) @@ -1104,7 +1108,7 @@ def collect_machine_types(self): resource_class=GCPMachineType, paginate_method_name="aggregatedList", search_map={ - "_zone": ["name", "zone"], + "zone": ["name", "zone"], }, attr_map={ "instance_cores": lambda r: float(r.get("guestCpus", 0)), @@ -1451,8 +1455,8 @@ def collect_databases(self): "tags": lambda r: r.get("settings", {}).get("userLabels", {}), }, search_map={ - "_region": ["name", "region"], - "_zone": ["name", "gceZone"], + "region": ["name", "region"], + "zone": ["name", "gceZone"], }, ) diff --git a/plugins/gcp/resoto_plugin_gcp/resources.py b/plugins/gcp/resoto_plugin_gcp/resources.py index c88856f6ef..64ca255884 100644 --- a/plugins/gcp/resoto_plugin_gcp/resources.py +++ b/plugins/gcp/resoto_plugin_gcp/resources.py @@ -199,13 +199,7 @@ class GCPRegion(GCPResource, BaseRegion): } api_identifier: ClassVar[str] = "region" region_status: Optional[str] = None - - def __attrs_post_init__(self, quotas: List[str]) -> None: - super().__attrs_post_init__() - if quotas is not None: - self._quotas = quotas - else: - self._quotas = [] + _quotas: Optional[List[str]] = field(factory=list, alias="_quotas") @define(eq=False, slots=False) @@ -274,24 +268,14 @@ class GCPInstance(GCPResource, BaseInstance): api_identifier: ClassVar[str] = "instance" network_interfaces: Optional[str] = None - - def __attrs_post_init__(self, machine_type_link: str, machine_type: BaseInstanceType) -> None: - super().__attrs_post_init__() - self._machine_type_link = machine_type_link - self._machine_type = machine_type - - @property - def _machine_type(self) -> Optional[BaseInstanceType]: - if hasattr(self, "__machine_type"): - return self.__machine_type - - @_machine_type.setter - def _machine_type(self, value: BaseInstanceType) -> None: - if isinstance(value, BaseInstanceType): - self.__machine_type = value - self.instance_cores = value.instance_cores - self.instance_memory = value.instance_memory - self.instance_type = value.name + _machine_type_link: Optional[str] = field(default=None, alias="_machine_type_link") + _machine_type: Optional[BaseInstanceType] = field(default=None, alias="_machine_type") + + def init_machine_type(self) -> None: + if isinstance(self._machine_type, BaseInstanceType): + self.instance_cores = self._machine_type.instance_cores + self.instance_memory = self._machine_type.instance_memory + self.instance_type = self._machine_type.name @define(eq=False, slots=False)