Skip to content

Commit

Permalink
[plugins/gcp][fix] Fix the GCP collection issues (#1389)
Browse files Browse the repository at this point in the history 
* [plugins/gcp][fix] Fix the GCP collection issues

* remove old comments

* quotas fix

* Refactor machine_type usage

* fix the quotas factory

* fix quotas kwarg

---------

Co-authored-by: Lukas Lösche <[email protected]>
  • Loading branch information
@meln1k @lloesche
meln1k and lloesche authored Jan 27, 2023
1 parent 40395ff commit 4fc338f
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 41 deletions.
1 change: 1 addition & 0 deletions plugins/cleanup_expired/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import pkg_resources
from setuptools import setup, find_packages


def read(file_name: str) -> str:
with open(os.path.join(os.path.dirname(__file__), file_name)) as of:
return of.read()
Expand Down
36 changes: 20 additions & 16 deletions plugins/gcp/resoto_plugin_gcp/collector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,7 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di
"ctime": iso2datetime(result.get("creationTimestamp")),
"link": result.get("selfLink"),
"label_fingerprint": result.get("labelFingerprint"),
"_account": self.project,
"account": self.project,
}

if attr_map is not None:
Expand All @@ -469,7 +469,7 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di
kwargs[map_to] = data

# By default we search for a resources region and/or zone
default_search_map = {"_region": ["link", "region"], "_zone": ["link", "zone"]}
default_search_map = {"region": ["link", "region"], "zone": ["link", "zone"]}
search_results = {}
if search_map is None:
search_map = dict(default_search_map)
Expand Down Expand Up @@ -505,12 +505,12 @@ def default_attributes(self, result: Dict, attr_map: Dict = None, search_map: Di
# region based on the zone information we found.
# E.g. if we know a disk is in zone us-central1-a then we can find
# the region us-central1 from that.
if "_zone" in kwargs and "_region" not in kwargs and isinstance(kwargs["_zone"], BaseResource):
region = kwargs["_zone"].region(self.graph)
if "zone" in kwargs and "region" not in kwargs and isinstance(kwargs["zone"], BaseResource):
region = kwargs["zone"].region(self.graph)
if region:
kwargs["_region"] = region
if "_region" in search_map.keys() and "_region" not in search_results:
search_results["_region"] = region
kwargs["region"] = region
if "region" in search_map.keys() and "region" not in search_results:
search_results["region"] = region

return kwargs, search_results

Expand Down Expand Up @@ -620,7 +620,7 @@ def collect_something(
log.debug(f"Parent resource for {r.rtdname} set to {pr.rtdname}")

if not isinstance(pr, BaseResource):
pr = kwargs.get("_zone", kwargs.get("_region", self.graph.root))
pr = kwargs.get("zone", kwargs.get("region", self.graph.root))
log.debug(f"Parent resource for {r.rtdname} automatically set to {pr.rtdname}")
self.graph.add_resource(pr, r, edge_type=EdgeType.default)

Expand Down Expand Up @@ -692,7 +692,7 @@ def post_process(resource: GCPRegion, graph: Graph):

self.collect_something(
resource_class=GCPRegion,
attr_map={"region_status": "status", "quotas": "quotas"},
attr_map={"region_status": "status", "_quotas": "quotas"},
post_process=post_process,
)

Expand Down Expand Up @@ -756,7 +756,8 @@ def post_process(resource: GCPInstance, graph: Graph):
Once added to the graph resoto will find it for successive
instances of the same machine type.
"""
if resource.instance_type == "" and "custom" in resource._machine_type_link:
resource.init_machine_type()
if resource.instance_type in ("", None) and "custom" in resource._machine_type_link:
if resource.instance_status == InstanceStatus.TERMINATED:
resource._cleaned = True
log.debug(f"Fetching custom instance type for {resource.rtdname}")
Expand All @@ -781,6 +782,9 @@ def post_process(resource: GCPInstance, graph: Graph):
graph.add_edge(machine_type, resource, edge_type=EdgeType.default)
self.post_process_machine_type(machine_type, graph)
resource._machine_type = machine_type
resource.init_machine_type()
resource._machine_type = None
resource._machine_type_link = None

instance_status_map: Dict[str, InstanceStatus] = {
"PROVISIONING": InstanceStatus.BUSY,
Expand Down Expand Up @@ -810,14 +814,14 @@ def post_process(resource: GCPInstance, graph: Graph):
"link",
(lambda r: next(iter(r.get("networkInterfaces", [])), {}).get("subnetwork")),
],
"machine_type": ["link", "machineType"],
"_machine_type": ["link", "machineType"],
},
attr_map={
"instance_status": lambda i: instance_status_map.get(i["status"], InstanceStatus.UNKNOWN),
"machine_type_link": "machineType",
"_machine_type_link": "machineType",
},
predecessors={
EdgeType.default: ["__network", "__subnetwork", "machine_type"],
EdgeType.default: ["__network", "__subnetwork", "_machine_type"],
EdgeType.delete: ["__network", "__subnetwork"],
},
)
Expand Down Expand Up @@ -1104,7 +1108,7 @@ def collect_machine_types(self):
resource_class=GCPMachineType,
paginate_method_name="aggregatedList",
search_map={
"_zone": ["name", "zone"],
"zone": ["name", "zone"],
},
attr_map={
"instance_cores": lambda r: float(r.get("guestCpus", 0)),
Expand Down Expand Up @@ -1451,8 +1455,8 @@ def collect_databases(self):
"tags": lambda r: r.get("settings", {}).get("userLabels", {}),
},
search_map={
"_region": ["name", "region"],
"_zone": ["name", "gceZone"],
"region": ["name", "region"],
"zone": ["name", "gceZone"],
},
)

Expand Down
34 changes: 9 additions & 25 deletions plugins/gcp/resoto_plugin_gcp/resources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,13 +199,7 @@ class GCPRegion(GCPResource, BaseRegion):
}
api_identifier: ClassVar[str] = "region"
region_status: Optional[str] = None

def __attrs_post_init__(self, quotas: List[str]) -> None:
super().__attrs_post_init__()
if quotas is not None:
self._quotas = quotas
else:
self._quotas = []
_quotas: Optional[List[str]] = field(factory=list, alias="_quotas")


@define(eq=False, slots=False)
Expand Down Expand Up @@ -274,24 +268,14 @@ class GCPInstance(GCPResource, BaseInstance):
api_identifier: ClassVar[str] = "instance"

network_interfaces: Optional[str] = None

def __attrs_post_init__(self, machine_type_link: str, machine_type: BaseInstanceType) -> None:
super().__attrs_post_init__()
self._machine_type_link = machine_type_link
self._machine_type = machine_type

@property
def _machine_type(self) -> Optional[BaseInstanceType]:
if hasattr(self, "__machine_type"):
return self.__machine_type

@_machine_type.setter
def _machine_type(self, value: BaseInstanceType) -> None:
if isinstance(value, BaseInstanceType):
self.__machine_type = value
self.instance_cores = value.instance_cores
self.instance_memory = value.instance_memory
self.instance_type = value.name
_machine_type_link: Optional[str] = field(default=None, alias="_machine_type_link")
_machine_type: Optional[BaseInstanceType] = field(default=None, alias="_machine_type")

def init_machine_type(self) -> None:
if isinstance(self._machine_type, BaseInstanceType):
self.instance_cores = self._machine_type.instance_cores
self.instance_memory = self._machine_type.instance_memory
self.instance_type = self._machine_type.name


@define(eq=False, slots=False)
Expand Down

0 comments on commit 4fc338f

Please sign in to comment.