From 6d23f4051bd97a2b09664d4e0926fd98a9c2da35 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 25 Sep 2024 11:22:05 +0200 Subject: [PATCH 1/3] [feat] Provide one summary for all kinds --- fixbackend/inventory/inventory_router.py | 17 +++----- fixbackend/inventory/inventory_schemas.py | 6 +++ fixbackend/inventory/inventory_service.py | 47 ++++++++++------------- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/fixbackend/inventory/inventory_router.py b/fixbackend/inventory/inventory_router.py index 05858b39..24c5cf06 100644 --- a/fixbackend/inventory/inventory_router.py +++ b/fixbackend/inventory/inventory_router.py @@ -37,6 +37,7 @@ SearchTableRequest, TimeseriesRequest, Scatters, + KindUsage, ) from fixbackend.streaming_response import streaming_response, StreamOnSuccessResponse from fixbackend.workspaces.dependencies import UserWorkspaceDependency @@ -167,6 +168,7 @@ async def summary(graph_db: CurrentGraphDbDependency, workspace: UserWorkspaceDe async def model( graph_db: CurrentGraphDbDependency, kind: Optional[List[str]] = Query(default=None, description="Kinds to return."), + kind_filter: Optional[List[str]] = Query(default=None, description="Kind filter to apply."), with_bases: bool = Query(default=False, description="Include base kinds."), with_property_kinds: bool = Query(default=False, description="Include property kinds."), aggregate_roots_only: bool = Query(default=True, description="Include only aggregate roots."), @@ -179,6 +181,7 @@ async def model( graph_db, result_format="simple", kind=kind, + kind_filter=kind_filter, with_bases=with_bases, with_property_kinds=with_property_kinds, aggregate_roots_only=aggregate_roots_only, @@ -410,16 +413,8 @@ async def timeseries(graph_db: CurrentGraphDbDependency, ts: TimeseriesRequest) aggregation=ts.aggregation, ) - @router.get("/descendant/summary/{level}", tags=["timeseries"]) - async def descendant_summary_account( - graph_db: CurrentGraphDbDependency, level: Literal["account", "region", "zone"] - ) -> Dict[str, Dict[str, int]]: - return await inventory().descendant_summary(graph_db, level) - - @router.get("/descendant/count/{level}", tags=["timeseries"]) - async def descendant_count_account( - graph_db: CurrentGraphDbDependency, level: Literal["account", "region", "zone"] - ) -> Dict[str, int]: - return await inventory().descendant_count_by(graph_db, level) + @router.get("/descendant/summary", tags=["search"]) + async def descendant_summary_account(graph_db: CurrentGraphDbDependency) -> Dict[str, KindUsage]: + return await inventory().descendant_summary(graph_db) return router diff --git a/fixbackend/inventory/inventory_schemas.py b/fixbackend/inventory/inventory_schemas.py index 9c38eb50..ff7c2717 100644 --- a/fixbackend/inventory/inventory_schemas.py +++ b/fixbackend/inventory/inventory_schemas.py @@ -270,3 +270,9 @@ class TimeseriesRequest(BaseModel): group: Optional[Set[str]] = None filter_group: Optional[List[str]] = None aggregation: Optional[str] = None + + +class KindUsage(BaseModel): + accounts: int = Field(default=0, description="The number of accounts using this kind.") + regions: int = Field(default=0, description="The number of regions using this kind.") + resources: int = Field(default=0, description="The number of resources of this kind.") diff --git a/fixbackend/inventory/inventory_service.py b/fixbackend/inventory/inventory_service.py index 4a2dd24c..383d8d1a 100644 --- a/fixbackend/inventory/inventory_service.py +++ b/fixbackend/inventory/inventory_service.py @@ -87,6 +87,7 @@ Scatter, Scatters, SearchTableRequest, + KindUsage, ) from fixbackend.logging_context import set_cloud_account_id, set_fix_cloud_account_id, set_workspace_id from fixbackend.types import Redis @@ -860,35 +861,29 @@ async def overall_score() -> Tuple[int, int]: return await self.cache.call(compute_inventory_info, key=str(dba.workspace_id))(duration) - async def descendant_summary( - self, dba: GraphDatabaseAccess, kind: Literal["account", "region", "zone"] - ) -> Dict[str, Dict[str, int]]: + async def descendant_summary(self, dba: GraphDatabaseAccess) -> Dict[str, KindUsage]: - async def compute_descendant_summary(on: str) -> Dict[str, Dict[str, int]]: - result: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) - async with self.client.search(dba, f"is({on}) and /metadata.descendant_count>0") as response: + async def compute_descendant_summary() -> Dict[str, KindUsage]: + account_usage: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) + region_usage: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) + async with self.client.search(dba, f"is(account) and /metadata.descendant_count>0") as response: async for acc in response: - if level_name := value_in_path(acc, "reported.id"): + if account_id := value_in_path(acc, "reported.id"): descendant_summary = value_in_path(acc, "metadata.descendant_summary") or {} for descendant_kind, count in descendant_summary.items(): - result[level_name][descendant_kind] += count - return result - - return await self.cache.call(compute_descendant_summary, key=str(dba.workspace_id))(kind) - - async def descendant_count_by( - self, dba: GraphDatabaseAccess, kind: Literal["account", "region", "zone"] - ) -> Dict[str, int]: - - async def compute_descendant_count_by(on: str) -> Dict[str, int]: - counter: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) - async with self.client.search(dba, f"is({on}) and /metadata.descendant_count>0") as response: + account_usage[descendant_kind][account_id] += count + async with self.client.search(dba, f"is(region) and /metadata.descendant_count>0") as response: async for acc in response: - if level_name := value_in_path(acc, "reported.id"): + if region_id := value_in_path(acc, "reported.id"): descendant_summary = value_in_path(acc, "metadata.descendant_summary") or {} - for descendant_kind in descendant_summary: - counter[descendant_kind][level_name] = 1 - - return {k: len(v) for k, v in counter.items()} - - return await self.cache.call(compute_descendant_count_by, key=str(dba.workspace_id))(kind) + for descendant_kind, count in descendant_summary.items(): + region_usage[descendant_kind][region_id] += count + kind_usage: Dict[str, KindUsage] = defaultdict(lambda: KindUsage()) + for kind, accounts in account_usage.items(): + kind_usage[kind].accounts = len(accounts) + kind_usage[kind].resources = sum(accounts.values()) + for kind, regions in region_usage.items(): + kind_usage[kind].regions = len(regions) + return kind_usage + + return await self.cache.call(compute_descendant_summary, key=str(dba.workspace_id))() From 7060b0becea2a65f63e7c34e4f06912e9cd3a886 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 25 Sep 2024 12:06:03 +0200 Subject: [PATCH 2/3] make linter happy --- fixbackend/inventory/inventory_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fixbackend/inventory/inventory_service.py b/fixbackend/inventory/inventory_service.py index 383d8d1a..1922f75d 100644 --- a/fixbackend/inventory/inventory_service.py +++ b/fixbackend/inventory/inventory_service.py @@ -866,13 +866,13 @@ async def descendant_summary(self, dba: GraphDatabaseAccess) -> Dict[str, KindUs async def compute_descendant_summary() -> Dict[str, KindUsage]: account_usage: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) region_usage: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int)) - async with self.client.search(dba, f"is(account) and /metadata.descendant_count>0") as response: + async with self.client.search(dba, "is(account) and /metadata.descendant_count>0") as response: async for acc in response: if account_id := value_in_path(acc, "reported.id"): descendant_summary = value_in_path(acc, "metadata.descendant_summary") or {} for descendant_kind, count in descendant_summary.items(): account_usage[descendant_kind][account_id] += count - async with self.client.search(dba, f"is(region) and /metadata.descendant_count>0") as response: + async with self.client.search(dba, "is(region) and /metadata.descendant_count>0") as response: async for acc in response: if region_id := value_in_path(acc, "reported.id"): descendant_summary = value_in_path(acc, "metadata.descendant_summary") or {} From 222c85b36a180389fa72ce333007b50deda33c63 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Wed, 25 Sep 2024 12:18:40 +0200 Subject: [PATCH 3/3] no lambda --- fixbackend/inventory/inventory_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fixbackend/inventory/inventory_service.py b/fixbackend/inventory/inventory_service.py index 1922f75d..56d3aa74 100644 --- a/fixbackend/inventory/inventory_service.py +++ b/fixbackend/inventory/inventory_service.py @@ -878,7 +878,7 @@ async def compute_descendant_summary() -> Dict[str, KindUsage]: descendant_summary = value_in_path(acc, "metadata.descendant_summary") or {} for descendant_kind, count in descendant_summary.items(): region_usage[descendant_kind][region_id] += count - kind_usage: Dict[str, KindUsage] = defaultdict(lambda: KindUsage()) + kind_usage: Dict[str, KindUsage] = defaultdict(KindUsage) for kind, accounts in account_usage.items(): kind_usage[kind].accounts = len(accounts) kind_usage[kind].resources = sum(accounts.values())