[feat] Report Summary: add resource counts to summaries (#185)

fixbackend/inventory/inventory_service.py

@@ -57,7 +57,7 @@
 # alias names for better readability
 BenchmarkById = Dict[str, BenchmarkSummary]
 ChecksByBenchmarkId = Dict[str, List[Dict[str, str]]]  # benchmark_id -> [{id: check_id, severity: medium}, ...]
-ChecksByAccountId = Dict[str, Set[str]]
+ChecksByAccountId = Dict[str, Dict[str, int]]  # account_id -> check_id -> count
 SeverityByCheckId = Dict[str, str]
 T = TypeVar("T")
 V = TypeVar("V")
@@ -216,16 +216,23 @@ async def issues_since(
 
         async def account_summary() -> Dict[str, AccountSummary]:
             return {
-                entry["reported"]["id"]: AccountSummary(
-                    id=entry["reported"]["id"],
-                    name=entry["reported"]["name"],
-                    cloud=entry["ancestors"]["cloud"]["reported"]["name"],
+                entry["group"]["account_id"]: AccountSummary(
+                    id=entry["group"]["account_id"],
+                    name=entry["group"]["account_name"],
+                    cloud=entry["group"]["cloud_name"],
+                    resource_count=entry["count"],
+                )
+                async for entry in await self.client.aggregate(
+                    db,
+                    "search /ancestors.account.reported.id!=null | aggregate "
+                    "/ancestors.account.reported.id as account_id, "
+                    "/ancestors.account.reported.name as account_name, "
+                    "/ancestors.cloud.reported.name as cloud_name: sum(1) as count",
                 )
-                async for entry in await self.client.search_list(db, "is (account)")
             }
 
         async def check_summary() -> Tuple[ChecksByAccountId, SeverityByCheckId]:
-            check_accounts: ChecksByAccountId = defaultdict(set)
+            check_accounts: ChecksByAccountId = defaultdict(dict)
             check_severity: Dict[str, str] = {}
 
             async for entry in await self.client.aggregate(
@@ -234,12 +241,13 @@ async def check_summary() -> Tuple[ChecksByAccountId, SeverityByCheckId]:
                 "/security.issues[].check as check_id,"
                 "/security.issues[].severity as severity,"
                 "/ancestors.account.reported.id as account_id"
-                ": sum(1)",
+                ": sum(1) as count",
             ):
                 group = entry["group"]
+                count = entry["count"]
                 check_id = group["check_id"]
                 if isinstance(account_id := group["account_id"], str):
-                    check_accounts[check_id].add(account_id)
+                    check_accounts[check_id][account_id] = count
                 check_severity[check_id] = group["severity"]
             return check_accounts, check_severity
 
@@ -298,29 +306,36 @@ def overall_score(accounts: Dict[str, AccountSummary]) -> int:
 
             # combine benchmark and account data
             account_counter: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
-            severity_counter: Dict[str, int] = defaultdict(int)
-            account_sum_count: Dict[str, int] = defaultdict(int)
+            severity_check_counter: Dict[str, int] = defaultdict(int)
+            severity_resource_counter: Dict[str, int] = defaultdict(int)
+            account_check_sum_count: Dict[str, int] = defaultdict(int)
             failed_checks_by_severity: Dict[str, Set[str]] = defaultdict(set)
             available_checks = 0
             for bid, bench in benchmarks.items():
-                benchmark_counter: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+                benchmark_account_issue_counter: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
+                benchmark_account_resource_counter: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
                 benchmark_severity_count: Dict[str, int] = defaultdict(int)
                 for check_info in checks.get(bid, []):
                     check_id = check_info["id"]
                     benchmark_severity_count[check_info["severity"]] += 1
                     available_checks += 1
                     if severity := severity_by_check_id.get(check_id):
-                        severity_counter[severity] += 1
-                        for account_id in failed_accounts_by_check_id.get(check_id, []):
-                            benchmark_counter[account_id][severity] += 1
+                        severity_check_counter[severity] += 1
+                        for account_id, failed_resource_count in failed_accounts_by_check_id[check_id].items():
+                            benchmark_account_issue_counter[account_id][severity] += 1
+                            benchmark_account_resource_counter[account_id][severity] += failed_resource_count
+                            severity_resource_counter[severity] += failed_resource_count
                             account_counter[account_id][severity] += 1
-                            account_sum_count[severity] += 1
+                            account_check_sum_count[severity] += 1
                             failed_checks_by_severity[severity].add(check_id)
                 for account_id, account in accounts.items():
                     if account.cloud in bench.clouds:
-                        failing = benchmark_counter.get(account_id)
+                        failing = benchmark_account_issue_counter.get(account_id)
+                        failed_resources = benchmark_account_resource_counter.get(account_id)
                         bench.account_summary[account_id] = BenchmarkAccountSummary(
-                            score=bench_account_score(failing or {}, benchmark_severity_count), failed_checks=failing
+                            score=bench_account_score(failing or {}, benchmark_severity_count),
+                            failed_checks=failing,
+                            failed_resources=failed_resources,
                         )
 
             # compute a score for every account by averaging the scores of all benchmark results
@@ -333,16 +348,26 @@ def overall_score(accounts: Dict[str, AccountSummary]) -> int:
             # get issues for the top 5 issue_ids
             tops = await top_issues(failed_checks_by_severity, num=5)
 
+            # sort top changed account by score
+            vulnerable_changed.accounts_selection.sort(key=lambda x: accounts[x].score if x in accounts else 100)
+            compliant_changed.accounts_selection.sort(key=lambda x: accounts[x].score if x in accounts else 100)
+
             return ReportSummary(
                 check_summary=CheckSummary(
                     available_checks=available_checks,
-                    failed_checks=sum(v for v in severity_counter.values()),
-                    failed_checks_by_severity=severity_counter,
+                    failed_checks=sum(v for v in severity_check_counter.values()),
+                    failed_checks_by_severity=severity_check_counter,
+                    available_resources=sum(v.resource_count for v in accounts.values()),
+                    failed_resources=sum(v for v in severity_resource_counter.values()),
+                    failed_resources_by_severity=severity_resource_counter,
                 ),
                 account_check_summary=CheckSummary(
                     available_checks=available_checks * len(accounts),
-                    failed_checks=sum(v for v in account_sum_count.values()),
-                    failed_checks_by_severity=account_sum_count,
+                    failed_checks=sum(v for v in account_check_sum_count.values()),
+                    failed_checks_by_severity=account_check_sum_count,
+                    available_resources=sum(v.resource_count for v in accounts.values()),
+                    failed_resources=sum(v for v in severity_resource_counter.values()),
+                    failed_resources_by_severity=severity_resource_counter,
                 ),
                 overall_score=overall_score(accounts),
                 accounts=sorted(list(accounts.values()), key=lambda x: x.score),
@@ -354,7 +379,14 @@ def overall_score(accounts: Dict[str, AccountSummary]) -> int:
 
         except GraphDatabaseNotAvailable:
             log.warning("Graph database not available yet. Returning empty summary.")
-            empty = CheckSummary(available_checks=0, failed_checks=0, failed_checks_by_severity={})
+            empty = CheckSummary(
+                available_checks=0,
+                failed_checks=0,
+                failed_checks_by_severity={},
+                available_resources=0,
+                failed_resources=0,
+                failed_resources_by_severity={},
+            )
             return ReportSummary(
                 check_summary=empty,
                 account_check_summary=empty,

fixbackend/inventory/schemas.py

@@ -23,12 +23,14 @@ class AccountSummary(BaseModel):
     id: str = Field(description="The account id")
     name: str = Field(description="The account name")
     cloud: str = Field(description="The name of the cloud provider")
+    resource_count: int = Field(description="The number of resources in the account")
     score: int = Field(description="The score of the account", default=100)
 
 
 class BenchmarkAccountSummary(BaseModel):
     score: int = Field(description="The score of the account", default=0)
     failed_checks: Optional[Dict[str, int]] = Field(description="The number of failed checks by severity.")
+    failed_resources: Optional[Dict[str, int]] = Field(description="The number of failed resources by severity.")
 
 
 class BenchmarkSummary(BaseModel):
@@ -48,6 +50,9 @@ class CheckSummary(BaseModel):
     available_checks: int = Field(description="The number of all available checks.")
     failed_checks: int = Field(description="The number of failed checks.")
     failed_checks_by_severity: Dict[str, int] = Field(description="The number of failed checks by severity.")
+    available_resources: int = Field("The number of all available resources.")
+    failed_resources: int = Field(description="The number of failed resources.")
+    failed_resources_by_severity: Dict[str, int] = Field(description="The number of failed resources by severity.")
 
 
 class VulnerabilitiesChanged(BaseModel):

tests/fixbackend/inventory/inventory_service_test.py

@@ -92,15 +92,15 @@ async def mock(request: Request) -> Response:
                 [{"clouds": ["aws"], "description": "Test AWS", "framework": "CIS", "id": "aws_test", "report_checks": [{"id": "aws_c1", "severity": "high"}, {"id": "aws_c2", "severity": "critical"}], "title": "AWS Test", "version": "0.1"},  # fmt: skip
                  {"clouds": ["gcp"], "description": "Test GCP", "framework": "CIS", "id": "gcp_test", "report_checks": [{"id": "gcp_c1", "severity": "low"}, {"id": "gcp_c2", "severity": "medium"}], "title": "GCP Test", "version": "0.2"}]  # fmt: skip
             )
-        elif request.url.path == "/graph/resoto/search/list" and content == "is (account)":
+        elif request.url.path == "/graph/resoto/search/aggregate" and content.startswith("search /ancestors.account.reported.id!=null"):  # fmt: skip
             return nd_json_response(
-                [{"id": "n1", "type": "node", "reported": {"id": "234", "name": "account 1"}, "ancestors": {"cloud": {"reported": {"name": "gcp", "id": "gcp"}}}},  # fmt: skip
-                 {"id": "n2", "type": "node", "reported": {"id": "123", "name": "account 2"}, "ancestors": {"cloud": {"reported": {"name": "aws", "id": "aws"}}}}]  # fmt: skip
+                [{"group": {"account_id": "123", "account_name": "account 2", "cloud_name": "aws"}, "count": 54321},  # fmt: skip
+                 {"group": {"account_id": "234", "account_name": "account 1", "cloud_name": "gcp"}, "count": 12345}]  # fmt: skip
             )
-        elif request.url.path == "/graph/resoto/search/aggregate":
+        elif request.url.path == "/graph/resoto/search/aggregate" and content.startswith("search /security.has_issues==true"):  # fmt: skip
             return nd_json_response(
-                [{"group": {"check_id": "aws_c1", "severity": "low", "account_id": "123", "account_name": "t1", "cloud": "aws"}, "sum_of_1": 8},  # fmt: skip
-                 {"group": {"check_id": "gcp_c2", "severity": "critical", "account_id": "234", "account_name": "t2", "cloud": "gcp"}, "sum_of_1": 2}]  # fmt: skip
+                [{"group": {"check_id": "aws_c1", "severity": "low", "account_id": "123", "account_name": "t1", "cloud": "aws"}, "count": 8},  # fmt: skip
+                 {"group": {"check_id": "gcp_c2", "severity": "critical", "account_id": "234", "account_name": "t2", "cloud": "gcp"}, "count": 2}]  # fmt: skip
             )
         elif request.url.path == "/graph/resoto/node/some_node_id":
             return json_response(azure_virtual_machine_resource_json)
@@ -128,18 +128,26 @@ async def test_summary(inventory_service: InventoryService, mocked_answers: Requ
     assert summary.check_summary.available_checks == 4
     assert summary.check_summary.failed_checks == 2
     assert summary.check_summary.failed_checks_by_severity == {"critical": 1, "low": 1}
+    assert summary.check_summary.failed_resources == 10
+    assert summary.check_summary.failed_resources_by_severity == {"critical": 2, "low": 8}
     # account checks summary
     assert summary.account_check_summary.available_checks == 8
     assert summary.account_check_summary.failed_checks == 2
     assert summary.account_check_summary.failed_checks_by_severity == {"critical": 1, "low": 1}
+    assert summary.account_check_summary.failed_resources == 10
+    assert summary.account_check_summary.failed_resources_by_severity == {"critical": 2, "low": 8}
     # check benchmarks
     b1, b2 = summary.benchmarks
     assert b1.id == "aws_test"
     assert b1.clouds == ["aws"]
-    assert b1.account_summary == {"123": BenchmarkAccountSummary(score=85, failed_checks={"low": 1})}
+    assert b1.account_summary == {
+        "123": BenchmarkAccountSummary(score=85, failed_checks={"low": 1}, failed_resources={"low": 8})
+    }
     assert b2.id == "gcp_test"
     assert b2.clouds == ["gcp"]
-    assert b2.account_summary == {"234": BenchmarkAccountSummary(score=0, failed_checks={"critical": 1})}
+    assert b2.account_summary == {
+        "234": BenchmarkAccountSummary(score=0, failed_checks={"critical": 1}, failed_resources={"critical": 2})
+    }
     assert len(summary.accounts) == 2
     # check accounts
     gcp, aws = summary.accounts
@@ -152,10 +160,10 @@ async def test_summary(inventory_service: InventoryService, mocked_answers: Requ
     assert aws.cloud == "aws"
     assert aws.score == 85
     # check becoming vulnerable
-    assert summary.changed_vulnerable.accounts_selection == ["123", "234"]
+    assert summary.changed_vulnerable.accounts_selection == ["234", "123"]
     assert summary.changed_vulnerable.resource_count_by_severity == {"critical": 1, "medium": 87}
     assert summary.changed_vulnerable.resource_count_by_kind_selection == {"aws_instance": 87, "gcp_disk": 1}
-    assert summary.changed_compliant.accounts_selection == ["123", "234"]
+    assert summary.changed_compliant.accounts_selection == ["234", "123"]
     assert summary.changed_compliant.resource_count_by_severity == {"critical": 1, "medium": 87}
     assert summary.changed_compliant.resource_count_by_kind_selection == {"aws_instance": 87, "gcp_disk": 1}
     # top checks
@@ -171,9 +179,17 @@ async def app(_: Request) -> Response:
     async_client = AsyncClient(transport=MockTransport(app))
     async with InventoryClient("http://localhost:8980", client=async_client) as client:
         async with InventoryService(client, graph_database_access_manager, domain_event_subscriber) as service:
+            empty = CheckSummary(
+                available_checks=0,
+                failed_checks=0,
+                failed_checks_by_severity={},
+                available_resources=0,
+                failed_resources=0,
+                failed_resources_by_severity={},
+            )
             assert await service.summary(db) == ReportSummary(
-                check_summary=CheckSummary(available_checks=0, failed_checks=0, failed_checks_by_severity={}),
-                account_check_summary=CheckSummary(available_checks=0, failed_checks=0, failed_checks_by_severity={}),
+                check_summary=empty,
+                account_check_summary=empty,
                 overall_score=0,
                 accounts=[],
                 benchmarks=[],