Alerts&AI Conditions CCU Consumption Report (#9)

* add alerts & AI fetching policies and conditions. add generate csv from a list of dict

* Update alertsai.json

* Update alertsai.json

* Update README for ccuconsumption usage

* add conditions ccu consumption report

* Updated fetch condition details for migrating/mutation policies and conditions

* Remove .DS_Store from repository and added it to gitignore file

README.md

@@ -22,7 +22,9 @@ The table below lists the scripts that can be used for different use cases.
 No. | Use Case | Scripts
 --- | ---------------- | -----------------------------------------------------------
 1. | Update Tag value | entitytags --delTagValues "key:value" --addTags "key:value"
-
+2. | Dashboards | 
+3. | Alerts AI |
+4. | CCU Consumption |
 ## Usage
 
 ### 1) python3 entitytags.py
@@ -37,7 +39,7 @@ addTags | Tags to be added : owner:Jack
 getAllInfraHostTags | pass to list all mutable tags for all infra hosts
 rmAllInfraHostTags | pass to delete all mutable tags for all infra hosts
 
-### 1) python3 dashboards.py
+### 2) python3 dashboards.py
 
 Supports two actions --download or --copy
 
@@ -54,6 +56,20 @@ toAccount | copy toAccount
 toApiKey | (optional) if not provided fromApiKey is used and assumed to work for toAccount
 toName | (optional) copy toName if not then copied as 'Copy of ' source dashboard name
 
+### 3) python3 alertsai.py
+
+
+### 4) python3 ccuconsumption.py
+This script will generate a report of all conditions CCU consumption between the given `SINCE` and `UNTIL` date from all the accounts the user has access to.
+
+#### Pre-requisites:
+Update the `ccuconsumption.json` file with the following details:
+- `nr_user_api_key`: User API key
+- `since`: The start date for the report in the format `YYYY-MM-DDTHH:MM:SSZ`
+- `until`: The end date for the report in the format `YYYY-MM-DDTHH:MM:SSZ`
+
+
+
 ### Logging
 
 Logs are stored in logs/nrpy.log Logging level can be set in nrpylogger.py. Default level for file and stdout is INFO

alertsai.py

@@ -43,29 +43,68 @@ def get_all_policy_conditions(nr_user_api_key, accountId, policyId, policyName,
  while True:
  result = alertsaiclient.get_policy_conditions_nrql(nr_user_api_key, accountId, policyId, policyName, nextCursor)
  if 'error' in result:
- logger.error(json.dumps(result))
  logger.info(str(policyId) + " is an invalid policy.")
  invalid_policy_list.append({'policyId': policyId, 'policyName': policyName})
- return conditions_list, empty_policy_list, invalid_policy_list
+ return conditions_list, invalid_policy_list, empty_policy_list
  else:
  conditions = result['response']['data']['actor']['account']['alerts']['nrqlConditionsSearch']
  nextCursor = conditions['nextCursor']
  # If there are no conditions, add the policy to the empty policy list
  if not conditions['nrqlConditions']:
  logger.info(str(policyId) + " has no conditions.")
  empty_policy_list.append({'policyId': policyId, 'policyName': policyName})
- return conditions_list, empty_policy_list, invalid_policy_list
+ return conditions_list, invalid_policy_list, empty_policy_list
  else:
  result_condition_list = conditions['nrqlConditions']
  for condition in result_condition_list:
  conditions_list.append({
  "policyId": policyId,
  "policyName": policyName,
  "conditionId": condition['id'],
+ "conditionType": condition['type'],
  "conditionName": condition['name'],
  "conditionQuery": condition['nrql']['query'],
- "enabled": condition['enabled']
+ "nrqlEvaluationOffset": condition['nrql']['evaluationOffset'],
+ "description": condition['description'],
+ "enabled": condition['enabled'],
+ "runbookUrl": condition['runbookUrl'],
+ "closeViolationsOnExpiration": condition['expiration']['closeViolationsOnExpiration'],
+ "expirationDuration": condition['expiration']['expirationDuration'],
+ "openViolationOnExpiration": condition['expiration']['openViolationOnExpiration'],
+ "aggregationDelay": condition['signal']['aggregationDelay'],
+ "aggregationMethod": condition['signal']['aggregationMethod'],
+ "aggregationTimer": condition['signal']['aggregationTimer'],
+ "aggregationWindow": condition['signal']['aggregationWindow'],
+ "evaluationDelay": condition['signal']['evaluationDelay'],
+ "evaluationOffset": condition['signal']['evaluationOffset'],
+ "fillOption": condition['signal']['fillOption'],
+ "fillValue": condition['signal']['fillValue'],
+ "slideBy": condition['signal']['slideBy'],
+ "violationTimeLimit": condition['violationTimeLimit'],
+ "violationTimeLimitSeconds": condition['violationTimeLimitSeconds']
  })
+ if len(condition['terms']) > 1:
+ conditions_list[-1]["operator1"] = condition['terms'][0]['operator']
+ conditions_list[-1]["priority1"] = condition['terms'][0]['priority']
+ conditions_list[-1]["threshold1"] = condition['terms'][0]['threshold']
+ conditions_list[-1]["thresholdDuration1"] = condition['terms'][0]['thresholdDuration']
+ conditions_list[-1]["thresholdOccurrences1"] = condition['terms'][0]['thresholdOccurrences']
+ conditions_list[-1]["operator2"] = condition['terms'][1]['operator']
+ conditions_list[-1]["priority2"] = condition['terms'][1]['priority']
+ conditions_list[-1]["threshold2"] = condition['terms'][1]['threshold']
+ conditions_list[-1]["thresholdDuration2"] = condition['terms'][1]['thresholdDuration']
+ conditions_list[-1]["thresholdOccurrences2"] = condition['terms'][1]['thresholdOccurrences']
+ else:
+ conditions_list[-1]["operator1"] = condition['terms'][0]['operator']
+ conditions_list[-1]["priority1"] = condition['terms'][0]['priority']
+ conditions_list[-1]["threshold1"] = condition['terms'][0]['threshold']
+ conditions_list[-1]["thresholdDuration1"] = condition['terms'][0]['thresholdDuration']
+ conditions_list[-1]["thresholdOccurrences1"] = condition['terms'][0]['thresholdOccurrences']
+ conditions_list[-1]["operator2"] = None
+ conditions_list[-1]["priority2"] = None
+ conditions_list[-1]["threshold2"] = None
+ conditions_list[-1]["thresholdDuration2"] = None
+ conditions_list[-1]["thresholdOccurrences2"] = None
 
  # Break the loop if there is no next cursor
  if not nextCursor:

ccuconsumption.json

@@ -0,0 +1,5 @@
+{
+ "nr_user_api_key": "USER_API_KEY",
+ "since": "2024-03-10T00:00:00Z",
+ "until": "2024-03-16T23:59:59Z"
+}

ccuconsumption.py

@@ -0,0 +1,96 @@
+import os
+import argparse
+import json
+import csv
+from datetime import date
+from library import utils
+import library.clients.ccuconsumptionclient as ccuconsumptionclient
+import library.localstore as store
+import library.nrpylogger as nrpylogger
+
+ccuconsumptionclient = ccuconsumptionclient.CCUConsumption()
+logger = nrpylogger.get_logger(os.path.basename(__file__))
+
+config = store.load_json_from_file(".", "ccuconsumption.json")
+nr_user_api_key = config['nr_user_api_key']
+since = config['since']
+until = config['until']
+
+
+def get_all_accounts(nr_user_api_key):
+ all_accounts_dicts = []
+ all_accounts_list = []
+ result = ccuconsumptionclient.get_current_user_all_accounts(nr_user_api_key)
+ accounts = result['response']['data']['actor']['accounts']
+ for account in accounts:
+ all_accounts_list.append(account['id'])
+ all_accounts_dicts.append({
+ "id": account['id'],
+ "name": account['name']
+ })
+ return all_accounts_list, all_accounts_dicts
+
+
+def generate_ccu_consumption_report_for_all_accounts(nr_user_api_key):
+ all_accounts_list, all_accounts_dicts = get_all_accounts(nr_user_api_key)
+ ccu_consumption_report = []
+ for account in all_accounts_list:
+ result = get_ccu_consumption_per_condition(nr_user_api_key, account, since, until)
+ for condition in result:
+ condition_details = get_condition_details(nr_user_api_key, account, condition["conditionId"])
+ if condition_details:
+ ccu_consumption_report.append({
+ "accountId": account,
+ "conditionId": condition["conditionId"],
+ "conditionName": condition_details["conditionName"],
+ "policyId": condition_details["policyId"],
+ "conditionQuery": condition_details["query"],
+ "ccuConsumption": condition["ccuConsumption"]
+ })
+ store.save_list_of_dict_as_csv(ccu_consumption_report, "ccu_consumption_report_%s_%s.csv" % (since, until))
+ logger.info("CCU consumption report has been generated.")
+
+
+def get_condition_details(nr_user_api_key, accountId, conditionId):
+ condition_details = {}
+ result = ccuconsumptionclient.get_condition_details(nr_user_api_key, accountId, conditionId)
+ if 'error' in result:
+ logger.error(json.dumps(result))
+ return
+ elif result['response']['data']['actor']['account']['alerts']['nrqlCondition']:
+ conditions_data = result['response']['data']['actor']['account']['alerts']['nrqlCondition']
+ if conditions_data['name']:
+ condition_details["conditionName"] = conditions_data['name']
+ else:
+ condition_details["conditionName"] = ""
+ condition_details["policyId"] = conditions_data['policyId']
+ condition_details["query"] = conditions_data['nrql']['query']
+ return condition_details
+ else:
+ logger.info("Condition id " + str(conditionId) + " is invalid.")
+ return
+
+def get_ccu_consumption_per_condition(nr_user_api_key, accountId, since, until):
+ condition_ccu_consumption = []
+ result = ccuconsumptionclient.get_ccu_consumption(nr_user_api_key, accountId, since, until)
+ ccu_per_conidition = result['response']['data']['actor']['nrql']['results']
+ for condition in ccu_per_conidition:
+ condition_ccu_consumption.append({
+ "conditionId": condition['dimension_conditionId'],
+ "ccuConsumption": condition['sum.usage']
+ })
+ return condition_ccu_consumption
+
+
+def load_ccu_tier_prices():
+ ccu_tier_list = store.load_csv_to_list_of_dicts("ccu_tier.csv")
+ return ccu_tier_list
+
+
+def load_ccu_discounts():
+ ccu_discount_list = store.load_csv_to_list_of_dicts("ccu_discount.csv")
+ return ccu_discount_list
+
+
+if __name__ == '__main__':
+ generate_ccu_consumption_report_for_all_accounts(nr_user_api_key)

library/clients/alertsaiclient.py

@@ -69,44 +69,106 @@ def get_all_policies_payload(accountId, nextCursor=None):
  def get_policy_conditions_payload(accountId, policyId, policyName, nextCursor=None):
  if nextCursor:
  conditions_query = """query ($accountId: Int!, $policyId: ID) {
- actor {
- account(id: $accountId) {
- alerts {
- nrqlConditionsSearch(searchCriteria: {policyId: $policyId}, cursor: "%s") {
- nextCursor
- nrqlConditions {
- enabled
- name
- id
- nrql {
- query
- }
- }
- }
- }
- }
- }
- }""" % (nextCursor)
+ actor {
+ account(id: $accountId) {
+ alerts {
+ nrqlConditionsSearch(searchCriteria: {policyId: $policyId}, cursor: "%s") {
+ nextCursor
+ nrqlConditions {
+ description
+ enabled
+ expiration {
+ closeViolationsOnExpiration
+ expirationDuration
+ openViolationOnExpiration
+ }
+ id
+ name
+ nrql {
+ evaluationOffset
+ query
+ }
+ policyId
+ runbookUrl
+ signal {
+ aggregationDelay
+ aggregationMethod
+ aggregationTimer
+ aggregationWindow
+ evaluationDelay
+ evaluationOffset
+ fillOption
+ fillValue
+ slideBy
+ }
+ terms {
+ operator
+ priority
+ threshold
+ thresholdDuration
+ thresholdOccurrences
+ }
+ type
+ violationTimeLimit
+ violationTimeLimitSeconds
+ }
+ totalCount
+ }
+ }
+ }
+ }
+ }""" % nextCursor
  variables = {'accountId': accountId, 'policyId': policyId}
  else:
  conditions_query = """query ($accountId: Int!, $policyId: ID) {
- actor {
- account(id: $accountId) {
- alerts {
- nrqlConditionsSearch(searchCriteria: {policyId: $policyId}) {
- nextCursor
- nrqlConditions {
- enabled
- name
- id
- nrql {
- query
+ actor {
+ account(id: $accountId) {
+ alerts {
+ nrqlConditionsSearch(searchCriteria: {policyId: $policyId}) {
+ nextCursor
+ nrqlConditions {
+ description
+ enabled
+ expiration {
+ closeViolationsOnExpiration
+ expirationDuration
+ openViolationOnExpiration
+ }
+ id
+ name
+ nrql {
+ evaluationOffset
+ query
+ }
+ policyId
+ runbookUrl
+ signal {
+ aggregationDelay
+ aggregationMethod
+ aggregationTimer
+ aggregationWindow
+ evaluationDelay
+ evaluationOffset
+ fillOption
+ fillValue
+ slideBy
+ }
+ terms {
+ operator
+ priority
+ threshold
+ thresholdDuration
+ thresholdOccurrences
+ }
+ type
+ violationTimeLimit
+ violationTimeLimitSeconds
+ }
+ totalCount
+ }
+ }
  }
  }
- }
- }
- }
- }
- }"""
+ }"""
  variables = {'accountId': accountId, 'policyId': policyId}
  return {'query': conditions_query, 'variables': variables}