Merge pull request #14 from OpenClassrooms/use-multiple-dimension

Use multiple dimension

app/exporter.py

@@ -3,7 +3,6 @@
 # Filename: exporter.py
 
 import logging
-import time
 from datetime import datetime
 
 import boto3
@@ -38,19 +37,17 @@ def __init__(
                 self.labels.add(group["label_name"])
         self.aws_daily_cost_usd = Gauge(self.metric_name, "Daily cost of an AWS account in USD", self.labels)
 
-    def run_metrics_loop(self):
-        while True:
-            # every time we clear up all the existing labels before setting new ones
-            self.aws_daily_cost_usd .clear()
-
-            for aws_account in self.targets:
-                logging.info("querying cost data for aws account %s" % aws_account["Publisher"])
-                try:
-                    self.fetch(aws_account)
-                except Exception as e:
-                    logging.error(e)
-                    continue
-            time.sleep(self.polling_interval_seconds)
+    def run_metrics(self):
+        # every time we clear up all the existing labels before setting new ones
+        self.aws_daily_cost_usd .clear()
+
+        for aws_account in self.targets:
+            logging.info("querying cost data for aws account %s" % aws_account["Publisher"])
+            try:
+                self.fetch(aws_account)
+            except Exception as e:
+                logging.error(e)
+                continue
 
     def get_aws_account_session_via_iam_user(self, account_id):
         sts_client = boto3.client(

exporter_config.yaml

@@ -1,28 +1,47 @@
 exporter_port: $EXPORTER_PORT|9090 # the port that exposes cost metrics
 polling_interval_seconds: $POLLING_INTERVAL_SECONDS|28800 # by default it is 8 hours because for daily cost, AWS only updates the data once per day
-metric_name: aws_daily_cost_usd # change the metric name if needed
 
 aws_access_key: $AWS_ACCESS_KEY|"" # for prod deployment, DO NOT put the actual value here or default is null ("") to use iam-role/irsa
 aws_access_secret: $AWS_ACCESS_SECRET|"" # for prod deployment, DO NOT put the actual value here or default is set null ("") to use iam-role/irsa
 aws_assumed_role_name: example-assumerole
 
-group_by:
-  enabled: true
-  # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY.
-  # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html
-  # note: label_name should be unique, and different from the labes in target_aws_accounts
-  groups:
-    - type: DIMENSION
-      key: SERVICE
-      label_name: ServiceName
-    - type: DIMENSION
-      key: REGION
-      label_name: RegionName
-  merge_minor_cost:
-    # if this is enabled, minor cost that is below the threshold will be merged into one group
+metrics:
+- metric_name: aws_daily_cost_by_service_by_account # change the metric name if needed
+  group_by:
+    enabled: true
+    # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY.
+    # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html
+    # note: label_name should be unique, and different from the labes in target_aws_accounts
+    groups:
+      - type: DIMENSION
+        key: SERVICE
+        label_name: ServiceName
+      - type: DIMENSION
+        key: LINKED_ACCOUNT
+        label_name: AccountName
+    merge_minor_cost:
+      # if this is enabled, minor cost that is below the threshold will be merged into one group
+      enabled: false
+      threshold: 10
+      tag_value: other
+- metric_name: aws_daily_cost_usd # change the metric name if needed
+  group_by:
     enabled: false
-    threshold: 10
-    tag_value: other
+    # Cost data can be groupped using up to two different groups: DIMENSION, TAG, COST_CATEGORY.
+    # ref: https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_GetCostAndUsageWithResources.html
+    # note: label_name should be unique, and different from the labes in target_aws_accounts
+    groups:
+      - type: DIMENSION
+        key: SERVICE
+        label_name: ServiceName
+      - type: DIMENSION
+        key: REGION
+        label_name: RegionName
+    merge_minor_cost:
+      # if this is enabled, minor cost that is below the threshold will be merged into one group
+      enabled: false
+      threshold: 10
+      tag_value: other
 
 target_aws_accounts:
   # here defines a list of target AWS accounts

main.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import sys
+import time
 
 from envyaml import EnvYAML
 from prometheus_client import start_http_server
@@ -32,26 +33,12 @@ def get_configs():
 
 
 def validate_configs(config):
-    if config["group_by.enabled"]:
-        if len(config["group_by.groups"]) < 1 or len(config["group_by.groups"]) > 2:
-            logging.error("If group_by is enabled, there should be at least one group, and at most two groups!")
-            sys.exit(1)
-        group_label_names = set()
-        for group in config["group_by.groups"]:
-            if group["label_name"] in group_label_names:
-                logging.error("Group label names should be unique!")
-                sys.exit(1)
-            else:
-                group_label_names.add(group["label_name"])
 
     if len(config["target_aws_accounts"]) == 0:
         logging.error("There should be at least one target AWS accounts defined in the config!")
         sys.exit(1)
 
     labels = config["target_aws_accounts"][0].keys()
-    if group_label_names and (group_label_names & set(labels)):
-        logging.error("Some label names in group_by are the same as AWS account labels!")
-        sys.exit(1)
 
     if "Publisher" not in labels:
         logging.error("Publisher is a mandatory key in target_aws_accounts!")
@@ -62,20 +49,41 @@ def validate_configs(config):
             logging.error("All the target AWS accounts should have the same set of keys (labels)!")
             sys.exit(1)
 
+    for config_metric in config["metrics"]:
+
+        if config_metric["group_by"]["enabled"]:
+            if len(config_metric["group_by"]["groups"]) < 1 or len(config_metric["group_by"]["groups"]) > 2:
+                logging.error("If group_by is enabled, there should be at least one group, and at most two groups!")
+                sys.exit(1)
+            group_label_names = set()
+            for group in config_metric["group_by"]["groups"]:
+                if group["label_name"] in group_label_names:
+                    logging.error("Group label names should be unique!")
+                    sys.exit(1)
+                else:
+                    group_label_names.add(group["label_name"])
+        if group_label_names and (group_label_names & set(labels)):
+            logging.error("Some label names in group_by are the same as AWS account labels!")
+            sys.exit(1)
+
+
 
 def main(config):
-    app_metrics = MetricExporter(
-        polling_interval_seconds=config["polling_interval_seconds"],
-        metric_name=config["metric_name"],
-        aws_access_key=config["aws_access_key"],
-        aws_access_secret=config["aws_access_secret"],
-        aws_assumed_role_name=config["aws_assumed_role_name"],
-        group_by=config["group_by"],
-        targets=config["target_aws_accounts"],
-    )
-    start_http_server(config["exporter_port"])
-    app_metrics.run_metrics_loop()
 
+    start_http_server(config["exporter_port"])
+    while True:
+        for config_metric in config["metrics"]:
+            app_metrics = MetricExporter(
+                polling_interval_seconds=config["polling_interval_seconds"],
+                aws_access_key=config["aws_access_key"],
+                aws_access_secret=config["aws_access_secret"],
+                aws_assumed_role_name=config["aws_assumed_role_name"],
+                targets=config["target_aws_accounts"],
+                metric_name=config_metric["metric_name"],
+                group_by=config_metric["group_by"],
+            )
+            app_metrics.run_metrics()
+        time.sleep(config["polling_interval_seconds"])
 
 if __name__ == "__main__":
     logger_format = "%(asctime)-15s %(levelname)-8s %(message)s"

requirements.txt

@@ -1,5 +1,5 @@
-boto3==1.28.62
-botocore==1.31.65
+boto3==1.35.8
+botocore==1.35.8
 envyaml==1.10.211231
 prometheus_client==0.17.0
 python_dateutil==2.8.2