Add tool to scrape prometheus endpoint to csv

collector/tidb/dashboards/overview/main.tf

@@ -0,0 +1,314 @@
+terraform {
+ required_providers {
+ lightstep = {
+ source = "lightstep/lightstep"
+ version = "~> 1.70.10"
+ }
+ }
+ required_version = ">= v1.0.11"
+}
+
+variable "lightstep_project" {
+ description = "Cloud Observability Project Name"
+ type = string
+}
+
+output "dashboard_url" {
+ value = "https://app.lightstep.com/${var.lightstep_project}/dashboard/${lightstep_dashboard.collector_pulsar_overview.id}"
+ description = "OpenTelemetry Collector Pulsar Dashboard URL"
+}
+
+resource "lightstep_dashboard" "collector_pulsar_overview" {
+ project_name = var.lightstep_project
+ dashboard_name = "OpenTelemetry Pulsar Dashboard"
+ dashboard_description = "Monitor Pulsar and metrics with this summary dashboard."
+
+ chart {
+ name = "CPU Time Spent"
+ rank = "0"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric process_cpu_seconds_total | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Resident Memory Size"
+ rank = "1"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric process_resident_memory_bytes | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Virtual Memory Size"
+ rank = "2"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric process_virtual_memory_bytes | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Process Max FDS"
+ rank = "2"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric process_max_fds | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Process Open FDS"
+ rank = "2"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric process_open_fds | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "UP"
+ rank = "2"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric up | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Active Connections"
+ rank = "10"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_active_connections | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Broker Message Backlog"
+ rank = "10"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_broker_msg_backlog | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Broker Incoming Message Rate"
+ rank = "11"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_broker_rate_in | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Broker Outgoing Message Rate"
+ rank = "12"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_broker_rate_out | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Broker Storage Size"
+ rank = "13"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_broker_storage_size | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Broker Topics Count"
+ rank = "14"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_broker_topics_count | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Message Backlog Across Topics"
+ rank = "15"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_msg_backlog | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Producers Count"
+ rank = "16"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_producers_count | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Consumers Count"
+ rank = "17"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_consumers_count | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Total Incoming Bytes"
+ rank = "18"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_in_bytes_total | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Total Outgoing Bytes"
+ rank = "19"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_out_bytes_total | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Storage Read Rate"
+ rank = "20"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_storage_read_rate | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Storage Write Rate"
+ rank = "21"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_storage_write_rate | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Total Storage Size"
+ rank = "22"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_storage_size | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Total Storage Size"
+ rank = "22"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_storage_logical_size | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Subscription Backlog"
+ rank = "30"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_subscription_back_log | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Subscription Consumers Count"
+ rank = "31"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_subscription_consumers_count | rate | group_by [], sum"
+ }
+ }
+
+ chart {
+ name = "Subscription Message Rate Out"
+ rank = "32"
+ type = "timeseries"
+ query {
+ query_name = "a"
+ display = "line"
+ hidden = false
+ query_string = "metric pulsar_subscription_msg_rate_out | rate | group_by [], sum"
+ }
+ }
+
+}

tools/python/prometheus-scrape2csv.py

@@ -0,0 +1,60 @@
+"""
+Generate CSV file by scraping Prometheus metrics endpoint
+"""
+import requests
+import csv
+from prometheus_client import parser
+
+
+def parse_prometheus_metrics(metrics_data: str):
+ metrics = parser.text_string_to_metric_families(metrics_data)
+ parsed_metrics = {}
+ for metric in metrics:
+ metric_name = metric.name
+ metric_description = metric.documentation
+ metric_type = metric.type
+ for sample in metric.samples:
+ metric_labels = sample.labels
+ if metric_name not in parsed_metrics:
+ parsed_metrics[metric_name] = {
+ "Name": metric_name,
+ "Description": metric_description,
+ "Unit": "",
+ "DataType": metric_type,
+ "Attributes": ",".join(metric_labels.keys())
+ }
+ return parsed_metrics.values()
+
+
+def fetch_prometheus_metrics(url):
+ response = requests.get(url)
+ if response.status_code == 200:
+ return response.text
+ else:
+ print(f"Failed to fetch data from {url}")
+ return None
+
+
+def write_metrics_to_csv(metrics_list, csv_filename):
+ with open(csv_filename, 'w', newline='', encoding='utf-8') as csvfile:
+ csv_writer = csv.writer(csvfile, quoting=csv.QUOTE_MINIMAL)
+ csv_writer.writerow(["Name", "Description", "Unit", "DataType", "Attributes"])
+ for metric in metrics_list:
+ csv_writer.writerow(
+ [metric["Name"], metric["Description"], metric["Unit"], metric["DataType"], metric["Attributes"]])
+
+
+# Example usage
+if __name__ == "__main__":
+ try:
+ default_prometheus_url = "http://localhost:9091/metrics"
+ prometheus_url = input("Enter Prometheus URL (default: %s): " % default_prometheus_url) or default_prometheus_url
+ md = fetch_prometheus_metrics(prometheus_url)
+ if md:
+ metrics_list = parse_prometheus_metrics(md)
+ write_metrics_to_csv(metrics_list, "metrics.csv")
+ print("Metrics data written to metrics.csv")
+ else:
+ print("Failed to fetch Prometheus metrics.")
+ except KeyboardInterrupt:
+ print("\nExiting...")

tools/python/requirements.txt

@@ -1 +1,3 @@
 docker==6.*
+prometheus-client==0.17.1
+requests==2.31.0