From 23af67114214dd8ea972079c703e5c45c965052b Mon Sep 17 00:00:00 2001 From: Stephen Lang Date: Thu, 7 Dec 2023 14:14:16 +0000 Subject: [PATCH] refactor: locally scope s3 metrics --- pkg/aws/s3/s3.go | 109 +++++++++++++++++++++---------------- pkg/aws/s3/s3_test.go | 123 ++++++++++++++++++++++++++---------------- 2 files changed, 140 insertions(+), 92 deletions(-) diff --git a/pkg/aws/s3/s3.go b/pkg/aws/s3/s3.go index edf36d29..ad44e686 100644 --- a/pkg/aws/s3/s3.go +++ b/pkg/aws/s3/s3.go @@ -64,42 +64,57 @@ var billingToRegionMap = map[string]string{ "AWS GovCloud (US)": "us-gov-west-1", } -// Create two metrics that are gauges -var ( +// Metrics exported by this collector. +type Metrics struct { // StorageGauge measures the cost of storage in $/GiB, per region and class. - StorageGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "aws_s3_storage_hourly_cost", - Help: "S3 storage hourly cost in GiB", - }, - []string{"region", "class"}, - ) + StorageGauge *prometheus.GaugeVec // OperationsGauge measures the cost of operations in $/1k requests - OperationsGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "aws_s3_operations_cost", - Help: "S3 operations cost per 1k requests", - }, - []string{"region", "class", "tier"}, - ) + OperationsGauge *prometheus.GaugeVec // RequestCount is a counter that tracks the number of requests made to the AWS Cost Explorer API - RequestCount = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "aws_cost_exporter_requests_total", - Help: "Total number of requests made to the AWS Cost Explorer API", - }) + RequestCount prometheus.Counter // RequestErrorsCount is a counter that tracks the number of errors when making requests to the AWS Cost Explorer API - RequestErrorsCount = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "aws_cost_exporter_request_errors_total", - Help: "Total number of errors when making requests to the AWS Cost Explorer API", - }) - - // NextScrapeGuage is a gauge that tracks the next time the exporter will scrape AWS billing data - NextScrapeGuage = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "aws_cost_exporter_next_scrape", - Help: "The next time the exporter will scrape AWS billing data. Can be used to trigger alerts if now - nextScrape > interval", - }) -) + RequestErrorsCount prometheus.Counter + + // NextScrapeGauge is a gauge that tracks the next time the exporter will scrape AWS billing data + NextScrapeGauge prometheus.Gauge +} + +// NewMetrics returns a new Metrics instance. +func NewMetrics() Metrics { + return Metrics{ + StorageGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "aws_s3_storage_hourly_cost", + Help: "S3 storage hourly cost in GiB", + }, + []string{"region", "class"}, + ), + + OperationsGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "aws_s3_operations_cost", + Help: "S3 operations cost per 1k requests", + }, + []string{"region", "class", "tier"}, + ), + + RequestCount: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "aws_cost_exporter_requests_total", + Help: "Total number of requests made to the AWS Cost Explorer API", + }), + + RequestErrorsCount: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "aws_cost_exporter_request_errors_total", + Help: "Total number of errors when making requests to the AWS Cost Explorer API", + }), + + NextScrapeGauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "aws_cost_exporter_next_scrape", + Help: "The next time the exporter will scrape AWS billing data. Can be used to trigger alerts if now - nextScrape > interval", + }), + } +} // Collector is the AWS implementation of the Collector interface // It is responsible for registering and collecting metrics @@ -107,6 +122,7 @@ type Collector struct { client costexplorer.CostExplorer interval time.Duration nextScrape time.Time + metrics Metrics } // New creates a new Collector with a client and scrape interval defined. @@ -116,6 +132,7 @@ func New(scrapeInterval time.Duration, client costexplorer.CostExplorer) (*Colle interval: scrapeInterval, // Initially Set nextScrape to the current time minus the scrape interval so that the first scrape will run immediately nextScrape: time.Now().Add(-scrapeInterval), + metrics: NewMetrics(), }, nil } @@ -125,11 +142,11 @@ func (r *Collector) Name() string { // Register is called prior to the first collection. It registers any custom metric that needs to be exported for AWS billing data func (r *Collector) Register(registry provider.Registry) error { - registry.MustRegister(StorageGauge) - registry.MustRegister(OperationsGauge) - registry.MustRegister(RequestCount) - registry.MustRegister(NextScrapeGuage) - registry.MustRegister(RequestErrorsCount) + registry.MustRegister(r.metrics.StorageGauge) + registry.MustRegister(r.metrics.OperationsGauge) + registry.MustRegister(r.metrics.RequestCount) + registry.MustRegister(r.metrics.NextScrapeGauge) + registry.MustRegister(r.metrics.RequestErrorsCount) return nil } @@ -143,8 +160,8 @@ func (r *Collector) Collect() error { return nil } r.nextScrape = time.Now().Add(r.interval) - NextScrapeGuage.Set(float64(r.nextScrape.Unix())) - return ExportBillingData(r.client) + r.metrics.NextScrapeGauge.Set(float64(r.nextScrape.Unix())) + return ExportBillingData(r.client, r.metrics) } // S3BillingData is the struct for the data we will be collecting @@ -229,7 +246,7 @@ func (s S3BillingData) AddMetricGroup(region string, component string, group typ // getBillingData is responsible for making the API call to the AWS Cost Explorer API and parsing the response // into a S3BillingData struct -func getBillingData(client costexplorer.CostExplorer, startDate time.Time, endDate time.Time) (S3BillingData, error) { +func getBillingData(client costexplorer.CostExplorer, startDate time.Time, endDate time.Time, m Metrics) (S3BillingData, error) { log.Printf("Getting billing data for %s to %s\n", startDate.Format("2006-01-02"), endDate.Format("2006-01-02")) input := &awscostexplorer.GetCostAndUsageInput{ TimePeriod: &types.DateInterval{ @@ -255,11 +272,11 @@ func getBillingData(client costexplorer.CostExplorer, startDate time.Time, endDa var outputs []*awscostexplorer.GetCostAndUsageOutput for { - RequestCount.Inc() + m.RequestCount.Inc() output, err := client.GetCostAndUsage(context.TODO(), input) if err != nil { log.Printf("Error getting cost and usage: %v\n", err) - RequestErrorsCount.Inc() + m.RequestErrorsCount.Inc() return S3BillingData{}, err } outputs = append(outputs, output) @@ -342,32 +359,32 @@ func getComponentFromKey(key string) string { } // ExportBillingData will query the previous 30 days of S3 billing data and export it to the prometheus metrics -func ExportBillingData(client costexplorer.CostExplorer) error { +func ExportBillingData(client costexplorer.CostExplorer, m Metrics) error { // We go one day into the past as the current days billing data has no guarantee of being complete endDate := time.Now().AddDate(0, 0, -1) // Current assumption is that we're going to pull 30 days worth of billing data startDate := endDate.AddDate(0, 0, -30) - s3BillingData, err := getBillingData(client, startDate, endDate) + s3BillingData, err := getBillingData(client, startDate, endDate, m) if err != nil { return err } - exportMetrics(s3BillingData) + exportMetrics(s3BillingData, m) return nil } // exportMetrics will iterate over the S3BillingData and export the metrics to prometheus -func exportMetrics(s3BillingData S3BillingData) { +func exportMetrics(s3BillingData S3BillingData, m Metrics) { log.Printf("Exporting metrics for %d regions\n", len(s3BillingData.Regions)) for region, pricingModel := range s3BillingData.Regions { for component, pricing := range pricingModel.Model { switch component { case "Requests-Tier1": - OperationsGauge.WithLabelValues(region, StandardLabel, "1").Set(pricing.UnitCost) + m.OperationsGauge.WithLabelValues(region, StandardLabel, "1").Set(pricing.UnitCost) case "Requests-Tier2": - OperationsGauge.WithLabelValues(region, StandardLabel, "2").Set(pricing.UnitCost) + m.OperationsGauge.WithLabelValues(region, StandardLabel, "2").Set(pricing.UnitCost) case "TimedStorage": - StorageGauge.WithLabelValues(region, StandardLabel).Set(pricing.UnitCost) + m.StorageGauge.WithLabelValues(region, StandardLabel).Set(pricing.UnitCost) } } } diff --git a/pkg/aws/s3/s3_test.go b/pkg/aws/s3/s3_test.go index 2c2c6d27..27bef5a7 100644 --- a/pkg/aws/s3/s3_test.go +++ b/pkg/aws/s3/s3_test.go @@ -220,17 +220,6 @@ func TestCollector_Collect(t *testing.T) { "aws_cost_exporter_request_errors_total", } - withoutNextScrapeAndRequestsTotal := []string{ - "aws_s3_storage_hourly_cost", - "aws_s3_operations_cost", - "aws_cost_exporter_request_errors_total", - } - - justCostMetrics := []string{ - "aws_s3_storage_hourly_cost", - "aws_s3_operations_cost", - } - for _, tc := range []struct { name string nextScrape time.Time @@ -275,15 +264,13 @@ aws_cost_exporter_requests_total 0 return &awscostexplorer.GetCostAndUsageOutput{}, nil }, metricNames: withoutNextScrape, - - // Requests total increases by one on each test case, so we just check it once here. expectedExposition: ` # HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API # TYPE aws_cost_exporter_request_errors_total counter -aws_cost_exporter_request_errors_total 1 +aws_cost_exporter_request_errors_total 0 # HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API # TYPE aws_cost_exporter_requests_total counter -aws_cost_exporter_requests_total 2 +aws_cost_exporter_requests_total 1 `, }, { @@ -298,14 +285,14 @@ aws_cost_exporter_requests_total 2 }}, }, nil }, - metricNames: withoutNextScrapeAndRequestsTotal, - - // Request errors total appears to always be "1" due to the error case above, so we check it for the last - // time here. + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API # TYPE aws_cost_exporter_request_errors_total counter -aws_cost_exporter_request_errors_total 1 +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 `, }, { @@ -320,7 +307,15 @@ aws_cost_exporter_request_errors_total 1 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, + expectedExposition: ` +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 +`, }, { name: "cost and usage output - one result with keys but special-case region", @@ -334,7 +329,15 @@ aws_cost_exporter_request_errors_total 1 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, + expectedExposition: ` +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 +`, }, { name: "cost and usage output - one result with keys and valid region with a hyphen", @@ -350,7 +353,15 @@ aws_cost_exporter_request_errors_total 1 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, + expectedExposition: ` +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 +`, }, { name: "cost and usage output - three results with keys and valid region without a hyphen", @@ -376,7 +387,7 @@ aws_cost_exporter_request_errors_total 1 }, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge @@ -385,6 +396,12 @@ aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} # HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB # TYPE aws_s3_storage_hourly_cost gauge aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 `, }, { @@ -410,15 +427,18 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="1"} 0 aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} 0 -# HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB -# TYPE aws_s3_storage_hourly_cost gauge -aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 2 `, }, { @@ -437,15 +457,17 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="1"} 0 -aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} 0 -# HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB -# TYPE aws_s3_storage_hourly_cost gauge -aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 `, }, { @@ -465,15 +487,17 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="1"} 0 -aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} 0 -# HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB -# TYPE aws_s3_storage_hourly_cost gauge -aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 `, }, { @@ -493,15 +517,17 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 }}, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="1"} 1000 -aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} 0 -# HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB -# TYPE aws_s3_storage_hourly_cost gauge -aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 `, }, { @@ -551,17 +577,21 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 }, }, nil }, - metricNames: justCostMetrics, + metricNames: withoutNextScrape, expectedExposition: ` +# HELP aws_cost_exporter_request_errors_total Total number of errors when making requests to the AWS Cost Explorer API +# TYPE aws_cost_exporter_request_errors_total counter +aws_cost_exporter_request_errors_total 0 +# HELP aws_cost_exporter_requests_total Total number of requests made to the AWS Cost Explorer API +# TYPE aws_cost_exporter_requests_total counter +aws_cost_exporter_requests_total 1 # HELP aws_s3_operations_cost S3 operations cost per 1k requests # TYPE aws_s3_operations_cost gauge aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="1"} 1000 aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-1",tier="2"} 1000 -aws_s3_operations_cost{class="StandardStorage",region="ap-northeast-2",tier="2"} 0 # HELP aws_s3_storage_hourly_cost S3 storage hourly cost in GiB # TYPE aws_s3_storage_hourly_cost gauge aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-1"} 0.0013689253935660506 -aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 `, }, } { @@ -583,6 +613,7 @@ aws_s3_storage_hourly_cost{class="StandardStorage",region="ap-northeast-3"} 0 c := &Collector{ client: ce, nextScrape: tc.nextScrape, + metrics: NewMetrics(), } err := c.Collect() if tc.expectedError != nil {