From b014f360809ebd53c1889152d1f5d16833309d90 Mon Sep 17 00:00:00 2001 From: Sam Kirsch Date: Fri, 22 Mar 2024 09:59:42 -0400 Subject: [PATCH 1/5] adding a new option to delay subsequent collections Signed-off-by: Sam Kirsch --- collectors/monitoring_collector.go | 49 ++++++++++++++++++++---------- stackdriver_exporter.go | 5 +++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 4f9ddb30..855390eb 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -42,6 +42,7 @@ type MonitoringCollector struct { metricsTypePrefixes []string metricsFilters []MetricFilter metricsInterval time.Duration + metricsDelay time.Duration metricsOffset time.Duration metricsIngestDelay bool monitoringService *monitoring.Service @@ -70,6 +71,9 @@ type MonitoringCollectorOptions struct { // RequestInterval is the time interval used in each request to get metrics. If there are many data points returned // during this interval, only the latest will be reported. RequestInterval time.Duration + // RequestDelay is the time interval between each subsequent request to get metrics. If you are receiving a rateLimited + // error, this can be increased. + RequestDelay time.Duration // RequestOffset is used to offset the requested interval into the past. RequestOffset time.Duration // IngestDelay decides if the ingestion delay specified in the metrics metadata is used when calculating the @@ -198,6 +202,7 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv metricsTypePrefixes: opts.MetricTypePrefixes, metricsFilters: opts.ExtraFilters, metricsInterval: opts.RequestInterval, + metricsDelay: opts.RequestDelay, metricsOffset: opts.RequestOffset, metricsIngestDelay: opts.IngestDelay, monitoringService: monitoringService, @@ -319,25 +324,37 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri IntervalEndTime(endTime.Format(time.RFC3339Nano)) for { - c.apiCallsTotalMetric.Inc() - page, err := timeSeriesListCall.Do() - if err != nil { - level.Error(c.logger).Log("msg", "error retrieving Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) - errChannel <- err - break - } - if page == nil { - break - } - if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch, begun); err != nil { - level.Error(c.logger).Log("msg", "error reporting Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) - errChannel <- err - break + t := time.NewTicker(c.metricsDelay) + var endLoop bool + select { + case <-t.C: + c.apiCallsTotalMetric.Inc() + page, err := timeSeriesListCall.Do() + if err != nil { + level.Error(c.logger).Log("msg", "error retrieving Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) + errChannel <- err + endLoop = true + break + } + if page == nil { + endLoop = true + break + } + if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch, begun); err != nil { + level.Error(c.logger).Log("msg", "error reporting Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) + errChannel <- err + endLoop = true + break + } + if page.NextPageToken == "" { + endLoop = true + break + } + timeSeriesListCall.PageToken(page.NextPageToken) } - if page.NextPageToken == "" { + if endLoop { break } - timeSeriesListCall.PageToken(page.NextPageToken) } }(metricDescriptor, ch, startTime, endTime) } diff --git a/stackdriver_exporter.go b/stackdriver_exporter.go index f71c4f43..0b21be8a 100644 --- a/stackdriver_exporter.go +++ b/stackdriver_exporter.go @@ -93,6 +93,10 @@ var ( "monitoring.metrics-interval", "Interval to request the Google Stackdriver Monitoring Metrics for. Only the most recent data point is used.", ).Default("5m").Duration() + monitoringMetricsDelay = kingpin.Flag( + "monitoring.metrics-delay", "Interval delay between metrics requests to the Google Stackdriver Monitoring.", + ).Default("0s").Duration() + monitoringMetricsOffset = kingpin.Flag( "monitoring.metrics-offset", "Offset for the Google Stackdriver Monitoring Metrics interval into the past.", ).Default("0s").Duration() @@ -215,6 +219,7 @@ func (h *handler) innerHandler(filters map[string]bool) http.Handler { MetricTypePrefixes: h.filterMetricTypePrefixes(filters), ExtraFilters: h.metricsExtraFilters, RequestInterval: *monitoringMetricsInterval, + RequestDelay: *monitoringMetricsDelay, RequestOffset: *monitoringMetricsOffset, IngestDelay: *monitoringMetricsIngestDelay, FillMissingLabels: *collectorFillMissingLabels, From 87de76f26e3aba30ca1ac8a486d4c93771ab95d7 Mon Sep 17 00:00:00 2001 From: Sam Kirsch Date: Fri, 22 Mar 2024 10:21:12 -0400 Subject: [PATCH 2/5] simplifiying case to block for timer Signed-off-by: Sam Kirsch --- collectors/monitoring_collector.go | 49 +++++++++++++----------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 855390eb..f43bf806 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -324,37 +324,30 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri IntervalEndTime(endTime.Format(time.RFC3339Nano)) for { - t := time.NewTicker(c.metricsDelay) - var endLoop bool - select { - case <-t.C: - c.apiCallsTotalMetric.Inc() - page, err := timeSeriesListCall.Do() - if err != nil { - level.Error(c.logger).Log("msg", "error retrieving Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) - errChannel <- err - endLoop = true - break - } - if page == nil { - endLoop = true - break - } - if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch, begun); err != nil { - level.Error(c.logger).Log("msg", "error reporting Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) - errChannel <- err - endLoop = true - break - } - if page.NextPageToken == "" { - endLoop = true - break - } - timeSeriesListCall.PageToken(page.NextPageToken) + if c.metricsDelay != 0 { + t := time.NewTicker(c.metricsDelay) + // Will block until timer goes off + <-t.C } - if endLoop { + c.apiCallsTotalMetric.Inc() + page, err := timeSeriesListCall.Do() + if err != nil { + level.Error(c.logger).Log("msg", "error retrieving Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) + errChannel <- err + break + } + if page == nil { + break + } + if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch, begun); err != nil { + level.Error(c.logger).Log("msg", "error reporting Time Series metrics for descriptor", "descriptor", metricDescriptor.Type, "err", err) + errChannel <- err + break + } + if page.NextPageToken == "" { break } + timeSeriesListCall.PageToken(page.NextPageToken) } }(metricDescriptor, ch, startTime, endTime) } From 9e69d9caab19e5a28d9800feb30f8e76a980f66b Mon Sep 17 00:00:00 2001 From: Sam Kirsch Date: Fri, 22 Mar 2024 10:22:13 -0400 Subject: [PATCH 3/5] simplifiying case to block for timer Signed-off-by: Sam Kirsch --- collectors/monitoring_collector.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index f43bf806..2270d0da 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -325,9 +325,8 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri for { if c.metricsDelay != 0 { - t := time.NewTicker(c.metricsDelay) // Will block until timer goes off - <-t.C + <-time.NewTicker(c.metricsDelay).C } c.apiCallsTotalMetric.Inc() page, err := timeSeriesListCall.Do() From ec60af0b6fa02e32ac26ba3826c62c4726d4d596 Mon Sep 17 00:00:00 2001 From: Sam Kirsch Date: Fri, 22 Mar 2024 10:25:29 -0400 Subject: [PATCH 4/5] simplifiying case to just a sleep... we dontt need a ticker Signed-off-by: Sam Kirsch --- collectors/monitoring_collector.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 2270d0da..33221e8a 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -325,8 +325,7 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri for { if c.metricsDelay != 0 { - // Will block until timer goes off - <-time.NewTicker(c.metricsDelay).C + time.Sleep(c.metricsDelay) } c.apiCallsTotalMetric.Inc() page, err := timeSeriesListCall.Do() From 211dd7008630c92e2e71ddb07c166db294fd95d3 Mon Sep 17 00:00:00 2001 From: Sam Kirsch Date: Fri, 22 Mar 2024 11:16:40 -0400 Subject: [PATCH 5/5] move delay to end of forked goroutine to delay between descriptors --- collectors/monitoring_collector.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 33221e8a..25a4d01a 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -324,9 +324,6 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri IntervalEndTime(endTime.Format(time.RFC3339Nano)) for { - if c.metricsDelay != 0 { - time.Sleep(c.metricsDelay) - } c.apiCallsTotalMetric.Inc() page, err := timeSeriesListCall.Do() if err != nil { @@ -348,6 +345,9 @@ func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metri timeSeriesListCall.PageToken(page.NextPageToken) } }(metricDescriptor, ch, startTime, endTime) + if c.metricsDelay != 0 { + time.Sleep(c.metricsDelay) + } } wg.Wait()