From 84d5f94baddd4709f13514e6d0be8309f474ff88 Mon Sep 17 00:00:00 2001 From: David Buckley Date: Wed, 21 Aug 2024 17:53:15 +0100 Subject: [PATCH] [#956] allow instances to go 25% over their hard max --- conf/carbon.conf.example | 24 ++++++++++++++++++++---- lib/carbon/client.py | 9 ++++++++- lib/carbon/conf.py | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/conf/carbon.conf.example b/conf/carbon.conf.example index ebb76e464..561c3e48c 100644 --- a/conf/carbon.conf.example +++ b/conf/carbon.conf.example @@ -472,8 +472,11 @@ DESTINATIONS = 127.0.0.1:2004 # This is the maximum number of datapoints that can be queued up # for a single destination. Once this limit is hit, we will -# stop accepting new data if USE_FLOW_CONTROL is True, otherwise -# we will drop any subsequently received datapoints. +# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and +# internally-generated datapoints will still be processed, and data +# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE +# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped +# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused. MAX_QUEUE_SIZE = 10000 # This is the factor that the queue must be empty before it will accept @@ -485,6 +488,11 @@ MAX_QUEUE_SIZE = 10000 # even before the relay incrementally clears more of the queue QUEUE_LOW_WATERMARK_PCT = 0.8 +# This is the factor of the max length of a queue before data will be dropped +# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data +# is still processed, which can send a queue slightly over the configured max. +MAX_QUEUE_SIZE_HARD_PCT = 1.25 + # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting @@ -619,8 +627,11 @@ REPLICATION_FACTOR = 1 # This is the maximum number of datapoints that can be queued up # for a single destination. Once this limit is hit, we will -# stop accepting new data if USE_FLOW_CONTROL is True, otherwise -# we will drop any subsequently received datapoints. +# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and +# internally-generated datapoints will still be processed, and data +# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE +# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped +# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused. MAX_QUEUE_SIZE = 10000 # This is the factor that the queue must be empty before it will accept @@ -632,6 +643,11 @@ MAX_QUEUE_SIZE = 10000 # even before the relay incrementally clears more of the queue QUEUE_LOW_WATERMARK_PCT = 0.8 +# This is the factor of the max length of a queue before data will be dropped +# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data +# is still processed, which can send a queue slightly over the configured max. +MAX_QUEUE_SIZE_HARD_PCT = 1.25 + # Set this to False to drop datapoints when any send queue (sending datapoints # to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the # default) then sockets over which metrics are received will temporarily stop accepting diff --git a/lib/carbon/client.py b/lib/carbon/client.py index d9ba5a99c..6bed7b7f5 100644 --- a/lib/carbon/client.py +++ b/lib/carbon/client.py @@ -35,6 +35,10 @@ SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * settings.QUEUE_LOW_WATERMARK_PCT +if settings.USE_FLOW_CONTROL: + SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE * settings.MAX_QUEUE_SIZE_HARD_PCT +else: + SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE class CarbonClientProtocol(object): @@ -350,7 +354,10 @@ def sendDatapoint(self, metric, datapoint): if self.queueSize >= settings.MAX_QUEUE_SIZE: if not self.queueFull.called: self.queueFull.callback(self.queueSize) - instrumentation.increment(self.fullQueueDrops) + if self.queueSize < SEND_QUEUE_HARD_MAX: + self.enqueue(metric, datapoint) + else: + instrumentation.increment(self.fullQueueDrops) else: self.enqueue(metric, datapoint) diff --git a/lib/carbon/conf.py b/lib/carbon/conf.py index c474cd918..8a99df06b 100644 --- a/lib/carbon/conf.py +++ b/lib/carbon/conf.py @@ -70,6 +70,7 @@ FORWARD_ALL=True, MAX_QUEUE_SIZE=10000, QUEUE_LOW_WATERMARK_PCT=0.8, + MAX_QUEUE_SIZE_HARD_PCT=1.25, TIME_TO_DEFER_SENDING=0.0001, ENABLE_AMQP=False, AMQP_METRIC_NAME_IN_BODY=False,