Skip to content

Commit

Permalink
[graphite-project#956] allow instances to go 25% over their hard max
Browse files Browse the repository at this point in the history
  • Loading branch information
bucko909 committed Aug 29, 2024
1 parent 1ae7548 commit 84d5f94
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 5 deletions.
24 changes: 20 additions & 4 deletions conf/carbon.conf.example
Original file line number Diff line number Diff line change
Expand Up @@ -472,8 +472,11 @@ DESTINATIONS = 127.0.0.1:2004

# This is the maximum number of datapoints that can be queued up
# for a single destination. Once this limit is hit, we will
# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
# we will drop any subsequently received datapoints.
# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and
# internally-generated datapoints will still be processed, and data
# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE
# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped
# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused.
MAX_QUEUE_SIZE = 10000

# This is the factor that the queue must be empty before it will accept
Expand All @@ -485,6 +488,11 @@ MAX_QUEUE_SIZE = 10000
# even before the relay incrementally clears more of the queue
QUEUE_LOW_WATERMARK_PCT = 0.8

# This is the factor of the max length of a queue before data will be dropped
# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data
# is still processed, which can send a queue slightly over the configured max.
MAX_QUEUE_SIZE_HARD_PCT = 1.25

# Set this to False to drop datapoints when any send queue (sending datapoints
# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
# default) then sockets over which metrics are received will temporarily stop accepting
Expand Down Expand Up @@ -619,8 +627,11 @@ REPLICATION_FACTOR = 1

# This is the maximum number of datapoints that can be queued up
# for a single destination. Once this limit is hit, we will
# stop accepting new data if USE_FLOW_CONTROL is True, otherwise
# we will drop any subsequently received datapoints.
# stop accepting new data if USE_FLOW_CONTROL is True. In-flight and
# internally-generated datapoints will still be processed, and data
# will still be dropped if MAX_QUEUE_SIZE_HARD_PCT * MAX_QUEUE_SIZE
# is hit. If USE_FLOW_CONTROL is False, metrics are immediately dropped
# after MAX_QUEUE_SIZE, and MAX_QUEUE_SIZE_HARD_PCT is unused.
MAX_QUEUE_SIZE = 10000

# This is the factor that the queue must be empty before it will accept
Expand All @@ -632,6 +643,11 @@ MAX_QUEUE_SIZE = 10000
# even before the relay incrementally clears more of the queue
QUEUE_LOW_WATERMARK_PCT = 0.8

# This is the factor of the max length of a queue before data will be dropped
# with USE_FLOW_CONTROL enabled. When incoming data is paused, in-flight data
# is still processed, which can send a queue slightly over the configured max.
MAX_QUEUE_SIZE_HARD_PCT = 1.25

# Set this to False to drop datapoints when any send queue (sending datapoints
# to a downstream carbon daemon) hits MAX_QUEUE_SIZE. If this is True (the
# default) then sockets over which metrics are received will temporarily stop accepting
Expand Down
9 changes: 8 additions & 1 deletion lib/carbon/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@


SEND_QUEUE_LOW_WATERMARK = settings.MAX_QUEUE_SIZE * settings.QUEUE_LOW_WATERMARK_PCT
if settings.USE_FLOW_CONTROL:
SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE * settings.MAX_QUEUE_SIZE_HARD_PCT
else:
SEND_QUEUE_HARD_MAX = settings.MAX_QUEUE_SIZE


class CarbonClientProtocol(object):
Expand Down Expand Up @@ -350,7 +354,10 @@ def sendDatapoint(self, metric, datapoint):
if self.queueSize >= settings.MAX_QUEUE_SIZE:
if not self.queueFull.called:
self.queueFull.callback(self.queueSize)
instrumentation.increment(self.fullQueueDrops)
if self.queueSize < SEND_QUEUE_HARD_MAX:
self.enqueue(metric, datapoint)
else:
instrumentation.increment(self.fullQueueDrops)
else:
self.enqueue(metric, datapoint)

Expand Down
1 change: 1 addition & 0 deletions lib/carbon/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
FORWARD_ALL=True,
MAX_QUEUE_SIZE=10000,
QUEUE_LOW_WATERMARK_PCT=0.8,
MAX_QUEUE_SIZE_HARD_PCT=1.25,
TIME_TO_DEFER_SENDING=0.0001,
ENABLE_AMQP=False,
AMQP_METRIC_NAME_IN_BODY=False,
Expand Down

0 comments on commit 84d5f94

Please sign in to comment.