diff --git a/server/docker/metrics/dashboards/block-node-server.json b/server/docker/metrics/dashboards/block-node-server.json index 4a7e0807..e28ff6b8 100644 --- a/server/docker/metrics/dashboards/block-node-server.json +++ b/server/docker/metrics/dashboards/block-node-server.json @@ -21,16 +21,14 @@ "links": [], "panels": [ { - "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 9, - "panels": [], - "title": "Live Stream", + "id": 16, + "title": "Errors", "type": "row" }, { @@ -50,22 +48,27 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 1 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 6, "x": 0, "y": 1 }, - "id": 3, + "id": 15, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -88,14 +91,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_live_block_items_total", + "expr": "hedera_block_node_live_block_stream_mediator_error_total", "instant": false, - "legendFormat": "BlockItems", + "legendFormat": "Block Item Errors", "range": true, "refId": "A" } ], - "title": "Live Block Item Counter", + "title": "Mediator Errors", "type": "stat" }, { @@ -137,7 +140,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [], @@ -145,30 +148,25 @@ "mode": "absolute", "steps": [ { - "color": "dark-red", + "color": "green", "value": null }, { - "color": "#EAB839", - "value": 5 - }, - { - "color": "green", - "value": 30 + "color": "red", + "value": 1 } ] - }, - "unit": "reqps" + } }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 18, "x": 6, "y": 1 }, - "id": 7, + "id": 14, "options": { "legend": { "calcs": [], @@ -181,7 +179,6 @@ "sort": "none" } }, - "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -189,16 +186,29 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate( hedera_block_node_live_block_items_total [$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_stream_mediator_error_total [$__rate_interval])", "instant": false, - "legendFormat": "BlockItems", + "legendFormat": "Mediator Errors", "range": true, "refId": "A" } ], - "title": "Live Block Item Counter", + "title": "Rate of Mediator Errors", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 9, + "panels": [], + "title": "Live Stream", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -218,20 +228,21 @@ "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 6, "x": 0, - "y": 8 + "y": 10 }, - "id": 11, + "id": 13, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -254,14 +265,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_live_block_items_consumed_total", + "expr": "hedera_block_node_live_block_items_received_total", "instant": false, - "legendFormat": "BlockItems Consumed", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Live Block Items Consumed Counter", + "title": "Live Block Items Received from Producer", "type": "stat" }, { @@ -269,6 +280,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -311,25 +323,30 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "#EAB839", + "value": 5 + }, + { + "color": "green", + "value": 30 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 18, "x": 6, - "y": 8 + "y": 10 }, - "id": 10, + "id": 12, "options": { "legend": { "calcs": [], @@ -349,14 +366,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate( hedera_block_node_live_block_items_consumed_total [$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_received_total [$__rate_interval])", "instant": false, - "legendFormat": "__auto", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Live Block Items Consumed", + "title": "Rate of Live Block Items Received from Producer", "type": "timeseries" }, { @@ -378,7 +395,8 @@ "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, @@ -386,12 +404,12 @@ "h": 7, "w": 6, "x": 0, - "y": 14 + "y": 18 }, - "id": 6, + "id": 3, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -414,14 +432,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_blocks_persisted_total", + "expr": "hedera_block_node_live_block_items_total", "instant": false, - "legendFormat": "Block Persistence Counter", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Block Persistence Counter", + "title": "Live Block Item Counter", "type": "stat" }, { @@ -471,20 +489,20 @@ "mode": "absolute", "steps": [ { - "color": "red", + "color": "dark-red", "value": null }, { "color": "#EAB839", - "value": 2 + "value": 5 }, { "color": "green", - "value": 4 + "value": 30 } ] }, - "unit": "wps" + "unit": "reqps" }, "overrides": [] }, @@ -492,9 +510,9 @@ "h": 7, "w": 18, "x": 6, - "y": 14 + "y": 18 }, - "id": 8, + "id": 7, "options": { "legend": { "calcs": [], @@ -515,14 +533,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(hedera_block_node_blocks_persisted_total[$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_total [$__rate_interval])", "instant": false, - "legendFormat": "Block Persistence Counter", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Block Persistence Counter", + "title": "Rate of Live Block Items Received by Mediator", "type": "timeseries" }, { @@ -535,40 +553,33 @@ "color": { "mode": "thresholds" }, - "displayName": "Subscribers", "mappings": [], - "max": 20, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 15 + "color": "green", + "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 9, + "h": 7, + "w": 6, "x": 0, - "y": 21 + "y": 25 }, - "id": 4, + "id": 11, "options": { - "minVizHeight": 75, - "minVizWidth": 75, + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -576,9 +587,9 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "11.1.4", "targets": [ @@ -588,15 +599,15 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_subscribers", + "expr": "hedera_block_node_live_block_items_consumed_total", "instant": false, - "legendFormat": "__auto", + "legendFormat": "BlockItems Consumed", "range": true, "refId": "A" } ], - "title": "Subscribers", - "type": "gauge" + "title": "Live Block Items Consumed Counter", + "type": "stat" }, { "datasource": { @@ -645,10 +656,15 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "red", + "value": null }, { - "color": "red", + "color": "#EAB839", + "value": 5 + }, + { + "color": "green", "value": 80 } ] @@ -658,12 +674,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 15, - "x": 9, - "y": 21 + "h": 7, + "w": 18, + "x": 6, + "y": 25 }, - "id": 5, + "id": 10, "options": { "legend": { "calcs": [], @@ -683,14 +699,613 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_consumed_total [$__rate_interval])", "instant": false, - "legendFormat": "RPS of Single Blocks Retrieval", + "legendFormat": "BlockItems", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Block Items Sent to Consumer(s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Subscribers", + "mappings": [], + "max": 20, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 15 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_subscribers", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Subscribers", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 18, + "panels": [], + "title": "Persistence", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 41 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_blocks_persisted_total", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Block Persistence Counter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 41 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_blocks_persisted_total[$__rate_interval])", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Blocks Written to the File System", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 17, + "panels": [], + "title": "Single Block", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_single_blocks_retrieved_total", + "instant": false, + "legendFormat": "Single Blocks Retrieved", + "range": true, + "refId": "A" + } + ], + "title": "Single Blocks Retrieved", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 49 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "instant": false, + "legendFormat": "RPS of Single Blocks Retrieval", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Single Blocks Retrieved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 57 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_single_blocks_not_found_total", + "instant": false, + "legendFormat": "Single Blocks Not Found", + "range": true, + "refId": "A" + } + ], + "title": "Single Blocks Not Found", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 57 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate( hedera_block_node_single_blocks_not_found_total [$__rate_interval])", + "instant": false, + "legendFormat": "Single Blocks Not Found", "range": true, "refId": "A" } ], - "title": "Rate of Single Blocks Retrieval", + "title": "Rate of Single Blocks Not Found", "type": "timeseries" } ], @@ -708,6 +1323,6 @@ "timezone": "browser", "title": "Block-Node Server Dashboard", "uid": "edu86nutnxts0c", - "version": 8, + "version": 1, "weekStart": "" } diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/BlockStreamService.java index 7d8b87b0..807630b6 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/BlockStreamService.java @@ -22,6 +22,7 @@ import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricNames.Counter.SingleBlocksNotFound; import static com.hedera.block.server.metrics.BlockNodeMetricNames.Counter.SingleBlocksRetrieved; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; @@ -139,7 +140,7 @@ StreamObserver protocPublishB LOGGER.log(DEBUG, "Executing bidirectional publishBlockStream gRPC method"); return new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); } void protocSubscribeBlockStream( @@ -208,6 +209,7 @@ private void singleBlock( } else { LOGGER.log(DEBUG, "Block number {0} not found", blockNumber); singleBlockResponseStreamObserver.onNext(buildSingleBlockNotFoundResponse()); + metricsService.get(SingleBlocksNotFound).increment(); } } catch (IOException e) { LOGGER.log(ERROR, "Error reading block number: {0}", blockNumber); diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 32d351f1..da5887d8 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -167,9 +167,6 @@ public void onEvent( // Refresh the producer liveness and pass the BlockItem to the downstream observer. producerLivenessMillis = currentMillis; - // Increment counter - metricsService.get(LiveBlockItemsConsumed).increment(); - final SubscribeStreamResponse subscribeStreamResponse = event.get(); final ResponseSender responseSender = getResponseSender(subscribeStreamResponse); responseSender.send(subscribeStreamResponse); @@ -219,6 +216,9 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) if (streamStarted) { LOGGER.log(DEBUG, "Sending BlockItem downstream: {0}", blockItem); + + // Increment counter + metricsService.get(LiveBlockItemsConsumed).increment(); subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); } } diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 02478630..e967740e 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -17,6 +17,7 @@ package com.hedera.block.server.mediator; import static com.hedera.block.server.metrics.BlockNodeMetricNames.Counter.LiveBlockItems; +import static com.hedera.block.server.metrics.BlockNodeMetricNames.Counter.LiveBlockStreamMediatorError; import static com.hedera.block.server.metrics.BlockNodeMetricNames.Gauge.Subscribers; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; @@ -126,15 +127,19 @@ public void publish(@NonNull final BlockItem blockItem) throws IOException { // Persist the BlockItem blockWriter.write(blockItem); } catch (IOException e) { + + // Increment the error counter + metricsService.get(LiveBlockStreamMediatorError).increment(); + // Disable BlockItem publication for upstream producers - serviceStatus.setRunning(false); + serviceStatus.stopRunning(this.getClass().getName()); LOGGER.log( ERROR, "An exception occurred while attempting to persist the BlockItem: " + blockItem, e); - LOGGER.log(DEBUG, "Send a response to end the stream"); + LOGGER.log(ERROR, "Send a response to end the stream"); // Publish the block for all subscribers to receive final SubscribeStreamResponse endStreamResponse = buildEndStreamResponse(); @@ -142,7 +147,7 @@ public void publish(@NonNull final BlockItem blockItem) throws IOException { // Unsubscribe all downstream consumers for (final var subscriber : subscribers.keySet()) { - LOGGER.log(DEBUG, "Unsubscribing: {0}", subscriber); + LOGGER.log(ERROR, String.format("Unsubscribing: %s", subscriber)); unsubscribe(subscriber); } diff --git a/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricNames.java b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricNames.java index e763f913..094e2de3 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricNames.java +++ b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricNames.java @@ -22,10 +22,17 @@ public final class BlockNodeMetricNames { private BlockNodeMetricNames() {} public enum Counter implements MetricMetadata { + // Standard counters + LiveBlockItemsReceived("live_block_items_received", "Live Block Items Received"), LiveBlockItems("live_block_items", "Live BlockItems"), BlocksPersisted("blocks_persisted", "Blocks Persisted"), LiveBlockItemsConsumed("live_block_items_consumed", "Live Block Items Consumed"), - SingleBlocksRetrieved("single_blocks_retrieved", "Single Blocks Retrieved"); + SingleBlocksRetrieved("single_blocks_retrieved", "Single Blocks Retrieved"), + SingleBlocksNotFound("single_blocks_not_found", "Single Blocks Not Found"), + + // Error counters + LiveBlockStreamMediatorError( + "live_block_stream_mediator_error", "Live Block Stream Mediator Error"); private final String grafanaLabel; private final String description; diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index d8bb5fb3..36dd6f6c 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -18,13 +18,16 @@ import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricNames.Counter.LiveBlockItemsReceived; import static com.hedera.block.server.producer.Util.getFakeHash; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; import com.hedera.block.server.ServiceStatus; +import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.mediator.Publisher; +import com.hedera.block.server.metrics.MetricsService; import com.hedera.hapi.block.Acknowledgement; import com.hedera.hapi.block.EndOfStream; import com.hedera.hapi.block.ItemAcknowledgement; @@ -53,6 +56,7 @@ public class ProducerBlockItemObserver publishStreamResponseObserver; private final Publisher publisher; private final ServiceStatus serviceStatus; + private final MetricsService metricsService; /** * Constructor for the ProducerBlockStreamObserver class. It is responsible for calling the @@ -72,10 +76,12 @@ public ProducerBlockItemObserver( @NonNull final StreamObserver publishStreamResponseObserver, + @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.publisher = publisher; this.publishStreamResponseObserver = publishStreamResponseObserver; + this.metricsService = blockNodeContext.metricsService(); this.serviceStatus = serviceStatus; } @@ -92,8 +98,12 @@ public void onNext( try { + LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); final BlockItem blockItem = toPbj(BlockItem.PROTOBUF, publishStreamRequest.getBlockItem().toByteArray()); + LOGGER.log(DEBUG, "Received block item: " + blockItem); + + metricsService.get(LiveBlockItemsReceived).increment(); // Publish the block to all the subscribers unless // there's an issue with the StreamMediator. @@ -113,10 +123,12 @@ public void onNext( } } else { + LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); + // Close the upstream connection to the producer(s) final var errorResponse = buildErrorStreamResponse(); publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log(DEBUG, "StreamMediator is not accepting BlockItems"); + LOGGER.log(ERROR, "Error PublishStreamResponse sent to upstream producer"); } } catch (IOException io) { final var errorResponse = buildErrorStreamResponse(); diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index bac879f4..848cd870 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -89,10 +89,14 @@ public class ProducerBlockItemObserverTest { @Test public void testProducerOnNext() throws IOException, NoSuchAlgorithmException { + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final List blockItems = generateBlockItems(1); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); when(serviceStatus.isRunning()).thenReturn(true); @@ -173,7 +177,10 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder().blockItem(blockItem).build(); @@ -196,10 +203,15 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) } @Test - public void testOnError() { + public void testOnError() throws IOException { + + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final Throwable t = new Throwable("Test error"); producerBlockItemObserver.onError(t); @@ -207,13 +219,17 @@ public void testOnError() { } @Test - public void testItemAckBuilderExceptionTest() { + public void testItemAckBuilderExceptionTest() throws IOException { when(serviceStatus.isRunning()).thenReturn(true); + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver testProducerBlockItemObserver = new TestProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final List blockItems = generateBlockItems(1); final BlockItem blockHeader = blockItems.getFirst(); @@ -232,10 +248,16 @@ public void testItemAckBuilderExceptionTest() { } @Test - public void testBlockItemThrowsParseException() throws InvalidProtocolBufferException { + public void testBlockItemThrowsParseException() + throws IOException, InvalidProtocolBufferException { + + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); // Create a pbj block item final List blockItems = generateBlockItems(1); @@ -276,11 +298,13 @@ public void testBlockItemThrowsParseException() throws InvalidProtocolBufferExce private static class TestProducerBlockItemObserver extends ProducerBlockItemObserver { public TestProducerBlockItemObserver( - StreamMediator> streamMediator, - StreamObserver + final StreamMediator> + streamMediator, + final StreamObserver publishStreamResponseObserver, - ServiceStatus serviceStatus) { - super(streamMediator, publishStreamResponseObserver, serviceStatus); + final BlockNodeContext blockNodeContext, + final ServiceStatus serviceStatus) { + super(streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); } @NonNull